diff --git a/fiaas_deploy_daemon/__init__.py b/fiaas_deploy_daemon/__init__.py index 3f52d231..96a5392a 100644 --- a/fiaas_deploy_daemon/__init__.py +++ b/fiaas_deploy_daemon/__init__.py @@ -58,10 +58,7 @@ def configure(self, bind): def provide_session(self, config): session = requests.Session() if config.proxy: - session.proxies = {scheme: config.proxy for scheme in ( - "http", - "https" - )} + session.proxies = {scheme: config.proxy for scheme in ("http", "https")} if config.debug: session.hooks["response"].append(log_request_response) return session @@ -76,12 +73,14 @@ def __init__(self, deployer, scheduler, crd_watcher, usage_reporter): pass def is_healthy(self): - return all(( - self._deployer.is_alive(), - self._scheduler.is_alive(), - self._crd_watcher.is_alive(), - self._usage_reporter.is_alive(), - )) + return all( + ( + self._deployer.is_alive(), + self._scheduler.is_alive(), + self._crd_watcher.is_alive(), + self._usage_reporter.is_alive(), + ) + ) class Main(object): @@ -138,58 +137,60 @@ def warn_if_env_variable_config(config, log): """temporary deprecation warning for https://github.com/fiaas/fiaas-deploy-daemon/issues/12""" configuration_env_variable_keys = { - 'SECRETS_DIRECTORY', - 'LOG_FORMAT', - 'http_proxy', - 'DEBUG', - 'FIAAS_ENVIRONMENT', - 'FIAAS_SERVICE_TYPE', - 'FIAAS_INFRASTRUCTURE', - 'PORT', - 'ENABLE_CRD_SUPPORT', - 'SECRETS_INIT_CONTAINER_IMAGE', - 'SECRETS_SERVICE_ACCOUNT_NAME', - 'DATADOG_CONTAINER_IMAGE', - 'DATADOG_CONTAINER_MEMORY', - 'FIAAS_DATADOG_GLOBAL_TAGS', - 'PRE_STOP_DELAY', - 'STRONGBOX_INIT_CONTAINER_IMAGE', - 'ENABLE_DEPRECATED_MULTI_NAMESPACE_SUPPORT', - 'USE_INGRESS_TLS', - 'TLS_CERTIFICATE_ISSUER', - 'USE_IN_MEMORY_EMPTYDIRS', - 'DEPLOYMENT_MAX_SURGE', - 'DEPLOYMENT_MAX_UNAVAILABLE', - 'READY_CHECK_TIMEOUT_MULTIPLIER', - 'DISABLE_DEPRECATED_MANAGED_ENV_VARS', - 'USAGE_REPORTING_CLUSTER_NAME', - 'USAGE_REPORTING_OPERATOR', - 'USAGE_REPORTING_ENDPOINT', - 'USAGE_REPORTING_TENANT', - 'USAGE_REPORTING_TEAM', - 'API_SERVER', - 'API_TOKEN', - 'API_CERT', - 'CLIENT_CERT', - 'CLIENT_KEY', - 'INGRESS_SUFFIXES', - 'HOST_REWRITE_RULES', - 'FIAAS_GLOBAL_ENV', - 'FIAAS_SECRET_INIT_CONTAINERS', + "SECRETS_DIRECTORY", + "LOG_FORMAT", + "http_proxy", + "DEBUG", + "FIAAS_ENVIRONMENT", + "FIAAS_SERVICE_TYPE", + "FIAAS_INFRASTRUCTURE", + "PORT", + "ENABLE_CRD_SUPPORT", + "SECRETS_INIT_CONTAINER_IMAGE", + "SECRETS_SERVICE_ACCOUNT_NAME", + "DATADOG_CONTAINER_IMAGE", + "DATADOG_CONTAINER_MEMORY", + "FIAAS_DATADOG_GLOBAL_TAGS", + "PRE_STOP_DELAY", + "STRONGBOX_INIT_CONTAINER_IMAGE", + "ENABLE_DEPRECATED_MULTI_NAMESPACE_SUPPORT", + "USE_INGRESS_TLS", + "TLS_CERTIFICATE_ISSUER", + "USE_IN_MEMORY_EMPTYDIRS", + "DEPLOYMENT_MAX_SURGE", + "DEPLOYMENT_MAX_UNAVAILABLE", + "READY_CHECK_TIMEOUT_MULTIPLIER", + "DISABLE_DEPRECATED_MANAGED_ENV_VARS", + "USAGE_REPORTING_CLUSTER_NAME", + "USAGE_REPORTING_OPERATOR", + "USAGE_REPORTING_ENDPOINT", + "USAGE_REPORTING_TENANT", + "USAGE_REPORTING_TEAM", + "API_SERVER", + "API_TOKEN", + "API_CERT", + "CLIENT_CERT", + "CLIENT_KEY", + "INGRESS_SUFFIXES", + "HOST_REWRITE_RULES", + "FIAAS_GLOBAL_ENV", + "FIAAS_SECRET_INIT_CONTAINERS", } environ_keys = set(os.environ.keys()) possible_config_env_variables = sorted(configuration_env_variable_keys & environ_keys) if len(possible_config_env_variables) > 0: - log.warn("found configuration environment variables %s. The ability to configure fiaas-deploy-daemon via environment variables " + - "has been removed. If you are trying to use these environment variables to configure fiaas-deploy-daemon, " + - "that configuration will not take effect. Please switch to configuring via a config file/ConfigMap or command-line " + - "flags. See https://github.com/fiaas/fiaas-deploy-daemon/issues/12 for more information.", - ', '.join(possible_config_env_variables)) + log.warn( + "found configuration environment variables %s. The ability to configure fiaas-deploy-daemon via environment variables " + + "has been removed. If you are trying to use these environment variables to configure fiaas-deploy-daemon, " + + "that configuration will not take effect. Please switch to configuring via a config file/ConfigMap or command-line " + + "flags. See https://github.com/fiaas/fiaas-deploy-daemon/issues/12 for more information.", + ", ".join(possible_config_env_variables), + ) def expose_fdd_version(config): - i = Info('fiaas_fdd_version', 'The tag of the running fiaas-deploy-daemon docker image.') - i.info({'fiaas_fdd_version': config.version}) + i = Info("fiaas_fdd_version", "The tag of the running fiaas-deploy-daemon docker image.") + i.info({"fiaas_fdd_version": config.version}) def main(): diff --git a/fiaas_deploy_daemon/base_thread.py b/fiaas_deploy_daemon/base_thread.py index 00f72d88..184a4855 100644 --- a/fiaas_deploy_daemon/base_thread.py +++ b/fiaas_deploy_daemon/base_thread.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 import logging + # Copyright 2017-2019 The FIAAS Authors # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/fiaas_deploy_daemon/bootstrap/__init__.py b/fiaas_deploy_daemon/bootstrap/__init__.py index 4855a8b6..0533bdf7 100644 --- a/fiaas_deploy_daemon/bootstrap/__init__.py +++ b/fiaas_deploy_daemon/bootstrap/__init__.py @@ -49,10 +49,7 @@ def configure(self, bind): def provide_session(self, config): session = requests.Session() if config.proxy: - session.proxies = {scheme: config.proxy for scheme in ( - "http", - "https" - )} + session.proxies = {scheme: config.proxy for scheme in ("http", "https")} return session @@ -88,5 +85,5 @@ def main(): sys.exit(1) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/fiaas_deploy_daemon/bootstrap/bootstrapper.py b/fiaas_deploy_daemon/bootstrap/bootstrapper.py index 0b5f3dd0..c7d8368d 100644 --- a/fiaas_deploy_daemon/bootstrap/bootstrapper.py +++ b/fiaas_deploy_daemon/bootstrap/bootstrapper.py @@ -42,8 +42,11 @@ def __init__(self): def store_status(self, status, app_name, namespace): with self._statuses_lock: self._statuses[(app_name, namespace)] = status - LOG.info("Received {status} for {name} in namespace {namespace}".format(status=status, name=app_name, - namespace=namespace)) + LOG.info( + "Received {status} for {name} in namespace {namespace}".format( + status=status, name=app_name, namespace=namespace + ) + ) def values(self): with self._statuses_lock: @@ -67,19 +70,22 @@ def __init__(self, config, deploy_queue, spec_factory, lifecycle): self._resource_class = FiaasApplication from ..crd.status import connect_signals else: - raise InvalidConfigurationException( - "Custom Resource Definition support must be enabled when bootstrapping") + raise InvalidConfigurationException("Custom Resource Definition support must be enabled when bootstrapping") connect_signals() signal(DEPLOY_STATUS_CHANGED).connect(self._store_status) def run(self): - for application in self._resource_class.find(name=None, namespace=self._namespace, - labels={"fiaas/bootstrap": "true"}): + for application in self._resource_class.find( + name=None, namespace=self._namespace, labels={"fiaas/bootstrap": "true"} + ): try: self._deploy(application) except BaseException: - LOG.exception("Caught exception when deploying {name} in namespace {namespace}".format( - name=application.metadata.name, namespace=application.metadata.namespace)) + LOG.exception( + "Caught exception when deploying {name} in namespace {namespace}".format( + name=application.metadata.name, namespace=application.metadata.namespace + ) + ) raise return self._wait_for_readiness(wait_time_seconds=5, timeout_seconds=120) @@ -88,20 +94,25 @@ def _deploy(self, application): LOG.debug("Deploying %s", application.spec.application) try: deployment_id = application.metadata.labels["fiaas/deployment_id"] - set_extras(app_name=application.spec.application, - namespace=application.metadata.namespace, - deployment_id=deployment_id) + set_extras( + app_name=application.spec.application, + namespace=application.metadata.namespace, + deployment_id=deployment_id, + ) except (AttributeError, KeyError, TypeError): - raise ValueError("The Application {} is missing the 'fiaas/deployment_id' label".format( - application.spec.application)) + raise ValueError( + "The Application {} is missing the 'fiaas/deployment_id' label".format(application.spec.application) + ) - lifecycle_subject = self._lifecycle.initiate(uid=application.metadata.uid, - app_name=application.spec.application, - namespace=application.metadata.namespace, - deployment_id=deployment_id, - repository=None, - labels=application.spec.additional_labels.status, - annotations=application.spec.additional_annotations.status) + lifecycle_subject = self._lifecycle.initiate( + uid=application.metadata.uid, + app_name=application.spec.application, + namespace=application.metadata.namespace, + deployment_id=deployment_id, + repository=None, + labels=application.spec.additional_labels.status, + annotations=application.spec.additional_annotations.status, + ) try: app_spec = self._spec_factory( @@ -118,8 +129,9 @@ def _deploy(self, application): ) self._store_status(None, DEPLOY_SCHEDULED, lifecycle_subject) self._deploy_queue.put(DeployerEvent("UPDATE", app_spec, lifecycle_subject)) - LOG.debug("Queued deployment for %s in namespace %s", application.spec.application, - application.metadata.namespace) + LOG.debug( + "Queued deployment for %s in namespace %s", application.spec.application, application.metadata.namespace + ) except (YAMLError, InvalidConfiguration): self._lifecycle.failed(lifecycle_subject) raise diff --git a/fiaas_deploy_daemon/config.py b/fiaas_deploy_daemon/config.py index 0f6bc71a..c13ca7f8 100644 --- a/fiaas_deploy_daemon/config.py +++ b/fiaas_deploy_daemon/config.py @@ -102,7 +102,9 @@ If an arg is specified in more than one place, then commandline values override config file values which override defaults. -""".format(DEFAULT_CONFIG_FILE) +""".format( + DEFAULT_CONFIG_FILE +) DATADOG_GLOBAL_TAGS_LONG_HELP = """ If you wish to send certain tags to datadog in every application @@ -155,137 +157,243 @@ def __init__(self, args=None, **kwargs): self.namespace = self._resolve_namespace() def _parse_args(self, args): - parser = configargparse.ArgParser(add_config_file_help=False, - add_env_var_help=False, - config_file_parser_class=configargparse.YAMLConfigFileParser, - default_config_files=[DEFAULT_CONFIG_FILE], - args_for_setting_config_path=["-c", "--config-file"], - ignore_unknown_config_file_keys=True, - description="%(prog)s deploys applications to Kubernetes", - epilog=EPILOG, - formatter_class=configargparse.ArgumentDefaultsHelpFormatter) - parser.add_argument("--secrets-directory", help="Load secrets from this directory (default: %(default)s)", - default=DEFAULT_SECRETS_DIR) - parser.add_argument("--log-format", help="Set logformat (default: %(default)s)", choices=self.VALID_LOG_FORMAT, - default="plain") + parser = configargparse.ArgParser( + add_config_file_help=False, + add_env_var_help=False, + config_file_parser_class=configargparse.YAMLConfigFileParser, + default_config_files=[DEFAULT_CONFIG_FILE], + args_for_setting_config_path=["-c", "--config-file"], + ignore_unknown_config_file_keys=True, + description="%(prog)s deploys applications to Kubernetes", + epilog=EPILOG, + formatter_class=configargparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--secrets-directory", + help="Load secrets from this directory (default: %(default)s)", + default=DEFAULT_SECRETS_DIR, + ) + parser.add_argument( + "--log-format", help="Set logformat (default: %(default)s)", choices=self.VALID_LOG_FORMAT, default="plain" + ) parser.add_argument("--proxy", help="Use http proxy (currently only used for for usage reporting)") - parser.add_argument("--debug", help="Enable a number of debugging options (including disable SSL-verification)", - action="store_true") + parser.add_argument( + "--debug", + help="Enable a number of debugging options (including disable SSL-verification)", + action="store_true", + ) parser.add_argument("--environment", help="Environment to deploy to", default="") - parser.add_argument("--service-type", help="Type of kubernetes Service to create", - choices=("ClusterIP", "NodePort", "LoadBalancer"), default="ClusterIP") - parser.add_argument("--infrastructure", - help="The underlying infrastructure of the cluster to deploy to. (default: %(default)s).", - choices=("diy", "gke"), default="diy") - parser.add_argument("--port", help="Port to use for the web-interface (default: %(default)s)", type=int, - default=5000) - parser.add_argument("--enable-crd-support", help="Enable Custom Resource Definition support.", - action="store_true") - parser.add_argument("--secrets-init-container-image", - help="Use specified docker image as init container for secrets (experimental)", - default=None) - parser.add_argument("--secrets-service-account-name", - help="The service account that is passed to secrets init containers", default=None) - parser.add_argument("--datadog-container-image", - help="Use specified docker image as datadog sidecar for apps", default=None) - parser.add_argument("--datadog-container-memory", - help="The amount of memory (request and limit) for the datadog sidecar", default="2Gi") + parser.add_argument( + "--service-type", + help="Type of kubernetes Service to create", + choices=("ClusterIP", "NodePort", "LoadBalancer"), + default="ClusterIP", + ) + parser.add_argument( + "--infrastructure", + help="The underlying infrastructure of the cluster to deploy to. (default: %(default)s).", + choices=("diy", "gke"), + default="diy", + ) + parser.add_argument( + "--port", help="Port to use for the web-interface (default: %(default)s)", type=int, default=5000 + ) + parser.add_argument( + "--enable-crd-support", help="Enable Custom Resource Definition support.", action="store_true" + ) + parser.add_argument( + "--secrets-init-container-image", + help="Use specified docker image as init container for secrets (experimental)", + default=None, + ) + parser.add_argument( + "--secrets-service-account-name", + help="The service account that is passed to secrets init containers", + default=None, + ) + parser.add_argument( + "--datadog-container-image", help="Use specified docker image as datadog sidecar for apps", default=None + ) + parser.add_argument( + "--datadog-container-memory", + help="The amount of memory (request and limit) for the datadog sidecar", + default="2Gi", + ) datadog_global_tags_parser = parser.add_argument_group("Datadog Global tags", DATADOG_GLOBAL_TAGS_LONG_HELP) - datadog_global_tags_parser.add_argument("--datadog-global-tags", default=[], - help="Various non-essential global tags to send to datadog for all applications", - action="append", type=KeyValue, dest="datadog_global_tags") - parser.add_argument("--datadog-activate-sleep", - help="Flag to activate the sleep in datadog when a pre-stop-delay is in the main container", - action="store_true", default=False) - parser.add_argument("--pre-stop-delay", type=int, - help="Add a pre-stop hook that sleeps for this amount of seconds (default: %(default)s)", - default=0) - parser.add_argument("--strongbox-init-container-image", - help="Use specified docker image as init container for apps that are configured to use Strongbox", - default=None) - parser.add_argument("--enable-deprecated-multi-namespace-support", help=MULTI_NAMESPACE_HELP, - action="store_true") - parser.add_argument("--use-in-memory-emptydirs", help="Use memory for emptydirs mounted in the deployed application", - action="store_true") - parser.add_argument("--deployment-max-surge", help="maximum number of extra pods that can be scheduled above the desired " - "number of pods during an update", - default="25%", type=_int_or_unicode) - parser.add_argument("--deployment-max-unavailable", help="The maximum number of pods that can be unavailable during an update", - default="0", type=_int_or_unicode) - parser.add_argument("--enable-deprecated-tls-entry-per-host", help=TLS_ENTRY_PER_HOST_HELP, - action="store_true") - parser.add_argument("--ready-check-timeout-multiplier", type=int, - help="Multiply default ready check timeout (replicas * initialDelaySeconds) with this " + - "number of seconds (default: %(default)s)", default=10) - parser.add_argument("--disable-deprecated-managed-env-vars", help=DISABLE_DEPRECATED_MANAGED_ENV_VARS, - action="store_true", default=False) + datadog_global_tags_parser.add_argument( + "--datadog-global-tags", + default=[], + help="Various non-essential global tags to send to datadog for all applications", + action="append", + type=KeyValue, + dest="datadog_global_tags", + ) + parser.add_argument( + "--datadog-activate-sleep", + help="Flag to activate the sleep in datadog when a pre-stop-delay is in the main container", + action="store_true", + default=False, + ) + parser.add_argument( + "--pre-stop-delay", + type=int, + help="Add a pre-stop hook that sleeps for this amount of seconds (default: %(default)s)", + default=0, + ) + parser.add_argument( + "--strongbox-init-container-image", + help="Use specified docker image as init container for apps that are configured to use Strongbox", + default=None, + ) + parser.add_argument( + "--enable-deprecated-multi-namespace-support", help=MULTI_NAMESPACE_HELP, action="store_true" + ) + parser.add_argument( + "--use-in-memory-emptydirs", + help="Use memory for emptydirs mounted in the deployed application", + action="store_true", + ) + parser.add_argument( + "--deployment-max-surge", + help="maximum number of extra pods that can be scheduled above the desired " + "number of pods during an update", + default="25%", + type=_int_or_unicode, + ) + parser.add_argument( + "--deployment-max-unavailable", + help="The maximum number of pods that can be unavailable during an update", + default="0", + type=_int_or_unicode, + ) + parser.add_argument("--enable-deprecated-tls-entry-per-host", help=TLS_ENTRY_PER_HOST_HELP, action="store_true") + parser.add_argument( + "--ready-check-timeout-multiplier", + type=int, + help="Multiply default ready check timeout (replicas * initialDelaySeconds) with this " + + "number of seconds (default: %(default)s)", + default=10, + ) + parser.add_argument( + "--disable-deprecated-managed-env-vars", + help=DISABLE_DEPRECATED_MANAGED_ENV_VARS, + action="store_true", + default=False, + ) parser.add_argument("--extension-hook-url", help=EXTENSION_HOOK_URL_HELP, default=None) - parser.add_argument("--enable-service-account-per-app", help=ENABLE_SERVICE_ACCOUNT_PER_APP, - action="store_true", default=False) - parser.add_argument("--use-networkingv1-ingress", - help="use ingress from apiversion networking.k8s.io instead of extensions/v1beta1", - action="store_true", default=False) - parser.add_argument("--use-apiextensionsv1-crd", - help="Use apiextensions/v1 CustomResourceDefinition instead of apiextensions/v1beta1", - action="store_true", default=False) - parser.add_argument("--disable-crd-creation", - help="Crd Watcher will skip the crd creation and it will go directly to watch the events", - action="store_true", default=False) + parser.add_argument( + "--enable-service-account-per-app", help=ENABLE_SERVICE_ACCOUNT_PER_APP, action="store_true", default=False + ) + parser.add_argument( + "--use-networkingv1-ingress", + help="use ingress from apiversion networking.k8s.io instead of extensions/v1beta1", + action="store_true", + default=False, + ) + parser.add_argument( + "--use-apiextensionsv1-crd", + help="Use apiextensions/v1 CustomResourceDefinition instead of apiextensions/v1beta1", + action="store_true", + default=False, + ) + parser.add_argument( + "--disable-crd-creation", + help="Crd Watcher will skip the crd creation and it will go directly to watch the events", + action="store_true", + default=False, + ) usage_reporting_parser = parser.add_argument_group("Usage Reporting", USAGE_REPORTING_LONG_HELP) - usage_reporting_parser.add_argument("--usage-reporting-cluster-name", - help="Name of the cluster where the fiaas-deploy-daemon instance resides") - usage_reporting_parser.add_argument("--usage-reporting-operator", - help="Identifier for the operator of the fiaas-deploy-daemon instance") + usage_reporting_parser.add_argument( + "--usage-reporting-cluster-name", help="Name of the cluster where the fiaas-deploy-daemon instance resides" + ) + usage_reporting_parser.add_argument( + "--usage-reporting-operator", help="Identifier for the operator of the fiaas-deploy-daemon instance" + ) usage_reporting_parser.add_argument("--usage-reporting-endpoint", help="Endpoint to POST usage data to") usage_reporting_parser.add_argument("--usage-reporting-tenant", help="Name of publisher of events") - usage_reporting_parser.add_argument("--usage-reporting-team", - help="""Name of team that is responsible for components deployed \ - "by the fiaas-deploy-daemon instance""") + usage_reporting_parser.add_argument( + "--usage-reporting-team", + help="""Name of team that is responsible for components deployed \ + "by the fiaas-deploy-daemon instance""", + ) api_parser = parser.add_argument_group("API server") - api_parser.add_argument("--api-server", help="Address of the api-server to use (IP or name)", - default="https://kubernetes.default.svc.cluster.local") + api_parser.add_argument( + "--api-server", + help="Address of the api-server to use (IP or name)", + default="https://kubernetes.default.svc.cluster.local", + ) api_parser.add_argument("--api-token", help="Token to use (default: lookup from service account)", default=None) - api_parser.add_argument("--api-cert", help="API server certificate (default: lookup from service account)", - default=None) + api_parser.add_argument( + "--api-cert", help="API server certificate (default: lookup from service account)", default=None + ) client_cert_parser = parser.add_argument_group("Client certificate") client_cert_parser.add_argument("--client-cert", help="Client certificate to use", default=None) client_cert_parser.add_argument("--client-key", help="Client certificate key to use", default=None) ingress_parser = parser.add_argument_group("Ingress suffix", INGRESS_SUFFIX_LONG_HELP) - ingress_parser.add_argument("--ingress-suffix", help="Suffix to use for ingress", action="append", - dest="ingress_suffixes", default=[]) + ingress_parser.add_argument( + "--ingress-suffix", help="Suffix to use for ingress", action="append", dest="ingress_suffixes", default=[] + ) host_rule_parser = parser.add_argument_group("Host rewrite rules", HOST_REWRITE_RULE_LONG_HELP) - host_rule_parser.add_argument("--host-rewrite-rule", help="Rule for rewriting host", action="append", - type=HostRewriteRule, dest="host_rewrite_rules", default=[]) + host_rule_parser.add_argument( + "--host-rewrite-rule", + help="Rule for rewriting host", + action="append", + type=HostRewriteRule, + dest="host_rewrite_rules", + default=[], + ) global_env_parser = parser.add_argument_group("Global environment variables", GLOBAL_ENV_LONG_HELP) - global_env_parser.add_argument("--global-env", default=[], - help="Various non-essential global variables to expose for all applications", - action="append", type=KeyValue, dest="global_env") + global_env_parser.add_argument( + "--global-env", + default=[], + help="Various non-essential global variables to expose for all applications", + action="append", + type=KeyValue, + dest="global_env", + ) secret_init_containers_parser = parser.add_argument_group("Secret init-containers", SECRET_CONTAINERS_LONG_HELP) - secret_init_containers_parser.add_argument("--secret-init-containers", default=[], - help="Images to use for secret init-containers by key", - action="append", type=KeyValue, dest="secret_init_containers") + secret_init_containers_parser.add_argument( + "--secret-init-containers", + default=[], + help="Images to use for secret init-containers by key", + action="append", + type=KeyValue, + dest="secret_init_containers", + ) tls_parser = parser.add_argument_group("Ingress TLS") - tls_parser.add_argument("--use-ingress-tls", help=TLS_HELP, - choices=("disabled", "default_off", "default_on"), - default="disabled") - tls_parser.add_argument("--tls-certificate-issuer", - help="Certificate issuer to use with cert-manager to provision certificates", - default=None) - tls_parser.add_argument("--tls-certificate-issuer-type-default", - help="The annotation to set for cert-manager to provision certificates", - default="certmanager.k8s.io/cluster-issuer") - tls_parser.add_argument("--tls-certificate-issuer-type-overrides", help="Issuers to use for specified domain suffixes", - default=[], - action="append", type=KeyValue, dest="tls_certificate_issuer_type_overrides") - tls_parser.add_argument("--tls-certificate-ready", help="Check whether certificates are ready", default=False, action="store_true") + tls_parser.add_argument( + "--use-ingress-tls", help=TLS_HELP, choices=("disabled", "default_off", "default_on"), default="disabled" + ) + tls_parser.add_argument( + "--tls-certificate-issuer", + help="Certificate issuer to use with cert-manager to provision certificates", + default=None, + ) + tls_parser.add_argument( + "--tls-certificate-issuer-type-default", + help="The annotation to set for cert-manager to provision certificates", + default="certmanager.k8s.io/cluster-issuer", + ) + tls_parser.add_argument( + "--tls-certificate-issuer-type-overrides", + help="Issuers to use for specified domain suffixes", + default=[], + action="append", + type=KeyValue, + dest="tls_certificate_issuer_type_overrides", + ) + tls_parser.add_argument( + "--tls-certificate-ready", help="Check whether certificates are ready", default=False, action="store_true" + ) parser.parse_args(args, namespace=self) self.global_env = {env_var.key: env_var.value for env_var in self.global_env} self.datadog_global_tags = {tag.key: tag.value for tag in self.datadog_global_tags} self.secret_init_containers = {provider.key: provider.value for provider in self.secret_init_containers} - self.tls_certificate_issuer_type_overrides = {issuer_type.key: issuer_type.value - for issuer_type in self.tls_certificate_issuer_type_overrides} + self.tls_certificate_issuer_type_overrides = { + issuer_type.key: issuer_type.value for issuer_type in self.tls_certificate_issuer_type_overrides + } def _resolve_env(self): image = os.getenv("IMAGE") @@ -305,7 +413,7 @@ def _resolve_namespace(): namespace_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" namespace_env_variable = "NAMESPACE" try: - with open(namespace_file_path, 'r') as fobj: + with open(namespace_file_path, "r") as fobj: namespace = fobj.read().strip() if namespace: return namespace @@ -315,12 +423,17 @@ def _resolve_namespace(): return namespace raise InvalidConfigurationException( "Could not determine namespace: could not read {path}, and ${env_var} was not set".format( - path=namespace_file_path, env_var=namespace_env_variable)) + path=namespace_file_path, env_var=namespace_env_variable + ) + ) def __repr__(self): return "Configuration({})".format( - ", ".join("{}={}".format(key, self.__dict__[key]) for key in vars(self) - if not key.startswith("_") and not key.isupper() and "token" not in key) + ", ".join( + "{}={}".format(key, self.__dict__[key]) + for key in vars(self) + if not key.startswith("_") and not key.isupper() and "token" not in key + ) ) diff --git a/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1.py b/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1.py index 1474c5f4..f60194a6 100644 --- a/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1.py +++ b/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1.py @@ -18,9 +18,15 @@ import logging from k8s.models.common import ObjectMeta -from k8s.models.apiextensions_v1_custom_resource_definition import CustomResourceConversion,\ - CustomResourceDefinitionNames, CustomResourceDefinitionSpec, CustomResourceDefinition,\ - CustomResourceDefinitionVersion, CustomResourceValidation, JSONSchemaProps +from k8s.models.apiextensions_v1_custom_resource_definition import ( + CustomResourceConversion, + CustomResourceDefinitionNames, + CustomResourceDefinitionSpec, + CustomResourceDefinition, + CustomResourceDefinitionVersion, + CustomResourceValidation, + JSONSchemaProps, +) from ..retry import retry_on_upsert_conflict @@ -42,7 +48,7 @@ def _create_or_update(kind, plural, short_names, group, schema_properties): versions=[version_v1], preserveUnknownFields=False, scope="Namespaced", - conversion=CustomResourceConversion(strategy="None") + conversion=CustomResourceConversion(strategy="None"), ) definition = CustomResourceDefinition.get_or_create(metadata=metadata, spec=spec) definition.save() @@ -73,7 +79,7 @@ def update_crd_resources(cls): "service_account": object_with_unknown_fields, "pod": object_with_unknown_fields, "status": object_with_unknown_fields, - } + }, }, "additional_annotations": { "type": "object", @@ -86,22 +92,22 @@ def update_crd_resources(cls): "service_account": object_with_unknown_fields, "pod": object_with_unknown_fields, "status": object_with_unknown_fields, - } - } - } + }, + }, + }, } } application_status_schema_properties = { - "result": { - "type": "string" - }, - "logs": { - "type": "array", - "items": { - "type": "string" - } - } + "result": {"type": "string"}, + "logs": {"type": "array", "items": {"type": "string"}}, } - cls._create_or_update("Application", "applications", ("app", "fa"), "fiaas.schibsted.io", application_schema_properties) - cls._create_or_update("ApplicationStatus", "application-statuses", ("status", "appstatus", "fs"), "fiaas.schibsted.io", - application_status_schema_properties) + cls._create_or_update( + "Application", "applications", ("app", "fa"), "fiaas.schibsted.io", application_schema_properties + ) + cls._create_or_update( + "ApplicationStatus", + "application-statuses", + ("status", "appstatus", "fs"), + "fiaas.schibsted.io", + application_status_schema_properties, + ) diff --git a/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1beta1.py b/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1beta1.py index 5fb4e108..84247a7d 100644 --- a/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1beta1.py +++ b/fiaas_deploy_daemon/crd/crd_resources_syncer_apiextensionsv1beta1.py @@ -18,8 +18,11 @@ import logging from k8s.models.common import ObjectMeta -from k8s.models.custom_resource_definition import CustomResourceDefinitionNames, CustomResourceDefinitionSpec, \ - CustomResourceDefinition +from k8s.models.custom_resource_definition import ( + CustomResourceDefinitionNames, + CustomResourceDefinitionSpec, + CustomResourceDefinition, +) from ..retry import retry_on_upsert_conflict @@ -41,4 +44,6 @@ def _create_or_update(kind, plural, short_names, group): @classmethod def update_crd_resources(cls): cls._create_or_update("Application", "applications", ("app", "fa"), "fiaas.schibsted.io") - cls._create_or_update("ApplicationStatus", "application-statuses", ("status", "appstatus", "fs"), "fiaas.schibsted.io") + cls._create_or_update( + "ApplicationStatus", "application-statuses", ("status", "appstatus", "fs"), "fiaas.schibsted.io" + ) diff --git a/fiaas_deploy_daemon/crd/status.py b/fiaas_deploy_daemon/crd/status.py index a4dde998..aab05573 100644 --- a/fiaas_deploy_daemon/crd/status.py +++ b/fiaas_deploy_daemon/crd/status.py @@ -1,4 +1,3 @@ - # Copyright 2017-2019 The FIAAS Authors # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -78,15 +77,24 @@ def _save_status(result, subject): status = FiaasApplicationStatus.get_or_create(metadata=metadata, result=result, logs=logs) resource_version = status.metadata.resourceVersion - LOG.debug("save()-ing %s for %s/%s deployment_id=%s resourceVersion=%s", result, namespace, app_name, - deployment_id, resource_version) + LOG.debug( + "save()-ing %s for %s/%s deployment_id=%s resourceVersion=%s", + result, + namespace, + app_name, + deployment_id, + resource_version, + ) _apply_owner_reference(status, subject) status.save() def _get_logs(app_name, namespace, deployment_id, result): - return get_running_logs(app_name, namespace, deployment_id) if result in ["RUNNING", "INITIATED"] else \ - get_final_logs(app_name, namespace, deployment_id) + return ( + get_running_logs(app_name, namespace, deployment_id) + if result in ["RUNNING", "INITIATED"] + else get_final_logs(app_name, namespace, deployment_id) + ) def _cleanup(app_name=None, namespace=None): @@ -110,17 +118,19 @@ def create_name(name, deployment_id): By convention, the names of Kubernetes resources should be up to maximum length of 253 characters and consist of lower case alphanumeric characters, '-', and '.'. """ - suffix = b32encode(struct.pack('q', hash27(deployment_id))).decode("utf-8").lower().strip("=") + suffix = b32encode(struct.pack("q", hash27(deployment_id))).decode("utf-8").lower().strip("=") return "{}-{}".format(name, suffix) def _apply_owner_reference(status, subject): if subject.uid: - owner_reference = OwnerReference(apiVersion="fiaas.schibsted.io/v1", - blockOwnerDeletion=True, - controller=True, - kind="Application", - name=subject.app_name, - uid=subject.uid) + owner_reference = OwnerReference( + apiVersion="fiaas.schibsted.io/v1", + blockOwnerDeletion=True, + controller=True, + kind="Application", + name=subject.app_name, + uid=subject.uid, + ) status.metadata.ownerReferences = [owner_reference] diff --git a/fiaas_deploy_daemon/crd/watcher.py b/fiaas_deploy_daemon/crd/watcher.py index 5773d422..195d6b0b 100644 --- a/fiaas_deploy_daemon/crd/watcher.py +++ b/fiaas_deploy_daemon/crd/watcher.py @@ -73,23 +73,22 @@ def _deploy(self, application): LOG.debug("Deploying %s", app_name) try: deployment_id = application.metadata.labels["fiaas/deployment_id"] - set_extras(app_name=app_name, - namespace=application.metadata.namespace, - deployment_id=deployment_id) + set_extras(app_name=app_name, namespace=application.metadata.namespace, deployment_id=deployment_id) except (AttributeError, KeyError, TypeError): - raise ValueError("The Application {} is missing the 'fiaas/deployment_id' label".format( - app_name)) + raise ValueError("The Application {} is missing the 'fiaas/deployment_id' label".format(app_name)) if self._already_deployed(app_name, application.metadata.namespace, deployment_id): LOG.debug("Have already deployed %s for app %s", deployment_id, app_name) return repository = _repository(application) - lifecycle_subject = self._lifecycle.initiate(uid=application.metadata.uid, - app_name=app_name, - namespace=application.metadata.namespace, - deployment_id=deployment_id, - repository=repository, - labels=application.spec.additional_labels.status, - annotations=application.spec.additional_annotations.status) + lifecycle_subject = self._lifecycle.initiate( + uid=application.metadata.uid, + app_name=app_name, + namespace=application.metadata.namespace, + deployment_id=deployment_id, + repository=repository, + labels=application.spec.additional_labels.status, + annotations=application.spec.additional_annotations.status, + ) try: app_spec = self._spec_factory( uid=application.metadata.uid, diff --git a/fiaas_deploy_daemon/deployer/__init__.py b/fiaas_deploy_daemon/deployer/__init__.py index b470c1de..60b1fb5d 100644 --- a/fiaas_deploy_daemon/deployer/__init__.py +++ b/fiaas_deploy_daemon/deployer/__init__.py @@ -35,4 +35,4 @@ def configure(self, bind, require): bind("deployer", to_class=Deployer) -DeployerEvent = namedtuple('DeployerEvent', ['action', 'app_spec', 'lifecycle_subject']) +DeployerEvent = namedtuple("DeployerEvent", ["action", "app_spec", "lifecycle_subject"]) diff --git a/fiaas_deploy_daemon/deployer/bookkeeper.py b/fiaas_deploy_daemon/deployer/bookkeeper.py index 601510f1..a7cf2e62 100644 --- a/fiaas_deploy_daemon/deployer/bookkeeper.py +++ b/fiaas_deploy_daemon/deployer/bookkeeper.py @@ -19,6 +19,7 @@ class Bookkeeper(object): """Measures time, fails and successes""" + deploy_gauge = Gauge("deployer_requests", "Request to deploy an app", ["app"]) error_counter = Counter("deployer_errors", "Deploy failed", ["app"]) success_counter = Counter("deployer_success", "Deploy successful", ["app"]) diff --git a/fiaas_deploy_daemon/deployer/deploy.py b/fiaas_deploy_daemon/deployer/deploy.py index 10395730..4adbfe6b 100644 --- a/fiaas_deploy_daemon/deployer/deploy.py +++ b/fiaas_deploy_daemon/deployer/deploy.py @@ -58,8 +58,16 @@ def _update(self, app_spec, lifecycle_subject): with self._bookkeeper.time(app_spec): self._adapter.deploy(app_spec) if app_spec.name != "fiaas-deploy-daemon": - self._scheduler.add(ReadyCheck(app_spec, self._bookkeeper, self._lifecycle, lifecycle_subject, - self._ingress_adapter, self._config)) + self._scheduler.add( + ReadyCheck( + app_spec, + self._bookkeeper, + self._lifecycle, + lifecycle_subject, + self._ingress_adapter, + self._config, + ) + ) else: self._lifecycle.success(lifecycle_subject) self._bookkeeper.success(app_spec) diff --git a/fiaas_deploy_daemon/deployer/kubernetes/adapter.py b/fiaas_deploy_daemon/deployer/kubernetes/adapter.py index 97ea6521..34d4f06c 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/adapter.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/adapter.py @@ -26,10 +26,11 @@ class K8s(object): - """Adapt from an AppSpec to the necessary definitions for a kubernetes cluster - """ + """Adapt from an AppSpec to the necessary definitions for a kubernetes cluster""" - def __init__(self, config, service_deployer, deployment_deployer, ingress_deployer, autoscaler, service_account_deployer): + def __init__( + self, config, service_deployer, deployment_deployer, ingress_deployer, autoscaler, service_account_deployer + ): self._version = config.version self._enable_service_account_per_app = config.enable_service_account_per_app self._service_deployer = service_deployer @@ -77,13 +78,11 @@ def _add_labels(prefix, labels, values): def _to_valid_label_value(value): - return value.lower() \ - .replace(" ", "-").replace("ø", "oe").replace("å", "aa").replace("æ", "ae") \ - .replace(":", "-") + return value.lower().replace(" ", "-").replace("ø", "oe").replace("å", "aa").replace("æ", "ae").replace(":", "-") def _make_selector(app_spec): - return {'app': app_spec.name} + return {"app": app_spec.name} def _remove_resource_requirements(app_spec): diff --git a/fiaas_deploy_daemon/deployer/kubernetes/autoscaler.py b/fiaas_deploy_daemon/deployer/kubernetes/autoscaler.py index 33cc0b8f..6bf9a3fa 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/autoscaler.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/autoscaler.py @@ -39,13 +39,19 @@ def deploy(self, app_spec, labels): if should_have_autoscaler(app_spec): LOG.info("Creating/updating %s for %s", self.name, app_spec.name) custom_labels = merge_dicts(app_spec.labels.horizontal_pod_autoscaler, labels) - metadata = ObjectMeta(name=app_spec.name, namespace=app_spec.namespace, labels=custom_labels, - annotations=app_spec.annotations.horizontal_pod_autoscaler) + metadata = ObjectMeta( + name=app_spec.name, + namespace=app_spec.namespace, + labels=custom_labels, + annotations=app_spec.annotations.horizontal_pod_autoscaler, + ) scale_target_ref = CrossVersionObjectReference(kind="Deployment", name=app_spec.name, apiVersion="apps/v1") - spec = HorizontalPodAutoscalerSpec(scaleTargetRef=scale_target_ref, - minReplicas=app_spec.autoscaler.min_replicas, - maxReplicas=app_spec.autoscaler.max_replicas, - targetCPUUtilizationPercentage=app_spec.autoscaler.cpu_threshold_percentage) + spec = HorizontalPodAutoscalerSpec( + scaleTargetRef=scale_target_ref, + minReplicas=app_spec.autoscaler.min_replicas, + maxReplicas=app_spec.autoscaler.max_replicas, + targetCPUUtilizationPercentage=app_spec.autoscaler.cpu_threshold_percentage, + ) autoscaler = HorizontalPodAutoscaler.get_or_create(metadata=metadata, spec=spec) self._owner_references.apply(autoscaler, app_spec) self._extension_hook.apply(autoscaler, app_spec) @@ -69,7 +75,9 @@ def should_have_autoscaler(app_spec): if not _autoscaler_enabled(app_spec.autoscaler): return False if not _enough_replicas_wanted(app_spec): - LOG.warn("Can't enable autoscaler for %s with only %d max replicas", app_spec.name, app_spec.autoscaler.max_replicas) + LOG.warn( + "Can't enable autoscaler for %s with only %d max replicas", app_spec.name, app_spec.autoscaler.max_replicas + ) return False if not _request_cpu_is_set(app_spec): LOG.warn("Can't enable autoscaler for %s without CPU requests", app_spec.name) diff --git a/fiaas_deploy_daemon/deployer/kubernetes/deployment/datadog.py b/fiaas_deploy_daemon/deployer/kubernetes/deployment/datadog.py index 9a254fa9..8fbbf35c 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/deployment/datadog.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/deployment/datadog.py @@ -14,8 +14,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from k8s.models.pod import ResourceRequirements, Container, EnvVar, EnvVarSource, ExecAction, Handler, Lifecycle, \ - SecretKeySelector +from k8s.models.pod import ( + ResourceRequirements, + Container, + EnvVar, + EnvVarSource, + ExecAction, + Handler, + Lifecycle, + SecretKeySelector, +) class DataDog(object): @@ -42,8 +50,10 @@ def _create_datadog_container(self, app_spec, besteffort_qos_is_required, pre_st if besteffort_qos_is_required: resource_requirements = ResourceRequirements() else: - resource_requirements = ResourceRequirements(limits={"cpu": "400m", "memory": self._datadog_container_memory}, - requests={"cpu": "200m", "memory": self._datadog_container_memory}) + resource_requirements = ResourceRequirements( + limits={"cpu": "400m", "memory": self._datadog_container_memory}, + requests={"cpu": "200m", "memory": self._datadog_container_memory}, + ) tags = {} if self._datadog_global_tags: @@ -70,20 +80,19 @@ def _create_datadog_container(self, app_spec, besteffort_qos_is_required, pre_st imagePullPolicy=image_pull_policy, env=[ EnvVar(name="DD_TAGS", value=dd_tags), - EnvVar(name="DD_API_KEY", - valueFrom=EnvVarSource(secretKeyRef=SecretKeySelector(name="datadog", key="apikey"))), + EnvVar( + name="DD_API_KEY", + valueFrom=EnvVarSource(secretKeyRef=SecretKeySelector(name="datadog", key="apikey")), + ), EnvVar(name="NON_LOCAL_TRAFFIC", value="false"), EnvVar(name="DD_LOGS_STDOUT", value="yes"), EnvVar(name="DD_EXPVAR_PORT", value="42622"), EnvVar(name="DD_CMD_PORT", value="42623"), ], lifecycle=lifecycle, - resources=resource_requirements + resources=resource_requirements, ) @staticmethod def _get_env_vars(): - return ( - EnvVar(name="STATSD_HOST", value="localhost"), - EnvVar(name="STATSD_PORT", value="8125") - ) + return (EnvVar(name="STATSD_HOST", value="localhost"), EnvVar(name="STATSD_PORT", value="8125")) diff --git a/fiaas_deploy_daemon/deployer/kubernetes/deployment/deployer.py b/fiaas_deploy_daemon/deployer/kubernetes/deployment/deployer.py index bf688f06..b83228a2 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/deployment/deployer.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/deployment/deployer.py @@ -19,12 +19,37 @@ from k8s.client import NotFound from k8s.models.common import ObjectMeta -from k8s.models.deployment import Deployment, DeploymentSpec, PodTemplateSpec, LabelSelector, DeploymentStrategy, \ - RollingUpdateDeployment -from k8s.models.pod import ContainerPort, EnvVar, HTTPGetAction, TCPSocketAction, ExecAction, HTTPHeader, Container, \ - PodSpec, VolumeMount, Volume, ResourceRequirements, Probe, ConfigMapEnvSource, \ - ConfigMapVolumeSource, EmptyDirVolumeSource, EnvFromSource, EnvVarSource, Lifecycle, Handler, \ - ResourceFieldSelector, ObjectFieldSelector +from k8s.models.deployment import ( + Deployment, + DeploymentSpec, + PodTemplateSpec, + LabelSelector, + DeploymentStrategy, + RollingUpdateDeployment, +) +from k8s.models.pod import ( + ContainerPort, + EnvVar, + HTTPGetAction, + TCPSocketAction, + ExecAction, + HTTPHeader, + Container, + PodSpec, + VolumeMount, + Volume, + ResourceRequirements, + Probe, + ConfigMapEnvSource, + ConfigMapVolumeSource, + EmptyDirVolumeSource, + EnvFromSource, + EnvVarSource, + Lifecycle, + Handler, + ResourceFieldSelector, + ObjectFieldSelector, +) from fiaas_deploy_daemon.deployer.kubernetes.autoscaler import should_have_autoscaler from fiaas_deploy_daemon.retry import retry_on_upsert_conflict @@ -50,8 +75,7 @@ def __init__(self, config, datadog, prometheus, deployment_secrets, owner_refere self._grace_period = self.MINIMUM_GRACE_PERIOD self._use_in_memory_emptydirs = config.use_in_memory_emptydirs if self._pre_stop_delay > 0: - self._lifecycle = Lifecycle(preStop=Handler( - _exec=ExecAction(command=["sleep", str(self._pre_stop_delay)]))) + self._lifecycle = Lifecycle(preStop=Handler(_exec=ExecAction(command=["sleep", str(self._pre_stop_delay)]))) self._grace_period += self._pre_stop_delay self._max_surge = config.deployment_max_surge self._max_unavailable = config.deployment_max_unavailable @@ -62,42 +86,52 @@ def __init__(self, config, datadog, prometheus, deployment_secrets, owner_refere def deploy(self, app_spec, selector, labels, besteffort_qos_is_required): LOG.info("Creating new deployment for %s", app_spec.name) deployment_labels = merge_dicts(app_spec.labels.deployment, labels) - metadata = ObjectMeta(name=app_spec.name, namespace=app_spec.namespace, labels=deployment_labels, - annotations=app_spec.annotations.deployment) - container_ports = [ContainerPort(name=port_spec.name, containerPort=port_spec.target_port) for port_spec in - app_spec.ports] + metadata = ObjectMeta( + name=app_spec.name, + namespace=app_spec.namespace, + labels=deployment_labels, + annotations=app_spec.annotations.deployment, + ) + container_ports = [ + ContainerPort(name=port_spec.name, containerPort=port_spec.target_port) for port_spec in app_spec.ports + ] env = self._make_env(app_spec) pull_policy = "IfNotPresent" if (":" in app_spec.image and ":latest" not in app_spec.image) else "Always" env_from = [EnvFromSource(configMapRef=ConfigMapEnvSource(name=app_spec.name, optional=True))] containers = [ - Container(name=app_spec.name, - image=app_spec.image, - ports=container_ports, - env=env, - envFrom=env_from, - lifecycle=self._lifecycle, - livenessProbe=_make_probe(app_spec.health_checks.liveness), - readinessProbe=_make_probe(app_spec.health_checks.readiness), - imagePullPolicy=pull_policy, - volumeMounts=self._make_volume_mounts(app_spec), - resources=_make_resource_requirements(app_spec.resources)) + Container( + name=app_spec.name, + image=app_spec.image, + ports=container_ports, + env=env, + envFrom=env_from, + lifecycle=self._lifecycle, + livenessProbe=_make_probe(app_spec.health_checks.liveness), + readinessProbe=_make_probe(app_spec.health_checks.readiness), + imagePullPolicy=pull_policy, + volumeMounts=self._make_volume_mounts(app_spec), + resources=_make_resource_requirements(app_spec.resources), + ) ] automount_service_account_token = app_spec.admin_access init_containers = [] service_account_name = app_spec.name if self._enable_service_account_per_app else "default" - pod_spec = PodSpec(containers=containers, - initContainers=init_containers, - volumes=self._make_volumes(app_spec), - serviceAccountName=service_account_name, - automountServiceAccountToken=automount_service_account_token, - terminationGracePeriodSeconds=self._grace_period) + pod_spec = PodSpec( + containers=containers, + initContainers=init_containers, + volumes=self._make_volumes(app_spec), + serviceAccountName=service_account_name, + automountServiceAccountToken=automount_service_account_token, + terminationGracePeriodSeconds=self._grace_period, + ) pod_labels = merge_dicts(app_spec.labels.pod, _add_status_label(labels)) - pod_metadata = ObjectMeta(name=app_spec.name, namespace=app_spec.namespace, labels=pod_labels, - annotations=app_spec.annotations.pod) + pod_metadata = ObjectMeta( + name=app_spec.name, namespace=app_spec.namespace, labels=pod_labels, annotations=app_spec.annotations.pod + ) pod_template_spec = PodTemplateSpec(metadata=pod_metadata, spec=pod_spec) replicas = app_spec.autoscaler.min_replicas # we must avoid that the deployment scales up to app_spec.autoscaler.min_replicas if autoscaler has set another value @@ -107,23 +141,35 @@ def deploy(self, app_spec, selector, labels, besteffort_qos_is_required): # the autoscaler won't scale up the deployment if the current number of replicas is 0 if deployment.spec.replicas > 0: replicas = deployment.spec.replicas - LOG.info("Configured replica size (%d) for deployment is being ignored, as current running replica size" - " is different (%d) for %s", app_spec.autoscaler.min_replicas, deployment.spec.replicas, app_spec.name) + LOG.info( + "Configured replica size (%d) for deployment is being ignored, as current running replica size" + " is different (%d) for %s", + app_spec.autoscaler.min_replicas, + deployment.spec.replicas, + app_spec.name, + ) except NotFound: pass deployment_strategy = DeploymentStrategy( - rollingUpdate=RollingUpdateDeployment(maxUnavailable=self._max_unavailable, - maxSurge=self._max_surge)) + rollingUpdate=RollingUpdateDeployment(maxUnavailable=self._max_unavailable, maxSurge=self._max_surge) + ) if app_spec.autoscaler.max_replicas == 1 and app_spec.singleton: deployment_strategy = DeploymentStrategy( - rollingUpdate=RollingUpdateDeployment(maxUnavailable=1, maxSurge=0)) - spec = DeploymentSpec(replicas=replicas, selector=LabelSelector(matchLabels=selector), - template=pod_template_spec, revisionHistoryLimit=5, - strategy=deployment_strategy) + rollingUpdate=RollingUpdateDeployment(maxUnavailable=1, maxSurge=0) + ) + spec = DeploymentSpec( + replicas=replicas, + selector=LabelSelector(matchLabels=selector), + template=pod_template_spec, + revisionHistoryLimit=5, + strategy=deployment_strategy, + ) deployment = Deployment.get_or_create(metadata=metadata, spec=spec) - self._datadog.apply(deployment, app_spec, besteffort_qos_is_required, self._pre_stop_delay + self.DATADOG_PRE_STOP_DELAY) + self._datadog.apply( + deployment, app_spec, besteffort_qos_is_required, self._pre_stop_delay + self.DATADOG_PRE_STOP_DELAY + ) self._prometheus.apply(deployment, app_spec) self._secrets.apply(deployment, app_spec) self._owner_references.apply(deployment, app_spec) @@ -140,8 +186,12 @@ def delete(self, app_spec): def _make_volumes(self, app_spec): volumes = [] - volumes.append(Volume(name="{}-config".format(app_spec.name), - configMap=ConfigMapVolumeSource(name=app_spec.name, optional=True))) + volumes.append( + Volume( + name="{}-config".format(app_spec.name), + configMap=ConfigMapVolumeSource(name=app_spec.name, optional=True), + ) + ) if self._use_in_memory_emptydirs: empty_dir_volume_source = EmptyDirVolumeSource(medium="Memory") else: @@ -152,22 +202,25 @@ def _make_volumes(self, app_spec): def _make_volume_mounts(self, app_spec): volume_mounts = [] volume_mounts.append( - VolumeMount(name="{}-config".format(app_spec.name), readOnly=True, mountPath="/var/run/config/fiaas/")) + VolumeMount(name="{}-config".format(app_spec.name), readOnly=True, mountPath="/var/run/config/fiaas/") + ) volume_mounts.append(VolumeMount(name="tmp", readOnly=False, mountPath="/tmp")) return volume_mounts def _make_env(self, app_spec): fiaas_managed_env = { - 'FIAAS_ARTIFACT_NAME': app_spec.name, - 'FIAAS_IMAGE': app_spec.image, - 'FIAAS_VERSION': app_spec.version, + "FIAAS_ARTIFACT_NAME": app_spec.name, + "FIAAS_IMAGE": app_spec.image, + "FIAAS_VERSION": app_spec.version, } if not self._disable_deprecated_managed_env_vars: - fiaas_managed_env.update({ - 'ARTIFACT_NAME': app_spec.name, - 'IMAGE': app_spec.image, - 'VERSION': app_spec.version, - }) + fiaas_managed_env.update( + { + "ARTIFACT_NAME": app_spec.name, + "IMAGE": app_spec.image, + "VERSION": app_spec.version, + } + ) # fiaas_managed_env overrides global_env overrides legacy_fiaas_env static_env = merge_dicts(self._legacy_fiaas_env, self._global_env, fiaas_managed_env) @@ -175,23 +228,50 @@ def _make_env(self, app_spec): env = [EnvVar(name=name, value=value) for name, value in list(static_env.items())] # FIAAS managed environment variables using the downward API - env.extend([ - EnvVar(name="FIAAS_REQUESTS_CPU", valueFrom=EnvVarSource( - resourceFieldRef=ResourceFieldSelector(containerName=app_spec.name, resource="requests.cpu", - divisor=1))), - EnvVar(name="FIAAS_REQUESTS_MEMORY", valueFrom=EnvVarSource( - resourceFieldRef=ResourceFieldSelector(containerName=app_spec.name, resource="requests.memory", - divisor=1))), - EnvVar(name="FIAAS_LIMITS_CPU", valueFrom=EnvVarSource( - resourceFieldRef=ResourceFieldSelector(containerName=app_spec.name, resource="limits.cpu", divisor=1))), - EnvVar(name="FIAAS_LIMITS_MEMORY", valueFrom=EnvVarSource( - resourceFieldRef=ResourceFieldSelector(containerName=app_spec.name, resource="limits.memory", - divisor=1))), - EnvVar(name="FIAAS_NAMESPACE", valueFrom=EnvVarSource( - fieldRef=ObjectFieldSelector(fieldPath="metadata.namespace"))), - EnvVar(name="FIAAS_POD_NAME", valueFrom=EnvVarSource( - fieldRef=ObjectFieldSelector(fieldPath="metadata.name"))), - ]) + env.extend( + [ + EnvVar( + name="FIAAS_REQUESTS_CPU", + valueFrom=EnvVarSource( + resourceFieldRef=ResourceFieldSelector( + containerName=app_spec.name, resource="requests.cpu", divisor=1 + ) + ), + ), + EnvVar( + name="FIAAS_REQUESTS_MEMORY", + valueFrom=EnvVarSource( + resourceFieldRef=ResourceFieldSelector( + containerName=app_spec.name, resource="requests.memory", divisor=1 + ) + ), + ), + EnvVar( + name="FIAAS_LIMITS_CPU", + valueFrom=EnvVarSource( + resourceFieldRef=ResourceFieldSelector( + containerName=app_spec.name, resource="limits.cpu", divisor=1 + ) + ), + ), + EnvVar( + name="FIAAS_LIMITS_MEMORY", + valueFrom=EnvVarSource( + resourceFieldRef=ResourceFieldSelector( + containerName=app_spec.name, resource="limits.memory", divisor=1 + ) + ), + ), + EnvVar( + name="FIAAS_NAMESPACE", + valueFrom=EnvVarSource(fieldRef=ObjectFieldSelector(fieldPath="metadata.namespace")), + ), + EnvVar( + name="FIAAS_POD_NAME", + valueFrom=EnvVarSource(fieldRef=ObjectFieldSelector(fieldPath="metadata.name")), + ), + ] + ) env.sort(key=lambda x: x.name) @@ -200,9 +280,7 @@ def _make_env(self, app_spec): def _add_status_label(labels): copy = labels.copy() - copy.update({ - "fiaas/status": "active" - }) + copy.update({"fiaas/status": "active"}) return copy @@ -214,15 +292,21 @@ def as_dict(resource_requirement_spec): def _make_probe(check_spec): - probe = Probe(initialDelaySeconds=check_spec.initial_delay_seconds, - timeoutSeconds=check_spec.timeout_seconds, - successThreshold=check_spec.success_threshold, - failureThreshold=check_spec.failure_threshold, - periodSeconds=check_spec.period_seconds) + probe = Probe( + initialDelaySeconds=check_spec.initial_delay_seconds, + timeoutSeconds=check_spec.timeout_seconds, + successThreshold=check_spec.success_threshold, + failureThreshold=check_spec.failure_threshold, + periodSeconds=check_spec.period_seconds, + ) if check_spec.http: - probe.httpGet = HTTPGetAction(path=check_spec.http.path, port=check_spec.http.port, - httpHeaders=[HTTPHeader(name=name, value=value) - for name, value in list(check_spec.http.http_headers.items())]) + probe.httpGet = HTTPGetAction( + path=check_spec.http.path, + port=check_spec.http.port, + httpHeaders=[ + HTTPHeader(name=name, value=value) for name, value in list(check_spec.http.http_headers.items()) + ], + ) elif check_spec.tcp: probe.tcpSocket = TCPSocketAction(port=check_spec.tcp.port) elif check_spec.execute: @@ -241,11 +325,15 @@ def _build_fiaas_env(config): "CONSTRETTO_TAGS": "kubernetes", # DEPRECATED. Remove in the future. } if config.environment: - env.update({ - "FINN_ENV": config.environment, # DEPRECATED. Remove in the future. - "FIAAS_ENVIRONMENT": config.environment, - "CONSTRETTO_TAGS": ",".join(("kubernetes-{}".format(config.environment), "kubernetes", config.environment)), - }) + env.update( + { + "FINN_ENV": config.environment, # DEPRECATED. Remove in the future. + "FIAAS_ENVIRONMENT": config.environment, + "CONSTRETTO_TAGS": ",".join( + ("kubernetes-{}".format(config.environment), "kubernetes", config.environment) + ), + } + ) return env @@ -254,5 +342,5 @@ def _build_global_env(global_env): global_env key/value are added as is and with the key prefix FIAAS_ """ _global_env_copy = global_env.copy() - _global_env_copy.update({'FIAAS_{}'.format(k): v for k, v in list(_global_env_copy.items())}) + _global_env_copy.update({"FIAAS_{}".format(k): v for k, v in list(_global_env_copy.items())}) return _global_env_copy diff --git a/fiaas_deploy_daemon/deployer/kubernetes/deployment/prometheus.py b/fiaas_deploy_daemon/deployer/kubernetes/deployment/prometheus.py index 408e6668..d5d915da 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/deployment/prometheus.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/deployment/prometheus.py @@ -43,5 +43,5 @@ def _make_prometheus_annotations(app_spec): return { "prometheus.io/scrape": str(prometheus_spec.enabled).lower(), "prometheus.io/port": str(port), - "prometheus.io/path": prometheus_spec.path + "prometheus.io/path": prometheus_spec.path, } diff --git a/fiaas_deploy_daemon/deployer/kubernetes/deployment/secrets.py b/fiaas_deploy_daemon/deployer/kubernetes/deployment/secrets.py index fcb83888..0c1faadb 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/deployment/secrets.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/deployment/secrets.py @@ -16,8 +16,18 @@ # limitations under the License. import logging -from k8s.models.pod import EnvFromSource, SecretEnvSource, EnvVar, Container, ConfigMapEnvSource, VolumeMount, Volume, \ - EmptyDirVolumeSource, ConfigMapVolumeSource, SecretVolumeSource +from k8s.models.pod import ( + EnvFromSource, + SecretEnvSource, + EnvVar, + Container, + ConfigMapEnvSource, + VolumeMount, + Volume, + EmptyDirVolumeSource, + ConfigMapVolumeSource, + SecretVolumeSource, +) from fiaas_deploy_daemon.tools import merge_dicts from fiaas_deploy_daemon.specs.models import SecretsSpec @@ -35,17 +45,11 @@ def _legacy_strongbox_secrets_spec(self, app_spec): "AWS_REGION": app_spec.strongbox.aws_region, "SECRET_GROUPS": ",".join(app_spec.strongbox.groups), } - strongbox_annotations = { - "iam.amazonaws.com/role": app_spec.strongbox.iam_role - } - return SecretsSpec(type="strongbox", - parameters=strongbox_params, - annotations=strongbox_annotations) + strongbox_annotations = {"iam.amazonaws.com/role": app_spec.strongbox.iam_role} + return SecretsSpec(type="strongbox", parameters=strongbox_params, annotations=strongbox_annotations) def _default_init_secret(self): - return SecretsSpec(type="default", - parameters={}, - annotations={}) + return SecretsSpec(type="default", parameters={}, annotations={}) def apply(self, deployment, app_spec): secret_specs = app_spec.secrets @@ -90,15 +94,20 @@ def _apply_mounts(self, app_spec, main_container): def _make_volumes(self, app_spec): volumes = [ - Volume(name="{}-secret".format(app_spec.name), - secret=SecretVolumeSource(secretName=app_spec.name, optional=True)), + Volume( + name="{}-secret".format(app_spec.name), + secret=SecretVolumeSource(secretName=app_spec.name, optional=True), + ), ] return volumes def _make_volume_mounts(self, app_spec, is_init_container=False): volume_mounts = [ - VolumeMount(name="{}-secret".format(app_spec.name), readOnly=not is_init_container, - mountPath="/var/run/secrets/fiaas/"), + VolumeMount( + name="{}-secret".format(app_spec.name), + readOnly=not is_init_container, + mountPath="/var/run/secrets/fiaas/", + ), ] return volume_mounts @@ -112,19 +121,23 @@ def __init__(self, config): self._automount_service_account_token = False if config.strongbox_init_container_image is not None: if "strongbox" in self._available_secrets_containers: - LOG.warn("Image %s for strongbox-init-container-image will be ignored.\ + LOG.warn( + "Image %s for strongbox-init-container-image will be ignored.\ secret-init-container strongbox=%s will be used instead", - config.strongbox_init_container_image, - self._available_secrets_containers["strongbox"]) + config.strongbox_init_container_image, + self._available_secrets_containers["strongbox"], + ) else: self._available_secrets_containers["strongbox"] = config.strongbox_init_container_image if config.secrets_init_container_image is not None: if "default" in self._available_secrets_containers: - LOG.warn("Image %s for secrets-init-container-image will be ignored.\ + LOG.warn( + "Image %s for secrets-init-container-image will be ignored.\ secret-init-container default=%s will be used instead", - config.secrets_init_container_image, - self._available_secrets_containers["default"]) + config.secrets_init_container_image, + self._available_secrets_containers["default"], + ) else: self._automount_service_account_token = True self._available_secrets_containers["default"] = config.secrets_init_container_image @@ -168,8 +181,10 @@ def _make_volumes(self, app_spec): empty_dir_volume_source = EmptyDirVolumeSource() volumes = [ Volume(name="{}-secret".format(app_spec.name), emptyDir=empty_dir_volume_source), - Volume(name="{}-config".format(self.SECRETS_INIT_CONTAINER_NAME), - configMap=ConfigMapVolumeSource(name=self.SECRETS_INIT_CONTAINER_NAME, optional=True)), + Volume( + name="{}-config".format(self.SECRETS_INIT_CONTAINER_NAME), + configMap=ConfigMapVolumeSource(name=self.SECRETS_INIT_CONTAINER_NAME, optional=True), + ), ] return volumes @@ -177,8 +192,11 @@ def _make_volume_mounts(self, app_spec, is_init_container=False): volume_mounts = super(GenericInitSecrets, self)._make_volume_mounts(app_spec, is_init_container) if is_init_container: init_container_mounts = [ - VolumeMount(name="{}-config".format(self.SECRETS_INIT_CONTAINER_NAME), readOnly=True, - mountPath="/var/run/config/{}/".format(self.SECRETS_INIT_CONTAINER_NAME)), + VolumeMount( + name="{}-config".format(self.SECRETS_INIT_CONTAINER_NAME), + readOnly=True, + mountPath="/var/run/config/{}/".format(self.SECRETS_INIT_CONTAINER_NAME), + ), VolumeMount(name="{}-config".format(app_spec.name), readOnly=True, mountPath="/var/run/config/fiaas/"), VolumeMount(name="tmp", readOnly=False, mountPath="/tmp"), ] @@ -190,13 +208,14 @@ def _make_secrets_init_container(self, app_spec, image, env_vars=None): env_vars = {} env_vars.update({"K8S_DEPLOYMENT": app_spec.name}) environment = [EnvVar(name=k, value=v) for k, v in list(env_vars.items())] - container = Container(name=self.SECRETS_INIT_CONTAINER_NAME, - image=image, - imagePullPolicy="IfNotPresent", - env=environment, - envFrom=[ - EnvFromSource(configMapRef=ConfigMapEnvSource(name=self.SECRETS_INIT_CONTAINER_NAME, - optional=True)) - ], - volumeMounts=self._make_volume_mounts(app_spec, is_init_container=True)) + container = Container( + name=self.SECRETS_INIT_CONTAINER_NAME, + image=image, + imagePullPolicy="IfNotPresent", + env=environment, + envFrom=[ + EnvFromSource(configMapRef=ConfigMapEnvSource(name=self.SECRETS_INIT_CONTAINER_NAME, optional=True)) + ], + volumeMounts=self._make_volume_mounts(app_spec, is_init_container=True), + ) return container diff --git a/fiaas_deploy_daemon/deployer/kubernetes/ingress.py b/fiaas_deploy_daemon/deployer/kubernetes/ingress.py index f00842c3..aa5bd6bf 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/ingress.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/ingress.py @@ -35,8 +35,9 @@ def __init__(self, config, default_app_spec, ingress_adapter): self._host_rewrite_rules = config.host_rewrite_rules self._ingress_adapter = ingress_adapter self._tls_issuer_type_default = config.tls_certificate_issuer_type_default - self._tls_issuer_type_overrides = sorted(iter(config.tls_certificate_issuer_type_overrides.items()), - key=lambda k_v: len(k_v[0]), reverse=True) + self._tls_issuer_type_overrides = sorted( + iter(config.tls_certificate_issuer_type_overrides.items()), key=lambda k_v: len(k_v[0]), reverse=True + ) def deploy(self, app_spec, labels): if self._should_have_ingress(app_spec): @@ -66,8 +67,11 @@ def _create(self, app_spec, labels): def _expand_default_hosts(self, app_spec): all_pathmappings = list( - deduplicate_in_order(chain.from_iterable( - ingress_item.pathmappings for ingress_item in app_spec.ingresses if not ingress_item.annotations)) + deduplicate_in_order( + chain.from_iterable( + ingress_item.pathmappings for ingress_item in app_spec.ingresses if not ingress_item.annotations + ) + ) ) if not all_pathmappings: @@ -76,8 +80,10 @@ def _expand_default_hosts(self, app_spec): default_path = self._resolve_default_path() all_pathmappings = [IngressPathMappingSpec(path=default_path, port=http_port)] - return [IngressItemSpec(host=host, pathmappings=all_pathmappings, annotations=None) - for host in self._generate_default_hosts(app_spec.name)] + return [ + IngressItemSpec(host=host, pathmappings=all_pathmappings, annotations=None) + for host in self._generate_default_hosts(app_spec.name) + ] @staticmethod def _resolve_http_port(app_spec): @@ -98,37 +104,53 @@ def _get_issuer_type(self, host): return self._tls_issuer_type_default def _group_ingresses(self, app_spec): - ''' Group the ingresses so that those with annotations are individual, and so that those using non-default TLS-issuers + """Group the ingresses so that those with annotations are individual, and so that those using non-default TLS-issuers are separated - ''' + """ explicit_host = _has_explicitly_set_host(app_spec.ingresses) - ingress_items = [item._replace(host=self._apply_host_rewrite_rules(item.host)) for item in app_spec.ingresses if item.host] + ingress_items = [ + item._replace(host=self._apply_host_rewrite_rules(item.host)) for item in app_spec.ingresses if item.host + ] ingress_items += self._expand_default_hosts(app_spec) - AnnotatedIngress = namedtuple("AnnotatedIngress", ["name", "ingress_items", "annotations", "explicit_host", - "issuer_type", "default"]) - default_ingress = AnnotatedIngress(name=app_spec.name, ingress_items=[], annotations={}, - explicit_host=explicit_host, issuer_type=self._tls_issuer_type_default, - default=True) + AnnotatedIngress = namedtuple( + "AnnotatedIngress", ["name", "ingress_items", "annotations", "explicit_host", "issuer_type", "default"] + ) + default_ingress = AnnotatedIngress( + name=app_spec.name, + ingress_items=[], + annotations={}, + explicit_host=explicit_host, + issuer_type=self._tls_issuer_type_default, + default=True, + ) ingresses = [default_ingress] override_issuer_ingresses = {} for ingress_item in ingress_items: issuer_type = self._get_issuer_type(ingress_item.host) next_name = "{}-{}".format(app_spec.name, len(ingresses) + len(override_issuer_ingresses)) if ingress_item.annotations: - annotated_ingresses = AnnotatedIngress(name=next_name, ingress_items=[ingress_item], - annotations=ingress_item.annotations, - explicit_host=True, issuer_type=issuer_type, - default=False) + annotated_ingresses = AnnotatedIngress( + name=next_name, + ingress_items=[ingress_item], + annotations=ingress_item.annotations, + explicit_host=True, + issuer_type=issuer_type, + default=False, + ) ingresses.append(annotated_ingresses) elif issuer_type != self._tls_issuer_type_default: - annotated_ingress = override_issuer_ingresses.setdefault(issuer_type, - AnnotatedIngress(name=next_name, - ingress_items=[], - annotations={}, - explicit_host=explicit_host, - issuer_type=issuer_type, - default=False)) + annotated_ingress = override_issuer_ingresses.setdefault( + issuer_type, + AnnotatedIngress( + name=next_name, + ingress_items=[], + annotations={}, + explicit_host=explicit_host, + issuer_type=issuer_type, + default=False, + ), + ) annotated_ingress.ingress_items.append(ingress_item) else: default_ingress.ingress_items.append(ingress_item) @@ -154,9 +176,11 @@ def _can_generate_host(self, app_spec): return len(self._ingress_suffixes) > 0 or _has_explicitly_set_host(app_spec.ingresses) def _get_hosts(self, app_spec): - return list(self._generate_default_hosts(app_spec.name)) + \ - [self._apply_host_rewrite_rules(ingress_item.host) - for ingress_item in app_spec.ingresses if ingress_item.host is not None] + return list(self._generate_default_hosts(app_spec.name)) + [ + self._apply_host_rewrite_rules(ingress_item.host) + for ingress_item in app_spec.ingresses + if ingress_item.host is not None + ] def _has_explicitly_set_host(ingress_items): @@ -191,13 +215,16 @@ def apply(self, ingress, app_spec, hosts, issuer_type, use_suffixes=True): if self._should_have_ingress_tls(app_spec): tls_annotations = {} if self._cert_issuer or app_spec.ingress_tls.certificate_issuer: - issuer = app_spec.ingress_tls.certificate_issuer if app_spec.ingress_tls.certificate_issuer else self._cert_issuer + issuer = ( + app_spec.ingress_tls.certificate_issuer + if app_spec.ingress_tls.certificate_issuer + else self._cert_issuer + ) tls_annotations[issuer_type] = issuer else: tls_annotations["kubernetes.io/tls-acme"] = "true" ingress.metadata.annotations = merge_dicts( - ingress.metadata.annotations if ingress.metadata.annotations else {}, - tls_annotations + ingress.metadata.annotations if ingress.metadata.annotations else {}, tls_annotations ) if self.enable_deprecated_tls_entry_per_host: @@ -212,7 +239,9 @@ def apply(self, ingress, app_spec, hosts, issuer_type, use_suffixes=True): # as the user doesn't control it we should generate a host we know will fit hosts = self._collapse_hosts(app_spec, hosts) - ingress.spec.tls.append(self.ingress_tls(hosts=hosts, secretName="{}-ingress-tls".format(ingress.metadata.name))) + ingress.spec.tls.append( + self.ingress_tls(hosts=hosts, secretName="{}-ingress-tls".format(ingress.metadata.name)) + ) def _collapse_hosts(self, app_spec, hosts): """The first hostname in the list will be used as Common Name in the certificate""" @@ -224,17 +253,17 @@ def _collapse_hosts(self, app_spec, hosts): return hosts def _should_have_ingress_tls(self, app_spec): - if self._use_ingress_tls == 'disabled' or app_spec.ingress_tls.enabled is False: + if self._use_ingress_tls == "disabled" or app_spec.ingress_tls.enabled is False: return False else: - return self._use_ingress_tls == 'default_on' or app_spec.ingress_tls.enabled is True + return self._use_ingress_tls == "default_on" or app_spec.ingress_tls.enabled is True def _generate_short_host(self, app_spec): h = hashlib.sha1() h.update(app_spec.name.encode("utf-8")) h.update(app_spec.namespace.encode("utf-8")) prefix = base64.b32encode(h.digest()).decode("utf-8").strip("=").lower() - short_prefix = prefix[:62 - len(self._shortest_suffix)] + short_prefix = prefix[: 62 - len(self._shortest_suffix)] short_name = "{}.{}".format(short_prefix, self._shortest_suffix) if len(short_name) > 63 or short_name[0] == ".": raise ValueError("Unable to generate a name short enough to be Common Name in certificate") diff --git a/fiaas_deploy_daemon/deployer/kubernetes/ingress_networkingv1.py b/fiaas_deploy_daemon/deployer/kubernetes/ingress_networkingv1.py index 07418ae1..d611f958 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/ingress_networkingv1.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/ingress_networkingv1.py @@ -18,8 +18,16 @@ from k8s.client import NotFound from k8s.models.common import ObjectMeta -from k8s.models.networking_v1_ingress import Ingress, IngressSpec, IngressRule, HTTPIngressRuleValue, HTTPIngressPath, IngressBackend, \ - IngressServiceBackend, ServiceBackendPort +from k8s.models.networking_v1_ingress import ( + Ingress, + IngressSpec, + IngressRule, + HTTPIngressRuleValue, + HTTPIngressPath, + IngressBackend, + IngressServiceBackend, + ServiceBackendPort, +) from k8s.base import Equality, Inequality, Exists from fiaas_deploy_daemon.retry import retry_on_upsert_conflict @@ -36,17 +44,19 @@ def __init__(self, ingress_tls_deployer, owner_references, extension_hook): @retry_on_upsert_conflict def create_ingress(self, app_spec, annotated_ingress, labels): - default_annotations = { - "fiaas/expose": "true" if annotated_ingress.explicit_host else "false" - } + default_annotations = {"fiaas/expose": "true" if annotated_ingress.explicit_host else "false"} annotations = merge_dicts(app_spec.annotations.ingress, annotated_ingress.annotations, default_annotations) - metadata = ObjectMeta(name=annotated_ingress.name, namespace=app_spec.namespace, labels=labels, - annotations=annotations) + metadata = ObjectMeta( + name=annotated_ingress.name, namespace=app_spec.namespace, labels=labels, annotations=annotations + ) per_host_ingress_rules = [ - IngressRule(host=ingress_item.host, http=self._make_http_ingress_rule_value(app_spec, ingress_item.pathmappings)) - for ingress_item in annotated_ingress.ingress_items if ingress_item.host is not None + IngressRule( + host=ingress_item.host, http=self._make_http_ingress_rule_value(app_spec, ingress_item.pathmappings) + ) + for ingress_item in annotated_ingress.ingress_items + if ingress_item.host is not None ] if annotated_ingress.default: use_suffixes = True @@ -58,7 +68,9 @@ def create_ingress(self, app_spec, annotated_ingress, labels): ingress = Ingress.get_or_create(metadata=metadata, spec=ingress_spec) hosts_for_tls = [rule.host for rule in per_host_ingress_rules] - self._ingress_tls_deployer.apply(ingress, app_spec, hosts_for_tls, annotated_ingress.issuer_type, use_suffixes=use_suffixes) + self._ingress_tls_deployer.apply( + ingress, app_spec, hosts_for_tls, annotated_ingress.issuer_type, use_suffixes=use_suffixes + ) self._owner_references.apply(ingress, app_spec) self._extension_hook.apply(ingress, app_spec) ingress.save() @@ -67,14 +79,15 @@ def delete_unused(self, app_spec, labels): filter_labels = [ ("app", Equality(labels["app"])), ("fiaas/deployment_id", Exists()), - ("fiaas/deployment_id", Inequality(labels["fiaas/deployment_id"])) + ("fiaas/deployment_id", Inequality(labels["fiaas/deployment_id"])), ] Ingress.delete_list(namespace=app_spec.namespace, labels=filter_labels) def delete_list(self, app_spec): try: - Ingress.delete_list(namespace=app_spec.namespace, - labels={"app": Equality(app_spec.name), "fiaas/deployment_id": Exists()}) + Ingress.delete_list( + namespace=app_spec.namespace, labels={"app": Equality(app_spec.name), "fiaas/deployment_id": Exists()} + ) except NotFound: pass @@ -86,8 +99,11 @@ def _make_http_ingress_rule_value(self, app_spec, pathmappings): HTTPIngressPath( path=pm.path, pathType="ImplementationSpecific", - backend=IngressBackend(service=IngressServiceBackend(name=app_spec.name, port=ServiceBackendPort(number=pm.port))), + backend=IngressBackend( + service=IngressServiceBackend(name=app_spec.name, port=ServiceBackendPort(number=pm.port)) + ), ) - for pm in deduplicate_in_order(pathmappings)] + for pm in deduplicate_in_order(pathmappings) + ] return HTTPIngressRuleValue(paths=http_ingress_paths) diff --git a/fiaas_deploy_daemon/deployer/kubernetes/ingress_v1beta1.py b/fiaas_deploy_daemon/deployer/kubernetes/ingress_v1beta1.py index 6d8cf11d..01896d87 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/ingress_v1beta1.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/ingress_v1beta1.py @@ -35,17 +35,19 @@ def __init__(self, ingress_tls_deployer, owner_references, extension_hook): @retry_on_upsert_conflict def create_ingress(self, app_spec, annotated_ingress, labels): - default_annotations = { - "fiaas/expose": "true" if annotated_ingress.explicit_host else "false" - } + default_annotations = {"fiaas/expose": "true" if annotated_ingress.explicit_host else "false"} annotations = merge_dicts(app_spec.annotations.ingress, annotated_ingress.annotations, default_annotations) - metadata = ObjectMeta(name=annotated_ingress.name, namespace=app_spec.namespace, labels=labels, - annotations=annotations) + metadata = ObjectMeta( + name=annotated_ingress.name, namespace=app_spec.namespace, labels=labels, annotations=annotations + ) per_host_ingress_rules = [ - IngressRule(host=ingress_item.host, http=self._make_http_ingress_rule_value(app_spec, ingress_item.pathmappings)) - for ingress_item in annotated_ingress.ingress_items if ingress_item.host is not None + IngressRule( + host=ingress_item.host, http=self._make_http_ingress_rule_value(app_spec, ingress_item.pathmappings) + ) + for ingress_item in annotated_ingress.ingress_items + if ingress_item.host is not None ] if annotated_ingress.default: use_suffixes = True @@ -57,7 +59,9 @@ def create_ingress(self, app_spec, annotated_ingress, labels): ingress = Ingress.get_or_create(metadata=metadata, spec=ingress_spec) hosts_for_tls = [rule.host for rule in per_host_ingress_rules] - self._ingress_tls_deployer.apply(ingress, app_spec, hosts_for_tls, annotated_ingress.issuer_type, use_suffixes=use_suffixes) + self._ingress_tls_deployer.apply( + ingress, app_spec, hosts_for_tls, annotated_ingress.issuer_type, use_suffixes=use_suffixes + ) self._owner_references.apply(ingress, app_spec) self._extension_hook.apply(ingress, app_spec) ingress.save() @@ -66,14 +70,15 @@ def delete_unused(self, app_spec, labels): filter_labels = [ ("app", Equality(labels["app"])), ("fiaas/deployment_id", Exists()), - ("fiaas/deployment_id", Inequality(labels["fiaas/deployment_id"])) + ("fiaas/deployment_id", Inequality(labels["fiaas/deployment_id"])), ] Ingress.delete_list(namespace=app_spec.namespace, labels=filter_labels) def delete_list(self, app_spec): try: - Ingress.delete_list(namespace=app_spec.namespace, - labels={"app": Equality(app_spec.name), "fiaas/deployment_id": Exists()}) + Ingress.delete_list( + namespace=app_spec.namespace, labels={"app": Equality(app_spec.name), "fiaas/deployment_id": Exists()} + ) except NotFound: pass @@ -83,6 +88,7 @@ def find(self, name, namespace): def _make_http_ingress_rule_value(self, app_spec, pathmappings): http_ingress_paths = [ HTTPIngressPath(path=pm.path, backend=IngressBackend(serviceName=app_spec.name, servicePort=pm.port)) - for pm in deduplicate_in_order(pathmappings)] + for pm in deduplicate_in_order(pathmappings) + ] return HTTPIngressRuleValue(paths=http_ingress_paths) diff --git a/fiaas_deploy_daemon/deployer/kubernetes/owner_references.py b/fiaas_deploy_daemon/deployer/kubernetes/owner_references.py index aa7b045a..e51454ed 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/owner_references.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/owner_references.py @@ -2,14 +2,15 @@ class OwnerReferences(object): - def apply(self, k8s_model, app_spec): if app_spec.uid: - owner_reference = OwnerReference(apiVersion="fiaas.schibsted.io/v1", - blockOwnerDeletion=True, - controller=True, - kind="Application", - name=app_spec.name, - uid=app_spec.uid) + owner_reference = OwnerReference( + apiVersion="fiaas.schibsted.io/v1", + blockOwnerDeletion=True, + controller=True, + kind="Application", + name=app_spec.name, + uid=app_spec.uid, + ) k8s_model.metadata.ownerReferences = [owner_reference] diff --git a/fiaas_deploy_daemon/deployer/kubernetes/ready_check.py b/fiaas_deploy_daemon/deployer/kubernetes/ready_check.py index 6b411e48..a3db73b9 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/ready_check.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/ready_check.py @@ -36,7 +36,7 @@ def __init__(self, app_spec, bookkeeper, lifecycle, lifecycle_subject, ingress_a self._fail_after_seconds = _calculate_fail_time( config.ready_check_timeout_multiplier, app_spec.autoscaler.max_replicas, - app_spec.health_checks.readiness.initial_delay_seconds + app_spec.health_checks.readiness.initial_delay_seconds, ) self._fail_after = time_monotonic() + self._fail_after_seconds self._should_check_ingress = config.tls_certificate_ready @@ -48,8 +48,11 @@ def __call__(self): self._bookkeeper.success(self._app_spec) return False if time_monotonic() >= self._fail_after: - LOG.error("Timed out after %d seconds waiting for %s to become ready", - self._fail_after_seconds, self._app_spec.name) + LOG.error( + "Timed out after %d seconds waiting for %s to become ready", + self._fail_after_seconds, + self._app_spec.name, + ) self._lifecycle.failed(self._lifecycle_subject) self._bookkeeper.failed(self._app_spec) return False @@ -62,10 +65,12 @@ def _deployment_ready(self): return False expected_value = dep.spec.replicas if dep.spec.replicas > 0 else None - return (dep.status.updatedReplicas == expected_value and - dep.status.replicas == expected_value and - dep.status.availableReplicas == expected_value and - dep.status.observedGeneration >= dep.metadata.generation) + return ( + dep.status.updatedReplicas == expected_value + and dep.status.replicas == expected_value + and dep.status.availableReplicas == expected_value + and dep.status.observedGeneration >= dep.metadata.generation + ) def _is_certificate_ready(self, cert): if cert.status.notAfter and (cert.status.notAfter < datetime.now(cert.status.notAfter.tzinfo)): @@ -100,9 +105,12 @@ def _ready(self): return self._deployment_ready() and self._ingress_ready() def __eq__(self, other): - return other._app_spec == self._app_spec and other._bookkeeper == self._bookkeeper \ - and other._lifecycle == self._lifecycle + return ( + other._app_spec == self._app_spec + and other._bookkeeper == self._bookkeeper + and other._lifecycle == self._lifecycle + ) def _calculate_fail_time(time_out, replicas, delay_seconds): - return time_out*delay_seconds if replicas == 0 else time_out*delay_seconds*replicas + return time_out * delay_seconds if replicas == 0 else time_out * delay_seconds * replicas diff --git a/fiaas_deploy_daemon/deployer/kubernetes/service.py b/fiaas_deploy_daemon/deployer/kubernetes/service.py index d7c8246c..bb4c7c37 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/service.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/service.py @@ -60,7 +60,9 @@ def _create(self, app_spec, selector, labels): service_name = app_spec.name custom_labels = merge_dicts(app_spec.labels.service, labels) custom_annotations = merge_dicts(app_spec.annotations.service, self._make_tcp_port_annotation(app_spec)) - metadata = ObjectMeta(name=service_name, namespace=app_spec.namespace, labels=custom_labels, annotations=custom_annotations) + metadata = ObjectMeta( + name=service_name, namespace=app_spec.namespace, labels=custom_labels, annotations=custom_annotations + ) spec = ServiceSpec(selector=selector, ports=ports, type=self._service_type) svc = Service.get_or_create(metadata=metadata, spec=spec) self._owner_references.apply(svc, app_spec) @@ -78,19 +80,12 @@ def _merge_ports(existing_ports, wanted_ports): @staticmethod def _make_service_port(port_spec): - return ServicePort( - protocol='TCP', - name=port_spec.name, - port=port_spec.port, - targetPort=port_spec.target_port) + return ServicePort(protocol="TCP", name=port_spec.name, port=port_spec.port, targetPort=port_spec.target_port) @staticmethod def _make_tcp_port_annotation(app_spec): - tcp_port_names = [port_spec.name for port_spec in app_spec.ports - if port_spec.protocol == "tcp"] - return { - 'fiaas/tcp_port_names': ','.join(map(str, tcp_port_names)) - } if tcp_port_names else {} + tcp_port_names = [port_spec.name for port_spec in app_spec.ports if port_spec.protocol == "tcp"] + return {"fiaas/tcp_port_names": ",".join(map(str, tcp_port_names))} if tcp_port_names else {} @staticmethod def _should_have_service(app_spec): diff --git a/fiaas_deploy_daemon/deployer/kubernetes/service_account.py b/fiaas_deploy_daemon/deployer/kubernetes/service_account.py index c43ac0f5..e7a90ca7 100644 --- a/fiaas_deploy_daemon/deployer/kubernetes/service_account.py +++ b/fiaas_deploy_daemon/deployer/kubernetes/service_account.py @@ -46,12 +46,18 @@ def _create(self, app_spec, labels): custom_labels = labels custom_labels = merge_dicts(app_spec.labels.service_account, custom_labels) custom_annotations = merge_dicts(app_spec.annotations.service_account, custom_annotations) - metadata = ObjectMeta(name=service_account_name, namespace=namespace, labels=custom_labels, annotations=custom_annotations) + metadata = ObjectMeta( + name=service_account_name, namespace=namespace, labels=custom_labels, annotations=custom_annotations + ) try: service_account = ServiceAccount.get(service_account_name, namespace) if not self._owned_by_fiaas(service_account): LOG.info("Found serviceAccount %s not managed by us.", service_account_name) - LOG.info("Aborting the creation of a serviceAccount for Application: %s with labels: %s", service_account_name, labels) + LOG.info( + "Aborting the creation of a serviceAccount for Application: %s with labels: %s", + service_account_name, + labels, + ) return except NotFound: service_account = ServiceAccount() @@ -70,5 +76,6 @@ def _create(self, app_spec, labels): def _owned_by_fiaas(self, service_account): return any( - ref.apiVersion == 'fiaas.schibsted.io/v1' and ref.kind == 'Application' for ref in service_account.metadata.ownerReferences + ref.apiVersion == "fiaas.schibsted.io/v1" and ref.kind == "Application" + for ref in service_account.metadata.ownerReferences ) diff --git a/fiaas_deploy_daemon/extension_hook_caller.py b/fiaas_deploy_daemon/extension_hook_caller.py index 8c9dd9b3..4c695aec 100644 --- a/fiaas_deploy_daemon/extension_hook_caller.py +++ b/fiaas_deploy_daemon/extension_hook_caller.py @@ -23,7 +23,6 @@ class ExtensionHookCaller(object): - def __init__(self, config, session): self._url = config.extension_hook_url self._session = session @@ -35,9 +34,7 @@ def apply(self, obj, app_spec): url = posixpath.join(url, type(obj).__name__) dump = json.dumps({"object": obj.as_dict(), "application": app_spec.app_config}) response = self._session.post( - url, - data=dump, - headers={'Content-Type': 'application/json', 'Accept': 'application/json'} + url, data=dump, headers={"Content-Type": "application/json", "Accept": "application/json"} ) if response.status_code == 200: data = response.json() diff --git a/fiaas_deploy_daemon/lifecycle.py b/fiaas_deploy_daemon/lifecycle.py index 41baba9e..1a810b5d 100644 --- a/fiaas_deploy_daemon/lifecycle.py +++ b/fiaas_deploy_daemon/lifecycle.py @@ -26,7 +26,9 @@ STATUS_INITIATED = "initiated" -Subject = namedtuple("Subject", ("uid", "app_name", "namespace", "deployment_id", "repository", "labels", "annotations")) +Subject = namedtuple( + "Subject", ("uid", "app_name", "namespace", "deployment_id", "repository", "labels", "annotations") +) class Lifecycle(object): diff --git a/fiaas_deploy_daemon/log_extras.py b/fiaas_deploy_daemon/log_extras.py index c6c5b22b..1c63f8d3 100644 --- a/fiaas_deploy_daemon/log_extras.py +++ b/fiaas_deploy_daemon/log_extras.py @@ -21,8 +21,9 @@ _LOGS = defaultdict(list) _LOG_EXTRAS = threading.local() -_LOG_FORMAT = "[%(asctime)s|%(levelname)7s] %(message)s " \ - "[%(name)s|%(threadName)s|%(extras_namespace)s/%(extras_app_name)s]" +_LOG_FORMAT = ( + "[%(asctime)s|%(levelname)7s] %(message)s " "[%(name)s|%(threadName)s|%(extras_namespace)s/%(extras_app_name)s]" +) class ExtraFilter(logging.Filter): diff --git a/fiaas_deploy_daemon/logsetup.py b/fiaas_deploy_daemon/logsetup.py index 27ad7d5a..355a5fef 100644 --- a/fiaas_deploy_daemon/logsetup.py +++ b/fiaas_deploy_daemon/logsetup.py @@ -27,8 +27,19 @@ class FiaasFormatter(logging.Formatter): UNWANTED = ( - "msg", "args", "exc_info", "exc_text", "levelno", "created", "msecs", "relativeCreated", "funcName", - "filename", "lineno", "module") + "msg", + "args", + "exc_info", + "exc_text", + "levelno", + "created", + "msecs", + "relativeCreated", + "funcName", + "filename", + "lineno", + "module", + ) RENAME = { "levelname": "level", "threadName": "thread", @@ -73,7 +84,7 @@ def _build_location(fields): "method": fields["funcName"], "file": fields["filename"], "line": fields["lineno"], - "module": fields["module"] + "module": fields["module"], } diff --git a/fiaas_deploy_daemon/retry.py b/fiaas_deploy_daemon/retry.py index 1b120482..522b45e4 100644 --- a/fiaas_deploy_daemon/retry.py +++ b/fiaas_deploy_daemon/retry.py @@ -1,4 +1,3 @@ - # Copyright 2017-2019 The FIAAS Authors # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,12 +23,12 @@ fiaas_upsert_conflict_retry_counter = Counter( "fiaas_upsert_conflict_retry", "Number of retries made due to 409 Conflict when upserting a Kubernetes resource", - ["target"] + ["target"], ) fiaas_upsert_conflict_failure_counter = Counter( "fiaas_upsert_conflict_failure", "Number of times max retries were exceeded due to 409 Conflict when upserting a Kubernetes resource", - ["target"] + ["target"], ) @@ -50,7 +49,7 @@ def __str__(self): method=self.response.request.method, url=self.response.request.url, message=message, - reason=reason + reason=reason, ) @@ -70,11 +69,14 @@ def retry_on_upsert_conflict(_func=None, max_value_seconds=CONFLICT_MAX_VALUE, m def _retry_decorator(func): target = canonical_name(func) - @backoff.on_exception(backoff.expo, UpsertConflict, - max_value=max_value_seconds, - max_tries=max_tries, - on_backoff=functools.partial(_count_retry, target), - on_giveup=functools.partial(_count_failure, target)) + @backoff.on_exception( + backoff.expo, + UpsertConflict, + max_value=max_value_seconds, + max_tries=max_tries, + on_backoff=functools.partial(_count_retry, target), + on_giveup=functools.partial(_count_failure, target), + ) @functools.wraps(func) def _wrap(*args, **kwargs): try: @@ -84,6 +86,7 @@ def _wrap(*args, **kwargs): raise UpsertConflict(e.response) from e else: raise + return _wrap if _func is None: diff --git a/fiaas_deploy_daemon/secrets.py b/fiaas_deploy_daemon/secrets.py index 5e62dc82..de4c839e 100644 --- a/fiaas_deploy_daemon/secrets.py +++ b/fiaas_deploy_daemon/secrets.py @@ -28,7 +28,7 @@ def resolve_secrets(secrets_directory): filename = field.replace("_", "-") fpath = os.path.join(secrets_directory, filename) if os.path.isfile(fpath): - with open(fpath, 'rb') as fobj: + with open(fpath, "rb") as fobj: kwargs[field] = fobj.read().strip() else: kwargs[field] = None diff --git a/fiaas_deploy_daemon/specs/__init__.py b/fiaas_deploy_daemon/specs/__init__.py index f7015c07..b43661b0 100644 --- a/fiaas_deploy_daemon/specs/__init__.py +++ b/fiaas_deploy_daemon/specs/__init__.py @@ -32,6 +32,7 @@ def configure(self, bind): def provide_factory(self): from .v3 import Factory + return Factory() def provide_transformers(self): diff --git a/fiaas_deploy_daemon/specs/default.py b/fiaas_deploy_daemon/specs/default.py index 679eac43..162eaa99 100644 --- a/fiaas_deploy_daemon/specs/default.py +++ b/fiaas_deploy_daemon/specs/default.py @@ -5,12 +5,13 @@ class DefaultAppSpec(object): DEFAULT_APP_CONFIG = { - 'version': 3, + "version": 3, } def __init__(self, spec_factory): - self.default_app_spec = spec_factory(None, None, None, self.DEFAULT_APP_CONFIG, None, None, None, None, None, - None) + self.default_app_spec = spec_factory( + None, None, None, self.DEFAULT_APP_CONFIG, None, None, None, None, None, None + ) def __call__(self, *args, **kwargs): return self.default_app_spec diff --git a/fiaas_deploy_daemon/specs/factory.py b/fiaas_deploy_daemon/specs/factory.py index 06ddf358..5a73d1ba 100644 --- a/fiaas_deploy_daemon/specs/factory.py +++ b/fiaas_deploy_daemon/specs/factory.py @@ -31,16 +31,37 @@ def __init__(self, factory, transformers, config): self._supported_versions = [factory.version] + list(transformers.keys()) self._fiaas_counter = Counter("fiaas_yml_version", "The version of fiaas.yml used", ["version", "app_name"]) - def __call__(self, uid, name, image, app_config, teams, tags, deployment_id, namespace, - additional_labels, additional_annotations): + def __call__( + self, + uid, + name, + image, + app_config, + teams, + tags, + deployment_id, + namespace, + additional_labels, + additional_annotations, + ): """Create an app_spec from app_config""" fiaas_version = app_config.get("version", 1) self._fiaas_counter.labels(fiaas_version, name).inc() LOG.info("Attempting to create app_spec for %s from fiaas.yml version %s", name, fiaas_version) try: app_config = self.transform(app_config) - app_spec = self._factory(uid, name, image, teams, tags, app_config, deployment_id, namespace, - additional_labels, additional_annotations) + app_spec = self._factory( + uid, + name, + image, + teams, + tags, + app_config, + deployment_id, + namespace, + additional_labels, + additional_annotations, + ) except InvalidConfiguration: raise except Exception as e: @@ -51,8 +72,10 @@ def __call__(self, uid, name, image, app_config, teams, tags, deployment_id, nam def transform(self, app_config, strip_defaults=False): fiaas_version = app_config.get("version", 1) if fiaas_version not in self._supported_versions: - raise InvalidConfiguration("Requested version %s, but the only supported versions are: %r" % - (fiaas_version, self._supported_versions)) + raise InvalidConfiguration( + "Requested version %s, but the only supported versions are: %r" + % (fiaas_version, self._supported_versions) + ) current_version = fiaas_version while current_version < self._factory.version: app_config = self._transformers[current_version](app_config, strip_defaults=strip_defaults) @@ -69,8 +92,9 @@ class BaseFactory(object): def version(self): raise NotImplementedError("Subclass must override version property") - def __call__(self, name, image, teams, tags, app_config, deployment_id, namespace, - additional_labels, additional_annotations): + def __call__( + self, name, image, teams, tags, app_config, deployment_id, namespace, additional_labels, additional_annotations + ): raise NotImplementedError("Subclass must override __call__") diff --git a/fiaas_deploy_daemon/specs/models.py b/fiaas_deploy_daemon/specs/models.py index c498fe64..95f46050 100644 --- a/fiaas_deploy_daemon/specs/models.py +++ b/fiaas_deploy_daemon/specs/models.py @@ -17,31 +17,36 @@ from collections import namedtuple -class AppSpec(namedtuple("AppSpec", [ - "uid", - "namespace", - "name", - "image", - "autoscaler", - "resources", - "admin_access", - "secrets_in_environment", - "prometheus", - "datadog", - "ports", - "health_checks", - "teams", - "tags", - "deployment_id", - "labels", - "annotations", - "ingresses", - "strongbox", - "singleton", - "ingress_tls", - "secrets", - "app_config" -])): +class AppSpec( + namedtuple( + "AppSpec", + [ + "uid", + "namespace", + "name", + "image", + "autoscaler", + "resources", + "admin_access", + "secrets_in_environment", + "prometheus", + "datadog", + "ports", + "health_checks", + "teams", + "tags", + "deployment_id", + "labels", + "annotations", + "ingresses", + "strongbox", + "singleton", + "ingress_tls", + "secrets", + "app_config", + ], + ) +): __slots__ = () @property @@ -49,100 +54,90 @@ def version(self): if ":" in self.image: return self.image.split(":")[-1] else: - raise RuntimeError('Version must be specified for docker image aka image:version') - - -ResourceRequirementSpec = namedtuple("ResourceRequirementSpec", [ - "cpu", - "memory"]) - -ResourcesSpec = namedtuple("ResourcesSpec", [ - "limits", - "requests"]) - -PrometheusSpec = namedtuple("PrometheusSpec", [ - "enabled", - "port", - "path"]) - -DatadogSpec = namedtuple("DatadogSpec", [ - "enabled", - "tags"]) - -PortSpec = namedtuple("PortSpec", [ - "protocol", - "name", - "port", - "target_port", -]) - -HealthCheckSpec = namedtuple("HealthCheckSpec", [ - "liveness", - "readiness"]) - -CheckSpec = namedtuple("CheckSpec", [ - "execute", - "http", - "tcp", - "initial_delay_seconds", - "period_seconds", - "success_threshold", - "failure_threshold", - "timeout_seconds"]) - -ExecCheckSpec = namedtuple("ExecCheckSpec", [ - "command"]) - -HttpCheckSpec = namedtuple("HttpCheckSpec", [ - "path", - "port", - "http_headers"]) - -TcpCheckSpec = namedtuple("TcpCheckSpec", [ - "port"]) - -AutoscalerSpec = namedtuple("AutoscalerSpec", [ - "enabled", - "min_replicas", - "max_replicas", - "cpu_threshold_percentage" -]) - -LabelAndAnnotationSpec = namedtuple("LabelAndAnnotationSpec", [ - "deployment", - "horizontal_pod_autoscaler", - "ingress", - "service", - "service_account", - "pod", - "status", -]) - -IngressItemSpec = namedtuple("IngressItemSpec", [ - "host", - "pathmappings", - "annotations" -]) - -IngressPathMappingSpec = namedtuple("IngressPathMappingSpec", [ - "path", - "port", -]) - -StrongboxSpec = namedtuple("StrongboxSpec", [ - "enabled", - "iam_role", - "aws_region", - "groups", -]) - -SecretsSpec = namedtuple("SecretsSpec", [ - "type", - "parameters", - "annotations" -]) - -IngressTLSSpec = namedtuple("IngressTLSSpec", [ - "enabled", - "certificate_issuer", -]) + raise RuntimeError("Version must be specified for docker image aka image:version") + + +ResourceRequirementSpec = namedtuple("ResourceRequirementSpec", ["cpu", "memory"]) + +ResourcesSpec = namedtuple("ResourcesSpec", ["limits", "requests"]) + +PrometheusSpec = namedtuple("PrometheusSpec", ["enabled", "port", "path"]) + +DatadogSpec = namedtuple("DatadogSpec", ["enabled", "tags"]) + +PortSpec = namedtuple( + "PortSpec", + [ + "protocol", + "name", + "port", + "target_port", + ], +) + +HealthCheckSpec = namedtuple("HealthCheckSpec", ["liveness", "readiness"]) + +CheckSpec = namedtuple( + "CheckSpec", + [ + "execute", + "http", + "tcp", + "initial_delay_seconds", + "period_seconds", + "success_threshold", + "failure_threshold", + "timeout_seconds", + ], +) + +ExecCheckSpec = namedtuple("ExecCheckSpec", ["command"]) + +HttpCheckSpec = namedtuple("HttpCheckSpec", ["path", "port", "http_headers"]) + +TcpCheckSpec = namedtuple("TcpCheckSpec", ["port"]) + +AutoscalerSpec = namedtuple("AutoscalerSpec", ["enabled", "min_replicas", "max_replicas", "cpu_threshold_percentage"]) + +LabelAndAnnotationSpec = namedtuple( + "LabelAndAnnotationSpec", + [ + "deployment", + "horizontal_pod_autoscaler", + "ingress", + "service", + "service_account", + "pod", + "status", + ], +) + +IngressItemSpec = namedtuple("IngressItemSpec", ["host", "pathmappings", "annotations"]) + +IngressPathMappingSpec = namedtuple( + "IngressPathMappingSpec", + [ + "path", + "port", + ], +) + +StrongboxSpec = namedtuple( + "StrongboxSpec", + [ + "enabled", + "iam_role", + "aws_region", + "groups", + ], +) + +SecretsSpec = namedtuple("SecretsSpec", ["type", "parameters", "annotations"]) + +IngressTLSSpec = namedtuple( + "IngressTLSSpec", + [ + "enabled", + "certificate_issuer", + ], +) diff --git a/fiaas_deploy_daemon/specs/v2/transformer.py b/fiaas_deploy_daemon/specs/v2/transformer.py index 0859eb51..5c9b9bb7 100644 --- a/fiaas_deploy_daemon/specs/v2/transformer.py +++ b/fiaas_deploy_daemon/specs/v2/transformer.py @@ -52,23 +52,24 @@ def __call__(self, app_config, strip_defaults=False): lookup = LookupMapping(app_config, self._defaults) liveness = self._health_check(lookup["healthchecks"]["liveness"], lookup["ports"]) - if not lookup["healthchecks"].get_config_value("readiness") \ - and lookup["healthchecks"].get_config_value("liveness"): + if not lookup["healthchecks"].get_config_value("readiness") and lookup["healthchecks"].get_config_value( + "liveness" + ): readiness = liveness else: readiness = self._health_check(lookup["healthchecks"]["readiness"], lookup["ports"]) new_config = { - 'version': 3, - 'replicas': { - 'minimum': lookup["replicas"], - 'maximum': lookup["replicas"], - 'cpu_threshold_percentage': lookup["autoscaler"]["cpu_threshold_percentage"], + "version": 3, + "replicas": { + "minimum": lookup["replicas"], + "maximum": lookup["replicas"], + "cpu_threshold_percentage": lookup["autoscaler"]["cpu_threshold_percentage"], }, "healthchecks": { "liveness": liveness, "readiness": readiness, - } + }, } for old, new in Transformer.COPY_MAPPING.items(): value = _get(lookup, old) @@ -78,7 +79,9 @@ def __call__(self, app_config, strip_defaults=False): new_config["resources"] = {} for requirement_type in ("limits", "requests"): - new_config["resources"][requirement_type] = self._resource_requirement(lookup["resources"][requirement_type]) + new_config["resources"][requirement_type] = self._resource_requirement( + lookup["resources"][requirement_type] + ) new_config.update(self._ports(lookup["ports"], lookup["host"])) new_config = _flatten(new_config) @@ -94,13 +97,11 @@ def _strip_v3_defaults(self, app_config): for resource in ("cpu", "memory"): if app_config["resources"][requirement_type][resource] == RESOURCE_UNDEFINED_UGLYHACK: del app_config["resources"][requirement_type][resource] - if not app_config['resources']: - del app_config['resources'] + if not app_config["resources"]: + del app_config["resources"] except KeyError: pass - return dict( - [("version", app_config["version"])] + - list(_remove_intersect(app_config, v3defaults).items())) + return dict([("version", app_config["version"])] + list(_remove_intersect(app_config, v3defaults).items())) @staticmethod def _health_check(lookup, ports_lookup): @@ -112,14 +113,9 @@ def _health_check(lookup, ports_lookup): if len(ports_lookup) > 1: raise InvalidConfiguration("Must specify health check when more than one ports defined") elif ports_lookup[0]["protocol"] == "http": - value["http"] = { - "path": ports_lookup[0]["path"], - "port": ports_lookup[0]["name"] - } + value["http"] = {"path": ports_lookup[0]["path"], "port": ports_lookup[0]["name"]} elif ports_lookup[0]["protocol"] == "tcp": - value["tcp"] = { - "port": ports_lookup[0]["name"] - } + value["tcp"] = {"port": ports_lookup[0]["name"]} return value @staticmethod @@ -128,24 +124,16 @@ def _ports(lookup, host): ports = [] for port in lookup: if port["protocol"] == "http": - paths.append({ - "path": port["path"], - "port": port["name"] - }) - ports.append({ - "protocol": port["protocol"], - "name": port["name"], - "port": port["port"], - "target_port": port["target_port"] - - }) - return { - "ingress": [{ - "host": host, - "paths": paths - }] if paths else [], - "ports": ports - } + paths.append({"path": port["path"], "port": port["name"]}) + ports.append( + { + "protocol": port["protocol"], + "name": port["name"], + "port": port["port"], + "target_port": port["target_port"], + } + ) + return {"ingress": [{"host": host, "paths": paths}] if paths else [], "ports": ports} @staticmethod def _resource_requirement(lookup): diff --git a/fiaas_deploy_daemon/specs/v3/factory.py b/fiaas_deploy_daemon/specs/v3/factory.py index 78b3705b..48fa72a5 100644 --- a/fiaas_deploy_daemon/specs/v3/factory.py +++ b/fiaas_deploy_daemon/specs/v3/factory.py @@ -22,9 +22,26 @@ from ..factory import BaseFactory, InvalidConfiguration from ..lookup import LookupMapping -from ..models import AppSpec, PrometheusSpec, DatadogSpec, ResourcesSpec, ResourceRequirementSpec, PortSpec, \ - HealthCheckSpec, CheckSpec, HttpCheckSpec, TcpCheckSpec, ExecCheckSpec, AutoscalerSpec, \ - LabelAndAnnotationSpec, IngressItemSpec, IngressPathMappingSpec, StrongboxSpec, IngressTLSSpec, SecretsSpec +from ..models import ( + AppSpec, + PrometheusSpec, + DatadogSpec, + ResourcesSpec, + ResourceRequirementSpec, + PortSpec, + HealthCheckSpec, + CheckSpec, + HttpCheckSpec, + TcpCheckSpec, + ExecCheckSpec, + AutoscalerSpec, + LabelAndAnnotationSpec, + IngressItemSpec, + IngressPathMappingSpec, + StrongboxSpec, + IngressTLSSpec, + SecretsSpec, +) from ..v2.transformer import RESOURCE_UNDEFINED_UGLYHACK from ...tools import merge_dicts @@ -37,10 +54,24 @@ def __init__(self, config=None): # Overwrite default value based on config flag for ingress_tls self._defaults["extensions"]["tls"]["enabled"] = config and config.use_ingress_tls == "default_on" - def __call__(self, uid, name, image, teams, tags, app_config, deployment_id, namespace, - additional_labels, additional_annotations): - if app_config.get("extensions") and app_config["extensions"].get("tls") and type( - app_config["extensions"]["tls"]) == bool: + def __call__( + self, + uid, + name, + image, + teams, + tags, + app_config, + deployment_id, + namespace, + additional_labels, + additional_annotations, + ): + if ( + app_config.get("extensions") + and app_config["extensions"].get("tls") + and type(app_config["extensions"]["tls"]) == bool + ): app_config["extensions"]["tls"] = {"enabled": app_config["extensions"]["tls"]} lookup = LookupMapping(config=app_config, defaults=self._defaults) app_spec = AppSpec( @@ -66,10 +97,10 @@ def __call__(self, uid, name, image, teams, tags, app_config, deployment_id, nam singleton=lookup["replicas"]["singleton"], ingress_tls=IngressTLSSpec( enabled=lookup["extensions"]["tls"]["enabled"], - certificate_issuer=lookup["extensions"]["tls"]["certificate_issuer"] + certificate_issuer=lookup["extensions"]["tls"]["certificate_issuer"], ), secrets=self._secrets_specs(lookup["extensions"]["secrets"]), - app_config=app_config + app_config=app_config, ) return app_spec @@ -84,13 +115,13 @@ def _autoscaler_spec(replicas_lookup): enabled=enabled, min_replicas=minimum, max_replicas=maximum, - cpu_threshold_percentage=cpu_threshold_percentage + cpu_threshold_percentage=cpu_threshold_percentage, ) def _resources_spec(self, resources_lookup): return ResourcesSpec( limits=self._resource_requirements_spec(resources_lookup["limits"]), - requests=self._resource_requirements_spec(resources_lookup["requests"]) + requests=self._resource_requirements_spec(resources_lookup["requests"]), ) @staticmethod @@ -102,25 +133,19 @@ def _resource_requirements_spec(resource_lookup): @staticmethod def _prometheus_spec(prometheus_lookup): return PrometheusSpec( - enabled=prometheus_lookup["enabled"], - port=prometheus_lookup["port"], - path=prometheus_lookup["path"] + enabled=prometheus_lookup["enabled"], port=prometheus_lookup["port"], path=prometheus_lookup["path"] ) @staticmethod def _datadog_spec(datadog_lookup): - return DatadogSpec( - enabled=datadog_lookup["enabled"], - tags=dict(datadog_lookup["tags"]) - ) + return DatadogSpec(enabled=datadog_lookup["enabled"], tags=dict(datadog_lookup["tags"])) @staticmethod def _port_specs(ports_lookup): - return [PortSpec(protocol=port["protocol"], - name=port["name"], - port=port["port"], - target_port=port["target_port"]) - for port in ports_lookup] + return [ + PortSpec(protocol=port["protocol"], name=port["name"], port=port["port"], target_port=port["target_port"]) + for port in ports_lookup + ] def _health_checks_spec(self, healthchecks_lookup, ports_lookup): liveness = self._check_spec(healthchecks_lookup["liveness"], ports_lookup, "liveness") @@ -154,7 +179,7 @@ def _check_spec(self, healthcheck_lookup, ports_lookup, check_type): healthcheck_lookup["period_seconds"], healthcheck_lookup["success_threshold"], healthcheck_lookup["failure_threshold"], - healthcheck_lookup["timeout_seconds"] + healthcheck_lookup["timeout_seconds"], ) @staticmethod @@ -188,13 +213,13 @@ def _exec_check_spec(check_lookup): @staticmethod def _labels_annotations_spec(labels_annotations_lookup, overrides): params = { - 'deployment': dict(labels_annotations_lookup["deployment"]), - 'horizontal_pod_autoscaler': dict(labels_annotations_lookup["horizontal_pod_autoscaler"]), - 'ingress': dict(labels_annotations_lookup["ingress"]), - 'service': dict(labels_annotations_lookup["service"]), - 'service_account': dict(labels_annotations_lookup["service_account"]), - 'pod': dict(labels_annotations_lookup["pod"]), - 'status': {} + "deployment": dict(labels_annotations_lookup["deployment"]), + "horizontal_pod_autoscaler": dict(labels_annotations_lookup["horizontal_pod_autoscaler"]), + "ingress": dict(labels_annotations_lookup["ingress"]), + "service": dict(labels_annotations_lookup["service"]), + "service_account": dict(labels_annotations_lookup["service_account"]), + "pod": dict(labels_annotations_lookup["pod"]), + "status": {}, } if overrides: globals = _get_value("_global", overrides) @@ -224,8 +249,10 @@ def ingress_item(host, paths, annotations): return IngressItemSpec(host=host, pathmappings=ingress_path_mapping_specs, annotations=annotations) if len(list(http_ports.items())) > 0: - return [ingress_item(host_path_mapping["host"], host_path_mapping["paths"], host_path_mapping["annotations"]) - for host_path_mapping in ingress_lookup] + return [ + ingress_item(host_path_mapping["host"], host_path_mapping["paths"], host_path_mapping["annotations"]) + for host_path_mapping in ingress_lookup + ] else: return [] @@ -234,15 +261,16 @@ def _strongbox(strongbox_lookup): iam_role = strongbox_lookup.get_config_value("iam_role") groups = strongbox_lookup.get_config_value("groups") enabled = iam_role is not None and groups is not None - return StrongboxSpec(enabled=enabled, iam_role=iam_role, - aws_region=strongbox_lookup["aws_region"], - groups=groups) + return StrongboxSpec( + enabled=enabled, iam_role=iam_role, aws_region=strongbox_lookup["aws_region"], groups=groups + ) @staticmethod def _secrets_specs(secrets_lookup): - return [SecretsSpec(type=k, - parameters=v["parameters"], - annotations=v["annotations"]) for (k, v) in secrets_lookup.items()] + return [ + SecretsSpec(type=k, parameters=v["parameters"], annotations=v["annotations"]) + for (k, v) in secrets_lookup.items() + ] def _get_value(key, overrides): diff --git a/fiaas_deploy_daemon/usage_reporting/dev_hose_auth.py b/fiaas_deploy_daemon/usage_reporting/dev_hose_auth.py index 478c5fb5..ca65a154 100644 --- a/fiaas_deploy_daemon/usage_reporting/dev_hose_auth.py +++ b/fiaas_deploy_daemon/usage_reporting/dev_hose_auth.py @@ -47,11 +47,13 @@ def _calculate_signature(self, r, timestamp, nonce): return base64.b64encode(signature.digest()).decode("utf-8") def _create_string_to_sign(self, r, timestamp, nonce): - return "\n".join(( - quote(r.path_url.encode("utf-8"), b".-~_@:!$&'()*+,;=/?"), - quote_plus(nonce.encode("utf-8")), - str(int(timestamp) * 1000), - quote_plus(self._auth_context), - quote_plus(r.body.encode("utf-8")), - "" - )) + return "\n".join( + ( + quote(r.path_url.encode("utf-8"), b".-~_@:!$&'()*+,;=/?"), + quote_plus(nonce.encode("utf-8")), + str(int(timestamp) * 1000), + quote_plus(self._auth_context), + quote_plus(r.body.encode("utf-8")), + "", + ) + ) diff --git a/fiaas_deploy_daemon/usage_reporting/transformer.py b/fiaas_deploy_daemon/usage_reporting/transformer.py index 97174d1a..ad0370b2 100644 --- a/fiaas_deploy_daemon/usage_reporting/transformer.py +++ b/fiaas_deploy_daemon/usage_reporting/transformer.py @@ -30,16 +30,18 @@ def namedtuple_with_defaults(typename, field_names, default_values=()): return T -DevhoseDeploymentEvent = namedtuple_with_defaults('DevhoseDeploymentEvent', - 'id application environment repository started_at timestamp target \ - status source_type facility details trigger team', - {'source_type': 'fiaas', 'facility': 'sdrn:schibsted:service:fiaas'}) +DevhoseDeploymentEvent = namedtuple_with_defaults( + "DevhoseDeploymentEvent", + "id application environment repository started_at timestamp target \ + status source_type facility details trigger team", + {"source_type": "fiaas", "facility": "sdrn:schibsted:service:fiaas"}, +) -status_map = {'STARTED': 'in_progress', 'SUCCESS': 'succeeded', 'FAILED': 'failed'} +status_map = {"STARTED": "in_progress", "SUCCESS": "succeeded", "FAILED": "failed"} class DevhoseDeploymentEventTransformer(object): - FIAAS_TRIGGER = {'type': 'fiaas'} + FIAAS_TRIGGER = {"type": "fiaas"} def __init__(self, config): self._environment = config.environment @@ -52,7 +54,7 @@ def __init__(self, config): def __call__(self, status, app_name, namespace, deployment_id, repository): timestamp = _timestamp() started_timestamp = None - if status == 'STARTED': + if status == "STARTED": self._deployments_started[(app_name, deployment_id)] = timestamp started_timestamp = timestamp else: @@ -62,25 +64,38 @@ def __call__(self, status, app_name, namespace, deployment_id, repository): # This can happen if deployment fails immediately such as in case of config parse errors pass - event = DevhoseDeploymentEvent(id=deployment_id, - application=app_name, - environment=_environment(self._environment[:3]), - repository=repository, - started_at=started_timestamp if started_timestamp else timestamp, - timestamp=timestamp, - target={'infrastructure': self._target_infrastructure, - 'provider': self._target_provider, - 'team': self._operator, - 'instance': namespace}, - status=status_map[status], - details={'environment': self._environment}, - trigger=DevhoseDeploymentEventTransformer.FIAAS_TRIGGER, - team=self._team) + event = DevhoseDeploymentEvent( + id=deployment_id, + application=app_name, + environment=_environment(self._environment[:3]), + repository=repository, + started_at=started_timestamp if started_timestamp else timestamp, + timestamp=timestamp, + target={ + "infrastructure": self._target_infrastructure, + "provider": self._target_provider, + "team": self._operator, + "instance": namespace, + }, + status=status_map[status], + details={"environment": self._environment}, + trigger=DevhoseDeploymentEventTransformer.FIAAS_TRIGGER, + team=self._team, + ) return event._asdict() def _environment(env): - return env if env in ('dev', 'pre', 'pro',) else 'other' + return ( + env + if env + in ( + "dev", + "pre", + "pro", + ) + else "other" + ) def _timestamp(): diff --git a/fiaas_deploy_daemon/usage_reporting/usage_reporter.py b/fiaas_deploy_daemon/usage_reporting/usage_reporter.py index 16b9a557..3d7b4708 100644 --- a/fiaas_deploy_daemon/usage_reporting/usage_reporter.py +++ b/fiaas_deploy_daemon/usage_reporting/usage_reporter.py @@ -62,13 +62,18 @@ def __init__(self, config, usage_transformer, session, usage_auth): LOG.info("Usage reporting enabled, sending events to %s", self._usage_reporting_endpoint) signal(DEPLOY_STATUS_CHANGED).connect(self._handle_signal) else: - LOG.debug("Usage reporting disabled: Endpoint: %r, UsageAuth: %r", - self._usage_reporting_endpoint, self._usage_auth) + LOG.debug( + "Usage reporting disabled: Endpoint: %r, UsageAuth: %r", + self._usage_reporting_endpoint, + self._usage_auth, + ) def _handle_signal(self, sender, status, subject): if status in [STATUS_STARTED, STATUS_SUCCESS, STATUS_FAILED]: status = status.upper() - self._event_queue.put(UsageEvent(status, subject.app_name, subject.namespace, subject.deployment_id, subject.repository)) + self._event_queue.put( + UsageEvent(status, subject.app_name, subject.namespace, subject.deployment_id, subject.repository) + ) def __call__(self): for event in self._event_queue: @@ -79,14 +84,16 @@ def _handle_event(self, event): try: self._send_data(data) except requests.exceptions.RequestException: - LOG.error('Unable to send usage reporting event', exc_info=True) - - @backoff.on_exception(backoff.expo, - requests.exceptions.RequestException, - max_tries=5, - on_success=_success_handler, - on_backoff=_retry_handler, - on_giveup=_failure_handler) + LOG.error("Unable to send usage reporting event", exc_info=True) + + @backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + max_tries=5, + on_success=_success_handler, + on_backoff=_retry_handler, + on_giveup=_failure_handler, + ) @reporting_histogram.time() def _send_data(self, data): resp = self._session.post(self._usage_reporting_endpoint, json=data, auth=self._usage_auth) diff --git a/fiaas_deploy_daemon/web/__init__.py b/fiaas_deploy_daemon/web/__init__.py index c8c1371e..e521499e 100644 --- a/fiaas_deploy_daemon/web/__init__.py +++ b/fiaas_deploy_daemon/web/__init__.py @@ -21,8 +21,18 @@ import pinject import yaml -from flask import Flask, Blueprint, current_app, render_template, make_response, request_started, request_finished, \ - got_request_exception, abort, request +from flask import ( + Flask, + Blueprint, + current_app, + render_template, + make_response, + request_started, + request_finished, + got_request_exception, + abort, + request, +) from flask_talisman import Talisman, DENY from prometheus_client import generate_latest, CONTENT_TYPE_LATEST, Counter, Histogram @@ -81,19 +91,19 @@ def healthz(): return "I don't feel so good...", 500 -@web.route("/transform", methods=['GET', 'POST']) +@web.route("/transform", methods=["GET", "POST"]) @transform_histogram.time() def transform(): - if request.method == 'GET': + if request.method == "GET": return render_template("transform.html") - elif request.method == 'POST': + elif request.method == "POST": return _transform(yaml.safe_load(request.get_data())) def _transform(app_config): try: data = current_app.transformer.transform(app_config) - return current_app.response_class(data, content_type='text/vnd.yaml; charset=utf-8') + return current_app.response_class(data, content_type="text/vnd.yaml; charset=utf-8") except InvalidConfiguration as err: abort(400, err.message) @@ -128,8 +138,13 @@ def provide_webapp(self, spec_factory, health_check): # TODO: These options are like this because we haven't set up TLS, but should be # configurable if the operator wants to. Even better would be to somehow auto-detect. - csp = {"default-src": "'self'", "style-src": ["'self'", "finncdn.no", "*.finncdn.no"], - "script-src": ["'none'"], "object-src": ["'none'"]} - Talisman(app, frame_options=DENY, content_security_policy=csp, - force_https=False, strict_transport_security=False) + csp = { + "default-src": "'self'", + "style-src": ["'self'", "finncdn.no", "*.finncdn.no"], + "script-src": ["'none'"], + "object-src": ["'none'"], + } + Talisman( + app, frame_options=DENY, content_security_policy=csp, force_https=False, strict_transport_security=False + ) return app diff --git a/fiaas_deploy_daemon/web/transformer.py b/fiaas_deploy_daemon/web/transformer.py index 7741ec29..bd4a3228 100644 --- a/fiaas_deploy_daemon/web/transformer.py +++ b/fiaas_deploy_daemon/web/transformer.py @@ -1,4 +1,3 @@ - # Copyright 2017-2019 The FIAAS Authors # # Licensed under the Apache License, Version 2.0 (the "License");