From 5e2a022ea6ebf95f89b413a7ac3004cc505827a5 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 3 May 2020 22:22:23 +0100 Subject: [PATCH 01/35] feat: use new TraceProvider --- python/aws_lambda_powertools/tracing/base.py | 220 ++++++++++++++ .../aws_lambda_powertools/tracing/tracer.py | 276 +++++++++--------- python/tests/unit/test_tracing.py | 146 ++++++++- 3 files changed, 508 insertions(+), 134 deletions(-) create mode 100644 python/aws_lambda_powertools/tracing/base.py diff --git a/python/aws_lambda_powertools/tracing/base.py b/python/aws_lambda_powertools/tracing/base.py new file mode 100644 index 0000000000..21b73c1017 --- /dev/null +++ b/python/aws_lambda_powertools/tracing/base.py @@ -0,0 +1,220 @@ +from __future__ import annotations + +import abc +import copy +import functools +import logging +import os +from typing import Any, Callable, List + +import aws_xray_sdk +import aws_xray_sdk.core + +is_cold_start = True +logger = logging.getLogger(__name__) + + +class TracerProvider(metaclass=abc.ABCMeta): + """Tracer provider abstract class + + Providers should be initialized independently. This + allows providers to control their config/initialization, + and only pass a class instance to + ```aws_lambda_powertools.tracing.tracer.Tracer```. + + Trace providers should implement the following methods: + + * **patch** + * **create_subsegment** + * **end_subsegment** + * **put_metadata** + * **put_annotation** + * **disable_tracing_provider** + + These methods will be called by + ```aws_lambda_powertools.tracing.tracer.Tracer```. + See ```aws_lambda_powertools.tracing.base.XrayProvider``` + for a reference implementation. + + Example + ------- + **Client using a custom tracing provider** + + from aws_lambda_powertools.tracing import Tracer + ... import ... ProviderX + custom_provider = ProviderX() + tracer = Tracer(service="greeting", provider=custom_provider) + """ + + def __init__(self): + """Trace provider initialization.""" + pass + + @abc.abstractmethod + def patch(self, modules: List[str] = None): + """Patch modules for instrumentation + + If modules are None, it should patch + all supported modules by the provider. + + Parameters + ---------- + modules : List[str], optional + List of modules to be pathced, by default None + e.g. `['boto3', 'requests']` + """ + raise NotImplementedError + + @abc.abstractmethod + def create_subsegment(self, name: str): + """Creates subsegment/span with a given name + + Parameters + ---------- + name : str + Subsegment/span name + """ + raise NotImplementedError + + @abc.abstractmethod + def end_subsegment(self): + """Ends an existing subsegment""" + raise NotImplementedError + + @abc.abstractmethod + def put_metadata(self, key: str, value: Any, namespace: str = None): + """Adds metadata to existing segment/span or subsegment + + Parameters + ---------- + key : str + Metadata key + value : Any + Metadata value + namespace : str, optional + Metadata namespace, by default None + """ + raise NotImplementedError + + @abc.abstractmethod + def put_annotation(self, key: str, value: Any): + """Adds annotation/label to existing segment/span or subsegment + + Parameters + ---------- + key : str + Annotation/label key + value : Any + Annotation/label value + """ + raise NotImplementedError + + @abc.abstractmethod + def disable_tracing_provider(self): + """Forcefully disables tracing provider""" + raise NotImplementedError + + +class XrayProvider(TracerProvider): + def __init__( + self, client: aws_xray_sdk.core.xray_recorder = aws_xray_sdk.core.xray_recorder + ): + self.client = client + + def create_subsegment(self, name: str) -> aws_xray_sdk.core.models.subsegment: + """Creates subsegment/span with a given name + + Parameters + ---------- + name : str + Subsegment name + + Example + ------- + + **Creates a subsegment** + + self.create_subsegment(name="a meaningful name") + + Returns + ------- + aws_xray_sdk.core.models.subsegment + AWS X-Ray Subsegment + """ + # Will no longer be needed once #155 is resolved + # https://github.com/aws/aws-xray-sdk-python/issues/155 + # if self.disabled: + # logger.debug("Tracing has been disabled, return dummy subsegment instead") + # return + subsegment = self.client.begin_subsegment(name=name) + global is_cold_start + if is_cold_start: + logger.debug("Annotating cold start") + subsegment.put_annotation("ColdStart", True) + is_cold_start = False + + return subsegment + + def end_subsegment(self): + """Ends an existing subsegment""" + self.client.end_subsegment() + + def put_annotation(self, key, value): + """Adds annotation to existing segment or subsegment + + Example + ------- + Custom annotation for a pseudo service named payment + + tracer = Tracer(service="payment") + tracer.put_annotation("PaymentStatus", "CONFIRMED") + + Parameters + ---------- + key : str + Annotation key (e.g. PaymentStatus) + value : Any + Value for annotation (e.g. "CONFIRMED") + """ + self.client.put_annotation(key=key, value=value) + + def put_metadata(self, key, value, namespace=None): + """Adds metadata to existing segment or subsegment + + Parameters + ---------- + key : str + Metadata key + value : object + Value for metadata + namespace : str, optional + Namespace that metadata will lie under, by default None + + Example + ------- + Custom metadata for a pseudo service named payment + + tracer = Tracer(service="payment") + response = collect_payment() + tracer.put_metadata("Payment collection", response) + """ + self.provider.put_metadata(key=key, value=value, namespace=namespace) + + def patch(self, modules: List[str] = None): + """Patch modules for instrumentation. + + Patches all supported modules by default if none are given. + + Parameters + ---------- + modules : List[str] + List of modules to be patched, optional by default + """ + if modules is None: + aws_xray_sdk.core.patch_all() + else: + aws_xray_sdk.core.patch(modules) + + def disable_tracing_provider(self): + """Forcefully disables X-Ray tracing globally""" + aws_xray_sdk.global_sdk_config.set_sdk_enabled(False) diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index 19cc319492..32b908aa56 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -3,9 +3,9 @@ import logging import os from distutils.util import strtobool -from typing import Any, Callable, Dict +from typing import Any, Callable, Dict, List -from aws_xray_sdk.core import models, patch_all, xray_recorder +from .base import TracerProvider, XrayProvider is_cold_start = True logger = logging.getLogger(__name__) @@ -116,26 +116,148 @@ def handler(event: dict, context: Any) -> Dict: Limitations ----------- * Async handler and methods not supported - """ - _default_config = {"service": "service_undefined", "disabled": False, "provider": xray_recorder, "auto_patch": True} + _default_config = { + "service": "service_undefined", + "disabled": False, + "provider": None, + "auto_patch": True, + "patch_modules": None, + "provider": None + } _config = copy.copy(_default_config) def __init__( - self, service: str = None, disabled: bool = None, provider: xray_recorder = None, auto_patch: bool = None + self, + service: str = None, + disabled: bool = None, + auto_patch: bool = None, + patch_modules: List = None, + provider: TracerProvider = None ): - self.__build_config(service=service, disabled=disabled, provider=provider, auto_patch=auto_patch) + self.__build_config( + service=service, + disabled=disabled, + auto_patch=auto_patch, + patch_modules=patch_modules, + provider=provider + ) self.provider = self._config["provider"] self.disabled = self._config["disabled"] self.service = self._config["service"] self.auto_patch = self._config["auto_patch"] if self.disabled: - self.__disable_tracing_provider() + self.disable_tracing_provider() if self.auto_patch: - self.patch() + self.patch(modules=patch_modules) + + def create_subsegment(self, name: str): + """Creates subsegment/span with a given name + + It also assumes Tracer would be instantiated statically so that cold starts are captured. + + Parameters + ---------- + name : str + Subsegment name + + Example + ------- + Creates a genuine subsegment + + self.create_subsegment(name="a meaningful name") + + # FIXME - Return Subsegment Any type + Returns + ------- + models.subsegment + AWS X-Ray Subsegment + """ + # Will no longer be needed once #155 is resolved + # https://github.com/aws/aws-xray-sdk-python/issues/155 + if self.disabled: + logger.debug("Tracing has been disabled, aborting create_segment") + return + + return self.provider.begin_subsegment(name=name) + + def end_subsegment(self): + """Ends an existing subsegment""" + if self.disabled: + logger.debug("Tracing has been disabled, aborting end_subsegment") + return + + self.provider.end_subsegment() + + def put_annotation(self, key: str, value: Any): + """Adds annotation to existing segment or subsegment + + Example + ------- + Custom annotation for a pseudo service named payment + + tracer = Tracer(service="payment") + tracer.put_annotation("PaymentStatus", "CONFIRMED") + + Parameters + ---------- + key : str + Annotation key (e.g. PaymentStatus) + value : Any + Value for annotation (e.g. "CONFIRMED") + """ + # Will no longer be needed once #155 is resolved + # https://github.com/aws/aws-xray-sdk-python/issues/155 + if self.disabled: + logger.debug("Tracing has been disabled, aborting put_annotation") + return + + logger.debug(f"Annotating on key '{key}'' with '{value}''") + self.provider.put_annotation(key=key, value=value) + + def put_metadata(self, key: str, value: object, namespace: str = None): + """Adds metadata to existing segment or subsegment + + Parameters + ---------- + key : str + Metadata key + value : object + Value for metadata + namespace : str, optional + Namespace that metadata will lie under, by default None + + Example + ------- + Custom metadata for a pseudo service named payment + + tracer = Tracer(service="payment") + response = collect_payment() + tracer.put_metadata("Payment collection", response) + """ + if self.disabled: + logger.debug("Tracing has been disabled, aborting put_metadata") + return + + namespace = namespace or self.service + logger.debug(f"Adding metadata on key '{key}'' with '{value}'' at namespace '{namespace}''") + self.provider.put_metadata(key=key, value=value, namespace=namespace) + + def patch(self, modules: List[str] = None): + """Patch modules for instrumentation""" + if self.disabled: + logger.debug("Tracing has been disabled, aborting patch") + return + + self.provider.patch(modules=modules) + + def disable_tracing_provider(self): + """Forcefully disables tracing""" + logger.debug("Disabling tracer provider...") + self.provider.disable_tracing_provider() def capture_lambda_handler(self, lambda_handler: Callable[[Dict, Any], Any] = None): """Decorator to create subsegment for lambda handlers @@ -232,129 +354,6 @@ def decorate(*args, **kwargs): return decorate - def put_annotation(self, key: str, value: Any): - """Adds annotation to existing segment or subsegment - - Example - ------- - Custom annotation for a pseudo service named payment - - tracer = Tracer(service="payment") - tracer.put_annotation("PaymentStatus", "CONFIRMED") - - Parameters - ---------- - key : str - Annotation key (e.g. PaymentStatus) - value : Any - Value for annotation (e.g. "CONFIRMED") - """ - # Will no longer be needed once #155 is resolved - # https://github.com/aws/aws-xray-sdk-python/issues/155 - if self.disabled: - return - - logger.debug(f"Annotating on key '{key}'' with '{value}''") - self.provider.put_annotation(key=key, value=value) - - def put_metadata(self, key: str, value: object, namespace: str = None): - """Adds metadata to existing segment or subsegment - - Parameters - ---------- - key : str - Metadata key - value : object - Value for metadata - namespace : str, optional - Namespace that metadata will lie under, by default None - - Example - ------- - Custom metadata for a pseudo service named payment - - tracer = Tracer(service="payment") - response = collect_payment() - tracer.put_metadata("Payment collection", response) - """ - # Will no longer be needed once #155 is resolved - # https://github.com/aws/aws-xray-sdk-python/issues/155 - if self.disabled: - return - - _namespace = namespace or self.service - logger.debug(f"Adding metadata on key '{key}'' with '{value}'' at namespace '{namespace}''") - self.provider.put_metadata(key=key, value=value, namespace=_namespace) - - def create_subsegment(self, name: str) -> models.subsegment: - """Creates subsegment or a dummy segment plus subsegment if tracing is disabled - - It also assumes Tracer would be instantiated statically so that cold starts are captured. - - Parameters - ---------- - name : str - Subsegment name - - Example - ------- - Creates a genuine subsegment - - self.create_subsegment(name="a meaningful name") - - Returns - ------- - models.subsegment - AWS X-Ray Subsegment - """ - # Will no longer be needed once #155 is resolved - # https://github.com/aws/aws-xray-sdk-python/issues/155 - subsegment = None - - if self.disabled: - logger.debug("Tracing has been disabled, return dummy subsegment instead") - segment = models.dummy_entities.DummySegment() - subsegment = models.dummy_entities.DummySubsegment(segment) - else: - subsegment = self.provider.begin_subsegment(name=name) - global is_cold_start - if is_cold_start: - logger.debug("Annotating cold start") - subsegment.put_annotation("ColdStart", True) - is_cold_start = False - - return subsegment - - def end_subsegment(self): - """Ends an existing subsegment - - Parameters - ---------- - subsegment : models.subsegment - Subsegment previously created - """ - if self.disabled: - logger.debug("Tracing has been disabled, return instead") - return - - self.provider.end_subsegment() - - def patch(self): - """Patch modules for instrumentation""" - logger.debug("Patching modules...") - - if self.disabled: - logger.debug("Tracing has been disabled, aborting patch") - return - - patch_all() # pragma: no cover - - def __disable_tracing_provider(self): - """Forcefully disables tracing and patching""" - from aws_xray_sdk import global_sdk_config - - global_sdk_config.set_sdk_enabled(False) - def __is_trace_disabled(self) -> bool: """Detects whether trace has been disabled @@ -384,16 +383,27 @@ def __is_trace_disabled(self) -> bool: return False def __build_config( - self, service: str = None, disabled: bool = None, provider: xray_recorder = None, auto_patch: bool = None + self, + service: str = None, + disabled: bool = None, + auto_patch: bool = None, + patch_modules: List = None, + provider: TracerProvider = None ): """ Populates Tracer config for new and existing initializations """ is_disabled = disabled if disabled is not None else self.__is_trace_disabled() is_service = service if service is not None else os.getenv("POWERTOOLS_SERVICE_NAME") - self._config["provider"] = provider if provider is not None else self._config["provider"] self._config["auto_patch"] = auto_patch if auto_patch is not None else self._config["auto_patch"] self._config["service"] = is_service if is_service else self._config["service"] self._config["disabled"] = is_disabled if is_disabled else self._config["disabled"] + self._config["patch_modules"] = patch_modules if patch_modules else self._config["patch_modules"] + + if provider is not None: + self._config["provider"] = provider + + if self._config["provider"] is None: + self._config["provider"] = XrayProvider() @classmethod def _reset_config(cls): diff --git a/python/tests/unit/test_tracing.py b/python/tests/unit/test_tracing.py index f8f43de0bf..4cc27faa6a 100644 --- a/python/tests/unit/test_tracing.py +++ b/python/tests/unit/test_tracing.py @@ -3,6 +3,7 @@ import pytest from aws_lambda_powertools.tracing import Tracer +from aws_lambda_powertools.tracing.base import TracerProvider @pytest.fixture @@ -19,11 +20,15 @@ def __init__( put_annotation_mock: mocker.MagicMock = None, begin_subsegment_mock: mocker.MagicMock = None, end_subsegment_mock: mocker.MagicMock = None, + patch_mock: mocker.MagicMock = None, + disable_tracing_provider_mock: mocker.MagicMock = None, ): self.put_metadata_mock = put_metadata_mock or mocker.MagicMock() self.put_annotation_mock = put_annotation_mock or mocker.MagicMock() self.begin_subsegment_mock = begin_subsegment_mock or mocker.MagicMock() self.end_subsegment_mock = end_subsegment_mock or mocker.MagicMock() + self.patch_mock = patch_mock or mocker.MagicMock() + self.disable_tracing_provider_mock = disable_tracing_provider_mock or mocker.MagicMock() def put_metadata(self, *args, **kwargs): return self.put_metadata_mock(*args, **kwargs) @@ -37,6 +42,12 @@ def begin_subsegment(self, *args, **kwargs): def end_subsegment(self, *args, **kwargs): return self.end_subsegment_mock(*args, **kwargs) + def patch(self, *args, **kwargs): + return self.patch_mock(*args, **kwargs) + + def disable_tracing_provider(self): + self.disable_tracing_provider_mock() + return XRayStub @@ -46,6 +57,30 @@ def reset_tracing_config(): yield +@pytest.fixture +def tracer_provider_stub(mocker): + class CustomTracerProvider(TracerProvider): + def create_subsegment(self, name): + pass + + def end_subsegment(self, name=None): + pass + + def patch(self): + pass + + def put_annotation(self, key, value): + pass + + def put_metadata(self, key, value, namespace=None): + pass + + def disable_tracing_provider(self): + pass + + return CustomTracerProvider + + def test_tracer_lambda_handler(mocker, dummy_response, xray_stub): put_metadata_mock = mocker.MagicMock() begin_subsegment_mock = mocker.MagicMock() @@ -78,7 +113,12 @@ def test_tracer_method(mocker, dummy_response, xray_stub): begin_subsegment_mock = mocker.MagicMock() end_subsegment_mock = mocker.MagicMock() - xray_provider = xray_stub(put_metadata_mock, put_annotation_mock, begin_subsegment_mock, end_subsegment_mock) + xray_provider = xray_stub( + put_metadata_mock=put_metadata_mock, + put_annotation_mock=put_annotation_mock, + begin_subsegment_mock=begin_subsegment_mock, + end_subsegment_mock=end_subsegment_mock, + ) tracer = Tracer(provider=xray_provider, service="booking") @tracer.capture_method @@ -181,3 +221,107 @@ def greeting(name, message): greeting(name="Foo", message="Bar") assert put_metadata_mock.call_count == 0 + + +def test_trace_provider_abc_init(mocker, xray_stub): + # GIVEN tracer is instantiated + # WHEN a custom provider that implements TracerProvider methods + # THEN it should run successfully + class XrayTracer(TracerProvider): + def create_subsegment(self, name): + pass + + def end_subsegment(self, name=None): + pass + + def patch(self): + pass + + def put_annotation(self, key, value): + pass + + def put_metadata(self, key, value, namespace=None): + pass + + def disable_tracing_provider(self): + pass + + with mock.patch.object(Tracer, "patch") as patch_mock: + tracer = Tracer(service="booking", provider=XrayTracer) + tracer2 = Tracer(auto_patch=False) + assert patch_mock.call_count == 1 + assert tracer.service == "booking" + assert tracer2.service == "booking" # inherited from tracer1 + assert tracer2.auto_patch == False # overriden in tracer2 + + +# def test_trace_provider_abc_decorators(mocker, dummy_response, xray_stub): +# # GIVEN tracer is instantiated +# # WHEN a new tracer provider implements TracerProvider methods +# # THEN it should inherit default decorators +# class XrayTracer(TracerProvider): + +# def create_subsegment(self, name): +# pass + +# def end_subsegment(self, name=None): +# pass + +# def patch(self): +# pass + +# def put_annotation(self, key, value): +# pass + +# def put_metadata(self, key, value, namespace=None): +# pass + +# put_metadata_mock = mocker.MagicMock() +# put_annotation_mock = mocker.MagicMock() +# begin_subsegment_mock = mocker.MagicMock() +# end_subsegment_mock = mocker.MagicMock() + +# xray_provider = xray_stub( +# put_metadata_mock=put_metadata_mock, +# put_annotation_mock=put_annotation_mock, +# begin_subsegment_mock=begin_subsegment_mock, +# end_subsegment_mock=end_subsegment_mock, +# ) + +# tracer = XrayTracer(provider=xray_provider, service="booking") +# annotation_key = "BookingId" +# annotation_value = "123456" + +# @tracer.capture_lambda_handler +# def handler(event, context): +# tracer.put_metadata(annotation_key, annotation_value) +# return dummy_response + +# handler({}, mocker.MagicMock()) + +# # It's 0 because it's calling base class method +# # and base class provider is different from ours +# # base class is calling ours methods though + +# # TODO - Create X-Ray Tracer class, and implement decorators there +# # TODO - Leave only docstring for Trace Provider +# # TODO - Update tests +# # FIXME - Create provider abc only, reuse Tracer class, optionally initialize + +# assert put_metadata_mock.call_count == 2 +# assert put_metadata_mock.call_args_list[0] == mocker.call( +# key=annotation_key, value=annotation_value, namespace="booking" +# ) + +# @tracer.capture_method +# def greeting(name, message): +# return dummy_response + +# greeting(name="Foo", message="Bar") + +# assert begin_subsegment_mock.call_count == 1 +# assert begin_subsegment_mock.call_args == mocker.call(name="## greeting") +# assert end_subsegment_mock.call_count == 1 +# assert put_metadata_mock.call_args == mocker.call( +# key="greeting response", value=dummy_response, namespace="booking" +# ) From 63a8ea9a6b69835d078c7920f85424fd21ffd7f3 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 4 May 2020 18:18:32 +0100 Subject: [PATCH 02/35] improv: update tests --- python/aws_lambda_powertools/tracing/base.py | 15 +- .../tracing/exceptions.py | 5 + .../aws_lambda_powertools/tracing/tracer.py | 36 ++- python/tests/unit/test_tracing.py | 273 ++++++++---------- 4 files changed, 161 insertions(+), 168 deletions(-) create mode 100644 python/aws_lambda_powertools/tracing/exceptions.py diff --git a/python/aws_lambda_powertools/tracing/base.py b/python/aws_lambda_powertools/tracing/base.py index 21b73c1017..811dd449db 100644 --- a/python/aws_lambda_powertools/tracing/base.py +++ b/python/aws_lambda_powertools/tracing/base.py @@ -46,10 +46,6 @@ class TracerProvider(metaclass=abc.ABCMeta): tracer = Tracer(service="greeting", provider=custom_provider) """ - def __init__(self): - """Trace provider initialization.""" - pass - @abc.abstractmethod def patch(self, modules: List[str] = None): """Patch modules for instrumentation @@ -116,9 +112,7 @@ def disable_tracing_provider(self): class XrayProvider(TracerProvider): - def __init__( - self, client: aws_xray_sdk.core.xray_recorder = aws_xray_sdk.core.xray_recorder - ): + def __init__(self, client: aws_xray_sdk.core.xray_recorder = aws_xray_sdk.core.xray_recorder): self.client = client def create_subsegment(self, name: str) -> aws_xray_sdk.core.models.subsegment: @@ -143,14 +137,11 @@ def create_subsegment(self, name: str) -> aws_xray_sdk.core.models.subsegment: """ # Will no longer be needed once #155 is resolved # https://github.com/aws/aws-xray-sdk-python/issues/155 - # if self.disabled: - # logger.debug("Tracing has been disabled, return dummy subsegment instead") - # return subsegment = self.client.begin_subsegment(name=name) global is_cold_start if is_cold_start: logger.debug("Annotating cold start") - subsegment.put_annotation("ColdStart", True) + subsegment.put_annotation(key="ColdStart", value=True) is_cold_start = False return subsegment @@ -198,7 +189,7 @@ def put_metadata(self, key, value, namespace=None): response = collect_payment() tracer.put_metadata("Payment collection", response) """ - self.provider.put_metadata(key=key, value=value, namespace=namespace) + self.client.put_metadata(key=key, value=value, namespace=namespace) def patch(self, modules: List[str] = None): """Patch modules for instrumentation. diff --git a/python/aws_lambda_powertools/tracing/exceptions.py b/python/aws_lambda_powertools/tracing/exceptions.py new file mode 100644 index 0000000000..12d2ad24e3 --- /dev/null +++ b/python/aws_lambda_powertools/tracing/exceptions.py @@ -0,0 +1,5 @@ +class InvalidTracerProviderError(Exception): + pass + +class TracerProviderNotInitializedError(Exception): + pass diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index 32b908aa56..fa5ca43cd0 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -1,11 +1,13 @@ import copy import functools +import inspect import logging import os from distutils.util import strtobool from typing import Any, Callable, Dict, List from .base import TracerProvider, XrayProvider +from .exceptions import InvalidTracerProviderError, TracerProviderNotInitializedError is_cold_start = True logger = logging.getLogger(__name__) @@ -124,7 +126,7 @@ def handler(event: dict, context: Any) -> Dict: "provider": None, "auto_patch": True, "patch_modules": None, - "provider": None + "provider": None, } _config = copy.copy(_default_config) @@ -134,14 +136,10 @@ def __init__( disabled: bool = None, auto_patch: bool = None, patch_modules: List = None, - provider: TracerProvider = None + provider: TracerProvider = None, ): self.__build_config( - service=service, - disabled=disabled, - auto_patch=auto_patch, - patch_modules=patch_modules, - provider=provider + service=service, disabled=disabled, auto_patch=auto_patch, patch_modules=patch_modules, provider=provider ) self.provider = self._config["provider"] self.disabled = self._config["disabled"] @@ -182,7 +180,7 @@ def create_subsegment(self, name: str): logger.debug("Tracing has been disabled, aborting create_segment") return - return self.provider.begin_subsegment(name=name) + return self.provider.create_subsegment(name=name) def end_subsegment(self): """Ends an existing subsegment""" @@ -388,7 +386,7 @@ def __build_config( disabled: bool = None, auto_patch: bool = None, patch_modules: List = None, - provider: TracerProvider = None + provider: TracerProvider = None, ): """ Populates Tracer config for new and existing initializations """ is_disabled = disabled if disabled is not None else self.__is_trace_disabled() @@ -400,11 +398,27 @@ def __build_config( self._config["patch_modules"] = patch_modules if patch_modules else self._config["patch_modules"] if provider is not None: + self._validate_provider(provider) self._config["provider"] = provider - - if self._config["provider"] is None: + elif self._config["provider"] is None: self._config["provider"] = XrayProvider() @classmethod def _reset_config(cls): cls._config = copy.copy(cls._default_config) + + def _validate_provider(self, provider) -> bool: + invalid_provider_msg = f"{provider} must implement TracerProvider interface" + # not bound + if inspect.isclass(provider): + if not issubclass(provider, TracerProvider): + raise InvalidTracerProviderError(invalid_provider_msg) + raise TracerProviderNotInitializedError( + f"Initialize {provider} and pass a class instance reference as the provider." + ) + + # bound + if not isinstance(provider, TracerProvider): + raise InvalidTracerProviderError(invalid_provider_msg) + + return True diff --git a/python/tests/unit/test_tracing.py b/python/tests/unit/test_tracing.py index 4cc27faa6a..0b153b7b62 100644 --- a/python/tests/unit/test_tracing.py +++ b/python/tests/unit/test_tracing.py @@ -3,7 +3,8 @@ import pytest from aws_lambda_powertools.tracing import Tracer -from aws_lambda_powertools.tracing.base import TracerProvider +from aws_lambda_powertools.tracing.base import TracerProvider, XrayProvider +from aws_lambda_powertools.tracing.exceptions import InvalidTracerProviderError, TracerProviderNotInitializedError @pytest.fixture @@ -12,20 +13,20 @@ def dummy_response(): @pytest.fixture -def xray_stub(mocker): - class XRayStub: +def provider_stub(mocker): + class CustomProvider(TracerProvider): def __init__( self, put_metadata_mock: mocker.MagicMock = None, put_annotation_mock: mocker.MagicMock = None, - begin_subsegment_mock: mocker.MagicMock = None, + create_subsegment_mock: mocker.MagicMock = None, end_subsegment_mock: mocker.MagicMock = None, patch_mock: mocker.MagicMock = None, disable_tracing_provider_mock: mocker.MagicMock = None, ): self.put_metadata_mock = put_metadata_mock or mocker.MagicMock() self.put_annotation_mock = put_annotation_mock or mocker.MagicMock() - self.begin_subsegment_mock = begin_subsegment_mock or mocker.MagicMock() + self.create_subsegment_mock = create_subsegment_mock or mocker.MagicMock() self.end_subsegment_mock = end_subsegment_mock or mocker.MagicMock() self.patch_mock = patch_mock or mocker.MagicMock() self.disable_tracing_provider_mock = disable_tracing_provider_mock or mocker.MagicMock() @@ -36,8 +37,8 @@ def put_metadata(self, *args, **kwargs): def put_annotation(self, *args, **kwargs): return self.put_annotation_mock(*args, **kwargs) - def begin_subsegment(self, *args, **kwargs): - return self.begin_subsegment_mock(*args, **kwargs) + def create_subsegment(self, *args, **kwargs): + return self.create_subsegment_mock(*args, **kwargs) def end_subsegment(self, *args, **kwargs): return self.end_subsegment_mock(*args, **kwargs) @@ -48,50 +49,28 @@ def patch(self, *args, **kwargs): def disable_tracing_provider(self): self.disable_tracing_provider_mock() - return XRayStub + return CustomProvider @pytest.fixture(scope="function", autouse=True) -def reset_tracing_config(): +def reset_tracing_config(mocker): Tracer._reset_config() + # reset global cold start module + mocker.patch("aws_lambda_powertools.tracing.base.is_cold_start", return_value=True) yield -@pytest.fixture -def tracer_provider_stub(mocker): - class CustomTracerProvider(TracerProvider): - def create_subsegment(self, name): - pass - - def end_subsegment(self, name=None): - pass - - def patch(self): - pass - - def put_annotation(self, key, value): - pass - - def put_metadata(self, key, value, namespace=None): - pass - - def disable_tracing_provider(self): - pass - - return CustomTracerProvider - - -def test_tracer_lambda_handler(mocker, dummy_response, xray_stub): +def test_tracer_lambda_handler(mocker, dummy_response, provider_stub): put_metadata_mock = mocker.MagicMock() - begin_subsegment_mock = mocker.MagicMock() + create_subsegment_mock = mocker.MagicMock() end_subsegment_mock = mocker.MagicMock() - xray_provider = xray_stub( + provider = provider_stub( put_metadata_mock=put_metadata_mock, - begin_subsegment_mock=begin_subsegment_mock, + create_subsegment_mock=create_subsegment_mock, end_subsegment_mock=end_subsegment_mock, ) - tracer = Tracer(provider=xray_provider, service="booking") + tracer = Tracer(provider=provider, service="booking") @tracer.capture_lambda_handler def handler(event, context): @@ -99,27 +78,27 @@ def handler(event, context): handler({}, mocker.MagicMock()) - assert begin_subsegment_mock.call_count == 1 - assert begin_subsegment_mock.call_args == mocker.call(name="## handler") + assert create_subsegment_mock.call_count == 1 + assert create_subsegment_mock.call_args == mocker.call(name="## handler") assert end_subsegment_mock.call_count == 1 assert put_metadata_mock.call_args == mocker.call( key="lambda handler response", value=dummy_response, namespace="booking" ) -def test_tracer_method(mocker, dummy_response, xray_stub): +def test_tracer_method(mocker, dummy_response, provider_stub): put_metadata_mock = mocker.MagicMock() put_annotation_mock = mocker.MagicMock() - begin_subsegment_mock = mocker.MagicMock() + create_subsegment_mock = mocker.MagicMock() end_subsegment_mock = mocker.MagicMock() - xray_provider = xray_stub( + provider = provider_stub( put_metadata_mock=put_metadata_mock, put_annotation_mock=put_annotation_mock, - begin_subsegment_mock=begin_subsegment_mock, + create_subsegment_mock=create_subsegment_mock, end_subsegment_mock=end_subsegment_mock, ) - tracer = Tracer(provider=xray_provider, service="booking") + tracer = Tracer(provider=provider, service="booking") @tracer.capture_method def greeting(name, message): @@ -127,20 +106,20 @@ def greeting(name, message): greeting(name="Foo", message="Bar") - assert begin_subsegment_mock.call_count == 1 - assert begin_subsegment_mock.call_args == mocker.call(name="## greeting") + assert create_subsegment_mock.call_count == 1 + assert create_subsegment_mock.call_args == mocker.call(name="## greeting") assert end_subsegment_mock.call_count == 1 assert put_metadata_mock.call_args == mocker.call( key="greeting response", value=dummy_response, namespace="booking" ) -def test_tracer_custom_metadata(mocker, dummy_response, xray_stub): +def test_tracer_custom_metadata(mocker, dummy_response, provider_stub): put_metadata_mock = mocker.MagicMock() - xray_provider = xray_stub(put_metadata_mock=put_metadata_mock) + provider = provider_stub(put_metadata_mock=put_metadata_mock) - tracer = Tracer(provider=xray_provider, service="booking") + tracer = Tracer(provider=provider, service="booking") annotation_key = "Booking response" annotation_value = {"bookingStatus": "CONFIRMED"} @@ -157,12 +136,12 @@ def handler(event, context): ) -def test_tracer_custom_annotation(mocker, dummy_response, xray_stub): +def test_tracer_custom_annotation(mocker, dummy_response, provider_stub): put_annotation_mock = mocker.MagicMock() - xray_provider = xray_stub(put_annotation_mock=put_annotation_mock) + provider = provider_stub(put_annotation_mock=put_annotation_mock) - tracer = Tracer(provider=xray_provider, service="booking") + tracer = Tracer(provider=provider, service="booking") annotation_key = "BookingId" annotation_value = "123456" @@ -195,10 +174,10 @@ def test_tracer_no_autopatch(patch_mock): assert patch_mock.call_count == 0 -def test_tracer_lambda_handler_empty_response_metadata(mocker, xray_stub): +def test_tracer_lambda_handler_empty_response_metadata(mocker, provider_stub): put_metadata_mock = mocker.MagicMock() - xray_provider = xray_stub(put_metadata_mock=put_metadata_mock) - tracer = Tracer(provider=xray_provider) + provider = provider_stub(put_metadata_mock=put_metadata_mock) + tracer = Tracer(provider=provider) @tracer.capture_lambda_handler def handler(event, context): @@ -209,10 +188,10 @@ def handler(event, context): assert put_metadata_mock.call_count == 0 -def test_tracer_method_empty_response_metadata(mocker, xray_stub): +def test_tracer_method_empty_response_metadata(mocker, provider_stub): put_metadata_mock = mocker.MagicMock() - xray_provider = xray_stub(put_metadata_mock=put_metadata_mock) - tracer = Tracer(provider=xray_provider) + provider = provider_stub(put_metadata_mock=put_metadata_mock) + tracer = Tracer(provider=provider) @tracer.capture_method def greeting(name, message): @@ -223,105 +202,109 @@ def greeting(name, message): assert put_metadata_mock.call_count == 0 -def test_trace_provider_abc_init(mocker, xray_stub): +@mock.patch("aws_lambda_powertools.tracing.base.aws_xray_sdk.core.patch") +@mock.patch("aws_lambda_powertools.tracing.base.aws_xray_sdk.core.patch_all") +def test_tracer_xray_provider(xray_patch_all_mock, xray_patch_mock, mocker): + # GIVEN tracer is instantiated + # WHEN default X-Ray provider client is mocked + # THEN tracer should run just fine + xray_client_mock = mock.MagicMock() + xray_client_mock.begin_subsegment = mock.MagicMock() + xray_client_mock.end_subsegment = mock.MagicMock() + xray_client_mock.put_annotation = mock.MagicMock() + xray_client_mock.put_metadata = mock.MagicMock() + xray_provider = XrayProvider(client=xray_client_mock) + + Tracer(provider=xray_provider) + assert xray_patch_all_mock.call_count == 1 + + modules = ["boto3"] + tracer = Tracer(service="booking", provider=xray_provider, patch_modules=modules) + assert xray_patch_mock.call_count == 1 + assert xray_patch_mock.call_args == mocker.call(modules) + + tracer.create_subsegment("test subsegment") + tracer.put_annotation(key="test_annotation", value="value") + tracer.put_metadata(key="test_metadata", value="value") + tracer.end_subsegment() + + assert xray_client_mock.begin_subsegment.call_count == 1 + assert xray_client_mock.begin_subsegment.call_args == mocker.call(name="test subsegment") + assert xray_client_mock.end_subsegment.call_count == 1 + assert xray_client_mock.put_annotation.call_count == 1 + assert xray_client_mock.put_annotation.call_args == mocker.call(key="test_annotation", value="value") + assert xray_client_mock.put_metadata.call_count == 1 + assert xray_client_mock.put_metadata.call_args == mocker.call( + key="test_metadata", value="value", namespace="booking" + ) + + +def test_tracer_xray_provider_cold_start(mocker): + # GIVEN tracer is instantiated + # WHEN multiple subsegments are created + # THEN tracer should record cold start only once + xray_client_mock = mock.MagicMock() + xray_client_mock.begin_subsegment.put_annotation = mock.MagicMock() + xray_provider = XrayProvider(client=xray_client_mock) + + tracer = Tracer(provider=xray_provider) + subsegment_mock = tracer.create_subsegment("test subsegment") + subsegment_mock = tracer.create_subsegment("test subsegment 2") + + assert subsegment_mock.put_annotation.call_count == 1 + assert subsegment_mock.put_annotation.call_args == mocker.call(key="ColdStart", value=True) + + +def test_trace_provider_abc_no_init(provider_stub): # GIVEN tracer is instantiated # WHEN a custom provider that implements TracerProvider methods # THEN it should run successfully - class XrayTracer(TracerProvider): - def create_subsegment(self, name): + + with pytest.raises(TracerProviderNotInitializedError): + Tracer(service="booking", provider=provider_stub) + + +def test_trace_invalid_provider(): + # GIVEN tracer is instantiated + # WHEN a custom provider does not implement TracerProvider + # THEN it should raise InvalidTracerProviderError + + class CustomProvider: + def __init__(self): pass - def end_subsegment(self, name=None): + def put_metadata(self, *args, **kwargs): + pass + + def put_annotation(self, *args, **kwargs): pass - def patch(self): + def create_subsegment(self, *args, **kwargs): pass - def put_annotation(self, key, value): + def end_subsegment(self, *args, **kwargs): pass - def put_metadata(self, key, value, namespace=None): + def patch(self, *args, **kwargs): pass def disable_tracing_provider(self): pass - with mock.patch.object(Tracer, "patch") as patch_mock: - tracer = Tracer(service="booking", provider=XrayTracer) - tracer2 = Tracer(auto_patch=False) - assert patch_mock.call_count == 1 - assert tracer.service == "booking" - assert tracer2.service == "booking" # inherited from tracer1 - assert tracer2.auto_patch == False # overriden in tracer2 - - -# def test_trace_provider_abc_decorators(mocker, dummy_response, xray_stub): -# # GIVEN tracer is instantiated -# # WHEN a new tracer provider implements TracerProvider methods -# # THEN it should inherit default decorators -# class XrayTracer(TracerProvider): - -# def create_subsegment(self, name): -# pass - -# def end_subsegment(self, name=None): -# pass - -# def patch(self): -# pass - -# def put_annotation(self, key, value): -# pass - -# def put_metadata(self, key, value, namespace=None): -# pass - -# put_metadata_mock = mocker.MagicMock() -# put_annotation_mock = mocker.MagicMock() -# begin_subsegment_mock = mocker.MagicMock() -# end_subsegment_mock = mocker.MagicMock() - -# xray_provider = xray_stub( -# put_metadata_mock=put_metadata_mock, -# put_annotation_mock=put_annotation_mock, -# begin_subsegment_mock=begin_subsegment_mock, -# end_subsegment_mock=end_subsegment_mock, -# ) - -# tracer = XrayTracer(provider=xray_provider, service="booking") -# annotation_key = "BookingId" -# annotation_value = "123456" - -# @tracer.capture_lambda_handler -# def handler(event, context): -# tracer.put_metadata(annotation_key, annotation_value) -# return dummy_response - -# handler({}, mocker.MagicMock()) - -# # It's 0 because it's calling base class method -# # and base class provider is different from ours -# # base class is calling ours methods though - -# # TODO - Create X-Ray Tracer class, and implement decorators there -# # TODO - Leave only docstring for Trace Provider -# # TODO - Update tests -# # FIXME - Create provider abc only, reuse Tracer class, optionally initialize - -# assert put_metadata_mock.call_count == 2 -# assert put_metadata_mock.call_args_list[0] == mocker.call( -# key=annotation_key, value=annotation_value, namespace="booking" -# ) - -# @tracer.capture_method -# def greeting(name, message): -# return dummy_response - -# greeting(name="Foo", message="Bar") - -# assert begin_subsegment_mock.call_count == 1 -# assert begin_subsegment_mock.call_args == mocker.call(name="## greeting") -# assert end_subsegment_mock.call_count == 1 -# assert put_metadata_mock.call_args == mocker.call( -# key="greeting response", value=dummy_response, namespace="booking" -# ) + with pytest.raises(InvalidTracerProviderError): + Tracer(service="booking", provider=CustomProvider) + + class InvalidProvider: + pass + + with pytest.raises(InvalidTracerProviderError): + Tracer(service="booking", provider=InvalidProvider) + + with pytest.raises(InvalidTracerProviderError): + Tracer(service="booking", provider=InvalidProvider()) + + with pytest.raises(InvalidTracerProviderError): + Tracer(service="booking", provider=True) + + with pytest.raises(InvalidTracerProviderError): + Tracer(service="booking", provider="provider") From e3fddaf992149eb746f91ce283e3e0a75c3063cc Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 4 May 2020 18:54:02 +0100 Subject: [PATCH 03/35] improv: update docs, linting --- .../metrics/exceptions.py | 8 +++++ .../middleware_factory/exceptions.py | 2 ++ python/aws_lambda_powertools/tracing/base.py | 36 +++++++++++++++---- .../tracing/exceptions.py | 5 +++ .../aws_lambda_powertools/tracing/tracer.py | 27 +++++++++----- 5 files changed, 62 insertions(+), 16 deletions(-) diff --git a/python/aws_lambda_powertools/metrics/exceptions.py b/python/aws_lambda_powertools/metrics/exceptions.py index b9b1107e74..88a38c2422 100644 --- a/python/aws_lambda_powertools/metrics/exceptions.py +++ b/python/aws_lambda_powertools/metrics/exceptions.py @@ -1,14 +1,22 @@ class MetricUnitError(Exception): + """When metric unit is not supported by CloudWatch""" + pass class SchemaValidationError(Exception): + """When serialization fail schema validation""" + pass class MetricValueError(Exception): + """When metric value isn't a valid number""" + pass class UniqueNamespaceError(Exception): + """When an additional namespace is set""" + pass diff --git a/python/aws_lambda_powertools/middleware_factory/exceptions.py b/python/aws_lambda_powertools/middleware_factory/exceptions.py index 55d5b2342b..4d807b8538 100644 --- a/python/aws_lambda_powertools/middleware_factory/exceptions.py +++ b/python/aws_lambda_powertools/middleware_factory/exceptions.py @@ -1,2 +1,4 @@ class MiddlewareInvalidArgumentError(Exception): + """When middleware receives non keyword=arguments""" + pass diff --git a/python/aws_lambda_powertools/tracing/base.py b/python/aws_lambda_powertools/tracing/base.py index 811dd449db..5c190fba23 100644 --- a/python/aws_lambda_powertools/tracing/base.py +++ b/python/aws_lambda_powertools/tracing/base.py @@ -1,11 +1,8 @@ from __future__ import annotations import abc -import copy -import functools import logging -import os -from typing import Any, Callable, List +from typing import Any, List import aws_xray_sdk import aws_xray_sdk.core @@ -20,7 +17,14 @@ class TracerProvider(metaclass=abc.ABCMeta): Providers should be initialized independently. This allows providers to control their config/initialization, and only pass a class instance to - ```aws_lambda_powertools.tracing.tracer.Tracer```. + `aws_lambda_powertools.tracing.tracer.Tracer`. + + It also allows custom providers to keep lean while Tracer provide: + + * a simplified UX + * decorators for Lambda handler and methods + * auto-patching, patch all modules by default or a subset + * disabling all tracing operations with a single parameter or env var Trace providers should implement the following methods: @@ -32,10 +36,17 @@ class TracerProvider(metaclass=abc.ABCMeta): * **disable_tracing_provider** These methods will be called by - ```aws_lambda_powertools.tracing.tracer.Tracer```. - See ```aws_lambda_powertools.tracing.base.XrayProvider``` + `aws_lambda_powertools.tracing.tracer.Tracer` - + See `aws_lambda_powertools.tracing.base.XrayProvider` for a reference implementation. + `aws_lambda_powertools.tracing.tracer.Tracer` decorators + for Lambda and methods use the following provider methods: + + * create_subsegment + * put_metadata + * end_subsegment + Example ------- **Client using a custom tracing provider** @@ -112,6 +123,17 @@ def disable_tracing_provider(self): class XrayProvider(TracerProvider): + """X-Ray Tracer provider + + It implements all basic ``aws_lambda_powertools.tracing.base.TracerProvider` methods, + and automatically annotates cold start on first subsegment created. + + Parameters + ---------- + client : aws_xray_sdk.core.xray_recorder + X-Ray recorder client + """ + def __init__(self, client: aws_xray_sdk.core.xray_recorder = aws_xray_sdk.core.xray_recorder): self.client = client diff --git a/python/aws_lambda_powertools/tracing/exceptions.py b/python/aws_lambda_powertools/tracing/exceptions.py index 12d2ad24e3..07cc68574c 100644 --- a/python/aws_lambda_powertools/tracing/exceptions.py +++ b/python/aws_lambda_powertools/tracing/exceptions.py @@ -1,5 +1,10 @@ class InvalidTracerProviderError(Exception): + """When given provider doesn't implement `aws_lambda_powertools.tracing.base.TracerProvider`""" + pass + class TracerProviderNotInitializedError(Exception): + """When given provider isn't initialized/bound""" + pass diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index fa5ca43cd0..c82290cb6b 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -4,11 +4,12 @@ import logging import os from distutils.util import strtobool -from typing import Any, Callable, Dict, List +from typing import Any, Callable, Dict, Generic, List, TypeVar from .base import TracerProvider, XrayProvider from .exceptions import InvalidTracerProviderError, TracerProviderNotInitializedError +subsegment = TypeVar("subsegment") is_cold_start = True logger = logging.getLogger(__name__) @@ -27,6 +28,9 @@ class Tracer: is useful when you are using your own middlewares and want to utilize an existing Tracer. Make sure to set `auto_patch=False` in subsequent Tracer instances to avoid double patching. + Tracer supports custom providers that implement + `aws_lambda_powertools.tracing.base.TracerProvider`. + Environment variables --------------------- POWERTOOLS_TRACE_DISABLED : str @@ -115,6 +119,13 @@ def handler(event: dict, context: Any) -> Dict: Tracer Tracer instance with imported modules patched + Raises + ------ + InvalidTracerProviderError + When given provider doesn't implement `aws_lambda_powertools.tracing.base.TracerProvider` + TracerProviderNotInitializedError + When given provider isn't initialized/bound + Limitations ----------- * Async handler and methods not supported @@ -152,7 +163,7 @@ def __init__( if self.auto_patch: self.patch(modules=patch_modules) - def create_subsegment(self, name: str): + def create_subsegment(self, name: str) -> Generic[subsegment]: """Creates subsegment/span with a given name It also assumes Tracer would be instantiated statically so that cold starts are captured. @@ -168,11 +179,10 @@ def create_subsegment(self, name: str): self.create_subsegment(name="a meaningful name") - # FIXME - Return Subsegment Any type Returns ------- - models.subsegment - AWS X-Ray Subsegment + subsegment + Trace provider subsegment """ # Will no longer be needed once #155 is resolved # https://github.com/aws/aws-xray-sdk-python/issues/155 @@ -204,7 +214,7 @@ def put_annotation(self, key: str, value: Any): ---------- key : str Annotation key (e.g. PaymentStatus) - value : Any + value : any Value for annotation (e.g. "CONFIRMED") """ # Will no longer be needed once #155 is resolved @@ -216,14 +226,14 @@ def put_annotation(self, key: str, value: Any): logger.debug(f"Annotating on key '{key}'' with '{value}''") self.provider.put_annotation(key=key, value=value) - def put_metadata(self, key: str, value: object, namespace: str = None): + def put_metadata(self, key: str, value: Any, namespace: str = None): """Adds metadata to existing segment or subsegment Parameters ---------- key : str Metadata key - value : object + value : any Value for metadata namespace : str, optional Namespace that metadata will lie under, by default None @@ -285,7 +295,6 @@ def handler(event, context) @functools.wraps(lambda_handler) def decorate(event, context): self.create_subsegment(name=f"## {lambda_handler.__name__}") - try: logger.debug("Calling lambda handler") response = lambda_handler(event, context) From 5ba54a8a0da2f52991ffc785ce3b5c4792a1ebe8 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 5 May 2020 10:05:12 +0100 Subject: [PATCH 04/35] improv: docstring readability and links --- .../aws_lambda_powertools/tracing/__init__.py | 2 ++ python/aws_lambda_powertools/tracing/base.py | 30 +++++++++++-------- .../aws_lambda_powertools/tracing/tracer.py | 9 ++++-- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/python/aws_lambda_powertools/tracing/__init__.py b/python/aws_lambda_powertools/tracing/__init__.py index 136fccce9f..d3e71dcd4d 100644 --- a/python/aws_lambda_powertools/tracing/__init__.py +++ b/python/aws_lambda_powertools/tracing/__init__.py @@ -1,5 +1,7 @@ """Tracing utility """ +from .base import TracerProvider, XrayProvider +from .exceptions import InvalidTracerProviderError, TracerProviderNotInitializedError from .tracer import Tracer __all__ = ["Tracer"] diff --git a/python/aws_lambda_powertools/tracing/base.py b/python/aws_lambda_powertools/tracing/base.py index 5c190fba23..45107ee57f 100644 --- a/python/aws_lambda_powertools/tracing/base.py +++ b/python/aws_lambda_powertools/tracing/base.py @@ -26,14 +26,14 @@ class TracerProvider(metaclass=abc.ABCMeta): * auto-patching, patch all modules by default or a subset * disabling all tracing operations with a single parameter or env var - Trace providers should implement the following methods: + Trace providers should implement - * **patch** - * **create_subsegment** - * **end_subsegment** - * **put_metadata** - * **put_annotation** - * **disable_tracing_provider** + * `aws_lambda_powertools.tracing.base.TracerProvider.patch` + * `aws_lambda_powertools.tracing.base.TracerProvider.create_subsegment` + * `aws_lambda_powertools.tracing.base.TracerProvider.end_subsegment` + * `aws_lambda_powertools.tracing.base.TracerProvider.put_metadata` + * `aws_lambda_powertools.tracing.base.TracerProvider.put_annotation` + * `aws_lambda_powertools.tracing.base.TracerProvider.disable_tracing_provider` These methods will be called by `aws_lambda_powertools.tracing.tracer.Tracer` - @@ -41,18 +41,20 @@ class TracerProvider(metaclass=abc.ABCMeta): for a reference implementation. `aws_lambda_powertools.tracing.tracer.Tracer` decorators - for Lambda and methods use the following provider methods: + `aws_lambda_powertools.tracing.Tracer.capture_lambda_handler`, + and `aws_lambda_powertools.tracing.Tracer.capture_method` + use the following provider methods: - * create_subsegment - * put_metadata - * end_subsegment + * `aws_lambda_powertools.tracing.base.TracerProvider.create_subsegment` + * `aws_lambda_powertools.tracing.base.TracerProvider.end_subsegment` + * `aws_lambda_powertools.tracing.base.TracerProvider.put_metadata` Example ------- **Client using a custom tracing provider** from aws_lambda_powertools.tracing import Tracer - ... import ... ProviderX + import ProviderX custom_provider = ProviderX() tracer = Tracer(service="greeting", provider=custom_provider) """ @@ -125,9 +127,11 @@ def disable_tracing_provider(self): class XrayProvider(TracerProvider): """X-Ray Tracer provider - It implements all basic ``aws_lambda_powertools.tracing.base.TracerProvider` methods, + It implements all basic `aws_lambda_powertools.tracing.base.TracerProvider` methods, and automatically annotates cold start on first subsegment created. + XrayProvider is the default provider for `aws_lambda_powertools.tracing.Tracer`. + Parameters ---------- client : aws_xray_sdk.core.xray_recorder diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index c82290cb6b..7f92896df6 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -29,7 +29,8 @@ class Tracer: Make sure to set `auto_patch=False` in subsequent Tracer instances to avoid double patching. Tracer supports custom providers that implement - `aws_lambda_powertools.tracing.base.TracerProvider`. + `aws_lambda_powertools.tracing.base.TracerProvider`. It defaults and initializes to + `aws_lambda_powertools.tracing.base.XrayProvider` if no custom provider is given. Environment variables --------------------- @@ -45,8 +46,12 @@ class Tracer: auto_patch: bool Patch existing imported modules during initialization, by default True disabled: bool - Flag to explicitly disable tracing, useful when running/testing locally. + Flag to explicitly disable tracing, useful when running/testing locally `Env POWERTOOLS_TRACE_DISABLED="true"` + patch_modules: List + List of modules supported by tracing provider to patch, by default all modules are patched + provider: TracerProvider + Tracing provider, by default `aws_lambda_powertools.tracing.base.XrayProvider` Example ------- From 0adc7a7f25b05405387e5b2dc2301633d73954d6 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 5 May 2020 14:30:04 +0100 Subject: [PATCH 05/35] improv: remove tracer provider --- .../aws_lambda_powertools/tracing/__init__.py | 2 - python/aws_lambda_powertools/tracing/base.py | 237 ------------------ .../tracing/exceptions.py | 10 - .../aws_lambda_powertools/tracing/tracer.py | 93 +++---- python/tests/unit/test_tracing.py | 134 +++------- 5 files changed, 66 insertions(+), 410 deletions(-) delete mode 100644 python/aws_lambda_powertools/tracing/base.py delete mode 100644 python/aws_lambda_powertools/tracing/exceptions.py diff --git a/python/aws_lambda_powertools/tracing/__init__.py b/python/aws_lambda_powertools/tracing/__init__.py index d3e71dcd4d..136fccce9f 100644 --- a/python/aws_lambda_powertools/tracing/__init__.py +++ b/python/aws_lambda_powertools/tracing/__init__.py @@ -1,7 +1,5 @@ """Tracing utility """ -from .base import TracerProvider, XrayProvider -from .exceptions import InvalidTracerProviderError, TracerProviderNotInitializedError from .tracer import Tracer __all__ = ["Tracer"] diff --git a/python/aws_lambda_powertools/tracing/base.py b/python/aws_lambda_powertools/tracing/base.py deleted file mode 100644 index 45107ee57f..0000000000 --- a/python/aws_lambda_powertools/tracing/base.py +++ /dev/null @@ -1,237 +0,0 @@ -from __future__ import annotations - -import abc -import logging -from typing import Any, List - -import aws_xray_sdk -import aws_xray_sdk.core - -is_cold_start = True -logger = logging.getLogger(__name__) - - -class TracerProvider(metaclass=abc.ABCMeta): - """Tracer provider abstract class - - Providers should be initialized independently. This - allows providers to control their config/initialization, - and only pass a class instance to - `aws_lambda_powertools.tracing.tracer.Tracer`. - - It also allows custom providers to keep lean while Tracer provide: - - * a simplified UX - * decorators for Lambda handler and methods - * auto-patching, patch all modules by default or a subset - * disabling all tracing operations with a single parameter or env var - - Trace providers should implement - - * `aws_lambda_powertools.tracing.base.TracerProvider.patch` - * `aws_lambda_powertools.tracing.base.TracerProvider.create_subsegment` - * `aws_lambda_powertools.tracing.base.TracerProvider.end_subsegment` - * `aws_lambda_powertools.tracing.base.TracerProvider.put_metadata` - * `aws_lambda_powertools.tracing.base.TracerProvider.put_annotation` - * `aws_lambda_powertools.tracing.base.TracerProvider.disable_tracing_provider` - - These methods will be called by - `aws_lambda_powertools.tracing.tracer.Tracer` - - See `aws_lambda_powertools.tracing.base.XrayProvider` - for a reference implementation. - - `aws_lambda_powertools.tracing.tracer.Tracer` decorators - `aws_lambda_powertools.tracing.Tracer.capture_lambda_handler`, - and `aws_lambda_powertools.tracing.Tracer.capture_method` - use the following provider methods: - - * `aws_lambda_powertools.tracing.base.TracerProvider.create_subsegment` - * `aws_lambda_powertools.tracing.base.TracerProvider.end_subsegment` - * `aws_lambda_powertools.tracing.base.TracerProvider.put_metadata` - - Example - ------- - **Client using a custom tracing provider** - - from aws_lambda_powertools.tracing import Tracer - import ProviderX - custom_provider = ProviderX() - tracer = Tracer(service="greeting", provider=custom_provider) - """ - - @abc.abstractmethod - def patch(self, modules: List[str] = None): - """Patch modules for instrumentation - - If modules are None, it should patch - all supported modules by the provider. - - Parameters - ---------- - modules : List[str], optional - List of modules to be pathced, by default None - e.g. `['boto3', 'requests']` - """ - raise NotImplementedError - - @abc.abstractmethod - def create_subsegment(self, name: str): - """Creates subsegment/span with a given name - - Parameters - ---------- - name : str - Subsegment/span name - """ - raise NotImplementedError - - @abc.abstractmethod - def end_subsegment(self): - """Ends an existing subsegment""" - raise NotImplementedError - - @abc.abstractmethod - def put_metadata(self, key: str, value: Any, namespace: str = None): - """Adds metadata to existing segment/span or subsegment - - Parameters - ---------- - key : str - Metadata key - value : Any - Metadata value - namespace : str, optional - Metadata namespace, by default None - """ - raise NotImplementedError - - @abc.abstractmethod - def put_annotation(self, key: str, value: Any): - """Adds annotation/label to existing segment/span or subsegment - - Parameters - ---------- - key : str - Annotation/label key - value : Any - Annotation/label value - """ - raise NotImplementedError - - @abc.abstractmethod - def disable_tracing_provider(self): - """Forcefully disables tracing provider""" - raise NotImplementedError - - -class XrayProvider(TracerProvider): - """X-Ray Tracer provider - - It implements all basic `aws_lambda_powertools.tracing.base.TracerProvider` methods, - and automatically annotates cold start on first subsegment created. - - XrayProvider is the default provider for `aws_lambda_powertools.tracing.Tracer`. - - Parameters - ---------- - client : aws_xray_sdk.core.xray_recorder - X-Ray recorder client - """ - - def __init__(self, client: aws_xray_sdk.core.xray_recorder = aws_xray_sdk.core.xray_recorder): - self.client = client - - def create_subsegment(self, name: str) -> aws_xray_sdk.core.models.subsegment: - """Creates subsegment/span with a given name - - Parameters - ---------- - name : str - Subsegment name - - Example - ------- - - **Creates a subsegment** - - self.create_subsegment(name="a meaningful name") - - Returns - ------- - aws_xray_sdk.core.models.subsegment - AWS X-Ray Subsegment - """ - # Will no longer be needed once #155 is resolved - # https://github.com/aws/aws-xray-sdk-python/issues/155 - subsegment = self.client.begin_subsegment(name=name) - global is_cold_start - if is_cold_start: - logger.debug("Annotating cold start") - subsegment.put_annotation(key="ColdStart", value=True) - is_cold_start = False - - return subsegment - - def end_subsegment(self): - """Ends an existing subsegment""" - self.client.end_subsegment() - - def put_annotation(self, key, value): - """Adds annotation to existing segment or subsegment - - Example - ------- - Custom annotation for a pseudo service named payment - - tracer = Tracer(service="payment") - tracer.put_annotation("PaymentStatus", "CONFIRMED") - - Parameters - ---------- - key : str - Annotation key (e.g. PaymentStatus) - value : Any - Value for annotation (e.g. "CONFIRMED") - """ - self.client.put_annotation(key=key, value=value) - - def put_metadata(self, key, value, namespace=None): - """Adds metadata to existing segment or subsegment - - Parameters - ---------- - key : str - Metadata key - value : object - Value for metadata - namespace : str, optional - Namespace that metadata will lie under, by default None - - Example - ------- - Custom metadata for a pseudo service named payment - - tracer = Tracer(service="payment") - response = collect_payment() - tracer.put_metadata("Payment collection", response) - """ - self.client.put_metadata(key=key, value=value, namespace=namespace) - - def patch(self, modules: List[str] = None): - """Patch modules for instrumentation. - - Patches all supported modules by default if none are given. - - Parameters - ---------- - modules : List[str] - List of modules to be patched, optional by default - """ - if modules is None: - aws_xray_sdk.core.patch_all() - else: - aws_xray_sdk.core.patch(modules) - - def disable_tracing_provider(self): - """Forcefully disables X-Ray tracing globally""" - aws_xray_sdk.global_sdk_config.set_sdk_enabled(False) diff --git a/python/aws_lambda_powertools/tracing/exceptions.py b/python/aws_lambda_powertools/tracing/exceptions.py deleted file mode 100644 index 07cc68574c..0000000000 --- a/python/aws_lambda_powertools/tracing/exceptions.py +++ /dev/null @@ -1,10 +0,0 @@ -class InvalidTracerProviderError(Exception): - """When given provider doesn't implement `aws_lambda_powertools.tracing.base.TracerProvider`""" - - pass - - -class TracerProviderNotInitializedError(Exception): - """When given provider isn't initialized/bound""" - - pass diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index 7f92896df6..44b9bca540 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -4,15 +4,16 @@ import logging import os from distutils.util import strtobool -from typing import Any, Callable, Dict, Generic, List, TypeVar +from typing import Any, Callable, Dict, Generic, List -from .base import TracerProvider, XrayProvider -from .exceptions import InvalidTracerProviderError, TracerProviderNotInitializedError +import aws_xray_sdk +import aws_xray_sdk.core -subsegment = TypeVar("subsegment") is_cold_start = True logger = logging.getLogger(__name__) +# FIXME - Add get current subsegment() +# FIXME - Add subsegment type class Tracer: """Tracer using AWS-XRay to provide decorators with known defaults for Lambda functions @@ -28,10 +29,6 @@ class Tracer: is useful when you are using your own middlewares and want to utilize an existing Tracer. Make sure to set `auto_patch=False` in subsequent Tracer instances to avoid double patching. - Tracer supports custom providers that implement - `aws_lambda_powertools.tracing.base.TracerProvider`. It defaults and initializes to - `aws_lambda_powertools.tracing.base.XrayProvider` if no custom provider is given. - Environment variables --------------------- POWERTOOLS_TRACE_DISABLED : str @@ -50,8 +47,6 @@ class Tracer: `Env POWERTOOLS_TRACE_DISABLED="true"` patch_modules: List List of modules supported by tracing provider to patch, by default all modules are patched - provider: TracerProvider - Tracing provider, by default `aws_lambda_powertools.tracing.base.XrayProvider` Example ------- @@ -124,13 +119,6 @@ def handler(event: dict, context: Any) -> Dict: Tracer Tracer instance with imported modules patched - Raises - ------ - InvalidTracerProviderError - When given provider doesn't implement `aws_lambda_powertools.tracing.base.TracerProvider` - TracerProviderNotInitializedError - When given provider isn't initialized/bound - Limitations ----------- * Async handler and methods not supported @@ -142,7 +130,7 @@ def handler(event: dict, context: Any) -> Dict: "provider": None, "auto_patch": True, "patch_modules": None, - "provider": None, + "provider": aws_xray_sdk.core.xray_recorder, } _config = copy.copy(_default_config) @@ -152,7 +140,7 @@ def __init__( disabled: bool = None, auto_patch: bool = None, patch_modules: List = None, - provider: TracerProvider = None, + provider: aws_xray_sdk.core.xray_recorder = None, ): self.__build_config( service=service, disabled=disabled, auto_patch=auto_patch, patch_modules=patch_modules, provider=provider @@ -168,11 +156,9 @@ def __init__( if self.auto_patch: self.patch(modules=patch_modules) - def create_subsegment(self, name: str) -> Generic[subsegment]: + def create_subsegment(self, name: str) -> aws_xray_sdk.core.models.subsegment: """Creates subsegment/span with a given name - It also assumes Tracer would be instantiated statically so that cold starts are captured. - Parameters ---------- name : str @@ -180,25 +166,30 @@ def create_subsegment(self, name: str) -> Generic[subsegment]: Example ------- - Creates a genuine subsegment + + **Creates a subsegment** self.create_subsegment(name="a meaningful name") Returns ------- - subsegment - Trace provider subsegment + aws_xray_sdk.core.models.subsegment + AWS X-Ray Subsegment """ # Will no longer be needed once #155 is resolved # https://github.com/aws/aws-xray-sdk-python/issues/155 - if self.disabled: - logger.debug("Tracing has been disabled, aborting create_segment") - return + subsegment = self.provider.begin_subsegment(name=name) + global is_cold_start + if is_cold_start: + logger.debug("Annotating cold start") + subsegment.put_annotation(key="ColdStart", value=True) + is_cold_start = False - return self.provider.create_subsegment(name=name) + return subsegment def end_subsegment(self): """Ends an existing subsegment""" + # FIXME - Receive subsegment to close if self.disabled: logger.debug("Tracing has been disabled, aborting end_subsegment") return @@ -260,17 +251,28 @@ def put_metadata(self, key: str, value: Any, namespace: str = None): self.provider.put_metadata(key=key, value=value, namespace=namespace) def patch(self, modules: List[str] = None): - """Patch modules for instrumentation""" + """Patch modules for instrumentation. + + Patches all supported modules by default if none are given. + + Parameters + ---------- + modules : List[str] + List of modules to be patched, optional by default + """ if self.disabled: logger.debug("Tracing has been disabled, aborting patch") return - self.provider.patch(modules=modules) + if modules is None: + aws_xray_sdk.core.patch_all() + else: + aws_xray_sdk.core.patch(modules) def disable_tracing_provider(self): """Forcefully disables tracing""" logger.debug("Disabling tracer provider...") - self.provider.disable_tracing_provider() + aws_xray_sdk.global_sdk_config.set_sdk_enabled(False) def capture_lambda_handler(self, lambda_handler: Callable[[Dict, Any], Any] = None): """Decorator to create subsegment for lambda handlers @@ -299,6 +301,7 @@ def handler(event, context) @functools.wraps(lambda_handler) def decorate(event, context): + # FIXME - Use newly created subsegment to avoid race conditions self.create_subsegment(name=f"## {lambda_handler.__name__}") try: logger.debug("Calling lambda handler") @@ -346,6 +349,7 @@ def some_function() @functools.wraps(method) def decorate(*args, **kwargs): method_name = f"{method.__name__}" + # FIXME - Use newly created subsegment to avoid race conditions self.create_subsegment(name=f"## {method_name}") try: @@ -400,39 +404,18 @@ def __build_config( disabled: bool = None, auto_patch: bool = None, patch_modules: List = None, - provider: TracerProvider = None, + provider: aws_xray_sdk.core.xray_recorder = None, ): """ Populates Tracer config for new and existing initializations """ is_disabled = disabled if disabled is not None else self.__is_trace_disabled() is_service = service if service is not None else os.getenv("POWERTOOLS_SERVICE_NAME") + self._config["provider"] = provider if provider is not None else self._config["provider"] self._config["auto_patch"] = auto_patch if auto_patch is not None else self._config["auto_patch"] self._config["service"] = is_service if is_service else self._config["service"] self._config["disabled"] = is_disabled if is_disabled else self._config["disabled"] self._config["patch_modules"] = patch_modules if patch_modules else self._config["patch_modules"] - if provider is not None: - self._validate_provider(provider) - self._config["provider"] = provider - elif self._config["provider"] is None: - self._config["provider"] = XrayProvider() - @classmethod def _reset_config(cls): cls._config = copy.copy(cls._default_config) - - def _validate_provider(self, provider) -> bool: - invalid_provider_msg = f"{provider} must implement TracerProvider interface" - # not bound - if inspect.isclass(provider): - if not issubclass(provider, TracerProvider): - raise InvalidTracerProviderError(invalid_provider_msg) - raise TracerProviderNotInitializedError( - f"Initialize {provider} and pass a class instance reference as the provider." - ) - - # bound - if not isinstance(provider, TracerProvider): - raise InvalidTracerProviderError(invalid_provider_msg) - - return True diff --git a/python/tests/unit/test_tracing.py b/python/tests/unit/test_tracing.py index 0b153b7b62..7d8cd9d42c 100644 --- a/python/tests/unit/test_tracing.py +++ b/python/tests/unit/test_tracing.py @@ -3,8 +3,6 @@ import pytest from aws_lambda_powertools.tracing import Tracer -from aws_lambda_powertools.tracing.base import TracerProvider, XrayProvider -from aws_lambda_powertools.tracing.exceptions import InvalidTracerProviderError, TracerProviderNotInitializedError @pytest.fixture @@ -14,19 +12,19 @@ def dummy_response(): @pytest.fixture def provider_stub(mocker): - class CustomProvider(TracerProvider): + class CustomProvider(): def __init__( self, put_metadata_mock: mocker.MagicMock = None, put_annotation_mock: mocker.MagicMock = None, - create_subsegment_mock: mocker.MagicMock = None, + begin_subsegment: mocker.MagicMock = None, end_subsegment_mock: mocker.MagicMock = None, patch_mock: mocker.MagicMock = None, disable_tracing_provider_mock: mocker.MagicMock = None, ): self.put_metadata_mock = put_metadata_mock or mocker.MagicMock() self.put_annotation_mock = put_annotation_mock or mocker.MagicMock() - self.create_subsegment_mock = create_subsegment_mock or mocker.MagicMock() + self.begin_subsegment = begin_subsegment or mocker.MagicMock() self.end_subsegment_mock = end_subsegment_mock or mocker.MagicMock() self.patch_mock = patch_mock or mocker.MagicMock() self.disable_tracing_provider_mock = disable_tracing_provider_mock or mocker.MagicMock() @@ -38,7 +36,7 @@ def put_annotation(self, *args, **kwargs): return self.put_annotation_mock(*args, **kwargs) def create_subsegment(self, *args, **kwargs): - return self.create_subsegment_mock(*args, **kwargs) + return self.begin_subsegment(*args, **kwargs) def end_subsegment(self, *args, **kwargs): return self.end_subsegment_mock(*args, **kwargs) @@ -46,9 +44,6 @@ def end_subsegment(self, *args, **kwargs): def patch(self, *args, **kwargs): return self.patch_mock(*args, **kwargs) - def disable_tracing_provider(self): - self.disable_tracing_provider_mock() - return CustomProvider @@ -56,18 +51,18 @@ def disable_tracing_provider(self): def reset_tracing_config(mocker): Tracer._reset_config() # reset global cold start module - mocker.patch("aws_lambda_powertools.tracing.base.is_cold_start", return_value=True) + mocker.patch("aws_lambda_powertools.tracing.tracer.is_cold_start", return_value=True) yield def test_tracer_lambda_handler(mocker, dummy_response, provider_stub): put_metadata_mock = mocker.MagicMock() - create_subsegment_mock = mocker.MagicMock() + begin_subsegment = mocker.MagicMock() end_subsegment_mock = mocker.MagicMock() provider = provider_stub( put_metadata_mock=put_metadata_mock, - create_subsegment_mock=create_subsegment_mock, + begin_subsegment=begin_subsegment, end_subsegment_mock=end_subsegment_mock, ) tracer = Tracer(provider=provider, service="booking") @@ -78,8 +73,8 @@ def handler(event, context): handler({}, mocker.MagicMock()) - assert create_subsegment_mock.call_count == 1 - assert create_subsegment_mock.call_args == mocker.call(name="## handler") + assert begin_subsegment.call_count == 1 + assert begin_subsegment.call_args == mocker.call(name="## handler") assert end_subsegment_mock.call_count == 1 assert put_metadata_mock.call_args == mocker.call( key="lambda handler response", value=dummy_response, namespace="booking" @@ -89,13 +84,13 @@ def handler(event, context): def test_tracer_method(mocker, dummy_response, provider_stub): put_metadata_mock = mocker.MagicMock() put_annotation_mock = mocker.MagicMock() - create_subsegment_mock = mocker.MagicMock() + begin_subsegment = mocker.MagicMock() end_subsegment_mock = mocker.MagicMock() provider = provider_stub( put_metadata_mock=put_metadata_mock, put_annotation_mock=put_annotation_mock, - create_subsegment_mock=create_subsegment_mock, + begin_subsegment=begin_subsegment, end_subsegment_mock=end_subsegment_mock, ) tracer = Tracer(provider=provider, service="booking") @@ -106,8 +101,8 @@ def greeting(name, message): greeting(name="Foo", message="Bar") - assert create_subsegment_mock.call_count == 1 - assert create_subsegment_mock.call_args == mocker.call(name="## greeting") + assert begin_subsegment.call_count == 1 + assert begin_subsegment.call_args == mocker.call(name="## greeting") assert end_subsegment_mock.call_count == 1 assert put_metadata_mock.call_args == mocker.call( key="greeting response", value=dummy_response, namespace="booking" @@ -202,109 +197,36 @@ def greeting(name, message): assert put_metadata_mock.call_count == 0 -@mock.patch("aws_lambda_powertools.tracing.base.aws_xray_sdk.core.patch") -@mock.patch("aws_lambda_powertools.tracing.base.aws_xray_sdk.core.patch_all") -def test_tracer_xray_provider(xray_patch_all_mock, xray_patch_mock, mocker): +@mock.patch("aws_lambda_powertools.tracing.tracer.aws_xray_sdk.core.patch") +@mock.patch("aws_lambda_powertools.tracing.tracer.aws_xray_sdk.core.patch_all") +def test_tracer_patch(xray_patch_all_mock, xray_patch_mock, mocker): # GIVEN tracer is instantiated # WHEN default X-Ray provider client is mocked # THEN tracer should run just fine - xray_client_mock = mock.MagicMock() - xray_client_mock.begin_subsegment = mock.MagicMock() - xray_client_mock.end_subsegment = mock.MagicMock() - xray_client_mock.put_annotation = mock.MagicMock() - xray_client_mock.put_metadata = mock.MagicMock() - xray_provider = XrayProvider(client=xray_client_mock) - - Tracer(provider=xray_provider) + + Tracer() assert xray_patch_all_mock.call_count == 1 modules = ["boto3"] - tracer = Tracer(service="booking", provider=xray_provider, patch_modules=modules) + tracer = Tracer(service="booking", patch_modules=modules) + assert xray_patch_mock.call_count == 1 assert xray_patch_mock.call_args == mocker.call(modules) - tracer.create_subsegment("test subsegment") - tracer.put_annotation(key="test_annotation", value="value") - tracer.put_metadata(key="test_metadata", value="value") - tracer.end_subsegment() - - assert xray_client_mock.begin_subsegment.call_count == 1 - assert xray_client_mock.begin_subsegment.call_args == mocker.call(name="test subsegment") - assert xray_client_mock.end_subsegment.call_count == 1 - assert xray_client_mock.put_annotation.call_count == 1 - assert xray_client_mock.put_annotation.call_args == mocker.call(key="test_annotation", value="value") - assert xray_client_mock.put_metadata.call_count == 1 - assert xray_client_mock.put_metadata.call_args == mocker.call( - key="test_metadata", value="value", namespace="booking" - ) - - -def test_tracer_xray_provider_cold_start(mocker): +def test_tracer_xray_provider_cold_start(mocker, provider_stub): # GIVEN tracer is instantiated # WHEN multiple subsegments are created # THEN tracer should record cold start only once - xray_client_mock = mock.MagicMock() - xray_client_mock.begin_subsegment.put_annotation = mock.MagicMock() - xray_provider = XrayProvider(client=xray_client_mock) + begin_subsegment_mock = mock.MagicMock() + begin_subsegment_mock.put_annotation = mock.MagicMock() + + provider = provider_stub( + begin_subsegment=begin_subsegment_mock, + ) + tracer = Tracer(provider=provider) - tracer = Tracer(provider=xray_provider) subsegment_mock = tracer.create_subsegment("test subsegment") subsegment_mock = tracer.create_subsegment("test subsegment 2") assert subsegment_mock.put_annotation.call_count == 1 assert subsegment_mock.put_annotation.call_args == mocker.call(key="ColdStart", value=True) - - -def test_trace_provider_abc_no_init(provider_stub): - # GIVEN tracer is instantiated - # WHEN a custom provider that implements TracerProvider methods - # THEN it should run successfully - - with pytest.raises(TracerProviderNotInitializedError): - Tracer(service="booking", provider=provider_stub) - - -def test_trace_invalid_provider(): - # GIVEN tracer is instantiated - # WHEN a custom provider does not implement TracerProvider - # THEN it should raise InvalidTracerProviderError - - class CustomProvider: - def __init__(self): - pass - - def put_metadata(self, *args, **kwargs): - pass - - def put_annotation(self, *args, **kwargs): - pass - - def create_subsegment(self, *args, **kwargs): - pass - - def end_subsegment(self, *args, **kwargs): - pass - - def patch(self, *args, **kwargs): - pass - - def disable_tracing_provider(self): - pass - - with pytest.raises(InvalidTracerProviderError): - Tracer(service="booking", provider=CustomProvider) - - class InvalidProvider: - pass - - with pytest.raises(InvalidTracerProviderError): - Tracer(service="booking", provider=InvalidProvider) - - with pytest.raises(InvalidTracerProviderError): - Tracer(service="booking", provider=InvalidProvider()) - - with pytest.raises(InvalidTracerProviderError): - Tracer(service="booking", provider=True) - - with pytest.raises(InvalidTracerProviderError): - Tracer(service="booking", provider="provider") From 46a11a28f4dc081d6920fedcade03258072113f9 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 5 May 2020 17:38:03 +0100 Subject: [PATCH 06/35] fix: patch modules type --- python/aws_lambda_powertools/tracing/tracer.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index 44b9bca540..f486480add 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -4,7 +4,7 @@ import logging import os from distutils.util import strtobool -from typing import Any, Callable, Dict, Generic, List +from typing import Any, Callable, Dict, List, Tuple import aws_xray_sdk import aws_xray_sdk.core @@ -45,8 +45,8 @@ class Tracer: disabled: bool Flag to explicitly disable tracing, useful when running/testing locally `Env POWERTOOLS_TRACE_DISABLED="true"` - patch_modules: List - List of modules supported by tracing provider to patch, by default all modules are patched + patch_modules: Tuple[str] + Tuple of modules supported by tracing provider to patch, by default all modules are patched Example ------- @@ -189,7 +189,6 @@ def create_subsegment(self, name: str) -> aws_xray_sdk.core.models.subsegment: def end_subsegment(self): """Ends an existing subsegment""" - # FIXME - Receive subsegment to close if self.disabled: logger.debug("Tracing has been disabled, aborting end_subsegment") return @@ -250,14 +249,14 @@ def put_metadata(self, key: str, value: Any, namespace: str = None): logger.debug(f"Adding metadata on key '{key}'' with '{value}'' at namespace '{namespace}''") self.provider.put_metadata(key=key, value=value, namespace=namespace) - def patch(self, modules: List[str] = None): + def patch(self, modules: Tuple[str] = None): """Patch modules for instrumentation. Patches all supported modules by default if none are given. Parameters ---------- - modules : List[str] + modules : Tuple[str] List of modules to be patched, optional by default """ if self.disabled: From c72e37cf95f4a487a9888e6a25a1359d2d0b4eab Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 5 May 2020 18:55:18 +0100 Subject: [PATCH 07/35] improv: use client ctx_manager for race conditions --- .../middleware_factory/factory.py | 5 +- .../aws_lambda_powertools/tracing/tracer.py | 111 ++++++------------ python/tests/unit/test_tracing.py | 83 +++++++------ 3 files changed, 84 insertions(+), 115 deletions(-) diff --git a/python/aws_lambda_powertools/middleware_factory/factory.py b/python/aws_lambda_powertools/middleware_factory/factory.py index 43c8e5ad9f..44b69dddbc 100644 --- a/python/aws_lambda_powertools/middleware_factory/factory.py +++ b/python/aws_lambda_powertools/middleware_factory/factory.py @@ -124,9 +124,8 @@ def wrapper(event, context): middleware = functools.partial(decorator, func, event, context, **kwargs) if trace_execution: tracer = Tracer(auto_patch=False) - tracer.create_subsegment(name=f"## {decorator.__qualname__}") - response = middleware() - tracer.end_subsegment() + with tracer.provider.in_subsegment(name=f"## {decorator.__qualname__}") as subsegment: + response = middleware() else: response = middleware() return response diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index f486480add..ac114c2f0f 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -3,6 +3,7 @@ import inspect import logging import os +from contextlib import contextmanager from distutils.util import strtobool from typing import Any, Callable, Dict, List, Tuple @@ -12,8 +13,6 @@ is_cold_start = True logger = logging.getLogger(__name__) -# FIXME - Add get current subsegment() -# FIXME - Add subsegment type class Tracer: """Tracer using AWS-XRay to provide decorators with known defaults for Lambda functions @@ -156,45 +155,6 @@ def __init__( if self.auto_patch: self.patch(modules=patch_modules) - def create_subsegment(self, name: str) -> aws_xray_sdk.core.models.subsegment: - """Creates subsegment/span with a given name - - Parameters - ---------- - name : str - Subsegment name - - Example - ------- - - **Creates a subsegment** - - self.create_subsegment(name="a meaningful name") - - Returns - ------- - aws_xray_sdk.core.models.subsegment - AWS X-Ray Subsegment - """ - # Will no longer be needed once #155 is resolved - # https://github.com/aws/aws-xray-sdk-python/issues/155 - subsegment = self.provider.begin_subsegment(name=name) - global is_cold_start - if is_cold_start: - logger.debug("Annotating cold start") - subsegment.put_annotation(key="ColdStart", value=True) - is_cold_start = False - - return subsegment - - def end_subsegment(self): - """Ends an existing subsegment""" - if self.disabled: - logger.debug("Tracing has been disabled, aborting end_subsegment") - return - - self.provider.end_subsegment() - def put_annotation(self, key: str, value: Any): """Adds annotation to existing segment or subsegment @@ -300,23 +260,26 @@ def handler(event, context) @functools.wraps(lambda_handler) def decorate(event, context): - # FIXME - Use newly created subsegment to avoid race conditions - self.create_subsegment(name=f"## {lambda_handler.__name__}") - try: - logger.debug("Calling lambda handler") - response = lambda_handler(event, context) - logger.debug("Received lambda handler response successfully") - logger.debug(response) - if response: - self.put_metadata("lambda handler response", response) - except Exception as err: - logger.exception("Exception received from lambda handler", exc_info=True) - self.put_metadata(f"{self.service}_error", err) - raise - finally: - self.end_subsegment() - - return response + with self.provider.in_subsegment(name=f"## {lambda_handler.__name__}") as subsegment: + global is_cold_start + if is_cold_start: + logger.debug("Annotating cold start") + self.put_annotation(key="ColdStart", value=True) + is_cold_start = False + + try: + logger.debug("Calling lambda handler") + response = lambda_handler(event, context) + logger.debug("Received lambda handler response successfully") + logger.debug(response) + if response: + self.put_metadata("lambda handler response", response) + except Exception as err: + logger.exception("Exception received from lambda handler", exc_info=True) + self.put_metadata(f"{self.service} error", err) + raise + + return response return decorate @@ -348,24 +311,20 @@ def some_function() @functools.wraps(method) def decorate(*args, **kwargs): method_name = f"{method.__name__}" - # FIXME - Use newly created subsegment to avoid race conditions - self.create_subsegment(name=f"## {method_name}") - - try: - logger.debug(f"Calling method: {method_name}") - response = method(*args, **kwargs) - logger.debug(f"Received {method_name} response successfully") - logger.debug(response) - if response is not None: - self.put_metadata(f"{method_name} response", response) - except Exception as err: - logger.exception(f"Exception received from '{method_name}'' method", exc_info=True) - self.put_metadata(f"{method_name} error", err) - raise - finally: - self.end_subsegment() - - return response + with self.provider.in_subsegment(name=f"## {method_name}") as subsegment: + try: + logger.debug(f"Calling method: {method_name}") + response = method(*args, **kwargs) + logger.debug(f"Received {method_name} response successfully") + logger.debug(response) + if response is not None: + self.put_metadata(f"{method_name} response", response) + except Exception as err: + logger.exception(f"Exception received from '{method_name}'' method", exc_info=True) + self.put_metadata(f"{method_name} error", err) + raise + + return response return decorate diff --git a/python/tests/unit/test_tracing.py b/python/tests/unit/test_tracing.py index 7d8cd9d42c..420aaacce2 100644 --- a/python/tests/unit/test_tracing.py +++ b/python/tests/unit/test_tracing.py @@ -17,15 +17,13 @@ def __init__( self, put_metadata_mock: mocker.MagicMock = None, put_annotation_mock: mocker.MagicMock = None, - begin_subsegment: mocker.MagicMock = None, - end_subsegment_mock: mocker.MagicMock = None, + in_subsegment: mocker.MagicMock = None, patch_mock: mocker.MagicMock = None, disable_tracing_provider_mock: mocker.MagicMock = None, ): self.put_metadata_mock = put_metadata_mock or mocker.MagicMock() self.put_annotation_mock = put_annotation_mock or mocker.MagicMock() - self.begin_subsegment = begin_subsegment or mocker.MagicMock() - self.end_subsegment_mock = end_subsegment_mock or mocker.MagicMock() + self.in_subsegment = in_subsegment or mocker.MagicMock() self.patch_mock = patch_mock or mocker.MagicMock() self.disable_tracing_provider_mock = disable_tracing_provider_mock or mocker.MagicMock() @@ -35,11 +33,8 @@ def put_metadata(self, *args, **kwargs): def put_annotation(self, *args, **kwargs): return self.put_annotation_mock(*args, **kwargs) - def create_subsegment(self, *args, **kwargs): - return self.begin_subsegment(*args, **kwargs) - - def end_subsegment(self, *args, **kwargs): - return self.end_subsegment_mock(*args, **kwargs) + def in_subsegment(self, *args, **kwargs): + return self.in_subsegment(*args, **kwargs) def patch(self, *args, **kwargs): return self.patch_mock(*args, **kwargs) @@ -57,13 +52,13 @@ def reset_tracing_config(mocker): def test_tracer_lambda_handler(mocker, dummy_response, provider_stub): put_metadata_mock = mocker.MagicMock() - begin_subsegment = mocker.MagicMock() - end_subsegment_mock = mocker.MagicMock() + in_subsegment = mocker.MagicMock() + put_annotation_mock = mocker.MagicMock() provider = provider_stub( put_metadata_mock=put_metadata_mock, - begin_subsegment=begin_subsegment, - end_subsegment_mock=end_subsegment_mock, + put_annotation_mock=put_annotation_mock, + in_subsegment=in_subsegment ) tracer = Tracer(provider=provider, service="booking") @@ -73,25 +68,24 @@ def handler(event, context): handler({}, mocker.MagicMock()) - assert begin_subsegment.call_count == 1 - assert begin_subsegment.call_args == mocker.call(name="## handler") - assert end_subsegment_mock.call_count == 1 + assert in_subsegment.call_count == 1 + assert in_subsegment.call_args == mocker.call(name="## handler") assert put_metadata_mock.call_args == mocker.call( key="lambda handler response", value=dummy_response, namespace="booking" ) + assert put_annotation_mock.call_count == 1 + assert put_annotation_mock.call_args == mocker.call(key="ColdStart", value=True) def test_tracer_method(mocker, dummy_response, provider_stub): put_metadata_mock = mocker.MagicMock() put_annotation_mock = mocker.MagicMock() - begin_subsegment = mocker.MagicMock() - end_subsegment_mock = mocker.MagicMock() + in_subsegment = mocker.MagicMock() provider = provider_stub( put_metadata_mock=put_metadata_mock, put_annotation_mock=put_annotation_mock, - begin_subsegment=begin_subsegment, - end_subsegment_mock=end_subsegment_mock, + in_subsegment=in_subsegment ) tracer = Tracer(provider=provider, service="booking") @@ -101,9 +95,8 @@ def greeting(name, message): greeting(name="Foo", message="Bar") - assert begin_subsegment.call_count == 1 - assert begin_subsegment.call_args == mocker.call(name="## greeting") - assert end_subsegment_mock.call_count == 1 + assert in_subsegment.call_count == 1 + assert in_subsegment.call_args == mocker.call(name="## greeting") assert put_metadata_mock.call_args == mocker.call( key="greeting response", value=dummy_response, namespace="booking" ) @@ -147,7 +140,7 @@ def handler(event, context): handler({}, mocker.MagicMock()) - assert put_annotation_mock.call_count == 1 + assert put_annotation_mock.call_count == 2 # cold_start + annotation assert put_annotation_mock.call_args == mocker.call(key=annotation_key, value=annotation_value) @@ -213,20 +206,38 @@ def test_tracer_patch(xray_patch_all_mock, xray_patch_mock, mocker): assert xray_patch_mock.call_count == 1 assert xray_patch_mock.call_args == mocker.call(modules) -def test_tracer_xray_provider_cold_start(mocker, provider_stub): - # GIVEN tracer is instantiated - # WHEN multiple subsegments are created - # THEN tracer should record cold start only once - begin_subsegment_mock = mock.MagicMock() - begin_subsegment_mock.put_annotation = mock.MagicMock() +def test_tracer_method_exception_metadata(mocker, provider_stub): + put_metadata_mock = mocker.MagicMock() provider = provider_stub( - begin_subsegment=begin_subsegment_mock, + put_metadata_mock=put_metadata_mock, ) - tracer = Tracer(provider=provider) + tracer = Tracer(provider=provider, service="booking") - subsegment_mock = tracer.create_subsegment("test subsegment") - subsegment_mock = tracer.create_subsegment("test subsegment 2") + @tracer.capture_method + def greeting(name, message): + raise ValueError("test") + + with pytest.raises(ValueError): + greeting(name="Foo", message="Bar") + assert put_metadata_mock.call_args == mocker.call( + key="greeting error", value=ValueError('test'), namespace="booking" + ) + +def test_tracer_lambda_handler_exception_metadata(mocker, provider_stub): + put_metadata_mock = mocker.MagicMock() + + provider = provider_stub( + put_metadata_mock=put_metadata_mock, + ) + tracer = Tracer(provider=provider, service="booking") + + @tracer.capture_lambda_handler + def handler(event, context): + raise ValueError("test") - assert subsegment_mock.put_annotation.call_count == 1 - assert subsegment_mock.put_annotation.call_args == mocker.call(key="ColdStart", value=True) + with pytest.raises(ValueError): + handler({}, mocker.MagicMock()) + assert put_metadata_mock.call_args == mocker.call( + key="booking error", value=ValueError('test'), namespace="booking" + ) From 022c373ea73e3c90f752738da8abfb632e54e019 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 5 May 2020 18:56:31 +0100 Subject: [PATCH 08/35] improv: make disabling provider private again --- python/aws_lambda_powertools/tracing/tracer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index ac114c2f0f..e7dfbb1e6b 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -150,7 +150,7 @@ def __init__( self.auto_patch = self._config["auto_patch"] if self.disabled: - self.disable_tracing_provider() + self.__disable_tracing_provider() if self.auto_patch: self.patch(modules=patch_modules) @@ -228,11 +228,6 @@ def patch(self, modules: Tuple[str] = None): else: aws_xray_sdk.core.patch(modules) - def disable_tracing_provider(self): - """Forcefully disables tracing""" - logger.debug("Disabling tracer provider...") - aws_xray_sdk.global_sdk_config.set_sdk_enabled(False) - def capture_lambda_handler(self, lambda_handler: Callable[[Dict, Any], Any] = None): """Decorator to create subsegment for lambda handlers @@ -328,6 +323,11 @@ def decorate(*args, **kwargs): return decorate + def __disable_tracing_provider(self): + """Forcefully disables tracing""" + logger.debug("Disabling tracer provider...") + aws_xray_sdk.global_sdk_config.set_sdk_enabled(False) + def __is_trace_disabled(self) -> bool: """Detects whether trace has been disabled From e038a724272e0fc12355b8563a7a3b0a8431e1c4 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 5 May 2020 18:56:47 +0100 Subject: [PATCH 09/35] chore: linting --- python/tests/unit/test_tracing.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/python/tests/unit/test_tracing.py b/python/tests/unit/test_tracing.py index 420aaacce2..4c476c0b46 100644 --- a/python/tests/unit/test_tracing.py +++ b/python/tests/unit/test_tracing.py @@ -12,7 +12,7 @@ def dummy_response(): @pytest.fixture def provider_stub(mocker): - class CustomProvider(): + class CustomProvider: def __init__( self, put_metadata_mock: mocker.MagicMock = None, @@ -56,9 +56,7 @@ def test_tracer_lambda_handler(mocker, dummy_response, provider_stub): put_annotation_mock = mocker.MagicMock() provider = provider_stub( - put_metadata_mock=put_metadata_mock, - put_annotation_mock=put_annotation_mock, - in_subsegment=in_subsegment + put_metadata_mock=put_metadata_mock, put_annotation_mock=put_annotation_mock, in_subsegment=in_subsegment ) tracer = Tracer(provider=provider, service="booking") @@ -83,9 +81,7 @@ def test_tracer_method(mocker, dummy_response, provider_stub): in_subsegment = mocker.MagicMock() provider = provider_stub( - put_metadata_mock=put_metadata_mock, - put_annotation_mock=put_annotation_mock, - in_subsegment=in_subsegment + put_metadata_mock=put_metadata_mock, put_annotation_mock=put_annotation_mock, in_subsegment=in_subsegment ) tracer = Tracer(provider=provider, service="booking") @@ -140,7 +136,7 @@ def handler(event, context): handler({}, mocker.MagicMock()) - assert put_annotation_mock.call_count == 2 # cold_start + annotation + assert put_annotation_mock.call_count == 2 # cold_start + annotation assert put_annotation_mock.call_args == mocker.call(key=annotation_key, value=annotation_value) @@ -202,16 +198,15 @@ def test_tracer_patch(xray_patch_all_mock, xray_patch_mock, mocker): modules = ["boto3"] tracer = Tracer(service="booking", patch_modules=modules) - + assert xray_patch_mock.call_count == 1 assert xray_patch_mock.call_args == mocker.call(modules) + def test_tracer_method_exception_metadata(mocker, provider_stub): put_metadata_mock = mocker.MagicMock() - provider = provider_stub( - put_metadata_mock=put_metadata_mock, - ) + provider = provider_stub(put_metadata_mock=put_metadata_mock,) tracer = Tracer(provider=provider, service="booking") @tracer.capture_method @@ -221,15 +216,14 @@ def greeting(name, message): with pytest.raises(ValueError): greeting(name="Foo", message="Bar") assert put_metadata_mock.call_args == mocker.call( - key="greeting error", value=ValueError('test'), namespace="booking" + key="greeting error", value=ValueError("test"), namespace="booking" ) + def test_tracer_lambda_handler_exception_metadata(mocker, provider_stub): put_metadata_mock = mocker.MagicMock() - provider = provider_stub( - put_metadata_mock=put_metadata_mock, - ) + provider = provider_stub(put_metadata_mock=put_metadata_mock,) tracer = Tracer(provider=provider, service="booking") @tracer.capture_lambda_handler @@ -239,5 +233,5 @@ def handler(event, context): with pytest.raises(ValueError): handler({}, mocker.MagicMock()) assert put_metadata_mock.call_args == mocker.call( - key="booking error", value=ValueError('test'), namespace="booking" + key="booking error", value=ValueError("test"), namespace="booking" ) From f58550625ad14b50370e56aa2d5db83c4c140a01 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 5 May 2020 20:32:59 +0100 Subject: [PATCH 10/35] fix: race condition annotation/metadata --- .../aws_lambda_powertools/tracing/tracer.py | 14 ++-- python/tests/unit/test_tracing.py | 68 +++++++++---------- 2 files changed, 43 insertions(+), 39 deletions(-) diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index e7dfbb1e6b..0d57132179 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -259,7 +259,7 @@ def decorate(event, context): global is_cold_start if is_cold_start: logger.debug("Annotating cold start") - self.put_annotation(key="ColdStart", value=True) + subsegment.put_annotation(key="ColdStart", value=True) is_cold_start = False try: @@ -268,10 +268,12 @@ def decorate(event, context): logger.debug("Received lambda handler response successfully") logger.debug(response) if response: - self.put_metadata("lambda handler response", response) + subsegment.put_metadata( + key="lambda handler response", value=response, namespace=self._config["service"] + ) except Exception as err: logger.exception("Exception received from lambda handler", exc_info=True) - self.put_metadata(f"{self.service} error", err) + subsegment.put_metadata(key=f"{self.service} error", value=err, namespace=self._config["service"]) raise return response @@ -313,10 +315,12 @@ def decorate(*args, **kwargs): logger.debug(f"Received {method_name} response successfully") logger.debug(response) if response is not None: - self.put_metadata(f"{method_name} response", response) + subsegment.put_metadata( + key=f"{method_name} response", value=response, namespace=self._config["service"] + ) except Exception as err: logger.exception(f"Exception received from '{method_name}'' method", exc_info=True) - self.put_metadata(f"{method_name} error", err) + subsegment.put_metadata(key=f"{method_name} error", value=err, namespace=self._config["service"]) raise return response diff --git a/python/tests/unit/test_tracing.py b/python/tests/unit/test_tracing.py index 4c476c0b46..c5fc2ab48d 100644 --- a/python/tests/unit/test_tracing.py +++ b/python/tests/unit/test_tracing.py @@ -50,14 +50,31 @@ def reset_tracing_config(mocker): yield +def mock_in_subsegment_annotation_metadata(): + """ Mock context manager in_subsegment, and its put_metadata/annotation methods + + Returns + ------- + in_subsegment_mock + in_subsegment_mock mock + put_annotation_mock + in_subsegment.put_annotation mock + put_metadata_mock + in_subsegment.put_metadata mock + """ + in_subsegment_mock = mock.MagicMock() + put_annotation_mock = mock.MagicMock() + put_metadata_mock = mock.MagicMock() + in_subsegment_mock.return_value.__enter__.return_value.put_annotation = put_annotation_mock + in_subsegment_mock.return_value.__enter__.return_value.put_metadata = put_metadata_mock + + return in_subsegment_mock, put_annotation_mock, put_metadata_mock + + def test_tracer_lambda_handler(mocker, dummy_response, provider_stub): - put_metadata_mock = mocker.MagicMock() - in_subsegment = mocker.MagicMock() - put_annotation_mock = mocker.MagicMock() + in_subsegment, put_annotation_mock, put_metadata_mock = mock_in_subsegment_annotation_metadata() - provider = provider_stub( - put_metadata_mock=put_metadata_mock, put_annotation_mock=put_annotation_mock, in_subsegment=in_subsegment - ) + provider = provider_stub(in_subsegment=in_subsegment) tracer = Tracer(provider=provider, service="booking") @tracer.capture_lambda_handler @@ -76,13 +93,9 @@ def handler(event, context): def test_tracer_method(mocker, dummy_response, provider_stub): - put_metadata_mock = mocker.MagicMock() - put_annotation_mock = mocker.MagicMock() - in_subsegment = mocker.MagicMock() + in_subsegment, _, put_metadata_mock = mock_in_subsegment_annotation_metadata() - provider = provider_stub( - put_metadata_mock=put_metadata_mock, put_annotation_mock=put_annotation_mock, in_subsegment=in_subsegment - ) + provider = provider_stub(in_subsegment=in_subsegment) tracer = Tracer(provider=provider, service="booking") @tracer.capture_method @@ -100,21 +113,14 @@ def greeting(name, message): def test_tracer_custom_metadata(mocker, dummy_response, provider_stub): put_metadata_mock = mocker.MagicMock() - - provider = provider_stub(put_metadata_mock=put_metadata_mock) - - tracer = Tracer(provider=provider, service="booking") annotation_key = "Booking response" annotation_value = {"bookingStatus": "CONFIRMED"} + + provider = provider_stub(put_metadata_mock=put_metadata_mock) + tracer = Tracer(provider=provider, service="booking") + tracer.put_metadata(annotation_key, annotation_value) - @tracer.capture_lambda_handler - def handler(event, context): - tracer.put_metadata(annotation_key, annotation_value) - return dummy_response - - handler({}, mocker.MagicMock()) - - assert put_metadata_mock.call_count == 2 + assert put_metadata_mock.call_count == 1 assert put_metadata_mock.call_args_list[0] == mocker.call( key=annotation_key, value=annotation_value, namespace="booking" ) @@ -122,21 +128,15 @@ def handler(event, context): def test_tracer_custom_annotation(mocker, dummy_response, provider_stub): put_annotation_mock = mocker.MagicMock() - - provider = provider_stub(put_annotation_mock=put_annotation_mock) - - tracer = Tracer(provider=provider, service="booking") annotation_key = "BookingId" annotation_value = "123456" - @tracer.capture_lambda_handler - def handler(event, context): - tracer.put_annotation(annotation_key, annotation_value) - return dummy_response + provider = provider_stub(put_annotation_mock=put_annotation_mock) + tracer = Tracer(provider=provider, service="booking") - handler({}, mocker.MagicMock()) + tracer.put_annotation(annotation_key, annotation_value) - assert put_annotation_mock.call_count == 2 # cold_start + annotation + assert put_annotation_mock.call_count == 1 assert put_annotation_mock.call_args == mocker.call(key=annotation_key, value=annotation_value) From be69d4f84b82581fbe959176859a9c2b28535e50 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 5 May 2020 20:34:44 +0100 Subject: [PATCH 11/35] chore: linting --- python/aws_lambda_powertools/middleware_factory/factory.py | 2 +- python/aws_lambda_powertools/tracing/tracer.py | 3 --- python/tests/unit/test_tracing.py | 4 ++-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/python/aws_lambda_powertools/middleware_factory/factory.py b/python/aws_lambda_powertools/middleware_factory/factory.py index 44b69dddbc..de781dc22c 100644 --- a/python/aws_lambda_powertools/middleware_factory/factory.py +++ b/python/aws_lambda_powertools/middleware_factory/factory.py @@ -124,7 +124,7 @@ def wrapper(event, context): middleware = functools.partial(decorator, func, event, context, **kwargs) if trace_execution: tracer = Tracer(auto_patch=False) - with tracer.provider.in_subsegment(name=f"## {decorator.__qualname__}") as subsegment: + with tracer.provider.in_subsegment(name=f"## {decorator.__qualname__}"): response = middleware() else: response = middleware() diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index 0d57132179..16b61cfd31 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -1,9 +1,7 @@ import copy import functools -import inspect import logging import os -from contextlib import contextmanager from distutils.util import strtobool from typing import Any, Callable, Dict, List, Tuple @@ -126,7 +124,6 @@ def handler(event: dict, context: Any) -> Dict: _default_config = { "service": "service_undefined", "disabled": False, - "provider": None, "auto_patch": True, "patch_modules": None, "provider": aws_xray_sdk.core.xray_recorder, diff --git a/python/tests/unit/test_tracing.py b/python/tests/unit/test_tracing.py index c5fc2ab48d..374ac02c18 100644 --- a/python/tests/unit/test_tracing.py +++ b/python/tests/unit/test_tracing.py @@ -115,7 +115,7 @@ def test_tracer_custom_metadata(mocker, dummy_response, provider_stub): put_metadata_mock = mocker.MagicMock() annotation_key = "Booking response" annotation_value = {"bookingStatus": "CONFIRMED"} - + provider = provider_stub(put_metadata_mock=put_metadata_mock) tracer = Tracer(provider=provider, service="booking") tracer.put_metadata(annotation_key, annotation_value) @@ -197,7 +197,7 @@ def test_tracer_patch(xray_patch_all_mock, xray_patch_mock, mocker): assert xray_patch_all_mock.call_count == 1 modules = ["boto3"] - tracer = Tracer(service="booking", patch_modules=modules) + Tracer(service="booking", patch_modules=modules) assert xray_patch_mock.call_count == 1 assert xray_patch_mock.call_args == mocker.call(modules) From 9e99f8ccb7448df61c19c67f60bc82614b482fa1 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Thu, 7 May 2020 18:00:29 +0100 Subject: [PATCH 12/35] feat: add async support for methods --- .../aws_lambda_powertools/tracing/tracer.py | 101 ++++++++--- python/poetry.lock | 20 ++- python/pyproject.toml | 1 + python/tests/unit/test_tracing.py | 158 +++++++++++++----- 4 files changed, 215 insertions(+), 65 deletions(-) diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index 16b61cfd31..ccdf173daa 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -1,5 +1,7 @@ +import asyncio import copy import functools +import inspect import logging import os from distutils.util import strtobool @@ -81,7 +83,7 @@ def confirm_booking(booking_id: str) -> Dict: @tracer.capture_lambda_handler def handler(event: dict, context: Any) -> Dict: print("Received event from Lambda...") - response = greeting(name="Heitor") + response = confirm_booking(booking_id=event["booking_id]) return response **A Lambda function using service name via POWERTOOLS_SERVICE_NAME** @@ -111,6 +113,23 @@ def handler(event: dict, context: Any) -> Dict: tracer = Tracer() ... + **Tracing an async method** + + from aws_lambda_powertools.tracing import Tracer + tracer = Tracer(service="booking") + + @tracer.capture_method + async def confirm_booking(booking_id: str) -> Dict: + resp = confirm_booking(booking_id=event["booking_id]) + + tracer.put_annotation("BookingConfirmation", resp['requestId']) + tracer.put_metadata("Booking confirmation", resp) + + return resp + + def lambda_handler(event: dict, context: Any) -> Dict: + asyncio.run(confirm_booking(booking=id)) + Returns ------- Tracer @@ -118,7 +137,7 @@ def handler(event: dict, context: Any) -> Dict: Limitations ----------- - * Async handler and methods not supported + * Async handler not supported """ _default_config = { @@ -301,26 +320,70 @@ def some_function() err Exception raised by method """ - - @functools.wraps(method) - def decorate(*args, **kwargs): - method_name = f"{method.__name__}" - with self.provider.in_subsegment(name=f"## {method_name}") as subsegment: - try: - logger.debug(f"Calling method: {method_name}") - response = method(*args, **kwargs) + method_name = f"{method.__name__}" + + async def decorate_logic( + decorated_method_with_args: functools.partial = None, + subsegment: aws_xray_sdk.core.models.subsegment = None, + coroutine: bool = False, + ) -> Any: + """Decorate logic runs both sync and async decorated methods + + Parameters + ---------- + decorated_method_with_args : functools.partial + Partial decorated method with arguments/keyword arguments + subsegment : aws_xray_sdk.core.models.subsegment + X-Ray subsegment to reuse + coroutine : bool, optional + Instruct whether partial decorated method is a wrapped coroutine, by default False + + Returns + ------- + Any + Returns method's response + """ + response = None + try: + logger.debug(f"Calling method: {method_name}") + if coroutine: + response = await decorated_method_with_args() + else: + response = decorated_method_with_args() logger.debug(f"Received {method_name} response successfully") logger.debug(response) - if response is not None: - subsegment.put_metadata( - key=f"{method_name} response", value=response, namespace=self._config["service"] - ) - except Exception as err: - logger.exception(f"Exception received from '{method_name}'' method", exc_info=True) - subsegment.put_metadata(key=f"{method_name} error", value=err, namespace=self._config["service"]) - raise + except Exception as err: + logger.exception(f"Exception received from '{method_name}'' method", exc_info=True) + subsegment.put_metadata(key=f"{method_name} error", value=err, namespace=self._config["service"]) + raise + finally: + if response is not None: + subsegment.put_metadata( # pragma: no cover + key=f"{method_name} response", value=response, namespace=self._config["service"] + ) - return response + return response + + if inspect.iscoroutinefunction(method): + + @functools.wraps(method) + async def decorate(*args, **kwargs): + decorated_method_with_args = functools.partial(method, *args, **kwargs) + async with self.provider.in_subsegment_async(name=f"## {method_name}") as subsegment: + return await decorate_logic( + decorated_method_with_args=decorated_method_with_args, subsegment=subsegment, coroutine=True + ) + + else: + + @functools.wraps(method) + def decorate(*args, **kwargs): + loop = asyncio.get_event_loop() + decorated_method_with_args = functools.partial(method, *args, **kwargs) + with self.provider.in_subsegment(name=f"## {method_name}") as subsegment: + return loop.run_until_complete( + decorate_logic(decorated_method_with_args=decorated_method_with_args, subsegment=subsegment) + ) return decorate diff --git a/python/poetry.lock b/python/poetry.lock index 2ca7efeb46..ec74004de0 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -586,6 +586,20 @@ version = ">=0.12" checkqa-mypy = ["mypy (v0.761)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +[[package]] +category = "dev" +description = "Pytest support for asyncio." +name = "pytest-asyncio" +optional = false +python-versions = ">= 3.5" +version = "0.12.0" + +[package.dependencies] +pytest = ">=5.4.0" + +[package.extras] +testing = ["async_generator (>=1.3)", "coverage", "hypothesis (>=5.7.1)"] + [[package]] category = "dev" description = "Pytest plugin for measuring coverage." @@ -738,7 +752,6 @@ version = "1.12.1" [[package]] category = "main" description = "Backport of pathlib-compatible object wrapper for zip files" -marker = "python_version < \"3.8\"" name = "zipp" optional = false python-versions = ">=3.6" @@ -749,7 +762,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "227b1d21877d1391dc50a8733d507226afd95471e77554328f9b2a3c2403b7fe" +content-hash = "286fd207f4e6ff9f2293e936682504383ad4778f81449bfd2feb9b2d2d158207" python-versions = "^3.6" [metadata.files] @@ -1002,6 +1015,9 @@ pytest = [ {file = "pytest-5.4.1-py3-none-any.whl", hash = "sha256:0e5b30f5cb04e887b91b1ee519fa3d89049595f428c1db76e73bd7f17b09b172"}, {file = "pytest-5.4.1.tar.gz", hash = "sha256:84dde37075b8805f3d1f392cc47e38a0e59518fb46a431cfdaf7cf1ce805f970"}, ] +pytest-asyncio = [ + {file = "pytest-asyncio-0.12.0.tar.gz", hash = "sha256:475bd2f3dc0bc11d2463656b3cbaafdbec5a47b47508ea0b329ee693040eebd2"}, +] pytest-cov = [ {file = "pytest-cov-2.8.1.tar.gz", hash = "sha256:cc6742d8bac45070217169f5f72ceee1e0e55b0221f54bcf24845972d3a47f2b"}, {file = "pytest_cov-2.8.1-py2.py3-none-any.whl", hash = "sha256:cdbdef4f870408ebdbfeb44e63e07eb18bb4619fae852f6e760645fa36172626"}, diff --git a/python/pyproject.toml b/python/pyproject.toml index 8b21c9dce8..a97957efb4 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -41,6 +41,7 @@ pre-commit = "^2.1.0" pytest-cov = "^2.8.1" pytest-mock = "^2.0.0" pdoc3 = "^0.7.5" +pytest-asyncio = "^0.12.0" [tool.coverage.run] source = ["aws_lambda_powertools"] diff --git a/python/tests/unit/test_tracing.py b/python/tests/unit/test_tracing.py index 374ac02c18..ffa8430c08 100644 --- a/python/tests/unit/test_tracing.py +++ b/python/tests/unit/test_tracing.py @@ -1,3 +1,4 @@ +from typing import NamedTuple from unittest import mock import pytest @@ -18,6 +19,7 @@ def __init__( put_metadata_mock: mocker.MagicMock = None, put_annotation_mock: mocker.MagicMock = None, in_subsegment: mocker.MagicMock = None, + in_subsegment_async: mocker.MagicMock = None, patch_mock: mocker.MagicMock = None, disable_tracing_provider_mock: mocker.MagicMock = None, ): @@ -26,6 +28,7 @@ def __init__( self.in_subsegment = in_subsegment or mocker.MagicMock() self.patch_mock = patch_mock or mocker.MagicMock() self.disable_tracing_provider_mock = disable_tracing_provider_mock or mocker.MagicMock() + self.in_subsegment_async = in_subsegment_async or mocker.MagicMock(spec=True) def put_metadata(self, *args, **kwargs): return self.put_metadata_mock(*args, **kwargs) @@ -50,31 +53,29 @@ def reset_tracing_config(mocker): yield -def mock_in_subsegment_annotation_metadata(): - """ Mock context manager in_subsegment, and its put_metadata/annotation methods +@pytest.fixture +def in_subsegment_mock(): + class Async_context_manager(mock.MagicMock): + async def __aenter__(self, *args, **kwargs): + return self.__enter__() + + async def __aexit__(self, *args, **kwargs): + return self.__exit__(*args, **kwargs) - Returns - ------- - in_subsegment_mock - in_subsegment_mock mock - put_annotation_mock - in_subsegment.put_annotation mock - put_metadata_mock - in_subsegment.put_metadata mock - """ - in_subsegment_mock = mock.MagicMock() - put_annotation_mock = mock.MagicMock() - put_metadata_mock = mock.MagicMock() - in_subsegment_mock.return_value.__enter__.return_value.put_annotation = put_annotation_mock - in_subsegment_mock.return_value.__enter__.return_value.put_metadata = put_metadata_mock + class In_subsegment(NamedTuple): + in_subsegment: mock.MagicMock = Async_context_manager() + put_annotation: mock.MagicMock = mock.MagicMock() + put_metadata: mock.MagicMock = mock.MagicMock() - return in_subsegment_mock, put_annotation_mock, put_metadata_mock + in_subsegment = In_subsegment() + in_subsegment.in_subsegment.return_value.__enter__.return_value.put_annotation = in_subsegment.put_annotation + in_subsegment.in_subsegment.return_value.__enter__.return_value.put_metadata = in_subsegment.put_metadata + yield in_subsegment -def test_tracer_lambda_handler(mocker, dummy_response, provider_stub): - in_subsegment, put_annotation_mock, put_metadata_mock = mock_in_subsegment_annotation_metadata() - provider = provider_stub(in_subsegment=in_subsegment) +def test_tracer_lambda_handler(mocker, dummy_response, provider_stub, in_subsegment_mock): + provider = provider_stub(in_subsegment=in_subsegment_mock.in_subsegment) tracer = Tracer(provider=provider, service="booking") @tracer.capture_lambda_handler @@ -83,19 +84,17 @@ def handler(event, context): handler({}, mocker.MagicMock()) - assert in_subsegment.call_count == 1 - assert in_subsegment.call_args == mocker.call(name="## handler") - assert put_metadata_mock.call_args == mocker.call( + assert in_subsegment_mock.in_subsegment.call_count == 1 + assert in_subsegment_mock.in_subsegment.call_args == mocker.call(name="## handler") + assert in_subsegment_mock.put_metadata.call_args == mocker.call( key="lambda handler response", value=dummy_response, namespace="booking" ) - assert put_annotation_mock.call_count == 1 - assert put_annotation_mock.call_args == mocker.call(key="ColdStart", value=True) + assert in_subsegment_mock.put_annotation.call_count == 1 + assert in_subsegment_mock.put_annotation.call_args == mocker.call(key="ColdStart", value=True) -def test_tracer_method(mocker, dummy_response, provider_stub): - in_subsegment, _, put_metadata_mock = mock_in_subsegment_annotation_metadata() - - provider = provider_stub(in_subsegment=in_subsegment) +def test_tracer_method(mocker, dummy_response, provider_stub, in_subsegment_mock): + provider = provider_stub(in_subsegment=in_subsegment_mock.in_subsegment) tracer = Tracer(provider=provider, service="booking") @tracer.capture_method @@ -104,9 +103,9 @@ def greeting(name, message): greeting(name="Foo", message="Bar") - assert in_subsegment.call_count == 1 - assert in_subsegment.call_args == mocker.call(name="## greeting") - assert put_metadata_mock.call_args == mocker.call( + assert in_subsegment_mock.in_subsegment.call_count == 1 + assert in_subsegment_mock.in_subsegment.call_args == mocker.call(name="## greeting") + assert in_subsegment_mock.put_metadata.call_args == mocker.call( key="greeting response", value=dummy_response, namespace="booking" ) @@ -203,10 +202,9 @@ def test_tracer_patch(xray_patch_all_mock, xray_patch_mock, mocker): assert xray_patch_mock.call_args == mocker.call(modules) -def test_tracer_method_exception_metadata(mocker, provider_stub): - put_metadata_mock = mocker.MagicMock() +def test_tracer_method_exception_metadata(mocker, provider_stub, in_subsegment_mock): - provider = provider_stub(put_metadata_mock=put_metadata_mock,) + provider = provider_stub(in_subsegment=in_subsegment_mock.in_subsegment) tracer = Tracer(provider=provider, service="booking") @tracer.capture_method @@ -215,15 +213,15 @@ def greeting(name, message): with pytest.raises(ValueError): greeting(name="Foo", message="Bar") - assert put_metadata_mock.call_args == mocker.call( - key="greeting error", value=ValueError("test"), namespace="booking" - ) + put_metadata_mock_args = in_subsegment_mock.put_metadata.call_args[1] + assert put_metadata_mock_args["key"] == "greeting error" + assert put_metadata_mock_args["namespace"] == "booking" -def test_tracer_lambda_handler_exception_metadata(mocker, provider_stub): - put_metadata_mock = mocker.MagicMock() - provider = provider_stub(put_metadata_mock=put_metadata_mock,) +def test_tracer_lambda_handler_exception_metadata(mocker, provider_stub, in_subsegment_mock): + + provider = provider_stub(in_subsegment=in_subsegment_mock.in_subsegment) tracer = Tracer(provider=provider, service="booking") @tracer.capture_lambda_handler @@ -232,6 +230,78 @@ def handler(event, context): with pytest.raises(ValueError): handler({}, mocker.MagicMock()) - assert put_metadata_mock.call_args == mocker.call( - key="booking error", value=ValueError("test"), namespace="booking" - ) + + put_metadata_mock_args = in_subsegment_mock.put_metadata.call_args[1] + assert put_metadata_mock_args["key"] == "booking error" + assert put_metadata_mock_args["namespace"] == "booking" + + +@pytest.mark.asyncio +async def test_tracer_method_nested_async(mocker, dummy_response, provider_stub, in_subsegment_mock): + provider = provider_stub(in_subsegment_async=in_subsegment_mock.in_subsegment) + tracer = Tracer(provider=provider, service="booking") + + @tracer.capture_method + async def greeting_2(name, message): + return dummy_response + + @tracer.capture_method + async def greeting(name, message): + await greeting_2(name, message) + return dummy_response + + await greeting(name="Foo", message="Bar") + + ( + in_subsegment_greeting_call_args, + in_subsegment_greeting2_call_args, + ) = in_subsegment_mock.in_subsegment.call_args_list + put_metadata_greeting2_call_args, put_metadata_greeting_call_args = in_subsegment_mock.put_metadata.call_args_list + + assert in_subsegment_mock.in_subsegment.call_count == 2 + assert in_subsegment_greeting_call_args == mocker.call(name="## greeting") + assert in_subsegment_greeting2_call_args == mocker.call(name="## greeting_2") + + assert in_subsegment_mock.put_metadata.call_count == 2 + assert put_metadata_greeting2_call_args == mocker.call( + key="greeting_2 response", value=dummy_response, namespace="booking" + ) + assert put_metadata_greeting_call_args == mocker.call( + key="greeting response", value=dummy_response, namespace="booking" + ) + + +@pytest.mark.asyncio +async def test_tracer_method_nested_async_disabled(dummy_response): + + tracer = Tracer(service="booking", disabled=True) + + @tracer.capture_method + async def greeting_2(name, message): + return dummy_response + + @tracer.capture_method + async def greeting(name, message): + await greeting_2(name, message) + return dummy_response + + ret = await greeting(name="Foo", message="Bar") + + assert ret == dummy_response + + +@pytest.mark.asyncio +async def test_tracer_method_exception_metadata_async(mocker, provider_stub, in_subsegment_mock): + provider = provider_stub(in_subsegment_async=in_subsegment_mock.in_subsegment) + tracer = Tracer(provider=provider, service="booking") + + @tracer.capture_method + async def greeting(name, message): + raise ValueError("test") + + with pytest.raises(ValueError): + await greeting(name="Foo", message="Bar") + + put_metadata_mock_args = in_subsegment_mock.put_metadata.call_args[1] + assert put_metadata_mock_args["key"] == "greeting error" + assert put_metadata_mock_args["namespace"] == "booking" From 9e7bd6d861324edb0f4062283963324cd0151a59 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 8 May 2020 20:18:44 +0100 Subject: [PATCH 13/35] improv: document async use cases, and edge cases --- .../aws_lambda_powertools/tracing/__init__.py | 3 +- .../aws_lambda_powertools/tracing/tracer.py | 81 +++++++++++++++---- 2 files changed, 66 insertions(+), 18 deletions(-) diff --git a/python/aws_lambda_powertools/tracing/__init__.py b/python/aws_lambda_powertools/tracing/__init__.py index 136fccce9f..64ab7c6a85 100644 --- a/python/aws_lambda_powertools/tracing/__init__.py +++ b/python/aws_lambda_powertools/tracing/__init__.py @@ -1,5 +1,6 @@ """Tracing utility """ from .tracer import Tracer +from aws_xray_sdk.ext.aiohttp.client import aws_xray_trace_config as aiohttp_trace_config -__all__ = ["Tracer"] +__all__ = ["Tracer", "aiohttp_trace_config"] diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index ccdf173daa..5cf9282805 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -113,23 +113,6 @@ def handler(event: dict, context: Any) -> Dict: tracer = Tracer() ... - **Tracing an async method** - - from aws_lambda_powertools.tracing import Tracer - tracer = Tracer(service="booking") - - @tracer.capture_method - async def confirm_booking(booking_id: str) -> Dict: - resp = confirm_booking(booking_id=event["booking_id]) - - tracer.put_annotation("BookingConfirmation", resp['requestId']) - tracer.put_metadata("Booking confirmation", resp) - - return resp - - def lambda_handler(event: dict, context: Any) -> Dict: - asyncio.run(confirm_booking(booking=id)) - Returns ------- Tracer @@ -302,6 +285,14 @@ def capture_method(self, method: Callable = None): It also captures both response and exceptions as metadata and creates a subsegment named `## ` + For concurrency async functions called via async.gather, + methods may impact each others subsegment and can trigger + and AlreadyEndedException from X-Ray due to async nature. + + When using async.gather, remember to set `return_exceptions`. + See example on how to best work around this using + an explicit context manager via the escape hatch mechanism. + Example ------- **Custom function using capture_method decorator** @@ -310,6 +301,62 @@ def capture_method(self, method: Callable = None): @tracer.capture_method def some_function() + **Custom async method using capture_method decorator** + + from aws_lambda_powertools.tracing import Tracer + tracer = Tracer(service="booking") + + @tracer.capture_method + async def confirm_booking(booking_id: str) -> Dict: + resp = confirm_booking(booking_id=event["booking_id]) + + tracer.put_annotation("BookingConfirmation", resp['requestId']) + tracer.put_metadata("Booking confirmation", resp) + + return resp + + def lambda_handler(event: dict, context: Any) -> Dict: + asyncio.run(confirm_booking(booking=id)) + + **Tracing nested async calls** + + from aws_lambda_powertools.tracing import Tracer + tracer = Tracer(service="booking") + + @tracer.capture_method + async def get_identity(): + ... + + @tracer.capture_method + async def long_async_call(): + ... + + @tracer.capture_method + async def async_tasks(): + await get_identity() + ret = await long_async_call() + + return { "task": "done", **ret } + + **Safely tracing multiple concurrent nested async calls** + + from aws_lambda_powertools.tracing import Tracer + tracer = Tracer(service="booking") + + async def get_identity(): + async with aioboto3.client("sts") as sts: + account = await sts.get_caller_identity() + return account + + async def long_async_call(): + ... + + @tracer.capture_method + async def async_tasks(): + _, ret = await asyncio.gather(get_identity(), long_async_call(), return_exceptions=True) + + return { "task": "done", **ret } + Parameters ---------- method : Callable From f79edffaf41fc59e87e6f96d72db23ce9c28cb5b Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 8 May 2020 20:19:18 +0100 Subject: [PATCH 14/35] improv: upgrade xray, flex pinning --- python/poetry.lock | 2 +- python/pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/poetry.lock b/python/poetry.lock index ec74004de0..7f024ccde0 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -762,7 +762,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "286fd207f4e6ff9f2293e936682504383ad4778f81449bfd2feb9b2d2d158207" +content-hash = "2ee5296385be8abadda22d87fe8548d0bd4f923147c35d45360bdc4cc64f8899" python-versions = "^3.6" [metadata.files] diff --git a/python/pyproject.toml b/python/pyproject.toml index a97957efb4..bbd9371d99 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -19,8 +19,8 @@ license = "MIT-0" [tool.poetry.dependencies] python = "^3.6" -aws-xray-sdk = "^2.4.3" -fastjsonschema = "^2.14.4" +aws-xray-sdk = "~=2.5.0" +fastjsonschema = "~=2.14.4" [tool.poetry.dev-dependencies] coverage = {extras = ["toml"], version = "^5.0.3"} From 4b22f7533a4eac632c3208ad21d53972ad126dff Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 8 May 2020 20:19:39 +0100 Subject: [PATCH 15/35] chore: linting --- python/aws_lambda_powertools/tracing/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/aws_lambda_powertools/tracing/__init__.py b/python/aws_lambda_powertools/tracing/__init__.py index 64ab7c6a85..dd67edfa95 100644 --- a/python/aws_lambda_powertools/tracing/__init__.py +++ b/python/aws_lambda_powertools/tracing/__init__.py @@ -1,6 +1,7 @@ """Tracing utility """ -from .tracer import Tracer from aws_xray_sdk.ext.aiohttp.client import aws_xray_trace_config as aiohttp_trace_config +from .tracer import Tracer + __all__ = ["Tracer", "aiohttp_trace_config"] From 44dbab6735c3179fb6d6171c09d776f2ff2e0a49 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 8 May 2020 20:27:32 +0100 Subject: [PATCH 16/35] improv: update example for async, escape hatch --- python/example/hello_world/app.py | 39 ++++++++++++++++++--- python/example/hello_world/requirements.txt | 4 ++- python/example/tests/test_handler.py | 1 + 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/python/example/hello_world/app.py b/python/example/hello_world/app.py index e503dc8362..ee185232df 100644 --- a/python/example/hello_world/app.py +++ b/python/example/hello_world/app.py @@ -1,14 +1,17 @@ +import asyncio import json import requests +import aioboto3 +import aiohttp from aws_lambda_powertools.logging import Logger +from aws_lambda_powertools.logging.logger import set_package_logger from aws_lambda_powertools.metrics import Metrics, MetricUnit, single_metric from aws_lambda_powertools.middleware_factory import lambda_handler_decorator -from aws_lambda_powertools.tracing import Tracer -from aws_lambda_powertools.logging.logger import set_package_logger +from aws_lambda_powertools.tracing import Tracer, aiohttp_trace_config -set_package_logger() # Enable package diagnostics (DEBUG log) +set_package_logger() # Enable package diagnostics (DEBUG log) tracer = Tracer() logger = Logger() @@ -19,6 +22,29 @@ metrics.add_dimension(name="operation", value="example") +async def aioboto_task(): + async with aioboto3.client("sts") as sts: + account = await sts.get_caller_identity() + return account + + +async def aiohttp_task(): + # You have full access to all xray_recorder methods via `tracer.provider` + # these include thread-safe methods, all context managers, x-ray configuration etc. + async with tracer.provider.in_subsegment_async("## aiohttp escape hatch"): + async with aiohttp.ClientSession(trace_configs=[aiohttp_trace_config()]) as session: + async with session.get("https://httpbin.org/json") as resp: + resp = await resp.json() + return resp + + +@tracer.capture_method +async def async_tasks(): + _, ret = await asyncio.gather(aioboto_task(), aiohttp_task(), return_exceptions=True) + + return {"task": "done", **ret} + + @lambda_handler_decorator(trace_execution=True) def my_middleware(handler, event, context, say_hello=False): if say_hello: @@ -56,6 +82,9 @@ def lambda_handler(event, context): Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html """ + + async_http_ret = asyncio.run(async_tasks()) + if "charge_id" in event: logger.structure_logs(append=True, payment_id="charge_id") @@ -77,8 +106,10 @@ def lambda_handler(event, context): with single_metric(name="UniqueMetricDimension", unit="Seconds", value=1) as metric: metric.add_dimension(name="unique_dimension", value="for_unique_metric") + resp = {"message": "hello world", "location": ip.text.replace("\n", ""), "async_http": async_http_ret} logger.info("Returning message to the caller") + return { "statusCode": 200, - "body": json.dumps({"message": "hello world", "location": ip.text.replace("\n", "")}), + "body": json.dumps(resp), } diff --git a/python/example/hello_world/requirements.txt b/python/example/hello_world/requirements.txt index 0241ab2efa..f89f0010c0 100644 --- a/python/example/hello_world/requirements.txt +++ b/python/example/hello_world/requirements.txt @@ -1,2 +1,4 @@ aws-lambda-powertools -requests \ No newline at end of file +requests +aioboto3 +aiohttp diff --git a/python/example/tests/test_handler.py b/python/example/tests/test_handler.py index f5447a8a81..909bb224c9 100644 --- a/python/example/tests/test_handler.py +++ b/python/example/tests/test_handler.py @@ -80,6 +80,7 @@ def test_lambda_handler(apigw_event, mocker, capsys): assert data["message"] == "hello world" assert "location" in data assert "message" in ret["body"] + assert "async_http" in data # assess custom metric was flushed in stdout/logs assert "SuccessfulLocations" in stdout_one_string From 107f572fd6271dce6311714d7d02c282062652e7 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 8 May 2020 20:32:24 +0100 Subject: [PATCH 17/35] fix: add example dev deps in project --- .../aws_lambda_powertools/tracing/tracer.py | 2 - python/poetry.lock | 263 +++++++++++++++++- python/pyproject.toml | 2 + 3 files changed, 261 insertions(+), 6 deletions(-) diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index 5cf9282805..dda1369f79 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -171,8 +171,6 @@ def put_annotation(self, key: str, value: Any): value : any Value for annotation (e.g. "CONFIRMED") """ - # Will no longer be needed once #155 is resolved - # https://github.com/aws/aws-xray-sdk-python/issues/155 if self.disabled: logger.debug("Tracing has been disabled, aborting put_annotation") return diff --git a/python/poetry.lock b/python/poetry.lock index 7f024ccde0..f548c47b37 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -1,3 +1,78 @@ +[[package]] +category = "dev" +description = "Async boto3 wrapper" +name = "aioboto3" +optional = false +python-versions = ">=3.6" +version = "8.0.3" + +[package.dependencies] +[package.dependencies.aiobotocore] +extras = ["boto3"] +version = "1.0.4" + +[package.extras] +s3cse = ["cryptography (>=2.3.1)"] + +[[package]] +category = "dev" +description = "Async client for aws services using botocore and aiohttp" +name = "aiobotocore" +optional = false +python-versions = ">=3.6" +version = "1.0.4" + +[package.dependencies] +aiohttp = ">=3.3.1" +aioitertools = ">=0.5.1" +botocore = ">=1.15.32,<1.15.33" +wrapt = ">=1.10.10" + +[package.dependencies.boto3] +optional = true +version = "1.12.32" + +[package.extras] +awscli = ["awscli (1.18.32)"] +boto3 = ["boto3 (1.12.32)"] + +[[package]] +category = "dev" +description = "Async http client/server framework (asyncio)" +name = "aiohttp" +optional = false +python-versions = ">=3.5.3" +version = "3.6.2" + +[package.dependencies] +async-timeout = ">=3.0,<4.0" +attrs = ">=17.3.0" +chardet = ">=2.0,<4.0" +multidict = ">=4.5,<5.0" +yarl = ">=1.0,<2.0" + +[package.dependencies.idna-ssl] +python = "<3.7" +version = ">=1.0" + +[package.dependencies.typing-extensions] +python = "<3.7" +version = ">=3.6.5" + +[package.extras] +speedups = ["aiodns", "brotlipy", "cchardet"] + +[[package]] +category = "dev" +description = "itertools and builtins for AsyncIO and mixed iterables" +name = "aioitertools" +optional = false +python-versions = ">=3.6" +version = "0.7.0" + +[package.dependencies] +typing_extensions = ">=3.7" + [[package]] category = "dev" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." @@ -6,6 +81,14 @@ optional = false python-versions = "*" version = "1.4.3" +[[package]] +category = "dev" +description = "Timeout context manager for asyncio programs" +name = "async-timeout" +optional = false +python-versions = ">=3.5.3" +version = "3.0.1" + [[package]] category = "dev" description = "Atomic file writes." @@ -63,13 +146,26 @@ typed-ast = ">=1.4.0" [package.extras] d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] +[[package]] +category = "dev" +description = "The AWS SDK for Python" +name = "boto3" +optional = false +python-versions = "*" +version = "1.12.32" + +[package.dependencies] +botocore = ">=1.15.32,<1.16.0" +jmespath = ">=0.7.1,<1.0.0" +s3transfer = ">=0.3.0,<0.4.0" + [[package]] category = "main" description = "Low-level, data-driven core of boto 3." name = "botocore" optional = false python-versions = "*" -version = "1.15.41" +version = "1.15.32" [package.dependencies] docutils = ">=0.10,<0.16" @@ -88,6 +184,14 @@ optional = false python-versions = ">=3.6" version = "3.0.0" +[[package]] +category = "dev" +description = "Universal encoding detector for Python 2 and 3" +name = "chardet" +optional = false +python-versions = "*" +version = "3.0.4" + [[package]] category = "dev" description = "Composable command line interface toolkit" @@ -321,6 +425,26 @@ version = "1.4.14" [package.extras] license = ["editdistance"] +[[package]] +category = "dev" +description = "Internationalized Domain Names in Applications (IDNA)" +name = "idna" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.9" + +[[package]] +category = "dev" +description = "Patch ssl.match_hostname for Unicode(idna) domains support" +marker = "python_version < \"3.7\"" +name = "idna-ssl" +optional = false +python-versions = "*" +version = "1.1.0" + +[package.dependencies] +idna = ">=2.0" + [[package]] category = "main" description = "Read metadata from Python packages" @@ -448,6 +572,14 @@ optional = false python-versions = ">=3.5" version = "8.2.0" +[[package]] +category = "dev" +description = "multidict implementation" +name = "multidict" +optional = false +python-versions = ">=3.5" +version = "4.7.5" + [[package]] category = "dev" description = "Node.js virtual environment builder" @@ -656,6 +788,17 @@ optional = false python-versions = "*" version = "2020.4.4" +[[package]] +category = "dev" +description = "An Amazon S3 Transfer Manager" +name = "s3transfer" +optional = false +python-versions = "*" +version = "0.3.3" + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + [[package]] category = "main" description = "Python 2 and 3 compatibility utilities" @@ -693,6 +836,14 @@ optional = false python-versions = "*" version = "1.4.1" +[[package]] +category = "dev" +description = "Backported and Experimental Type Hints for Python 3.5+" +name = "typing-extensions" +optional = false +python-versions = "*" +version = "3.7.4.2" + [[package]] category = "main" description = "HTTP library with thread-safe connection pooling, file post, and more." @@ -749,6 +900,18 @@ optional = false python-versions = "*" version = "1.12.1" +[[package]] +category = "dev" +description = "Yet another URL library" +name = "yarl" +optional = false +python-versions = ">=3.5" +version = "1.4.2" + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + [[package]] category = "main" description = "Backport of pathlib-compatible object wrapper for zip files" @@ -762,14 +925,44 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "2ee5296385be8abadda22d87fe8548d0bd4f923147c35d45360bdc4cc64f8899" +content-hash = "2f09044a57a6e75afb5f6fb30c4b48b9dbe1814f8d658f640fc7c777ce6155fb" python-versions = "^3.6" [metadata.files] +aioboto3 = [ + {file = "aioboto3-8.0.3-py2.py3-none-any.whl", hash = "sha256:b3fd112406dac77cbc4ec6457bd53bff6fb9ef13d58e440a66bd60e405d229ef"}, + {file = "aioboto3-8.0.3.tar.gz", hash = "sha256:1650a9c478d2d11cf7d48a2b72754b27713154675084d0c837c8d99ff8b070fc"}, +] +aiobotocore = [ + {file = "aiobotocore-1.0.4-py3-none-any.whl", hash = "sha256:1e89ef97c52eb77d89c7c4a9130cab162ae3b89d2709c6e45da30824163ed8d3"}, + {file = "aiobotocore-1.0.4.tar.gz", hash = "sha256:4103d90b9e162176203dc5295124b15f56c37eee0ddbcddc6929760443714ff8"}, +] +aiohttp = [ + {file = "aiohttp-3.6.2-cp35-cp35m-macosx_10_13_x86_64.whl", hash = "sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e"}, + {file = "aiohttp-3.6.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec"}, + {file = "aiohttp-3.6.2-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48"}, + {file = "aiohttp-3.6.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59"}, + {file = "aiohttp-3.6.2-cp36-cp36m-win32.whl", hash = "sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a"}, + {file = "aiohttp-3.6.2-cp36-cp36m-win_amd64.whl", hash = "sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17"}, + {file = "aiohttp-3.6.2-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a"}, + {file = "aiohttp-3.6.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd"}, + {file = "aiohttp-3.6.2-cp37-cp37m-win32.whl", hash = "sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965"}, + {file = "aiohttp-3.6.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654"}, + {file = "aiohttp-3.6.2-py3-none-any.whl", hash = "sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4"}, + {file = "aiohttp-3.6.2.tar.gz", hash = "sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326"}, +] +aioitertools = [ + {file = "aioitertools-0.7.0-py3-none-any.whl", hash = "sha256:e931a2f0dcabd4a8446b5cc2fc71b8bb14716e6adf37728a70869213f1f741cd"}, + {file = "aioitertools-0.7.0.tar.gz", hash = "sha256:341cb05a0903177ef1b73d4cc12c92aee18e81c364e0138f4efc7ec3c47b8177"}, +] appdirs = [ {file = "appdirs-1.4.3-py2.py3-none-any.whl", hash = "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e"}, {file = "appdirs-1.4.3.tar.gz", hash = "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92"}, ] +async-timeout = [ + {file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"}, + {file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"}, +] atomicwrites = [ {file = "atomicwrites-1.3.0-py2.py3-none-any.whl", hash = "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4"}, {file = "atomicwrites-1.3.0.tar.gz", hash = "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"}, @@ -786,14 +979,22 @@ black = [ {file = "black-19.10b0-py36-none-any.whl", hash = "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b"}, {file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"}, ] +boto3 = [ + {file = "boto3-1.12.32-py2.py3-none-any.whl", hash = "sha256:57398de1b5e074e715c866441e69f90c9468959d5743a021d8aeed04fbaa1078"}, + {file = "boto3-1.12.32.tar.gz", hash = "sha256:60ac1124597231ed36a7320547cd0d16a001bb92333ab30ad20514f77e585225"}, +] botocore = [ - {file = "botocore-1.15.41-py2.py3-none-any.whl", hash = "sha256:b12a5b642aa210a72d84204da18618276eeae052fbff58958f57d28ef3193034"}, - {file = "botocore-1.15.41.tar.gz", hash = "sha256:a45a65ba036bc980decfc3ce6c2688a2d5fffd76e4b02ea4d59e63ff0f6896d4"}, + {file = "botocore-1.15.32-py2.py3-none-any.whl", hash = "sha256:a963af564d94107787ff3d2c534e8b7aed7f12e014cdd609f8fcb17bf9d9b19a"}, + {file = "botocore-1.15.32.tar.gz", hash = "sha256:3ea89601ee452b65084005278bd832be854cfde5166685dcb14b6c8f19d3fc6d"}, ] cfgv = [ {file = "cfgv-3.0.0-py2.py3-none-any.whl", hash = "sha256:f22b426ed59cd2ab2b54ff96608d846c33dfb8766a67f0b4a6ce130ce244414f"}, {file = "cfgv-3.0.0.tar.gz", hash = "sha256:04b093b14ddf9fd4d17c53ebfd55582d27b76ed30050193c14e560770c5360eb"}, ] +chardet = [ + {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, + {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, +] click = [ {file = "click-7.1.1-py2.py3-none-any.whl", hash = "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a"}, {file = "click-7.1.1.tar.gz", hash = "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc"}, @@ -902,6 +1103,13 @@ identify = [ {file = "identify-1.4.14-py2.py3-none-any.whl", hash = "sha256:2bb8760d97d8df4408f4e805883dad26a2d076f04be92a10a3e43f09c6060742"}, {file = "identify-1.4.14.tar.gz", hash = "sha256:faffea0fd8ec86bb146ac538ac350ed0c73908326426d387eded0bcc9d077522"}, ] +idna = [ + {file = "idna-2.9-py2.py3-none-any.whl", hash = "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"}, + {file = "idna-2.9.tar.gz", hash = "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb"}, +] +idna-ssl = [ + {file = "idna-ssl-1.1.0.tar.gz", hash = "sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c"}, +] importlib-metadata = [ {file = "importlib_metadata-1.6.0-py2.py3-none-any.whl", hash = "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f"}, {file = "importlib_metadata-1.6.0.tar.gz", hash = "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e"}, @@ -973,6 +1181,25 @@ more-itertools = [ {file = "more-itertools-8.2.0.tar.gz", hash = "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507"}, {file = "more_itertools-8.2.0-py3-none-any.whl", hash = "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c"}, ] +multidict = [ + {file = "multidict-4.7.5-cp35-cp35m-macosx_10_13_x86_64.whl", hash = "sha256:fc3b4adc2ee8474cb3cd2a155305d5f8eda0a9c91320f83e55748e1fcb68f8e3"}, + {file = "multidict-4.7.5-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:42f56542166040b4474c0c608ed051732033cd821126493cf25b6c276df7dd35"}, + {file = "multidict-4.7.5-cp35-cp35m-win32.whl", hash = "sha256:7774e9f6c9af3f12f296131453f7b81dabb7ebdb948483362f5afcaac8a826f1"}, + {file = "multidict-4.7.5-cp35-cp35m-win_amd64.whl", hash = "sha256:c2c37185fb0af79d5c117b8d2764f4321eeb12ba8c141a95d0aa8c2c1d0a11dd"}, + {file = "multidict-4.7.5-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:e439c9a10a95cb32abd708bb8be83b2134fa93790a4fb0535ca36db3dda94d20"}, + {file = "multidict-4.7.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:85cb26c38c96f76b7ff38b86c9d560dea10cf3459bb5f4caf72fc1bb932c7136"}, + {file = "multidict-4.7.5-cp36-cp36m-win32.whl", hash = "sha256:620b37c3fea181dab09267cd5a84b0f23fa043beb8bc50d8474dd9694de1fa6e"}, + {file = "multidict-4.7.5-cp36-cp36m-win_amd64.whl", hash = "sha256:6e6fef114741c4d7ca46da8449038ec8b1e880bbe68674c01ceeb1ac8a648e78"}, + {file = "multidict-4.7.5-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:a326f4240123a2ac66bb163eeba99578e9d63a8654a59f4688a79198f9aa10f8"}, + {file = "multidict-4.7.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:dc561313279f9d05a3d0ffa89cd15ae477528ea37aa9795c4654588a3287a9ab"}, + {file = "multidict-4.7.5-cp37-cp37m-win32.whl", hash = "sha256:4b7df040fb5fe826d689204f9b544af469593fb3ff3a069a6ad3409f742f5928"}, + {file = "multidict-4.7.5-cp37-cp37m-win_amd64.whl", hash = "sha256:317f96bc0950d249e96d8d29ab556d01dd38888fbe68324f46fd834b430169f1"}, + {file = "multidict-4.7.5-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:b51249fdd2923739cd3efc95a3d6c363b67bbf779208e9f37fd5e68540d1a4d4"}, + {file = "multidict-4.7.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:ae402f43604e3b2bc41e8ea8b8526c7fa7139ed76b0d64fc48e28125925275b2"}, + {file = "multidict-4.7.5-cp38-cp38-win32.whl", hash = "sha256:bb519becc46275c594410c6c28a8a0adc66fe24fef154a9addea54c1adb006f5"}, + {file = "multidict-4.7.5-cp38-cp38-win_amd64.whl", hash = "sha256:544fae9261232a97102e27a926019100a9db75bec7b37feedd74b3aa82f29969"}, + {file = "multidict-4.7.5.tar.gz", hash = "sha256:aee283c49601fa4c13adc64c09c978838a7e812f85377ae130a24d7198c0331e"}, +] nodeenv = [ {file = "nodeenv-1.3.5-py2.py3-none-any.whl", hash = "sha256:5b2438f2e42af54ca968dd1b374d14a1194848955187b0e5e4be1f73813a5212"}, ] @@ -1066,6 +1293,10 @@ regex = [ {file = "regex-2020.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3"}, {file = "regex-2020.4.4.tar.gz", hash = "sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142"}, ] +s3transfer = [ + {file = "s3transfer-0.3.3-py2.py3-none-any.whl", hash = "sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13"}, + {file = "s3transfer-0.3.3.tar.gz", hash = "sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db"}, +] six = [ {file = "six-1.14.0-py2.py3-none-any.whl", hash = "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"}, {file = "six-1.14.0.tar.gz", hash = "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a"}, @@ -1102,6 +1333,11 @@ typed-ast = [ {file = "typed_ast-1.4.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34"}, {file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"}, ] +typing-extensions = [ + {file = "typing_extensions-3.7.4.2-py2-none-any.whl", hash = "sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392"}, + {file = "typing_extensions-3.7.4.2-py3-none-any.whl", hash = "sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5"}, + {file = "typing_extensions-3.7.4.2.tar.gz", hash = "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae"}, +] urllib3 = [ {file = "urllib3-1.25.9-py2.py3-none-any.whl", hash = "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"}, {file = "urllib3-1.25.9.tar.gz", hash = "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527"}, @@ -1117,6 +1353,25 @@ wcwidth = [ wrapt = [ {file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"}, ] +yarl = [ + {file = "yarl-1.4.2-cp35-cp35m-macosx_10_13_x86_64.whl", hash = "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b"}, + {file = "yarl-1.4.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1"}, + {file = "yarl-1.4.2-cp35-cp35m-win32.whl", hash = "sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080"}, + {file = "yarl-1.4.2-cp35-cp35m-win_amd64.whl", hash = "sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a"}, + {file = "yarl-1.4.2-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f"}, + {file = "yarl-1.4.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea"}, + {file = "yarl-1.4.2-cp36-cp36m-win32.whl", hash = "sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb"}, + {file = "yarl-1.4.2-cp36-cp36m-win_amd64.whl", hash = "sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70"}, + {file = "yarl-1.4.2-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d"}, + {file = "yarl-1.4.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce"}, + {file = "yarl-1.4.2-cp37-cp37m-win32.whl", hash = "sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2"}, + {file = "yarl-1.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce"}, + {file = "yarl-1.4.2-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b"}, + {file = "yarl-1.4.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae"}, + {file = "yarl-1.4.2-cp38-cp38-win32.whl", hash = "sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462"}, + {file = "yarl-1.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6"}, + {file = "yarl-1.4.2.tar.gz", hash = "sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b"}, +] zipp = [ {file = "zipp-3.1.0-py3-none-any.whl", hash = "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b"}, {file = "zipp-3.1.0.tar.gz", hash = "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96"}, diff --git a/python/pyproject.toml b/python/pyproject.toml index bbd9371d99..7bc7cd951c 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -42,6 +42,8 @@ pytest-cov = "^2.8.1" pytest-mock = "^2.0.0" pdoc3 = "^0.7.5" pytest-asyncio = "^0.12.0" +aioboto3 = "^8.0.3" +aiohttp = "^3.6.2" [tool.coverage.run] source = ["aws_lambda_powertools"] From fbfc759c4466b5d3380f51872033faf09e8d579e Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 8 May 2020 20:41:09 +0100 Subject: [PATCH 18/35] improv: add patch_modules example, formatting --- python/example/hello_world/app.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/python/example/hello_world/app.py b/python/example/hello_world/app.py index ee185232df..323ad55add 100644 --- a/python/example/hello_world/app.py +++ b/python/example/hello_world/app.py @@ -1,19 +1,20 @@ -import asyncio import json import requests -import aioboto3 -import aiohttp from aws_lambda_powertools.logging import Logger -from aws_lambda_powertools.logging.logger import set_package_logger from aws_lambda_powertools.metrics import Metrics, MetricUnit, single_metric from aws_lambda_powertools.middleware_factory import lambda_handler_decorator from aws_lambda_powertools.tracing import Tracer, aiohttp_trace_config +from aws_lambda_powertools.logging.logger import set_package_logger +import asyncio +import aioboto3 +import aiohttp -set_package_logger() # Enable package diagnostics (DEBUG log) +set_package_logger() # Enable package diagnostics (DEBUG log) -tracer = Tracer() +# tracer = Tracer() # patches all available modules +tracer = Tracer(patch_modules=("aioboto3", "boto3", "requests")) # ~90-100ms faster in perf depending on set of libs logger = Logger() metrics = Metrics() @@ -21,13 +22,11 @@ metrics.add_dimension(name="operation", value="example") - async def aioboto_task(): async with aioboto3.client("sts") as sts: account = await sts.get_caller_identity() return account - async def aiohttp_task(): # You have full access to all xray_recorder methods via `tracer.provider` # these include thread-safe methods, all context managers, x-ray configuration etc. @@ -42,7 +41,10 @@ async def aiohttp_task(): async def async_tasks(): _, ret = await asyncio.gather(aioboto_task(), aiohttp_task(), return_exceptions=True) - return {"task": "done", **ret} + return { + "task": "done", + **ret + } @lambda_handler_decorator(trace_execution=True) @@ -106,9 +108,13 @@ def lambda_handler(event, context): with single_metric(name="UniqueMetricDimension", unit="Seconds", value=1) as metric: metric.add_dimension(name="unique_dimension", value="for_unique_metric") - resp = {"message": "hello world", "location": ip.text.replace("\n", ""), "async_http": async_http_ret} + resp = { + "message": "hello world", + "location": ip.text.replace("\n", ""), + "async_http": async_http_ret + } logger.info("Returning message to the caller") - + return { "statusCode": 200, "body": json.dumps(resp), From b3447e3923c9d86bc79efc4039f865ea0cab2442 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 10 May 2020 15:00:35 +0100 Subject: [PATCH 19/35] improv: break down concurrent async calls example --- .../aws_lambda_powertools/tracing/tracer.py | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index dda1369f79..cf39878af8 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -283,13 +283,13 @@ def capture_method(self, method: Callable = None): It also captures both response and exceptions as metadata and creates a subsegment named `## ` - For concurrency async functions called via async.gather, - methods may impact each others subsegment and can trigger + When running [async functions concurrently](https://docs.python.org/3/library/asyncio-task.html#id6), + methods may impact each others subsegment, and can trigger and AlreadyEndedException from X-Ray due to async nature. - - When using async.gather, remember to set `return_exceptions`. - See example on how to best work around this using - an explicit context manager via the escape hatch mechanism. + + For this use case, either use `capture_method` only where + `async.gather` is called, or use `in_subsegment_async` + context manager via our escape hatch mechanism - See examples. Example ------- @@ -336,7 +336,7 @@ async def async_tasks(): return { "task": "done", **ret } - **Safely tracing multiple concurrent nested async calls** + **Safely tracing concurrent async calls with decorator** from aws_lambda_powertools.tracing import Tracer tracer = Tracer(service="booking") @@ -354,6 +354,25 @@ async def async_tasks(): _, ret = await asyncio.gather(get_identity(), long_async_call(), return_exceptions=True) return { "task": "done", **ret } + + **Safely tracing each concurrent async calls with escape hatch** + + from aws_lambda_powertools.tracing import Tracer + tracer = Tracer(service="booking") + + async def get_identity(): + async tracer.provider.in_subsegment_async("## get_identity"): + ... + + async def long_async_call(): + async tracer.provider.in_subsegment_async("## long_async_call"): + ... + + @tracer.capture_method + async def async_tasks(): + _, ret = await asyncio.gather(get_identity(), long_async_call(), return_exceptions=True) + + return { "task": "done", **ret } Parameters ---------- From c87bf626138e85f23ebeea2f7ead2195dfabae79 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 10 May 2020 15:39:46 +0100 Subject: [PATCH 20/35] docs: main doc clean up --- python/README.md | 125 +++++++++++++++++------------------------------ 1 file changed, 45 insertions(+), 80 deletions(-) diff --git a/python/README.md b/python/README.md index ff22f79907..9de33379d7 100644 --- a/python/README.md +++ b/python/README.md @@ -2,38 +2,38 @@ ![PackageStatus](https://img.shields.io/static/v1?label=status&message=beta&color=blueviolet?style=flat-square) ![PythonSupport](https://img.shields.io/static/v1?label=python&message=3.6%20|%203.7|%203.8&color=blue?style=flat-square&logo=python) ![PyPI version](https://badge.fury.io/py/aws-lambda-powertools.svg) ![PyPi monthly downloads](https://img.shields.io/pypi/dm/aws-lambda-powertools) ![Build](https://github.com/awslabs/aws-lambda-powertools/workflows/Powertools%20Python/badge.svg?branch=master) -A suite of utilities for AWS Lambda Functions that makes tracing with AWS X-Ray, structured logging and creating custom metrics asynchronously easier - Currently available for Python only and compatible with Python >=3.6. +A suite of utilities for AWS Lambda Functions that makes tracing with AWS X-Ray, structured logging, and creating custom metrics asynchronously easier - Compatible with Python >=3.6. -**Status**: Beta +> During beta, this library may change its API/methods, or environment variables as it receives feedback from customers. -## Features +* **Status**: Beta +* **How long until GA?**: [Current progress](https://github.com/awslabs/aws-lambda-powertools/projects/1) -**Tracing** +## Features -> It currently uses AWS X-Ray +**[Tracing](###Tracing)** -* Decorators that capture cold start as annotation, and response and exceptions as metadata +* Capture cold start as annotation, and response and exceptions as metadata * Run functions locally with SAM CLI without code change to disable tracing * Explicitly disable tracing via env var `POWERTOOLS_TRACE_DISABLED="true"` +* Support tracing async methods -**Logging** +**[Logging](###Logging)** -* Decorators that capture key fields from Lambda context, cold start and structures logging output as JSON -* Optionally log Lambda request when instructed (disabled by default) +* Capture key fields from Lambda context, cold start and structures logging output as JSON +* Log Lambda event when instructed (disabled by default) - Enable via `POWERTOOLS_LOGGER_LOG_EVENT="true"` or explicitly via decorator param -* Logs canonical custom metric line to logs that can be consumed asynchronously * Log sampling enables DEBUG log level for a percentage of requests (disabled by default) - Enable via `POWERTOOLS_LOGGER_SAMPLE_RATE=0.1`, ranges from 0 to 1, where 0.1 is 10% and 1 is 100% -* Append additional keys to structured log at any point in time so they're available across log statements +* Append additional keys to structured log at any point in time -**Metrics** +**[Metrics](###Metrics)** * Aggregate up to 100 metrics using a single CloudWatch Embedded Metric Format object (large JSON blob) * Context manager to create an one off metric with a different dimension than metrics already aggregated * Validate against common metric definitions mistakes (metric unit, values, max dimensions, max metrics, etc) -* No stack, custom resource, data collection needed — Metrics are created async by CloudWatch EMF -**Bring your own middleware** +**[Bring your own middleware](###Bring-your-own-middleware)** * Utility to easily create your own middleware * Run logic before, after, and handle exceptions @@ -45,12 +45,12 @@ A suite of utilities for AWS Lambda Functions that makes tracing with AWS X-Ray, Environment variable | Description | Default | Utility ------------------------------------------------- | --------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | ------------------------------------------------- POWERTOOLS_SERVICE_NAME | Sets service name used for tracing namespace, metrics dimensions and structured logging | "service_undefined" | all -POWERTOOLS_TRACE_DISABLED | Disables tracing | "false" | tracing -POWERTOOLS_TRACE_MIDDLEWARES | Creates sub-segment for each middleware created by lambda_handler_decorator | "false" | middleware_factory -POWERTOOLS_LOGGER_LOG_EVENT | Logs incoming event | "false" | logging -POWERTOOLS_LOGGER_SAMPLE_RATE | Debug log sampling | 0 | logging -POWERTOOLS_METRICS_NAMESPACE | Metrics namespace | None | metrics -LOG_LEVEL | Sets logging level | "INFO" | logging +POWERTOOLS_TRACE_DISABLED | Disables tracing | "false" | [Tracing](###Tracing) +POWERTOOLS_TRACE_MIDDLEWARES | Creates sub-segment for each middleware created by lambda_handler_decorator | "false" | [middleware_factory](###Bring-your-own-middleware) +POWERTOOLS_LOGGER_LOG_EVENT | Logs incoming event | "false" | [Logging](###Logging) +POWERTOOLS_LOGGER_SAMPLE_RATE | Debug log sampling | 0 | [Logging](###Logging) +POWERTOOLS_METRICS_NAMESPACE | Metrics namespace | None | [Metrics](###Metrics) +LOG_LEVEL | Sets logging level | "INFO" | [Logging](###Logging) ## Usage @@ -58,21 +58,11 @@ LOG_LEVEL | Sets logging level | "INFO" | logging With [pip](https://pip.pypa.io/en/latest/index.html) installed, run: ``pip install aws-lambda-powertools`` -### Tracing +See **[example](./example/README.md)** of all features, testing, and a SAM template with all Powertools env vars. All features also provide full docs, and code completion for VSCode and PyCharm. -**Example SAM template using supported environment variables** - -```yaml -Globals: - Function: - Tracing: Active # can also be enabled per function - Environment: - Variables: - POWERTOOLS_SERVICE_NAME: "payment" - POWERTOOLS_TRACE_DISABLED: "false" -``` +### Tracing -**Pseudo Python Lambda code** +#### Tracing Lambda handler and a custom method ```python from aws_lambda_powertools.tracing import Tracer @@ -81,10 +71,8 @@ tracer = Tracer() @tracer.capture_method def collect_payment(charge_id): - # logic - ret = requests.post(PAYMENT_ENDPOINT) - # custom annotation - tracer.put_annotation("PAYMENT_STATUS", "SUCCESS") + ret = requests.post(PAYMENT_ENDPOINT) # logic + tracer.put_annotation("PAYMENT_STATUS", "SUCCESS") # custom annotation return ret @tracer.capture_lambda_handler @@ -94,7 +82,7 @@ def handler(event, context) ... ``` -**Fetching a pre-configured tracer anywhere** +#### Using a pre-configured tracer anywhere ```python # handler.py @@ -114,21 +102,7 @@ tracer = Tracer(auto_patch=False) # new instance using existing configuration wi ### Logging -> **NOTE** `logger_setup` and `logger_inject_lambda_context` are deprecated and will be completely removed once it's GA. - -**Example SAM template using supported environment variables** - -```yaml -Globals: - Function: - Environment: - Variables: - POWERTOOLS_SERVICE_NAME: "payment" - POWERTOOLS_LOGGER_SAMPLE_RATE: 0.1 # enable debug logging for 1% of requests, 0% by default - LOG_LEVEL: "INFO" -``` - -**Pseudo Python Lambda code** +#### Structuring logs with Lambda context info ```python from aws_lambda_powertools.logging import Logger @@ -148,7 +122,8 @@ def handler(event, context) ... ``` -**Exerpt output in CloudWatch Logs** +
+Exerpt output in CloudWatch Logs ```json { @@ -182,8 +157,9 @@ def handler(event, context) } } ``` +
-**Append additional keys to structured log** +#### Appending additional keys to current logger ```python from aws_lambda_powertools.logging import Logger @@ -198,7 +174,8 @@ def handler(event, context) ... ``` -**Exerpt output in CloudWatch Logs** +
+Exerpt output in CloudWatch Logs ```json { @@ -216,14 +193,11 @@ def handler(event, context) "message": "Collecting payment" } ``` +
-### Custom Metrics async - -> **NOTE** `log_metric` will be removed once it's GA. +### Metrics -This feature makes use of CloudWatch Embedded Metric Format (EMF) and metrics are created asynchronously by CloudWatch service - -> Contrary to `log_metric`, you don't need any custom resource or additional CloudFormation stack anymore. +This feature makes use of CloudWatch Embedded Metric Format (EMF), and metrics are created asynchronously by CloudWatch service. Metrics middleware validates against the minimum necessary for a metric to be published: @@ -232,9 +206,9 @@ Metrics middleware validates against the minimum necessary for a metric to be pu * Only one Namespace * [Any Metric unit supported by CloudWatch](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDatum.html) -**Creating multiple metrics** +#### Creating multiple metrics -`log_metrics` decorator calls the decorated function, so leave that for last decorator or will fail with `SchemaValidationError` if no metrics are recorded. +If using multiple middlewares, use `log_metrics` as the last decorator, or else it will fail with `SchemaValidationError` if no metrics are recorded. ```python from aws_lambda_powertools.metrics import Metrics, MetricUnit @@ -267,11 +241,9 @@ with single_metric(name="ColdStart", unit=MetricUnit.Count, value=1) as metric: metric.add_dimension(name="function_context", value="$LATEST") ``` -> **NOTE**: If you want to instantiate Metrics() in multiple places in your code, make sure to use `POWERTOOLS_METRICS_NAMESPACE` env var as we don't keep a copy of that across instances. - -### Utilities +> **NOTE**: When using Metrics() in multiple places in your code, make sure to use `POWERTOOLS_METRICS_NAMESPACE` env var, or setting namespace param. -#### Bring your own middleware +### Bring your own middleware This feature allows you to create your own middleware as a decorator with ease by following a simple signature. @@ -279,7 +251,7 @@ This feature allows you to create your own middleware as a decorator with ease b * Always return the handler with event/context or response if executed - Supports nested middleware/decorators use case -**Middleware with no params** +#### Middleware with no params ```python from aws_lambda_powertools.middleware_factory import lambda_handler_decorator @@ -307,7 +279,7 @@ def lambda_handler(event, context): return True ``` -**Middleware with params** +#### Middleware with params ```python @lambda_handler_decorator @@ -325,9 +297,9 @@ def lambda_handler(event, context): return True ``` -**Optionally trace middleware execution** +#### Tracing middleware execution -This makes use of an existing Tracer instance that you may have initialized anywhere in your code, otherwise it'll initialize one using default options and provider (X-Ray). +This makes use of an existing Tracer instance that you may have initialized anywhere in your code. If no Tracer instance is found, it'll initialize one using default options. ```python from aws_lambda_powertools.middleware_factory import lambda_handler_decorator @@ -359,19 +331,12 @@ def lambda_handler(event, context): return True ``` - ### Debug mode -By default, all debug log statements from AWS Lambda Powertools package are suppressed. If you'd like to enable them, use `set_package_logger` utility: +By default, all log statements from AWS Lambda Powertools package are suppressed. If you'd like to enable them, use `set_package_logger` utility: ```python import aws_lambda_powertools aws_lambda_powertools.logging.logger.set_package_logger() ... ``` - -## Beta - -This library may change its API/methods or environment variables as it receives feedback from customers - -**[Progress towards GA](https://github.com/awslabs/aws-lambda-powertools/projects/1)** From e1b79eadf24623b9a8418bcb06804a606a0962c2 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 10 May 2020 15:57:44 +0100 Subject: [PATCH 21/35] docs: document async, escape hatch usage --- python/README.md | 62 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/python/README.md b/python/README.md index 9de33379d7..8abc6f3e19 100644 --- a/python/README.md +++ b/python/README.md @@ -54,15 +54,15 @@ LOG_LEVEL | Sets logging level | "INFO" | [Logging](###Logging) ## Usage +See **[example](./example/README.md)** of all features, testing, and a SAM template with all Powertools env vars. All features also provide full docs, and code completion for VSCode and PyCharm. + ### Installation With [pip](https://pip.pypa.io/en/latest/index.html) installed, run: ``pip install aws-lambda-powertools`` -See **[example](./example/README.md)** of all features, testing, and a SAM template with all Powertools env vars. All features also provide full docs, and code completion for VSCode and PyCharm. - ### Tracing -#### Tracing Lambda handler and a custom method +#### Tracing Lambda handler and a function ```python from aws_lambda_powertools.tracing import Tracer @@ -82,6 +82,62 @@ def handler(event, context) ... ``` +#### Tracing asynchronous functions + +```python +import asyncio + +from aws_lambda_powertools.tracing import Tracer +tracer = Tracer() +# tracer = Tracer(service="payment") # can also be explicitly defined + +@tracer.capture_method +async def collect_payment(charge_id): + ... + +@tracer.capture_lambda_handler +def handler(event, context) + charge_id = event.get('charge_id') + payment = asyncio.run(collect_payment(charge_id)) # python 3.7+ + ... +``` + +#### Using escape hatch mechanisms + +You can use `tracer.provider` attribute to access all methods provided by `xray_recorder`. This is useful when you need a feature available in X-Ray that is not available in the Tracer middleware, for example [thread-safe](https://github.com/aws/aws-xray-sdk-python/#user-content-trace-threadpoolexecutor), or [context managers](https://github.com/aws/aws-xray-sdk-python/#user-content-start-a-custom-segmentsubsegment). + +**Example using aiohttp with an async context manager** + +```python +import asyncio + +from aws_lambda_powertools.tracing import Tracer, aiohttp_trace_config +tracer = Tracer() + +async def aiohttp_task(): + # Async context manager as opposed to `@tracer.capture_method` + async with tracer.provider.in_subsegment_async("## aiohttp escape hatch"): + async with aiohttp.ClientSession(trace_configs=[aiohttp_trace_config()]) as session: + async with session.get("https://httpbin.org/json") as resp: + resp = await resp.json() + return resp + +@tracer.capture_method +async def async_tasks(): + ret = await aiohttp_task() + ... + + return { + "task": "done", + **ret + } + +@tracer.capture_lambda_handler +def handler(event, context) + ret = asyncio.run(async_tasks()) # python 3.7+ + ... +``` + #### Using a pre-configured tracer anywhere ```python From c58a4f1c33f43d0f932637067a06ba3a35ca5646 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 10 May 2020 15:58:47 +0100 Subject: [PATCH 22/35] chore: lint --- python/aws_lambda_powertools/tracing/tracer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index cf39878af8..54fdd7afa3 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -286,7 +286,7 @@ def capture_method(self, method: Callable = None): When running [async functions concurrently](https://docs.python.org/3/library/asyncio-task.html#id6), methods may impact each others subsegment, and can trigger and AlreadyEndedException from X-Ray due to async nature. - + For this use case, either use `capture_method` only where `async.gather` is called, or use `in_subsegment_async` context manager via our escape hatch mechanism - See examples. @@ -354,7 +354,7 @@ async def async_tasks(): _, ret = await asyncio.gather(get_identity(), long_async_call(), return_exceptions=True) return { "task": "done", **ret } - + **Safely tracing each concurrent async calls with escape hatch** from aws_lambda_powertools.tracing import Tracer From c91fe32e6c34d48b25b31a3a25284608adfd90f8 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 10 May 2020 16:32:51 +0100 Subject: [PATCH 23/35] docs: update example SAM template comments --- python/example/template.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/example/template.yaml b/python/example/template.yaml index 485641e5f6..e2104dd045 100644 --- a/python/example/template.yaml +++ b/python/example/template.yaml @@ -8,7 +8,7 @@ Description: > # More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst Globals: Function: - Timeout: 3 + Timeout: 7 Resources: HelloWorldFunction: @@ -17,15 +17,15 @@ Resources: CodeUri: hello_world/ Handler: app.lambda_handler Runtime: python3.7 - Tracing: Active + Tracing: Active # enables X-Ray tracing Environment: Variables: - POWERTOOLS_SERVICE_NAME: example # Sets service name used for tracing namespace, metrics dimensions and structured logging - POWERTOOLS_TRACE_DISABLED: "false" # Explicitly disables tracing - POWERTOOLS_LOGGER_LOG_EVENT: "false" # Logs incoming event - POWERTOOLS_LOGGER_SAMPLE_RATE: "0" # Debug log sampling percentage + POWERTOOLS_SERVICE_NAME: example # Sets service name used for all middlewares, "service_undefined" by default + POWERTOOLS_TRACE_DISABLED: "false" # Explicitly disables tracing, default + POWERTOOLS_LOGGER_LOG_EVENT: "false" # Logs incoming event, default + POWERTOOLS_LOGGER_SAMPLE_RATE: "0" # Debug log sampling percentage, default POWERTOOLS_METRICS_NAMESPACE: "Example" # Metric Namespace - LOG_LEVEL: INFO # Log level (INFO, DEBUG, etc.) + LOG_LEVEL: INFO # Log level for Logger (INFO, DEBUG, etc.), default Events: HelloWorld: Type: Api # More info about API Event Source: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#api From 7f0c59c0c61e1859d1c0109b46c2ac7211628bd2 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 10 May 2020 16:33:03 +0100 Subject: [PATCH 24/35] chore: updates poetry lock file --- python/poetry.lock | 101 +++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/python/poetry.lock b/python/poetry.lock index f548c47b37..368079b69a 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -96,7 +96,7 @@ marker = "sys_platform == \"win32\"" name = "atomicwrites" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "1.3.0" +version = "1.4.0" [[package]] category = "dev" @@ -198,7 +198,7 @@ description = "Composable command line interface toolkit" name = "click" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "7.1.1" +version = "7.1.2" [[package]] category = "dev" @@ -420,7 +420,7 @@ description = "File identification library for Python" name = "identify" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" -version = "1.4.14" +version = "1.4.15" [package.extras] license = ["editdistance"] @@ -467,7 +467,7 @@ marker = "python_version < \"3.7\"" name = "importlib-resources" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -version = "1.4.0" +version = "1.5.0" [package.dependencies] [package.dependencies.importlib-metadata] @@ -509,7 +509,7 @@ description = "Python library for serializing any arbitrary object graph into JS name = "jsonpickle" optional = false python-versions = ">=2.7" -version = "1.4" +version = "1.4.1" [package.dependencies] importlib-metadata = "*" @@ -540,10 +540,12 @@ description = "Python implementation of Markdown." name = "markdown" optional = false python-versions = ">=3.5" -version = "3.2.1" +version = "3.2.2" [package.dependencies] -setuptools = ">=36" +[package.dependencies.importlib-metadata] +python = "<3.8" +version = "*" [package.extras] testing = ["coverage", "pyyaml"] @@ -698,7 +700,7 @@ description = "pytest: simple powerful testing with Python" name = "pytest" optional = false python-versions = ">=3.5" -version = "5.4.1" +version = "5.4.2" [package.dependencies] atomicwrites = ">=1.0" @@ -786,7 +788,7 @@ description = "Alternative regular expression module, to replace re." name = "regex" optional = false python-versions = "*" -version = "2020.4.4" +version = "2020.5.7" [[package]] category = "dev" @@ -864,7 +866,7 @@ description = "Virtual Python Environment builder" name = "virtualenv" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" -version = "20.0.18" +version = "20.0.20" [package.dependencies] appdirs = ">=1.4.3,<2" @@ -881,8 +883,8 @@ python = "<3.7" version = ">=1.0,<2" [package.extras] -docs = ["sphinx (>=2.0.0,<3)", "sphinx-argparse (>=0.2.5,<1)", "sphinx-rtd-theme (>=0.4.3,<1)", "towncrier (>=19.9.0rc1)", "proselint (>=0.10.2,<1)"] -testing = ["pytest (>=4.0.0,<6)", "coverage (>=4.5.1,<6)", "pytest-mock (>=2.0.0,<3)", "pytest-env (>=0.6.2,<1)", "pytest-timeout (>=1.3.4,<2)", "packaging (>=20.0)", "xonsh (>=0.9.16,<1)"] +docs = ["sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=19.9.0rc1)", "proselint (>=0.10.2)"] +testing = ["pytest (>=4)", "coverage (>=5)", "coverage-enable-subprocess (>=1)", "pytest-xdist (>=1.31.0)", "pytest-mock (>=2)", "pytest-env (>=0.6.2)", "pytest-randomly (>=1)", "pytest-timeout", "packaging (>=20.0)", "xonsh (>=0.9.16)"] [[package]] category = "dev" @@ -915,6 +917,7 @@ multidict = ">=4.0" [[package]] category = "main" description = "Backport of pathlib-compatible object wrapper for zip files" +marker = "python_version < \"3.8\"" name = "zipp" optional = false python-versions = ">=3.6" @@ -964,8 +967,8 @@ async-timeout = [ {file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"}, ] atomicwrites = [ - {file = "atomicwrites-1.3.0-py2.py3-none-any.whl", hash = "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4"}, - {file = "atomicwrites-1.3.0.tar.gz", hash = "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"}, + {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, + {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] attrs = [ {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"}, @@ -996,8 +999,8 @@ chardet = [ {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, ] click = [ - {file = "click-7.1.1-py2.py3-none-any.whl", hash = "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a"}, - {file = "click-7.1.1.tar.gz", hash = "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc"}, + {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, + {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] colorama = [ {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, @@ -1100,8 +1103,8 @@ future = [ {file = "future-0.18.2.tar.gz", hash = "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"}, ] identify = [ - {file = "identify-1.4.14-py2.py3-none-any.whl", hash = "sha256:2bb8760d97d8df4408f4e805883dad26a2d076f04be92a10a3e43f09c6060742"}, - {file = "identify-1.4.14.tar.gz", hash = "sha256:faffea0fd8ec86bb146ac538ac350ed0c73908326426d387eded0bcc9d077522"}, + {file = "identify-1.4.15-py2.py3-none-any.whl", hash = "sha256:88ed90632023e52a6495749c6732e61e08ec9f4f04e95484a5c37b9caf40283c"}, + {file = "identify-1.4.15.tar.gz", hash = "sha256:23c18d97bb50e05be1a54917ee45cc61d57cb96aedc06aabb2b02331edf0dbf0"}, ] idna = [ {file = "idna-2.9-py2.py3-none-any.whl", hash = "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"}, @@ -1115,8 +1118,8 @@ importlib-metadata = [ {file = "importlib_metadata-1.6.0.tar.gz", hash = "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e"}, ] importlib-resources = [ - {file = "importlib_resources-1.4.0-py2.py3-none-any.whl", hash = "sha256:dd98ceeef3f5ad2ef4cc287b8586da4ebad15877f351e9688987ad663a0a29b8"}, - {file = "importlib_resources-1.4.0.tar.gz", hash = "sha256:4019b6a9082d8ada9def02bece4a76b131518866790d58fdda0b5f8c603b36c2"}, + {file = "importlib_resources-1.5.0-py2.py3-none-any.whl", hash = "sha256:85dc0b9b325ff78c8bef2e4ff42616094e16b98ebd5e3b50fe7e2f0bbcdcde49"}, + {file = "importlib_resources-1.5.0.tar.gz", hash = "sha256:6f87df66833e1942667108628ec48900e02a4ab4ad850e25fbf07cb17cf734ca"}, ] isort = [ {file = "isort-4.3.21-py2.py3-none-any.whl", hash = "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"}, @@ -1127,16 +1130,16 @@ jmespath = [ {file = "jmespath-0.9.5.tar.gz", hash = "sha256:cca55c8d153173e21baa59983015ad0daf603f9cb799904ff057bfb8ff8dc2d9"}, ] jsonpickle = [ - {file = "jsonpickle-1.4-py2.py3-none-any.whl", hash = "sha256:3d71018794242f6b1640f779a94a192500f73ceed9ef579b4f01799171ec3fb3"}, - {file = "jsonpickle-1.4.tar.gz", hash = "sha256:e8ca6ec3f379f5eee6e11380d48db220aacc282b480dea46b11cc6f6009d1cdb"}, + {file = "jsonpickle-1.4.1-py2.py3-none-any.whl", hash = "sha256:8919c166bac0574e3d74425c7559434062002d9dfc0ac2afa6dc746ba4a19439"}, + {file = "jsonpickle-1.4.1.tar.gz", hash = "sha256:e8d4b7cd0bd6826001a74377df1079a76ad8bae0f909282de2554164c837c8ba"}, ] mako = [ {file = "Mako-1.1.2-py2.py3-none-any.whl", hash = "sha256:8e8b53c71c7e59f3de716b6832c4e401d903af574f6962edbbbf6ecc2a5fe6c9"}, {file = "Mako-1.1.2.tar.gz", hash = "sha256:3139c5d64aa5d175dbafb95027057128b5fbd05a40c53999f3905ceb53366d9d"}, ] markdown = [ - {file = "Markdown-3.2.1-py2.py3-none-any.whl", hash = "sha256:e4795399163109457d4c5af2183fbe6b60326c17cfdf25ce6e7474c6624f725d"}, - {file = "Markdown-3.2.1.tar.gz", hash = "sha256:90fee683eeabe1a92e149f7ba74e5ccdc81cd397bd6c516d93a8da0ef90b6902"}, + {file = "Markdown-3.2.2-py3-none-any.whl", hash = "sha256:c467cd6233885534bf0fe96e62e3cf46cfc1605112356c4f9981512b8174de59"}, + {file = "Markdown-3.2.2.tar.gz", hash = "sha256:1fafe3f1ecabfb514a5285fca634a53c1b32a81cb0feb154264d55bf2ff22c17"}, ] markupsafe = [ {file = "MarkupSafe-1.1.1-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161"}, @@ -1239,8 +1242,8 @@ pyparsing = [ {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] pytest = [ - {file = "pytest-5.4.1-py3-none-any.whl", hash = "sha256:0e5b30f5cb04e887b91b1ee519fa3d89049595f428c1db76e73bd7f17b09b172"}, - {file = "pytest-5.4.1.tar.gz", hash = "sha256:84dde37075b8805f3d1f392cc47e38a0e59518fb46a431cfdaf7cf1ce805f970"}, + {file = "pytest-5.4.2-py3-none-any.whl", hash = "sha256:95c710d0a72d91c13fae35dce195633c929c3792f54125919847fdcdf7caa0d3"}, + {file = "pytest-5.4.2.tar.gz", hash = "sha256:eb2b5e935f6a019317e455b6da83dd8650ac9ffd2ee73a7b657a30873d67a698"}, ] pytest-asyncio = [ {file = "pytest-asyncio-0.12.0.tar.gz", hash = "sha256:475bd2f3dc0bc11d2463656b3cbaafdbec5a47b47508ea0b329ee693040eebd2"}, @@ -1271,27 +1274,27 @@ pyyaml = [ {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] regex = [ - {file = "regex-2020.4.4-cp27-cp27m-win32.whl", hash = "sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f"}, - {file = "regex-2020.4.4-cp27-cp27m-win_amd64.whl", hash = "sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1"}, - {file = "regex-2020.4.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b"}, - {file = "regex-2020.4.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db"}, - {file = "regex-2020.4.4-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156"}, - {file = "regex-2020.4.4-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3"}, - {file = "regex-2020.4.4-cp36-cp36m-win32.whl", hash = "sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8"}, - {file = "regex-2020.4.4-cp36-cp36m-win_amd64.whl", hash = "sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a"}, - {file = "regex-2020.4.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468"}, - {file = "regex-2020.4.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6"}, - {file = "regex-2020.4.4-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd"}, - {file = "regex-2020.4.4-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948"}, - {file = "regex-2020.4.4-cp37-cp37m-win32.whl", hash = "sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e"}, - {file = "regex-2020.4.4-cp37-cp37m-win_amd64.whl", hash = "sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a"}, - {file = "regex-2020.4.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e"}, - {file = "regex-2020.4.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683"}, - {file = "regex-2020.4.4-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b"}, - {file = "regex-2020.4.4-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89"}, - {file = "regex-2020.4.4-cp38-cp38-win32.whl", hash = "sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3"}, - {file = "regex-2020.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3"}, - {file = "regex-2020.4.4.tar.gz", hash = "sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142"}, + {file = "regex-2020.5.7-cp27-cp27m-win32.whl", hash = "sha256:5493a02c1882d2acaaf17be81a3b65408ff541c922bfd002535c5f148aa29f74"}, + {file = "regex-2020.5.7-cp27-cp27m-win_amd64.whl", hash = "sha256:021a0ae4d2baeeb60a3014805a2096cb329bd6d9f30669b7ad0da51a9cb73349"}, + {file = "regex-2020.5.7-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:4df91094ced6f53e71f695c909d9bad1cca8761d96fd9f23db12245b5521136e"}, + {file = "regex-2020.5.7-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:7ce4a213a96d6c25eeae2f7d60d4dad89ac2b8134ec3e69db9bc522e2c0f9388"}, + {file = "regex-2020.5.7-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:3b059e2476b327b9794c792c855aa05531a3f3044737e455d283c7539bd7534d"}, + {file = "regex-2020.5.7-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:652ab4836cd5531d64a34403c00ada4077bb91112e8bcdae933e2eae232cf4a8"}, + {file = "regex-2020.5.7-cp36-cp36m-win32.whl", hash = "sha256:1e2255ae938a36e9bd7db3b93618796d90c07e5f64dd6a6750c55f51f8b76918"}, + {file = "regex-2020.5.7-cp36-cp36m-win_amd64.whl", hash = "sha256:8127ca2bf9539d6a64d03686fd9e789e8c194fc19af49b69b081f8c7e6ecb1bc"}, + {file = "regex-2020.5.7-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f7f2f4226db6acd1da228adf433c5c3792858474e49d80668ea82ac87cf74a03"}, + {file = "regex-2020.5.7-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:2bc6a17a7fa8afd33c02d51b6f417fc271538990297167f68a98cae1c9e5c945"}, + {file = "regex-2020.5.7-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:b7c9f65524ff06bf70c945cd8d8d1fd90853e27ccf86026af2afb4d9a63d06b1"}, + {file = "regex-2020.5.7-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:fa09da4af4e5b15c0e8b4986a083f3fd159302ea115a6cc0649cd163435538b8"}, + {file = "regex-2020.5.7-cp37-cp37m-win32.whl", hash = "sha256:669a8d46764a09f198f2e91fc0d5acdac8e6b620376757a04682846ae28879c4"}, + {file = "regex-2020.5.7-cp37-cp37m-win_amd64.whl", hash = "sha256:b5b5b2e95f761a88d4c93691716ce01dc55f288a153face1654f868a8034f494"}, + {file = "regex-2020.5.7-cp38-cp38-manylinux1_i686.whl", hash = "sha256:0ff50843535593ee93acab662663cb2f52af8e31c3f525f630f1dc6156247938"}, + {file = "regex-2020.5.7-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:1b17bf37c2aefc4cac8436971fe6ee52542ae4225cfc7762017f7e97a63ca998"}, + {file = "regex-2020.5.7-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:04d6e948ef34d3eac133bedc0098364a9e635a7914f050edb61272d2ddae3608"}, + {file = "regex-2020.5.7-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:5b741ecc3ad3e463d2ba32dce512b412c319993c1bb3d999be49e6092a769fb2"}, + {file = "regex-2020.5.7-cp38-cp38-win32.whl", hash = "sha256:099568b372bda492be09c4f291b398475587d49937c659824f891182df728cdf"}, + {file = "regex-2020.5.7-cp38-cp38-win_amd64.whl", hash = "sha256:3ab5e41c4ed7cd4fa426c50add2892eb0f04ae4e73162155cd668257d02259dd"}, + {file = "regex-2020.5.7.tar.gz", hash = "sha256:73a10404867b835f1b8a64253e4621908f0d71150eb4e97ab2e7e441b53e9451"}, ] s3transfer = [ {file = "s3transfer-0.3.3-py2.py3-none-any.whl", hash = "sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13"}, @@ -1343,8 +1346,8 @@ urllib3 = [ {file = "urllib3-1.25.9.tar.gz", hash = "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527"}, ] virtualenv = [ - {file = "virtualenv-20.0.18-py2.py3-none-any.whl", hash = "sha256:5021396e8f03d0d002a770da90e31e61159684db2859d0ba4850fbea752aa675"}, - {file = "virtualenv-20.0.18.tar.gz", hash = "sha256:ac53ade75ca189bc97b6c1d9ec0f1a50efe33cbf178ae09452dcd9fd309013c1"}, + {file = "virtualenv-20.0.20-py2.py3-none-any.whl", hash = "sha256:b4c14d4d73a0c23db267095383c4276ef60e161f94fde0427f2f21a0132dde74"}, + {file = "virtualenv-20.0.20.tar.gz", hash = "sha256:fd0e54dec8ac96c1c7c87daba85f0a59a7c37fe38748e154306ca21c73244637"}, ] wcwidth = [ {file = "wcwidth-0.1.9-py2.py3-none-any.whl", hash = "sha256:cafe2186b3c009a04067022ce1dcd79cb38d8d65ee4f4791b8888d6599d1bbe1"}, From 252e35be3b2c4dfcbb355de1f3ae5d0cbcee2c39 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 10 May 2020 17:35:44 +0100 Subject: [PATCH 25/35] improv: example to use py 3.8 --- python/example/template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/example/template.yaml b/python/example/template.yaml index e2104dd045..47267d729f 100644 --- a/python/example/template.yaml +++ b/python/example/template.yaml @@ -16,7 +16,7 @@ Resources: Properties: CodeUri: hello_world/ Handler: app.lambda_handler - Runtime: python3.7 + Runtime: python3.8 Tracing: Active # enables X-Ray tracing Environment: Variables: From adb244e504598521f5b2522e50986adda592f695 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 10 May 2020 17:36:39 +0100 Subject: [PATCH 26/35] fix: AsyncMockMixin not being awaitable in 3.8 --- python/tests/unit/test_tracing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/tests/unit/test_tracing.py b/python/tests/unit/test_tracing.py index ffa8430c08..d68b52fea2 100644 --- a/python/tests/unit/test_tracing.py +++ b/python/tests/unit/test_tracing.py @@ -70,6 +70,7 @@ class In_subsegment(NamedTuple): in_subsegment = In_subsegment() in_subsegment.in_subsegment.return_value.__enter__.return_value.put_annotation = in_subsegment.put_annotation in_subsegment.in_subsegment.return_value.__enter__.return_value.put_metadata = in_subsegment.put_metadata + in_subsegment.in_subsegment.return_value.__aenter__.return_value.put_metadata = in_subsegment.put_metadata yield in_subsegment From ea38254d741d2e96a04c23e00bddde49c40d51ae Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 10 May 2020 18:11:04 +0100 Subject: [PATCH 27/35] fix: 3.8 defaulting to AsyncMock --- python/tests/unit/test_tracing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/tests/unit/test_tracing.py b/python/tests/unit/test_tracing.py index d68b52fea2..4774607389 100644 --- a/python/tests/unit/test_tracing.py +++ b/python/tests/unit/test_tracing.py @@ -2,6 +2,7 @@ from unittest import mock import pytest +import sys from aws_lambda_powertools.tracing import Tracer @@ -70,7 +71,9 @@ class In_subsegment(NamedTuple): in_subsegment = In_subsegment() in_subsegment.in_subsegment.return_value.__enter__.return_value.put_annotation = in_subsegment.put_annotation in_subsegment.in_subsegment.return_value.__enter__.return_value.put_metadata = in_subsegment.put_metadata - in_subsegment.in_subsegment.return_value.__aenter__.return_value.put_metadata = in_subsegment.put_metadata + + if sys.version_info >= (3, 8): # 3.8 introduced AsyncMock + in_subsegment.in_subsegment.return_value.__aenter__.return_value.put_metadata = in_subsegment.put_metadata yield in_subsegment From b7507e42e9417fb40b69251fd714820670014295 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 11 May 2020 09:46:56 +0100 Subject: [PATCH 28/35] improv: include x-ray bug for concurrent async calls --- python/README.md | 36 +++++++++++++++++++ .../aws_lambda_powertools/tracing/__init__.py | 2 ++ .../aws_lambda_powertools/tracing/tracer.py | 4 +++ python/example/hello_world/app.py | 28 +++++++-------- 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/python/README.md b/python/README.md index 8abc6f3e19..e7f9b2c634 100644 --- a/python/README.md +++ b/python/README.md @@ -102,6 +102,40 @@ def handler(event, context) ... ``` +#### Tracing concurrent asynchronous with gather + +:warning: This will no longer be necessary after [this X-Ray recorder issue is resolved](https://github.com/aws/aws-xray-sdk-python/issues/164) as it's an edge case. :warning: + +To safely workaround this issue, use `@tracer.capture_method` on functions not being run with `async.gather`, and instead use `in_subsegment_async` context manager escape hatch to have the same tracing effect. + + +```python +import asyncio + +from aws_lambda_powertools.tracing import Tracer +tracer = Tracer() +# tracer = Tracer(service="payment") # can also be explicitly defined + +async def another_async_task(): + async with tracer.provider.in_subsegment_async("## another_async_task"): + ... + +async def another_async_task_2(): + async with tracer.provider.in_subsegment_async("## another_async_task_2"): + ... + +@tracer.capture_method +async def collect_payment(charge_id): + asyncio.gather(another_async_task(), another_async_task_2()) + ... + +@tracer.capture_lambda_handler +def handler(event, context) + charge_id = event.get('charge_id') + payment = asyncio.run(collect_payment(charge_id)) # python 3.7+ + ... +``` + #### Using escape hatch mechanisms You can use `tracer.provider` attribute to access all methods provided by `xray_recorder`. This is useful when you need a feature available in X-Ray that is not available in the Tracer middleware, for example [thread-safe](https://github.com/aws/aws-xray-sdk-python/#user-content-trace-threadpoolexecutor), or [context managers](https://github.com/aws/aws-xray-sdk-python/#user-content-start-a-custom-segmentsubsegment). @@ -114,6 +148,8 @@ import asyncio from aws_lambda_powertools.tracing import Tracer, aiohttp_trace_config tracer = Tracer() +# aiohttp_trace_config is x-ray extension for aiohttp trace config known as aws_xray_trace_config + async def aiohttp_task(): # Async context manager as opposed to `@tracer.capture_method` async with tracer.provider.in_subsegment_async("## aiohttp escape hatch"): diff --git a/python/aws_lambda_powertools/tracing/__init__.py b/python/aws_lambda_powertools/tracing/__init__.py index dd67edfa95..bbe312bcd7 100644 --- a/python/aws_lambda_powertools/tracing/__init__.py +++ b/python/aws_lambda_powertools/tracing/__init__.py @@ -2,6 +2,8 @@ """ from aws_xray_sdk.ext.aiohttp.client import aws_xray_trace_config as aiohttp_trace_config +aiohttp_trace_config.__doc__ = "aiohttp extension for X-Ray (aws_xray_trace_config)" + from .tracer import Tracer __all__ = ["Tracer", "aiohttp_trace_config"] diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index 54fdd7afa3..9328cfcfb0 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -337,6 +337,8 @@ async def async_tasks(): return { "task": "done", **ret } **Safely tracing concurrent async calls with decorator** + + This may not needed once [this bug is closed](https://github.com/aws/aws-xray-sdk-python/issues/164) from aws_lambda_powertools.tracing import Tracer tracer = Tracer(service="booking") @@ -357,6 +359,8 @@ async def async_tasks(): **Safely tracing each concurrent async calls with escape hatch** + This may not needed once [this bug is closed](https://github.com/aws/aws-xray-sdk-python/issues/164) + from aws_lambda_powertools.tracing import Tracer tracer = Tracer(service="booking") diff --git a/python/example/hello_world/app.py b/python/example/hello_world/app.py index 323ad55add..35f44de67d 100644 --- a/python/example/hello_world/app.py +++ b/python/example/hello_world/app.py @@ -1,20 +1,20 @@ +import asyncio import json +import aioboto3 +import aiohttp import requests from aws_lambda_powertools.logging import Logger +from aws_lambda_powertools.logging.logger import set_package_logger from aws_lambda_powertools.metrics import Metrics, MetricUnit, single_metric from aws_lambda_powertools.middleware_factory import lambda_handler_decorator from aws_lambda_powertools.tracing import Tracer, aiohttp_trace_config -from aws_lambda_powertools.logging.logger import set_package_logger -import asyncio -import aioboto3 -import aiohttp -set_package_logger() # Enable package diagnostics (DEBUG log) +set_package_logger() # Enable package diagnostics (DEBUG log) # tracer = Tracer() # patches all available modules -tracer = Tracer(patch_modules=("aioboto3", "boto3", "requests")) # ~90-100ms faster in perf depending on set of libs +tracer = Tracer(patch_modules=("aioboto3", "boto3", "requests")) # ~90-100ms faster in perf depending on set of libs logger = Logger() metrics = Metrics() @@ -22,14 +22,17 @@ metrics.add_dimension(name="operation", value="example") + async def aioboto_task(): async with aioboto3.client("sts") as sts: account = await sts.get_caller_identity() return account + async def aiohttp_task(): # You have full access to all xray_recorder methods via `tracer.provider` # these include thread-safe methods, all context managers, x-ray configuration etc. + # see https://github.com/aws/aws-xray-sdk-python/issues/164 async with tracer.provider.in_subsegment_async("## aiohttp escape hatch"): async with aiohttp.ClientSession(trace_configs=[aiohttp_trace_config()]) as session: async with session.get("https://httpbin.org/json") as resp: @@ -41,10 +44,7 @@ async def aiohttp_task(): async def async_tasks(): _, ret = await asyncio.gather(aioboto_task(), aiohttp_task(), return_exceptions=True) - return { - "task": "done", - **ret - } + return {"task": "done", **ret} @lambda_handler_decorator(trace_execution=True) @@ -108,13 +108,9 @@ def lambda_handler(event, context): with single_metric(name="UniqueMetricDimension", unit="Seconds", value=1) as metric: metric.add_dimension(name="unique_dimension", value="for_unique_metric") - resp = { - "message": "hello world", - "location": ip.text.replace("\n", ""), - "async_http": async_http_ret - } + resp = {"message": "hello world", "location": ip.text.replace("\n", ""), "async_http": async_http_ret} logger.info("Returning message to the caller") - + return { "statusCode": 200, "body": json.dumps(resp), From b8ed79f8dca21cc94418ce6be0f52b0e1cd1599f Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 12 May 2020 09:54:37 +0100 Subject: [PATCH 29/35] fix: address nicolas's feedback --- python/README.md | 1 + .../aws_lambda_powertools/tracing/__init__.py | 3 ++- .../aws_lambda_powertools/tracing/tracer.py | 20 ++++++++++--------- python/tests/unit/test_tracing.py | 2 +- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/python/README.md b/python/README.md index e7f9b2c634..ab0efc45db 100644 --- a/python/README.md +++ b/python/README.md @@ -144,6 +144,7 @@ You can use `tracer.provider` attribute to access all methods provided by `xray_ ```python import asyncio +import aiohttp from aws_lambda_powertools.tracing import Tracer, aiohttp_trace_config tracer = Tracer() diff --git a/python/aws_lambda_powertools/tracing/__init__.py b/python/aws_lambda_powertools/tracing/__init__.py index bbe312bcd7..ece90f7d1b 100644 --- a/python/aws_lambda_powertools/tracing/__init__.py +++ b/python/aws_lambda_powertools/tracing/__init__.py @@ -2,8 +2,9 @@ """ from aws_xray_sdk.ext.aiohttp.client import aws_xray_trace_config as aiohttp_trace_config +from .tracer import Tracer + aiohttp_trace_config.__doc__ = "aiohttp extension for X-Ray (aws_xray_trace_config)" -from .tracer import Tracer __all__ = ["Tracer", "aiohttp_trace_config"] diff --git a/python/aws_lambda_powertools/tracing/tracer.py b/python/aws_lambda_powertools/tracing/tracer.py index 9328cfcfb0..022d8ef89a 100644 --- a/python/aws_lambda_powertools/tracing/tracer.py +++ b/python/aws_lambda_powertools/tracing/tracer.py @@ -75,7 +75,7 @@ def handler(event: dict, context: Any) -> Dict: def confirm_booking(booking_id: str) -> Dict: resp = add_confirmation(booking_id) - tracer.put_annotation("BookingConfirmation", resp['requestId']) + tracer.put_annotation("BookingConfirmation", resp["requestId"]) tracer.put_metadata("Booking confirmation", resp) return resp @@ -83,7 +83,8 @@ def confirm_booking(booking_id: str) -> Dict: @tracer.capture_lambda_handler def handler(event: dict, context: Any) -> Dict: print("Received event from Lambda...") - response = confirm_booking(booking_id=event["booking_id]) + booking_id = event.get("booking_id") + response = confirm_booking(booking_id=booking_id) return response **A Lambda function using service name via POWERTOOLS_SERVICE_NAME** @@ -175,7 +176,7 @@ def put_annotation(self, key: str, value: Any): logger.debug("Tracing has been disabled, aborting put_annotation") return - logger.debug(f"Annotating on key '{key}'' with '{value}''") + logger.debug(f"Annotating on key '{key}' with '{value}'") self.provider.put_annotation(key=key, value=value) def put_metadata(self, key: str, value: Any, namespace: str = None): @@ -203,7 +204,7 @@ def put_metadata(self, key: str, value: Any, namespace: str = None): return namespace = namespace or self.service - logger.debug(f"Adding metadata on key '{key}'' with '{value}'' at namespace '{namespace}''") + logger.debug(f"Adding metadata on key '{key}' with '{value}' at namespace '{namespace}'") self.provider.put_metadata(key=key, value=value, namespace=namespace) def patch(self, modules: Tuple[str] = None): @@ -306,15 +307,16 @@ def some_function() @tracer.capture_method async def confirm_booking(booking_id: str) -> Dict: - resp = confirm_booking(booking_id=event["booking_id]) + resp = call_to_booking_service() - tracer.put_annotation("BookingConfirmation", resp['requestId']) + tracer.put_annotation("BookingConfirmation", resp["requestId"]) tracer.put_metadata("Booking confirmation", resp) return resp def lambda_handler(event: dict, context: Any) -> Dict: - asyncio.run(confirm_booking(booking=id)) + booking_id = event.get("booking_id") + asyncio.run(confirm_booking(booking_id=booking_id)) **Tracing nested async calls** @@ -337,7 +339,7 @@ async def async_tasks(): return { "task": "done", **ret } **Safely tracing concurrent async calls with decorator** - + This may not needed once [this bug is closed](https://github.com/aws/aws-xray-sdk-python/issues/164) from aws_lambda_powertools.tracing import Tracer @@ -421,7 +423,7 @@ async def decorate_logic( logger.debug(f"Received {method_name} response successfully") logger.debug(response) except Exception as err: - logger.exception(f"Exception received from '{method_name}'' method", exc_info=True) + logger.exception(f"Exception received from '{method_name}' method", exc_info=True) subsegment.put_metadata(key=f"{method_name} error", value=err, namespace=self._config["service"]) raise finally: diff --git a/python/tests/unit/test_tracing.py b/python/tests/unit/test_tracing.py index 4774607389..f5ad586cb0 100644 --- a/python/tests/unit/test_tracing.py +++ b/python/tests/unit/test_tracing.py @@ -1,8 +1,8 @@ +import sys from typing import NamedTuple from unittest import mock import pytest -import sys from aws_lambda_powertools.tracing import Tracer From b767567ad1a02e044540ef1782cc3abbcdb9c425 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 12 May 2020 10:06:56 +0100 Subject: [PATCH 30/35] improv: add security baseline as part of PR process --- python/Makefile | 5 +- python/bandit.baseline | 226 +++++++++++++++++++++++++++++++++++++++++ python/poetry.lock | 94 ++++++++++++++++- python/pyproject.toml | 1 + 4 files changed, 322 insertions(+), 4 deletions(-) create mode 100644 python/bandit.baseline diff --git a/python/Makefile b/python/Makefile index fac2a8af79..2dfda981e2 100644 --- a/python/Makefile +++ b/python/Makefile @@ -20,7 +20,7 @@ test: coverage-html: poetry run pytest --cov-report html -pr: lint test +pr: lint test security-baseline build: pr poetry run build @@ -31,6 +31,9 @@ docs: dev docs-dev: poetry run pdoc --http : aws_lambda_powertools +security-baseline: + poetry run bandit --baseline bandit.baseline -r aws_lambda_powertools + # # Use `poetry version /` for version bump # diff --git a/python/bandit.baseline b/python/bandit.baseline new file mode 100644 index 0000000000..a989733b93 --- /dev/null +++ b/python/bandit.baseline @@ -0,0 +1,226 @@ +{ + "errors": [], + "generated_at": "2020-05-12T08:59:59Z", + "metrics": { + "_totals": { + "CONFIDENCE.HIGH": 1.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 1.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 1375, + "nosec": 0 + }, + "aws_lambda_powertools/__init__.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 6, + "nosec": 0 + }, + "aws_lambda_powertools/helper/__init__.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 2, + "nosec": 0 + }, + "aws_lambda_powertools/helper/models.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 108, + "nosec": 0 + }, + "aws_lambda_powertools/logging/__init__.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 5, + "nosec": 0 + }, + "aws_lambda_powertools/logging/exceptions.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 2, + "nosec": 0 + }, + "aws_lambda_powertools/logging/logger.py": { + "CONFIDENCE.HIGH": 1.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 1.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 375, + "nosec": 0 + }, + "aws_lambda_powertools/metrics/__init__.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 15, + "nosec": 0 + }, + "aws_lambda_powertools/metrics/base.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 162, + "nosec": 0 + }, + "aws_lambda_powertools/metrics/exceptions.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 12, + "nosec": 0 + }, + "aws_lambda_powertools/metrics/metric.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 90, + "nosec": 0 + }, + "aws_lambda_powertools/metrics/metrics.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 82, + "nosec": 0 + }, + "aws_lambda_powertools/middleware_factory/__init__.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 3, + "nosec": 0 + }, + "aws_lambda_powertools/middleware_factory/exceptions.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 3, + "nosec": 0 + }, + "aws_lambda_powertools/middleware_factory/factory.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 103, + "nosec": 0 + }, + "aws_lambda_powertools/tracing/__init__.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 6, + "nosec": 0 + }, + "aws_lambda_powertools/tracing/tracer.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 401, + "nosec": 0 + } + }, + "results": [ + { + "code": "369 try:\n370 if self.sampling_rate and random.random() <= float(self.sampling_rate):\n371 logger.debug(\"Setting log level to Debug due to sampling rate\")\n", + "filename": "aws_lambda_powertools/logging/logger.py", + "issue_confidence": "HIGH", + "issue_severity": "LOW", + "issue_text": "Standard pseudo-random generators are not suitable for security/cryptographic purposes.", + "line_number": 370, + "line_range": [ + 370 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_calls.html#b311-random", + "test_id": "B311", + "test_name": "blacklist" + } + ] +} \ No newline at end of file diff --git a/python/poetry.lock b/python/poetry.lock index 368079b69a..a806c8ec55 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -126,6 +126,21 @@ future = "*" jsonpickle = "*" wrapt = "*" +[[package]] +category = "dev" +description = "Security oriented static analyser for python code." +name = "bandit" +optional = false +python-versions = "*" +version = "1.6.2" + +[package.dependencies] +GitPython = ">=1.0.1" +PyYAML = ">=3.13" +colorama = ">=0.3.9" +six = ">=1.10.0" +stevedore = ">=1.20.0" + [[package]] category = "dev" description = "The uncompromising code formatter." @@ -203,7 +218,7 @@ version = "7.1.2" [[package]] category = "dev" description = "Cross-platform colored terminal text." -marker = "sys_platform == \"win32\"" +marker = "sys_platform == \"win32\" or platform_system == \"Windows\"" name = "colorama" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -414,6 +429,28 @@ optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" version = "0.18.2" +[[package]] +category = "dev" +description = "Git Object Database" +name = "gitdb" +optional = false +python-versions = ">=3.4" +version = "4.0.5" + +[package.dependencies] +smmap = ">=3.0.1,<4" + +[[package]] +category = "dev" +description = "Python Git Library" +name = "gitpython" +optional = false +python-versions = ">=3.4" +version = "3.1.2" + +[package.dependencies] +gitdb = ">=4.0.1,<5" + [[package]] category = "dev" description = "File identification library for Python" @@ -610,6 +647,14 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" version = "0.8.0" +[[package]] +category = "dev" +description = "Python Build Reasonableness" +name = "pbr" +optional = false +python-versions = "*" +version = "5.4.5" + [[package]] category = "dev" description = "Auto-generate API documentation for Python projects." @@ -809,6 +854,26 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" version = "1.14.0" +[[package]] +category = "dev" +description = "A pure Python implementation of a sliding window memory map manager" +name = "smmap" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "3.0.4" + +[[package]] +category = "dev" +description = "Manage dynamic plugins for Python applications" +name = "stevedore" +optional = false +python-versions = "*" +version = "1.32.0" + +[package.dependencies] +pbr = ">=2.0.0,<2.1.0 || >2.1.0" +six = ">=1.10.0" + [[package]] category = "dev" description = "A collection of helpers and mock objects for unit tests and doc tests." @@ -917,7 +982,6 @@ multidict = ">=4.0" [[package]] category = "main" description = "Backport of pathlib-compatible object wrapper for zip files" -marker = "python_version < \"3.8\"" name = "zipp" optional = false python-versions = ">=3.6" @@ -928,7 +992,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "2f09044a57a6e75afb5f6fb30c4b48b9dbe1814f8d658f640fc7c777ce6155fb" +content-hash = "cd5b1de52781f1830a029ddb8d9bf551978e7603e1dc820118f0c3c70a135a46" python-versions = "^3.6" [metadata.files] @@ -978,6 +1042,10 @@ aws-xray-sdk = [ {file = "aws-xray-sdk-2.5.0.tar.gz", hash = "sha256:8dfa785305fc8dc720d8d4c2ec6a58e85e467ddc3a53b1506a2ed8b5801c8fc7"}, {file = "aws_xray_sdk-2.5.0-py2.py3-none-any.whl", hash = "sha256:ae57baeb175993bdbf31f83843e2c0958dd5aa8cb691ab5628aafb6ccc78a0fc"}, ] +bandit = [ + {file = "bandit-1.6.2-py2.py3-none-any.whl", hash = "sha256:336620e220cf2d3115877685e264477ff9d9abaeb0afe3dc7264f55fa17a3952"}, + {file = "bandit-1.6.2.tar.gz", hash = "sha256:41e75315853507aa145d62a78a2a6c5e3240fe14ee7c601459d0df9418196065"}, +] black = [ {file = "black-19.10b0-py36-none-any.whl", hash = "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b"}, {file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"}, @@ -1102,6 +1170,14 @@ flake8-variables-names = [ future = [ {file = "future-0.18.2.tar.gz", hash = "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"}, ] +gitdb = [ + {file = "gitdb-4.0.5-py3-none-any.whl", hash = "sha256:91f36bfb1ab7949b3b40e23736db18231bf7593edada2ba5c3a174a7b23657ac"}, + {file = "gitdb-4.0.5.tar.gz", hash = "sha256:c9e1f2d0db7ddb9a704c2a0217be31214e91a4fe1dea1efad19ae42ba0c285c9"}, +] +gitpython = [ + {file = "GitPython-3.1.2-py3-none-any.whl", hash = "sha256:da3b2cf819974789da34f95ac218ef99f515a928685db141327c09b73dd69c09"}, + {file = "GitPython-3.1.2.tar.gz", hash = "sha256:864a47472548f3ba716ca202e034c1900f197c0fb3a08f641c20c3cafd15ed94"}, +] identify = [ {file = "identify-1.4.15-py2.py3-none-any.whl", hash = "sha256:88ed90632023e52a6495749c6732e61e08ec9f4f04e95484a5c37b9caf40283c"}, {file = "identify-1.4.15.tar.gz", hash = "sha256:23c18d97bb50e05be1a54917ee45cc61d57cb96aedc06aabb2b02331edf0dbf0"}, @@ -1214,6 +1290,10 @@ pathspec = [ {file = "pathspec-0.8.0-py2.py3-none-any.whl", hash = "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0"}, {file = "pathspec-0.8.0.tar.gz", hash = "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061"}, ] +pbr = [ + {file = "pbr-5.4.5-py2.py3-none-any.whl", hash = "sha256:579170e23f8e0c2f24b0de612f71f648eccb79fb1322c814ae6b3c07b5ba23e8"}, + {file = "pbr-5.4.5.tar.gz", hash = "sha256:07f558fece33b05caf857474a366dfcc00562bca13dd8b47b2b3e22d9f9bf55c"}, +] pdoc3 = [ {file = "pdoc3-0.7.5.tar.gz", hash = "sha256:ebca75b7fcf23f3b4320abe23339834d3f08c28517718e9d29e555fc38eeb33c"}, ] @@ -1304,6 +1384,14 @@ six = [ {file = "six-1.14.0-py2.py3-none-any.whl", hash = "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"}, {file = "six-1.14.0.tar.gz", hash = "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a"}, ] +smmap = [ + {file = "smmap-3.0.4-py2.py3-none-any.whl", hash = "sha256:54c44c197c819d5ef1991799a7e30b662d1e520f2ac75c9efbeb54a742214cf4"}, + {file = "smmap-3.0.4.tar.gz", hash = "sha256:9c98bbd1f9786d22f14b3d4126894d56befb835ec90cef151af566c7e19b5d24"}, +] +stevedore = [ + {file = "stevedore-1.32.0-py2.py3-none-any.whl", hash = "sha256:a4e7dc759fb0f2e3e2f7d8ffe2358c19d45b9b8297f393ef1256858d82f69c9b"}, + {file = "stevedore-1.32.0.tar.gz", hash = "sha256:18afaf1d623af5950cc0f7e75e70f917784c73b652a34a12d90b309451b5500b"}, +] testfixtures = [ {file = "testfixtures-6.14.1-py2.py3-none-any.whl", hash = "sha256:30566e24a1b34e4d3f8c13abf62557d01eeb4480bcb8f1745467bfb0d415a7d9"}, {file = "testfixtures-6.14.1.tar.gz", hash = "sha256:58d2b3146d93bc5ddb0cd24e0ccacb13e29bdb61e5c81235c58f7b8ee4470366"}, diff --git a/python/pyproject.toml b/python/pyproject.toml index 7bc7cd951c..8356bf4cd5 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -44,6 +44,7 @@ pdoc3 = "^0.7.5" pytest-asyncio = "^0.12.0" aioboto3 = "^8.0.3" aiohttp = "^3.6.2" +bandit = "^1.6.2" [tool.coverage.run] source = ["aws_lambda_powertools"] From 3ddcf168c6507c57a4c04721f3a43da37248c5ad Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 12 May 2020 10:57:17 +0100 Subject: [PATCH 31/35] improv: enforce lower code complexity --- python/.flake8 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/.flake8 b/python/.flake8 index e55ad0bdf3..d5490be789 100644 --- a/python/.flake8 +++ b/python/.flake8 @@ -2,7 +2,7 @@ exclude = docs, .eggs, setup.py, example, .aws-sam ignore = E203, E266, W503, BLK100, W291, I004 max-line-length = 120 -max-complexity = 18 +max-complexity = 15 [isort] multi_line_output = 3 @@ -10,4 +10,3 @@ include_trailing_comma = true force_grid_wrap = 0 use_parentheses = true line_length = 120 - From a5407d503c4ca0c28a77ef8b86b81c9eaafdcb6e Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 12 May 2020 10:57:33 +0100 Subject: [PATCH 32/35] chore: whitespace --- python/aws_lambda_powertools/logging/logger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/aws_lambda_powertools/logging/logger.py b/python/aws_lambda_powertools/logging/logger.py index 02c0e912b1..bbcf658906 100644 --- a/python/aws_lambda_powertools/logging/logger.py +++ b/python/aws_lambda_powertools/logging/logger.py @@ -402,7 +402,7 @@ def inject_lambda_context(self, lambda_handler: Callable[[Dict, Any], Any] = Non @logger.inject_lambda_context def handler(event, context): - logger.info("Hello") + logger.info("Hello") **Captures Lambda contextual runtime info and logs incoming request** @@ -412,7 +412,7 @@ def handler(event, context): @logger.inject_lambda_context(log_event=True) def handler(event, context): - logger.info("Hello") + logger.info("Hello") Returns ------- From 7647193b9872cfd6a289fb8ac7c0e42eee14db6c Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 12 May 2020 10:58:52 +0100 Subject: [PATCH 33/35] improv: add complexity baseline --- python/Makefile | 8 +- python/poetry.lock | 174 ++++++++++++++++++++++++++++++++---------- python/pyproject.toml | 6 +- 3 files changed, 144 insertions(+), 44 deletions(-) diff --git a/python/Makefile b/python/Makefile index 2dfda981e2..91dca5e658 100644 --- a/python/Makefile +++ b/python/Makefile @@ -20,7 +20,7 @@ test: coverage-html: poetry run pytest --cov-report html -pr: lint test security-baseline +pr: lint test security-baseline complexity-baseline build: pr poetry run build @@ -34,6 +34,12 @@ docs-dev: security-baseline: poetry run bandit --baseline bandit.baseline -r aws_lambda_powertools +complexity-baseline: + $(info Maintenability index) + poetry run radon mi aws_lambda_powertools + $(info Cyclomatic complexity index) + poetry run xenon --max-absolute C --max-modules A --max-average A aws_lambda_powertools + # # Use `poetry version /` for version bump # diff --git a/python/poetry.lock b/python/poetry.lock index a806c8ec55..07143b50fa 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -79,7 +79,7 @@ description = "A small Python module for determining appropriate platform-specif name = "appdirs" optional = false python-versions = "*" -version = "1.4.3" +version = "1.4.4" [[package]] category = "dev" @@ -191,6 +191,14 @@ python-dateutil = ">=2.1,<3.0.0" python = "<3.4.0 || >=3.5.0" version = ">=1.20,<1.26" +[[package]] +category = "dev" +description = "Python package for providing Mozilla's CA Bundle." +name = "certifi" +optional = false +python-versions = "*" +version = "2020.4.5.1" + [[package]] category = "dev" description = "Validate configuration and produce human readable error messages." @@ -218,11 +226,10 @@ version = "7.1.2" [[package]] category = "dev" description = "Cross-platform colored terminal text." -marker = "sys_platform == \"win32\" or platform_system == \"Windows\"" name = "colorama" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "0.4.3" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.4.1" [[package]] category = "dev" @@ -256,14 +263,6 @@ optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" version = "0.15.2" -[[package]] -category = "dev" -description = "Discover and load entry points from installed packages." -name = "entrypoints" -optional = false -python-versions = ">=2.7" -version = "0.3" - [[package]] category = "dev" description = "Removes commented-out code." @@ -293,17 +292,20 @@ version = "3.0.12" [[package]] category = "dev" -description = "the modular source code checker: pep8, pyflakes and co" +description = "the modular source code checker: pep8 pyflakes and co" name = "flake8" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "3.7.9" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +version = "3.8.1" [package.dependencies] -entrypoints = ">=0.3.0,<0.4.0" mccabe = ">=0.6.0,<0.7.0" -pycodestyle = ">=2.5.0,<2.6.0" -pyflakes = ">=2.1.0,<2.2.0" +pycodestyle = ">=2.6.0a1,<2.7.0" +pyflakes = ">=2.2.0,<2.3.0" + +[package.dependencies.importlib-metadata] +python = "<3.8" +version = "*" [[package]] category = "dev" @@ -376,11 +378,11 @@ description = "Flake8 plugin to find commented out code" name = "flake8-eradicate" optional = false python-versions = ">=3.6,<4.0" -version = "0.2.4" +version = "0.3.0" [package.dependencies] -attrs = ">=18.2,<20.0" -eradicate = ">=0.2.1,<1.1.0" +attrs = "*" +eradicate = ">=1.0,<2.0" flake8 = ">=3.5,<4.0" [[package]] @@ -410,6 +412,17 @@ version = ">=4.3.5" [package.extras] test = ["pytest"] +[[package]] +category = "dev" +description = "Polyfill package for Flake8 plugins" +name = "flake8-polyfill" +optional = false +python-versions = "*" +version = "1.0.2" + +[package.dependencies] +flake8 = "*" + [[package]] category = "dev" description = "A flake8 extension that helps to make more readable variables names" @@ -571,6 +584,20 @@ MarkupSafe = ">=0.9.2" babel = ["babel"] lingua = ["lingua"] +[[package]] +category = "dev" +description = "Create Python CLI apps with little to no effort at all!" +name = "mando" +optional = false +python-versions = "*" +version = "0.6.4" + +[package.dependencies] +six = "*" + +[package.extras] +restructuredText = ["rst2ansi"] + [[package]] category = "dev" description = "Python implementation of Markdown." @@ -721,7 +748,7 @@ description = "Python style guide checker" name = "pycodestyle" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "2.5.0" +version = "2.6.0" [[package]] category = "dev" @@ -729,7 +756,7 @@ description = "passive checker of Python programs" name = "pyflakes" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "2.1.1" +version = "2.2.0" [[package]] category = "dev" @@ -827,6 +854,20 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" version = "5.3.1" +[[package]] +category = "dev" +description = "Code Metrics in Python" +name = "radon" +optional = false +python-versions = "*" +version = "4.1.0" + +[package.dependencies] +colorama = "0.4.1" +flake8-polyfill = "*" +future = "*" +mando = ">=0.6,<0.7" + [[package]] category = "dev" description = "Alternative regular expression module, to replace re." @@ -835,6 +876,24 @@ optional = false python-versions = "*" version = "2020.5.7" +[[package]] +category = "dev" +description = "Python HTTP for Humans." +name = "requests" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "2.23.0" + +[package.dependencies] +certifi = ">=2017.4.17" +chardet = ">=3.0.2,<4" +idna = ">=2.5,<3" +urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" + +[package.extras] +security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] + [[package]] category = "dev" description = "An Amazon S3 Transfer Manager" @@ -914,7 +973,6 @@ version = "3.7.4.2" [[package]] category = "main" description = "HTTP library with thread-safe connection pooling, file post, and more." -marker = "python_version != \"3.4\"" name = "urllib3" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" @@ -967,6 +1025,19 @@ optional = false python-versions = "*" version = "1.12.1" +[[package]] +category = "dev" +description = "Monitor code metrics for Python on your CI server" +name = "xenon" +optional = false +python-versions = "*" +version = "0.7.0" + +[package.dependencies] +PyYAML = ">=4.2b1,<6.0" +radon = ">=4,<5" +requests = ">=2.0,<3.0" + [[package]] category = "dev" description = "Yet another URL library" @@ -982,6 +1053,7 @@ multidict = ">=4.0" [[package]] category = "main" description = "Backport of pathlib-compatible object wrapper for zip files" +marker = "python_version < \"3.8\"" name = "zipp" optional = false python-versions = ">=3.6" @@ -992,7 +1064,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "cd5b1de52781f1830a029ddb8d9bf551978e7603e1dc820118f0c3c70a135a46" +content-hash = "525f4150dc764e0fa82b790ada43514e328c26e0e3e90e26103b038ce0bd896e" python-versions = "^3.6" [metadata.files] @@ -1023,8 +1095,8 @@ aioitertools = [ {file = "aioitertools-0.7.0.tar.gz", hash = "sha256:341cb05a0903177ef1b73d4cc12c92aee18e81c364e0138f4efc7ec3c47b8177"}, ] appdirs = [ - {file = "appdirs-1.4.3-py2.py3-none-any.whl", hash = "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e"}, - {file = "appdirs-1.4.3.tar.gz", hash = "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92"}, + {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, + {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, ] async-timeout = [ {file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"}, @@ -1058,6 +1130,10 @@ botocore = [ {file = "botocore-1.15.32-py2.py3-none-any.whl", hash = "sha256:a963af564d94107787ff3d2c534e8b7aed7f12e014cdd609f8fcb17bf9d9b19a"}, {file = "botocore-1.15.32.tar.gz", hash = "sha256:3ea89601ee452b65084005278bd832be854cfde5166685dcb14b6c8f19d3fc6d"}, ] +certifi = [ + {file = "certifi-2020.4.5.1-py2.py3-none-any.whl", hash = "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304"}, + {file = "certifi-2020.4.5.1.tar.gz", hash = "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519"}, +] cfgv = [ {file = "cfgv-3.0.0-py2.py3-none-any.whl", hash = "sha256:f22b426ed59cd2ab2b54ff96608d846c33dfb8766a67f0b4a6ce130ce244414f"}, {file = "cfgv-3.0.0.tar.gz", hash = "sha256:04b093b14ddf9fd4d17c53ebfd55582d27b76ed30050193c14e560770c5360eb"}, @@ -1071,8 +1147,8 @@ click = [ {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] colorama = [ - {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, - {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"}, + {file = "colorama-0.4.1-py2.py3-none-any.whl", hash = "sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48"}, + {file = "colorama-0.4.1.tar.gz", hash = "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d"}, ] coverage = [ {file = "coverage-5.1-cp27-cp27m-macosx_10_12_x86_64.whl", hash = "sha256:0cb4be7e784dcdc050fc58ef05b71aa8e89b7e6636b99967fadbdba694cf2b65"}, @@ -1115,10 +1191,6 @@ docutils = [ {file = "docutils-0.15.2-py3-none-any.whl", hash = "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0"}, {file = "docutils-0.15.2.tar.gz", hash = "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99"}, ] -entrypoints = [ - {file = "entrypoints-0.3-py2.py3-none-any.whl", hash = "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19"}, - {file = "entrypoints-0.3.tar.gz", hash = "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"}, -] eradicate = [ {file = "eradicate-1.0.tar.gz", hash = "sha256:4ffda82aae6fd49dfffa777a857cb758d77502a1f2e0f54c9ac5155a39d2d01a"}, ] @@ -1131,8 +1203,8 @@ filelock = [ {file = "filelock-3.0.12.tar.gz", hash = "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59"}, ] flake8 = [ - {file = "flake8-3.7.9-py2.py3-none-any.whl", hash = "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca"}, - {file = "flake8-3.7.9.tar.gz", hash = "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb"}, + {file = "flake8-3.8.1-py2.py3-none-any.whl", hash = "sha256:6c1193b0c3f853ef763969238f6c81e9e63ace9d024518edc020d5f1d6d93195"}, + {file = "flake8-3.8.1.tar.gz", hash = "sha256:ea6623797bf9a52f4c9577d780da0bb17d65f870213f7b5bcc9fca82540c31d5"}, ] flake8-black = [ {file = "flake8-black-0.1.1.tar.gz", hash = "sha256:56f85aaa5a83f06a3f61e680e3b50f156b5e557ebdcb964d823d86f4c108b0c8"}, @@ -1153,8 +1225,8 @@ flake8-debugger = [ {file = "flake8-debugger-3.2.1.tar.gz", hash = "sha256:712d7c1ff69ddf3f0130e94cc88c2519e720760bce45e8c330bfdcb61ab4090d"}, ] flake8-eradicate = [ - {file = "flake8-eradicate-0.2.4.tar.gz", hash = "sha256:b693e9dfe6da42dbc7fb75af8486495b9414d1ab0372d15efcf85a2ac85fd368"}, - {file = "flake8_eradicate-0.2.4-py3-none-any.whl", hash = "sha256:b0bcdbb70a489fb799f9ee11fefc57bd0d3251e1ea9bdc5bf454443cccfd620c"}, + {file = "flake8-eradicate-0.3.0.tar.gz", hash = "sha256:d0b3d283d85079917acbfe39b9d637385cd82cba3ae3d76c1278c07ddcf0d9b9"}, + {file = "flake8_eradicate-0.3.0-py3-none-any.whl", hash = "sha256:e8b32b32300bfb407fe7ef74667c8d2d3a6a81bdf6f09c14a7bcc82b7b870f8b"}, ] flake8-fixme = [ {file = "flake8-fixme-1.1.1.tar.gz", hash = "sha256:50cade07d27a4c30d4f12351478df87339e67640c83041b664724bda6d16f33a"}, @@ -1164,6 +1236,10 @@ flake8-isort = [ {file = "flake8-isort-2.9.1.tar.gz", hash = "sha256:0d34b266080e1748412b203a1690792245011706b1858c203476b43460bf3652"}, {file = "flake8_isort-2.9.1-py2.py3-none-any.whl", hash = "sha256:a77df28778a1ac6ac4153339ebd9d252935f3ed4379872d4f8b84986296d8cc3"}, ] +flake8-polyfill = [ + {file = "flake8-polyfill-1.0.2.tar.gz", hash = "sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda"}, + {file = "flake8_polyfill-1.0.2-py2.py3-none-any.whl", hash = "sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9"}, +] flake8-variables-names = [ {file = "flake8_variables_names-0.0.3.tar.gz", hash = "sha256:d109f5a8fe8c20d64e165287330f1b0160b442d7f96e1527124ba1b63c438347"}, ] @@ -1213,6 +1289,10 @@ mako = [ {file = "Mako-1.1.2-py2.py3-none-any.whl", hash = "sha256:8e8b53c71c7e59f3de716b6832c4e401d903af574f6962edbbbf6ecc2a5fe6c9"}, {file = "Mako-1.1.2.tar.gz", hash = "sha256:3139c5d64aa5d175dbafb95027057128b5fbd05a40c53999f3905ceb53366d9d"}, ] +mando = [ + {file = "mando-0.6.4-py2.py3-none-any.whl", hash = "sha256:4ce09faec7e5192ffc3c57830e26acba0fd6cd11e1ee81af0d4df0657463bd1c"}, + {file = "mando-0.6.4.tar.gz", hash = "sha256:79feb19dc0f097daa64a1243db578e7674909b75f88ac2220f1c065c10a0d960"}, +] markdown = [ {file = "Markdown-3.2.2-py3-none-any.whl", hash = "sha256:c467cd6233885534bf0fe96e62e3cf46cfc1605112356c4f9981512b8174de59"}, {file = "Markdown-3.2.2.tar.gz", hash = "sha256:1fafe3f1ecabfb514a5285fca634a53c1b32a81cb0feb154264d55bf2ff22c17"}, @@ -1310,12 +1390,12 @@ py = [ {file = "py-1.8.1.tar.gz", hash = "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa"}, ] pycodestyle = [ - {file = "pycodestyle-2.5.0-py2.py3-none-any.whl", hash = "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56"}, - {file = "pycodestyle-2.5.0.tar.gz", hash = "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"}, + {file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"}, + {file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"}, ] pyflakes = [ - {file = "pyflakes-2.1.1-py2.py3-none-any.whl", hash = "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0"}, - {file = "pyflakes-2.1.1.tar.gz", hash = "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"}, + {file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"}, + {file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"}, ] pyparsing = [ {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, @@ -1353,6 +1433,10 @@ pyyaml = [ {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] +radon = [ + {file = "radon-4.1.0-py2.py3-none-any.whl", hash = "sha256:0c18111ec6cfe7f664bf9db6c51586714ac8c6d9741542706df8a85aca39b99a"}, + {file = "radon-4.1.0.tar.gz", hash = "sha256:56082c52206db45027d4a73612e1b21663c4cc2be3760fee769d966fd7efdd6d"}, +] regex = [ {file = "regex-2020.5.7-cp27-cp27m-win32.whl", hash = "sha256:5493a02c1882d2acaaf17be81a3b65408ff541c922bfd002535c5f148aa29f74"}, {file = "regex-2020.5.7-cp27-cp27m-win_amd64.whl", hash = "sha256:021a0ae4d2baeeb60a3014805a2096cb329bd6d9f30669b7ad0da51a9cb73349"}, @@ -1376,6 +1460,10 @@ regex = [ {file = "regex-2020.5.7-cp38-cp38-win_amd64.whl", hash = "sha256:3ab5e41c4ed7cd4fa426c50add2892eb0f04ae4e73162155cd668257d02259dd"}, {file = "regex-2020.5.7.tar.gz", hash = "sha256:73a10404867b835f1b8a64253e4621908f0d71150eb4e97ab2e7e441b53e9451"}, ] +requests = [ + {file = "requests-2.23.0-py2.py3-none-any.whl", hash = "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"}, + {file = "requests-2.23.0.tar.gz", hash = "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"}, +] s3transfer = [ {file = "s3transfer-0.3.3-py2.py3-none-any.whl", hash = "sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13"}, {file = "s3transfer-0.3.3.tar.gz", hash = "sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db"}, @@ -1444,6 +1532,10 @@ wcwidth = [ wrapt = [ {file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"}, ] +xenon = [ + {file = "xenon-0.7.0-py2.py3-none-any.whl", hash = "sha256:83e98f67b7077c95c25c3402aea6203dd2ed6256708b76ed9751e9dbf1aba125"}, + {file = "xenon-0.7.0.tar.gz", hash = "sha256:5e6433c9297d965bf666256a0a030b6e13660ab87680220c4eb07241f101625b"}, +] yarl = [ {file = "yarl-1.4.2-cp35-cp35m-macosx_10_13_x86_64.whl", hash = "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b"}, {file = "yarl-1.4.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1"}, diff --git a/python/pyproject.toml b/python/pyproject.toml index 8356bf4cd5..cbe9b47640 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -28,11 +28,9 @@ pytest = "^5.2" black = "^19.10b0" flake8 = "^3.7.9" flake8-black = "^0.1.1" -flake8-bugbear = "^20.1.4" flake8-builtins = "^1.4.2" flake8-comprehensions = "^3.2.2" flake8-debugger = "^3.2.1" -flake8-eradicate = "^0.2.4" flake8-fixme = "^1.1.1" flake8-isort = "^2.8.0" flake8-variables-names = "^0.0.3" @@ -45,6 +43,10 @@ pytest-asyncio = "^0.12.0" aioboto3 = "^8.0.3" aiohttp = "^3.6.2" bandit = "^1.6.2" +radon = "^4.1.0" +xenon = "^0.7.0" +flake8-bugbear = "^20.1.4" +flake8-eradicate = "^0.3.0" [tool.coverage.run] source = ["aws_lambda_powertools"] From 238a401e1f25b9ecd1da26d78cb4d776599a53bd Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 12 May 2020 13:16:43 +0100 Subject: [PATCH 34/35] chore: bump version to 0.9.0 --- python/HISTORY.md | 7 +++++++ python/pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/python/HISTORY.md b/python/HISTORY.md index 8001c9bba2..2f64eec56f 100644 --- a/python/HISTORY.md +++ b/python/HISTORY.md @@ -1,5 +1,12 @@ # HISTORY +## May 12th + +* **Tracer**: Support for async functions in `Tracer` via `capture_method` decorator +* **Tracer**: Support for `aiohttp` via `aiohttp_trace_config` trace config +* **Tracer**: Support for patching specific modules via `patch_modules` param +* **Tracer**: Document escape hatch mechanisms via `tracer.provider` + ## April 24th * Introduces `Logger` for stuctured logging as a replacement for `logger_setup` diff --git a/python/pyproject.toml b/python/pyproject.toml index cbe9b47640..95845cc9f9 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "aws_lambda_powertools" -version = "0.8.0" +version = "0.9.0" description = "Python utilities for AWS Lambda functions including but not limited to tracing, logging and custom metric" authors = ["Amazon Web Services"] classifiers=[ From 8fa07e6b3b5843db214a8478647b8905609f66d9 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 12 May 2020 13:20:24 +0100 Subject: [PATCH 35/35] chore: clean up history changes --- python/HISTORY.md | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/python/HISTORY.md b/python/HISTORY.md index 2f64eec56f..bce7ac6268 100644 --- a/python/HISTORY.md +++ b/python/HISTORY.md @@ -2,42 +2,52 @@ ## May 12th +**0.9.0** + * **Tracer**: Support for async functions in `Tracer` via `capture_method` decorator * **Tracer**: Support for `aiohttp` via `aiohttp_trace_config` trace config * **Tracer**: Support for patching specific modules via `patch_modules` param * **Tracer**: Document escape hatch mechanisms via `tracer.provider` +## May 1st + +**0.8.1** + +* **Metrics**: Fix incorrect metric units enum values for `*PerSecond` e.g. CountPerSecond + ## April 24th -* Introduces `Logger` for stuctured logging as a replacement for `logger_setup` -* Introduces `Logger.inject_lambda_context` decorator as a replacement for `logger_inject_lambda_context` -* Raise `DeprecationWarning` exception for both `logger_setup`, `logger_inject_lambda_context` +**0.8.0** + +* **Logger**: Introduces `Logger` class for stuctured logging as a replacement for `logger_setup` +* **Logger**: Introduces `Logger.inject_lambda_context` decorator as a replacement for `logger_inject_lambda_context` +* **Logger**: Raise `DeprecationWarning` exception for both `logger_setup`, `logger_inject_lambda_context` ## April 20th, 2020 **0.7.0** -* Introduces Middleware Factory to build your own middleware -* Fixes Metrics dimensions not being included correctly in EMF +* **Middleware factory**: Introduces Middleware Factory to build your own middleware via `lambda_handler_decorator` +* **Metrics**: Fixes metrics dimensions not being included correctly in EMF ## April 9th, 2020 **0.6.3** -* Fix `log_metrics` decorator logic not calling the decorated function, and exception handling +* **Logger**: Fix `log_metrics` decorator logic not calling the decorated function, and exception handling ## April 8th, 2020 **0.6.1** -* Introduces Metrics middleware to utilise CloudWatch Embedded Metric Format -* Adds deprecation warning for `log_metrics` +* **Metrics**: Introduces Metrics middleware to utilise CloudWatch Embedded Metric Format +* **Metrics**: Adds deprecation warning for `log_metrics` ## February 20th, 2020 **0.5.0** -* Introduces log sampling for debug - Thanks to [Danilo's contribution](https://github.com/awslabs/aws-lambda-powertools/pull/7) +* **Logger**: Introduces log sampling for debug - Thanks to [Danilo's contribution](https://github.com/awslabs/aws-lambda-powertools/pull/7) ## November 15th, 2019