Feat: Lambda advanced logging (#6282)

* Add support for Custom Log Group (#532) * Add support for custom Log group in samcli * Added more unit test * Updated pr to use f string * SAM INIT support for Advanced Logging (#556) * SAM Init support for Advanced Logging * feature links added * SAM Init support for Advanced Logging * change class named from LoggingConfigTemplateModifier to LoggingConfigTemplateModifier * update file name to match structured_logging_template_modifier * Sam local Invoke support for Advanced logging (#552) * Sam local support for Advanced logging * Sam local support for Advanced logging * pass AWS_LAMBDA_LOG_FORMAT to runtime * remove unnecessary if statement * Update test_init_command.py --------- Co-authored-by: jonife <[email protected]>
aws · Nov 16, 2023 · 2a527fd · 2a527fd
1 parent 366b3b7
commit 2a527fd
Show file tree

Hide file tree

Showing 19 changed files with 478 additions and 21 deletions.
diff --git a/samcli/commands/init/command.py b/samcli/commands/init/command.py
@@ -226,6 +226,11 @@ def wrapped(*args, **kwargs):
     default=None,
     help="Enable CloudWatch Application Insights monitoring for application.",
 )
+@click.option(
+    "--structured-logging/--no-structured-logging",
+    default=None,
+    help="Enable Structured Logging for application.",
+)
 @common_options
 @save_params_option
 @non_interactive_validation
@@ -249,6 +254,7 @@ def cli(
     extra_context,
     tracing,
     application_insights,
+    structured_logging,
     save_params,
     config_file,
     config_env,
@@ -273,6 +279,7 @@ def cli(
         extra_context,
         tracing,
         application_insights,
+        structured_logging,
     )  # pragma: no cover
 
 
@@ -294,6 +301,7 @@ def do_cli(
     extra_context,
     tracing,
     application_insights,
+    structured_logging,
 ):
     """
     Implementation of the ``cli`` method
@@ -346,6 +354,7 @@ def do_cli(
             extra_context,
             tracing,
             application_insights,
+            structured_logging,
         )
     else:
         if not (pt_explicit or runtime or dependency_manager or base_image or architecture):
@@ -366,6 +375,7 @@ def do_cli(
             no_input,
             tracing,
             application_insights,
+            structured_logging,
         )
 
 

diff --git a/samcli/commands/init/core/options.py b/samcli/commands/init/core/options.py
@@ -25,10 +25,7 @@
 
 CONFIGURATION_OPTION_NAMES: List[str] = ["config_env", "config_file"] + SAVE_PARAMS_OPTIONS
 
-ADDITIONAL_OPTIONS: List[str] = [
-    "tracing",
-    "application_insights",
-]
+ADDITIONAL_OPTIONS: List[str] = ["tracing", "application_insights", "structured_logging"]
 
 ALL_OPTIONS: List[str] = (
     APPLICATION_OPTIONS + NON_INTERACTIVE_OPTIONS + CONFIGURATION_OPTION_NAMES + ADDITIONAL_OPTIONS + ALL_COMMON_OPTIONS

diff --git a/samcli/commands/init/init_generator.py b/samcli/commands/init/init_generator.py
@@ -18,6 +18,7 @@ def do_generate(
     extra_context,
     tracing,
     application_insights,
+    structured_logging,
 ):
     try:
         generate_project(
@@ -31,6 +32,7 @@ def do_generate(
             extra_context,
             tracing,
             application_insights,
+            structured_logging,
         )
     except InitErrorException as e:
         raise UserException(str(e), wrapped_from=e.__class__.__name__) from e
diff --git a/samcli/commands/init/interactive_init_flow.py b/samcli/commands/init/interactive_init_flow.py
@@ -53,6 +53,7 @@ def do_interactive(
     no_input,
     tracing,
     application_insights,
+    structured_logging,
 ):
     """
     Implementation of the ``cli`` method when --interactive is provided.
@@ -80,6 +81,7 @@ def do_interactive(
         location_opt_choice,
         tracing,
         application_insights,
+        structured_logging,
     )
 
 
@@ -98,6 +100,7 @@ def generate_application(
     location_opt_choice,
     tracing,
     application_insights,
+    structured_logging,
 ):  # pylint: disable=too-many-arguments
     """
     The method holds the decision logic for generating an application
@@ -132,6 +135,8 @@ def generate_application(
         boolen value to determine if X-Ray tracing show be activated or not
     application_insights : bool
         boolean value to determine if AppInsights monitoring should be enabled or not
+    structured_logging: bool
+        boolean value to determine if Json structured logging should be enabled or not
     """
     if location_opt_choice == "1":
         _generate_from_use_case(
@@ -147,6 +152,7 @@ def generate_application(
             architecture,
             tracing,
             application_insights,
+            structured_logging,
         )
 
     else:
@@ -160,12 +166,22 @@ def generate_application(
             no_input,
             tracing,
             application_insights,
+            structured_logging,
         )
 
 
 # pylint: disable=too-many-statements
 def _generate_from_location(
-    location, package_type, runtime, dependency_manager, output_dir, name, no_input, tracing, application_insights
+    location,
+    package_type,
+    runtime,
+    dependency_manager,
+    output_dir,
+    name,
+    no_input,
+    tracing,
+    application_insights,
+    structured_logging,
 ):
     location = click.prompt("\nTemplate location (git, mercurial, http(s), zip, path)", type=str)
     summary_msg = """
@@ -189,6 +205,7 @@ def _generate_from_location(
         None,
         tracing,
         application_insights,
+        structured_logging,
     )
 
 
@@ -206,6 +223,7 @@ def _generate_from_use_case(
     architecture: Optional[str],
     tracing: Optional[bool],
     application_insights: Optional[bool],
+    structured_logging: Optional[bool],
 ) -> None:
     templates = InitTemplates()
     runtime_or_base_image = runtime if runtime else base_image
@@ -236,6 +254,9 @@ def _generate_from_use_case(
     if application_insights is None:
         application_insights = prompt_user_to_enable_application_insights()
 
+    if structured_logging is None:
+        structured_logging = prompt_user_to_enable_structured_logging()
+
     app_template = template_chosen["appTemplate"]
     base_image = (
         LAMBDA_IMAGES_RUNTIMES_MAP.get(str(runtime)) if not base_image and package_type == IMAGE else base_image
@@ -292,6 +313,7 @@ def _generate_from_use_case(
         extra_context,
         tracing,
         application_insights,
+        structured_logging,
     )
     # executing event_bridge logic if call is for Schema dynamic template
     if is_dynamic_schemas_template:
@@ -426,6 +448,23 @@ def prompt_user_to_enable_application_insights():
     return False
 
 
+def prompt_user_to_enable_structured_logging():
+    """
+    Prompt user to choose if structured loggingConfig should activated
+    for their functions in the SAM template and vice versa
+    """
+    if click.confirm("\nWould you like to set Structured Logging in JSON format on your Lambda functions? "):
+        doc_link = (
+            "https://docs.aws.amazon.com/lambda/latest/dg/"
+            "monitoring-cloudwatchlogs.html#monitoring-cloudwatchlogs-pricing"
+        )
+        click.echo(
+            f"Structured Logging in JSON format might incur an additional cost. View {doc_link} for more details"
+        )
+        return True
+    return False
+
+
 def _get_choice_from_options(chosen, options, question, msg):
     if chosen:
         return chosen

diff --git a/samcli/commands/local/lib/local_lambda.py b/samcli/commands/local/lib/local_lambda.py
@@ -296,6 +296,7 @@ def _make_env_vars(self, function: Function) -> EnvironmentVariables:
             function.memory,
             function.timeout,
             function.handler,
+            function.logging_config,
             variables=variables,
             shell_env_values=shell_env,
             override_values=overrides,

diff --git a/samcli/lib/init/__init__.py b/samcli/lib/init/__init__.py
@@ -15,6 +15,7 @@
 from samcli.lib.init.template_modifiers.application_insights_template_modifier import (
     ApplicationInsightsTemplateModifier,
 )
+from samcli.lib.init.template_modifiers.structured_logging_template_modifier import StructuredLoggingTemplateModifier
 from samcli.lib.init.template_modifiers.xray_tracing_template_modifier import XRayTracingTemplateModifier
 from samcli.lib.telemetry.event import EventName, EventTracker, UsedFeature
 from samcli.lib.utils import osutils
@@ -38,6 +39,7 @@ def generate_project(
     extra_context=None,
     tracing=False,
     application_insights=False,
+    structured_logging=False,
 ):
     """Generates project using cookiecutter and options given
 
@@ -70,6 +72,8 @@ def generate_project(
         Enable or disable X-Ray Tracing
     application_insights: Optional[str]
             Enable or disable AppInsights Monitoring
+    structured_logging: Optional[bool]
+        boolean value to determine if Json structured logging should be enabled or not
 
     Raises
     ------
@@ -132,6 +136,8 @@ def generate_project(
 
     _enable_application_insights(application_insights, output_dir, name)
 
+    _enable_structured_logging(structured_logging, output_dir, name)
+
     _create_default_samconfig(package_type, output_dir, name)
 
 
@@ -150,6 +156,13 @@ def _enable_application_insights(application_insights: bool, output_dir: str, na
         EventTracker.track_event(EventName.USED_FEATURE.value, UsedFeature.INIT_WITH_APPLICATION_INSIGHTS.value)
 
 
+def _enable_structured_logging(structured_logging, output_dir, name):
+    if structured_logging:
+        template_file_path = f"{output_dir}/{name}/template.yaml"
+        template_modifier = StructuredLoggingTemplateModifier(template_file_path)
+        template_modifier.modify_template()
+
+
 def _create_default_samconfig(package_type: str, output_dir: str, name: str) -> None:
     """
     Init post-processing function to create a default samconfig.toml

diff --git a/samcli/lib/init/template_modifiers/structured_logging_template_modifier.py b/samcli/lib/init/template_modifiers/structured_logging_template_modifier.py
@@ -0,0 +1,98 @@
+"""
+Class used to parse and update template when structured logging is enabled
+"""
+import logging
+from typing import Any
+
+from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap
+from ruamel.yaml.representer import RoundTripRepresenter
+
+from samcli.lib.init.template_modifiers.cli_template_modifier import TemplateModifier
+
+LOG = logging.getLogger(__name__)
+
+
+class StructuredLoggingTemplateModifier(TemplateModifier):
+    GLOBALS = "Globals"
+    RESOURCE = "Resources"
+    FUNCTION = "Function"
+    LOGGING_CONFIG = "LoggingConfig"
+    JSON_LOGFORMAT = {"LogFormat": "JSON"}
+    DOC_LINK = (
+        "https://docs.aws.amazon.com/serverless-application-model/latest/"
+        "developerguide/sam-resource-function.html#sam-function-loggingconfig"
+    )
+
+    COMMENT = (
+        "More info about Globals: "
+        "https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst\n"
+    )
+
+    MESSAGE = (
+        "You can add LoggingConfig parameters such as the Logformat, "
+        "Log Group, and SystemLogLevel or ApplicationLogLevel. "
+        f"Learn more here {DOC_LINK}.\n"
+    )
+
+    # set ignore aliases to true. This configuration avoids usage yaml aliases which is not parsed by CloudFormation.
+    class NonAliasingRTRepresenter(RoundTripRepresenter):
+        def ignore_aliases(self, data):
+            return True
+
+    def __init__(self, location):
+        self.yaml = YAML()
+        self.yaml.Representer = StructuredLoggingTemplateModifier.NonAliasingRTRepresenter
+        super().__init__(location)
+
+    def _get_template(self) -> Any:
+        with open(self.template_location) as file:
+            return self.yaml.load(file)
+
+    def _update_template_fields(self):
+        """
+        Add new field to SAM template
+        """
+        if self.template.get(self.GLOBALS):
+            template_globals = self.template.get(self.GLOBALS)
+
+            function_globals = template_globals.get(self.FUNCTION, {})
+            if not function_globals:
+                template_globals[self.FUNCTION] = {}
+            template_globals[self.FUNCTION][self.LOGGING_CONFIG] = CommentedMap(self.JSON_LOGFORMAT)
+            template_globals[self.FUNCTION].yaml_set_comment_before_after_key(
+                self.LOGGING_CONFIG, before=self.MESSAGE, indent=4
+            )
+
+        else:
+            self._add_logging_config_with_globals()
+
+    def _add_logging_config_with_globals(self):
+        """Adds Globals and LoggingConfig fields"""
+        global_section = {
+            self.FUNCTION: {self.LOGGING_CONFIG: self.JSON_LOGFORMAT},
+        }
+
+        self.template = CommentedMap(self.template)
+        self.template[self.GLOBALS] = CommentedMap(global_section)
+        self.template[self.GLOBALS].yaml_set_comment_before_after_key(self.LOGGING_CONFIG, before=self.MESSAGE)
+        self.template.yaml_set_comment_before_after_key(self.GLOBALS, before=self.COMMENT)
+
+    def _print_sanity_check_error(self):
+        message = (
+            "Warning: Unable to add LoggingConfig to the project. "
+            "To learn more about LoggingConfig visit {self.DOC_LINK}"
+        )
+        LOG.warning(message)
+
+    def _write(self, template: list):
+        """
+        write generated template into SAM template
+
+        Parameters
+        ----------
+        template : list
+            array with updated template data
+        """
+        with open(self.template_location, "w") as file:
+            self.yaml.dump(self.template, file)