From fca42eb3ad6b1e5350631bfb52c37123eb69f07c Mon Sep 17 00:00:00 2001 From: Georgi Date: Mon, 26 Feb 2024 11:55:33 +0100 Subject: [PATCH 1/8] [AWSX-678] Step1: restructure files in the lambda forwarder --- .../enhanced_lambda_metrics.py | 2 - aws/logs_monitoring/enrichment.py | 205 ++++++++++ aws/logs_monitoring/exceptions.py | 6 + aws/logs_monitoring/forwarder.py | 119 ++++++ aws/logs_monitoring/lambda_function.py | 368 +----------------- aws/logs_monitoring/logs.py | 132 +------ aws/logs_monitoring/logs_helpers.py | 71 ++++ aws/logs_monitoring/parsing.py | 164 -------- aws/logs_monitoring/splitting.py | 74 ++++ .../tests/test_cloudtrail_s3.py | 4 +- aws/logs_monitoring/tests/test_enrichment.py | 142 +++++++ .../tests/test_lambda_function.py | 235 ++--------- aws/logs_monitoring/tests/test_logs.py | 7 +- aws/logs_monitoring/tests/test_parsing.py | 217 ----------- aws/logs_monitoring/tests/test_splitting.py | 55 +++ .../tests/test_transformation.py | 225 +++++++++++ aws/logs_monitoring/transformation.py | 188 +++++++++ 17 files changed, 1141 insertions(+), 1073 deletions(-) create mode 100644 aws/logs_monitoring/enrichment.py create mode 100644 aws/logs_monitoring/exceptions.py create mode 100644 aws/logs_monitoring/forwarder.py create mode 100644 aws/logs_monitoring/logs_helpers.py create mode 100644 aws/logs_monitoring/splitting.py create mode 100644 aws/logs_monitoring/tests/test_enrichment.py create mode 100644 aws/logs_monitoring/tests/test_splitting.py create mode 100644 aws/logs_monitoring/tests/test_transformation.py create mode 100644 aws/logs_monitoring/transformation.py diff --git a/aws/logs_monitoring/enhanced_lambda_metrics.py b/aws/logs_monitoring/enhanced_lambda_metrics.py index aa2ec4d2a..2521d074c 100644 --- a/aws/logs_monitoring/enhanced_lambda_metrics.py +++ b/aws/logs_monitoring/enhanced_lambda_metrics.py @@ -6,9 +6,7 @@ import logging import re import datetime - from time import time - from lambda_cache import LambdaTagsCache ENHANCED_METRICS_NAMESPACE_PREFIX = "aws.lambda.enhanced" diff --git a/aws/logs_monitoring/enrichment.py b/aws/logs_monitoring/enrichment.py new file mode 100644 index 000000000..f3f1bee97 --- /dev/null +++ b/aws/logs_monitoring/enrichment.py @@ -0,0 +1,205 @@ +import logging +import json +import os +import re +from settings import ( + DD_SOURCE, + DD_SERVICE, + DD_HOST, + DD_CUSTOM_TAGS, +) +from enhanced_lambda_metrics import get_enriched_lambda_log_tags + +HOST_IDENTITY_REGEXP = re.compile( + r"^arn:aws:sts::.*?:assumed-role\/(?P.*?)/(?Pi-([0-9a-f]{8}|[0-9a-f]{17}))$" +) + +logger = logging.getLogger() +logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper())) + + +def enrich(events): + """Adds event-specific tags and attributes to each event + + Args: + events (dict[]): the list of event dicts we want to enrich + """ + for event in events: + add_metadata_to_lambda_log(event) + extract_ddtags_from_message(event) + extract_host_from_cloudtrails(event) + extract_host_from_guardduty(event) + extract_host_from_route53(event) + + return events + + +def add_metadata_to_lambda_log(event): + """Mutate log dict to add tags, host, and service metadata + + * tags for functionname, aws_account, region + * host from the Lambda ARN + * service from the Lambda name + + If the event arg is not a Lambda log then this returns without doing anything + + Args: + event (dict): the event we are adding Lambda metadata to + """ + lambda_log_metadata = event.get("lambda", {}) + lambda_log_arn = lambda_log_metadata.get("arn") + + # Do not mutate the event if it's not from Lambda + if not lambda_log_arn: + return + + # Set Lambda ARN to "host" + event[DD_HOST] = lambda_log_arn + + # Function name is the seventh piece of the ARN + function_name = lambda_log_arn.split(":")[6] + tags = [f"functionname:{function_name}"] + + # Get custom tags of the Lambda function + custom_lambda_tags = get_enriched_lambda_log_tags(event) + + # If not set during parsing or has a default value + # then set the service tag from lambda tags cache or using the function name + # otherwise, remove the service tag from the custom lambda tags if exists to avoid duplication + if not event[DD_SERVICE] or event[DD_SERVICE] == event[DD_SOURCE]: + service_tag = next( + (tag for tag in custom_lambda_tags if tag.startswith("service:")), + f"service:{function_name}", + ) + if service_tag: + tags.append(service_tag) + event[DD_SERVICE] = service_tag.split(":")[1] + else: + custom_lambda_tags = [ + tag for tag in custom_lambda_tags if not tag.startswith("service:") + ] + + # Check if one of the Lambda's custom tags is env + # If an env tag exists, remove the env:none placeholder + custom_env_tag = next( + (tag for tag in custom_lambda_tags if tag.startswith("env:")), None + ) + if custom_env_tag is not None: + event[DD_CUSTOM_TAGS] = event[DD_CUSTOM_TAGS].replace("env:none", "") + + tags += custom_lambda_tags + + # Dedup tags, so we don't end up with functionname twice + tags = list(set(tags)) + tags.sort() # Keep order deterministic + + event[DD_CUSTOM_TAGS] = ",".join([event[DD_CUSTOM_TAGS]] + tags) + + +def extract_ddtags_from_message(event): + """When the logs intake pipeline detects a `message` field with a + JSON content, it extracts the content to the top-level. The fields + of same name from the top-level will be overridden. + + E.g. the application adds some tags to the log, which appear in the + `message.ddtags` field, and the forwarder adds some common tags, such + as `aws_account`, which appear in the top-level `ddtags` field: + + { + "message": { + "ddtags": "mytag:value", # tags added by the application + ... + }, + "ddtags": "env:xxx,aws_account", # tags added by the forwarder + ... + } + + Only the custom tags added by the application will be kept. + + We might want to change the intake pipeline to "merge" the conflicting + fields rather than "overridding" in the future, but for now we should + extract `message.ddtags` and merge it with the top-level `ddtags` field. + """ + if "message" in event and DD_CUSTOM_TAGS in event["message"]: + if isinstance(event["message"], dict): + extracted_ddtags = event["message"].pop(DD_CUSTOM_TAGS) + if isinstance(event["message"], str): + try: + message_dict = json.loads(event["message"]) + extracted_ddtags = message_dict.pop(DD_CUSTOM_TAGS) + event["message"] = json.dumps(message_dict) + except Exception: + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Failed to extract ddtags from: {event}") + return + + # Extract service tag from message.ddtags if exists + if "service" in extracted_ddtags: + event[DD_SERVICE] = next( + tag[8:] + for tag in extracted_ddtags.split(",") + if tag.startswith("service:") + ) + event[DD_CUSTOM_TAGS] = ",".join( + [ + tag + for tag in event[DD_CUSTOM_TAGS].split(",") + if not tag.startswith("service") + ] + ) + + event[DD_CUSTOM_TAGS] = f"{event[DD_CUSTOM_TAGS]},{extracted_ddtags}" + + +def extract_host_from_cloudtrails(event): + """Extract the hostname from cloudtrail events userIdentity.arn field if it + matches AWS hostnames. + + In case of s3 events the fields of the event are not encoded in the + "message" field, but in the event object itself. + """ + + if event is not None and event.get(DD_SOURCE) == "cloudtrail": + message = event.get("message", {}) + if isinstance(message, str): + try: + message = json.loads(message) + except json.JSONDecodeError: + logger.debug("Failed to decode cloudtrail message") + return + + # deal with s3 input type events + if not message: + message = event + + if isinstance(message, dict): + arn = message.get("userIdentity", {}).get("arn") + if arn is not None: + match = HOST_IDENTITY_REGEXP.match(arn) + if match is not None: + event[DD_HOST] = match.group("host") + + +def extract_host_from_guardduty(event): + if event is not None and event.get(DD_SOURCE) == "guardduty": + host = event.get("detail", {}).get("resource") + if isinstance(host, dict): + host = host.get("instanceDetails", {}).get("instanceId") + if host is not None: + event[DD_HOST] = host + + +def extract_host_from_route53(event): + if event is not None and event.get(DD_SOURCE) == "route53": + message = event.get("message", {}) + if isinstance(message, str): + try: + message = json.loads(message) + except json.JSONDecodeError: + logger.debug("Failed to decode Route53 message") + return + + if isinstance(message, dict): + host = message.get("srcids", {}).get("instance") + if host is not None: + event[DD_HOST] = host diff --git a/aws/logs_monitoring/exceptions.py b/aws/logs_monitoring/exceptions.py new file mode 100644 index 000000000..b5586a500 --- /dev/null +++ b/aws/logs_monitoring/exceptions.py @@ -0,0 +1,6 @@ +class RetriableException(Exception): + pass + + +class ScrubbingException(Exception): + pass diff --git a/aws/logs_monitoring/forwarder.py b/aws/logs_monitoring/forwarder.py new file mode 100644 index 000000000..f4f5ab681 --- /dev/null +++ b/aws/logs_monitoring/forwarder.py @@ -0,0 +1,119 @@ +import logging +import json +import os + +from telemetry import ( + DD_FORWARDER_TELEMETRY_NAMESPACE_PREFIX, + get_forwarder_telemetry_tags, +) +from datadog_lambda.metric import lambda_stats +from trace_forwarder.connection import TraceConnection +from logs import ( + DatadogScrubber, + DatadogBatcher, + DatadogClient, + DatadogHTTPClient, + DatadogTCPClient, +) +from logs_helpers import filter_logs +from settings import ( + DD_API_KEY, + DD_USE_TCP, + DD_NO_SSL, + DD_SKIP_SSL_VALIDATION, + DD_URL, + DD_PORT, + SCRUBBING_RULE_CONFIGS, + INCLUDE_AT_MATCH, + EXCLUDE_AT_MATCH, + DD_TRACE_INTAKE_URL, +) + +logger = logging.getLogger() +logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper())) +trace_connection = TraceConnection( + DD_TRACE_INTAKE_URL, DD_API_KEY, DD_SKIP_SSL_VALIDATION +) + + +def forward_logs(logs): + """Forward logs to Datadog""" + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Forwarding {len(logs)} logs") + logs_to_forward = filter_logs( + [json.dumps(log, ensure_ascii=False) for log in logs], + include_pattern=INCLUDE_AT_MATCH, + exclude_pattern=EXCLUDE_AT_MATCH, + ) + scrubber = DatadogScrubber(SCRUBBING_RULE_CONFIGS) + if DD_USE_TCP: + batcher = DatadogBatcher(256 * 1000, 256 * 1000, 1) + cli = DatadogTCPClient(DD_URL, DD_PORT, DD_NO_SSL, DD_API_KEY, scrubber) + else: + batcher = DatadogBatcher(512 * 1000, 4 * 1000 * 1000, 400) + cli = DatadogHTTPClient( + DD_URL, DD_PORT, DD_NO_SSL, DD_SKIP_SSL_VALIDATION, DD_API_KEY, scrubber + ) + + with DatadogClient(cli) as client: + for batch in batcher.batch(logs_to_forward): + try: + client.send(batch) + except Exception: + logger.exception(f"Exception while forwarding log batch {batch}") + else: + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Forwarded log batch: {json.dumps(batch)}") + + lambda_stats.distribution( + "{}.logs_forwarded".format(DD_FORWARDER_TELEMETRY_NAMESPACE_PREFIX), + len(logs_to_forward), + tags=get_forwarder_telemetry_tags(), + ) + + +def forward_metrics(metrics): + """ + Forward custom metrics submitted via logs to Datadog in a background thread + using `lambda_stats` that is provided by the Datadog Python Lambda Layer. + """ + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Forwarding {len(metrics)} metrics") + + for metric in metrics: + try: + lambda_stats.distribution( + metric["m"], metric["v"], timestamp=metric["e"], tags=metric["t"] + ) + except Exception: + logger.exception(f"Exception while forwarding metric {json.dumps(metric)}") + else: + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Forwarded metric: {json.dumps(metric)}") + + lambda_stats.distribution( + "{}.metrics_forwarded".format(DD_FORWARDER_TELEMETRY_NAMESPACE_PREFIX), + len(metrics), + tags=get_forwarder_telemetry_tags(), + ) + + +def forward_traces(trace_payloads): + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Forwarding {len(trace_payloads)} traces") + + try: + trace_connection.send_traces(trace_payloads) + except Exception: + logger.exception( + f"Exception while forwarding traces {json.dumps(trace_payloads)}" + ) + else: + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Forwarded traces: {json.dumps(trace_payloads)}") + + lambda_stats.distribution( + "{}.traces_forwarded".format(DD_FORWARDER_TELEMETRY_NAMESPACE_PREFIX), + len(trace_payloads), + tags=get_forwarder_telemetry_tags(), + ) diff --git a/aws/logs_monitoring/lambda_function.py b/aws/logs_monitoring/lambda_function.py index 86df8453e..da5530a9d 100644 --- a/aws/logs_monitoring/lambda_function.py +++ b/aws/logs_monitoring/lambda_function.py @@ -6,46 +6,31 @@ import json import os import boto3 -import re import logging - -logger = logging.getLogger() -logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper())) - import requests from datadog_lambda.wrapper import datadog_lambda_wrapper -from datadog_lambda.metric import lambda_stats from datadog import api - -from trace_forwarder.connection import TraceConnection -from enhanced_lambda_metrics import ( - get_enriched_lambda_log_tags, - parse_and_submit_enhanced_metrics, -) -from logs import forward_logs -from parsing import ( - parse, - separate_security_hub_findings, - parse_aws_waf_logs, -) -from telemetry import ( - DD_FORWARDER_TELEMETRY_NAMESPACE_PREFIX, - get_forwarder_telemetry_tags, +from enhanced_lambda_metrics import parse_and_submit_enhanced_metrics +from parsing import parse +from enrichment import enrich +from transformation import transform +from splitting import split +from forwarder import ( + forward_metrics, + forward_traces, + forward_logs, ) from settings import ( DD_API_KEY, DD_FORWARD_LOG, DD_SKIP_SSL_VALIDATION, DD_API_URL, - DD_TRACE_INTAKE_URL, - DD_SOURCE, - DD_CUSTOM_TAGS, - DD_SERVICE, - DD_HOST, DD_FORWARDER_VERSION, DD_ADDITIONAL_TARGET_LAMBDAS, ) +logger = logging.getLogger() +logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper())) # DD_API_KEY must be set if DD_API_KEY == "" or DD_API_KEY == "": @@ -71,14 +56,6 @@ api._api_host = DD_API_URL api._cacert = not DD_SKIP_SSL_VALIDATION -trace_connection = TraceConnection( - DD_TRACE_INTAKE_URL, DD_API_KEY, DD_SKIP_SSL_VALIDATION -) - -HOST_IDENTITY_REGEXP = re.compile( - r"^arn:aws:sts::.*?:assumed-role\/(?P.*?)/(?Pi-([0-9a-f]{8}|[0-9a-f]{17}))$" -) - def datadog_forwarder(event, context): """The actual lambda function entry point""" @@ -102,9 +79,6 @@ def datadog_forwarder(event, context): parse_and_submit_enhanced_metrics(logs) -lambda_handler = datadog_lambda_wrapper(datadog_forwarder) - - def invoke_additional_target_lambdas(event): lambda_client = boto3.client("lambda") lambda_arns = DD_ADDITIONAL_TARGET_LAMBDAS.split(",") @@ -125,322 +99,4 @@ def invoke_additional_target_lambdas(event): return -def split(events): - """Split events into metrics, logs, and trace payloads""" - metrics, logs, trace_payloads = [], [], [] - for event in events: - metric = extract_metric(event) - trace_payload = extract_trace_payload(event) - if metric: - metrics.append(metric) - elif trace_payload: - trace_payloads.append(trace_payload) - else: - logs.append(event) - - if logger.isEnabledFor(logging.DEBUG): - logger.debug( - f"Extracted {len(metrics)} metrics, {len(trace_payloads)} traces, and {len(logs)} logs" - ) - - return metrics, logs, trace_payloads - - -def extract_metric(event): - """Extract metric from an event if possible""" - try: - metric = json.loads(event["message"]) - required_attrs = {"m", "v", "e", "t"} - if not all(attr in metric for attr in required_attrs): - return None - if not isinstance(metric["t"], list): - return None - if not (isinstance(metric["v"], int) or isinstance(metric["v"], float)): - return None - - lambda_log_metadata = event.get("lambda", {}) - lambda_log_arn = lambda_log_metadata.get("arn") - - if lambda_log_arn: - metric["t"] += [f"function_arn:{lambda_log_arn.lower()}"] - - metric["t"] += event[DD_CUSTOM_TAGS].split(",") - return metric - except Exception: - return None - - -def extract_trace_payload(event): - """Extract trace payload from an event if possible""" - try: - message = event["message"] - obj = json.loads(event["message"]) - - obj_has_traces = "traces" in obj - traces_is_a_list = isinstance(obj["traces"], list) - # check that the log is not containing a trace array unrelated to Datadog - trace_id_found = ( - len(obj["traces"]) > 0 - and len(obj["traces"][0]) > 0 - and obj["traces"][0][0]["trace_id"] is not None - ) - - if obj_has_traces and traces_is_a_list and trace_id_found: - return {"message": message, "tags": event[DD_CUSTOM_TAGS]} - return None - except Exception: - return None - - -def transform(events): - """Performs transformations on complex events - - Ex: handles special cases with nested arrays of JSON objects - Args: - events (dict[]): the list of event dicts we want to transform - """ - for event in reversed(events): - findings = separate_security_hub_findings(event) - if findings: - events.remove(event) - events.extend(findings) - - waf = parse_aws_waf_logs(event) - if waf != event: - events.remove(event) - events.append(waf) - return events - - -def enrich(events): - """Adds event-specific tags and attributes to each event - - Args: - events (dict[]): the list of event dicts we want to enrich - """ - for event in events: - add_metadata_to_lambda_log(event) - extract_ddtags_from_message(event) - extract_host_from_cloudtrails(event) - extract_host_from_guardduty(event) - extract_host_from_route53(event) - - return events - - -def add_metadata_to_lambda_log(event): - """Mutate log dict to add tags, host, and service metadata - - * tags for functionname, aws_account, region - * host from the Lambda ARN - * service from the Lambda name - - If the event arg is not a Lambda log then this returns without doing anything - - Args: - event (dict): the event we are adding Lambda metadata to - """ - lambda_log_metadata = event.get("lambda", {}) - lambda_log_arn = lambda_log_metadata.get("arn") - - # Do not mutate the event if it's not from Lambda - if not lambda_log_arn: - return - - # Set Lambda ARN to "host" - event[DD_HOST] = lambda_log_arn - - # Function name is the seventh piece of the ARN - function_name = lambda_log_arn.split(":")[6] - tags = [f"functionname:{function_name}"] - - # Get custom tags of the Lambda function - custom_lambda_tags = get_enriched_lambda_log_tags(event) - - # If not set during parsing or has a default value - # then set the service tag from lambda tags cache or using the function name - # otherwise, remove the service tag from the custom lambda tags if exists to avoid duplication - if not event[DD_SERVICE] or event[DD_SERVICE] == event[DD_SOURCE]: - service_tag = next( - (tag for tag in custom_lambda_tags if tag.startswith("service:")), - f"service:{function_name}", - ) - if service_tag: - tags.append(service_tag) - event[DD_SERVICE] = service_tag.split(":")[1] - else: - custom_lambda_tags = [ - tag for tag in custom_lambda_tags if not tag.startswith("service:") - ] - - # Check if one of the Lambda's custom tags is env - # If an env tag exists, remove the env:none placeholder - custom_env_tag = next( - (tag for tag in custom_lambda_tags if tag.startswith("env:")), None - ) - if custom_env_tag is not None: - event[DD_CUSTOM_TAGS] = event[DD_CUSTOM_TAGS].replace("env:none", "") - - tags += custom_lambda_tags - - # Dedup tags, so we don't end up with functionname twice - tags = list(set(tags)) - tags.sort() # Keep order deterministic - - event[DD_CUSTOM_TAGS] = ",".join([event[DD_CUSTOM_TAGS]] + tags) - - -def extract_ddtags_from_message(event): - """When the logs intake pipeline detects a `message` field with a - JSON content, it extracts the content to the top-level. The fields - of same name from the top-level will be overridden. - - E.g. the application adds some tags to the log, which appear in the - `message.ddtags` field, and the forwarder adds some common tags, such - as `aws_account`, which appear in the top-level `ddtags` field: - - { - "message": { - "ddtags": "mytag:value", # tags added by the application - ... - }, - "ddtags": "env:xxx,aws_account", # tags added by the forwarder - ... - } - - Only the custom tags added by the application will be kept. - - We might want to change the intake pipeline to "merge" the conflicting - fields rather than "overridding" in the future, but for now we should - extract `message.ddtags` and merge it with the top-level `ddtags` field. - """ - if "message" in event and DD_CUSTOM_TAGS in event["message"]: - if isinstance(event["message"], dict): - extracted_ddtags = event["message"].pop(DD_CUSTOM_TAGS) - if isinstance(event["message"], str): - try: - message_dict = json.loads(event["message"]) - extracted_ddtags = message_dict.pop(DD_CUSTOM_TAGS) - event["message"] = json.dumps(message_dict) - except Exception: - if logger.isEnabledFor(logging.DEBUG): - logger.debug(f"Failed to extract ddtags from: {event}") - return - - # Extract service tag from message.ddtags if exists - if "service" in extracted_ddtags: - event[DD_SERVICE] = next( - tag[8:] - for tag in extracted_ddtags.split(",") - if tag.startswith("service:") - ) - event[DD_CUSTOM_TAGS] = ",".join( - [ - tag - for tag in event[DD_CUSTOM_TAGS].split(",") - if not tag.startswith("service") - ] - ) - - event[DD_CUSTOM_TAGS] = f"{event[DD_CUSTOM_TAGS]},{extracted_ddtags}" - - -def extract_host_from_cloudtrails(event): - """Extract the hostname from cloudtrail events userIdentity.arn field if it - matches AWS hostnames. - - In case of s3 events the fields of the event are not encoded in the - "message" field, but in the event object itself. - """ - - if event is not None and event.get(DD_SOURCE) == "cloudtrail": - message = event.get("message", {}) - if isinstance(message, str): - try: - message = json.loads(message) - except json.JSONDecodeError: - logger.debug("Failed to decode cloudtrail message") - return - - # deal with s3 input type events - if not message: - message = event - - if isinstance(message, dict): - arn = message.get("userIdentity", {}).get("arn") - if arn is not None: - match = HOST_IDENTITY_REGEXP.match(arn) - if match is not None: - event[DD_HOST] = match.group("host") - - -def extract_host_from_guardduty(event): - if event is not None and event.get(DD_SOURCE) == "guardduty": - host = event.get("detail", {}).get("resource") - if isinstance(host, dict): - host = host.get("instanceDetails", {}).get("instanceId") - if host is not None: - event[DD_HOST] = host - - -def extract_host_from_route53(event): - if event is not None and event.get(DD_SOURCE) == "route53": - message = event.get("message", {}) - if isinstance(message, str): - try: - message = json.loads(message) - except json.JSONDecodeError: - logger.debug("Failed to decode Route53 message") - return - - if isinstance(message, dict): - host = message.get("srcids", {}).get("instance") - if host is not None: - event[DD_HOST] = host - - -def forward_metrics(metrics): - """ - Forward custom metrics submitted via logs to Datadog in a background thread - using `lambda_stats` that is provided by the Datadog Python Lambda Layer. - """ - if logger.isEnabledFor(logging.DEBUG): - logger.debug(f"Forwarding {len(metrics)} metrics") - - for metric in metrics: - try: - lambda_stats.distribution( - metric["m"], metric["v"], timestamp=metric["e"], tags=metric["t"] - ) - except Exception: - logger.exception(f"Exception while forwarding metric {json.dumps(metric)}") - else: - if logger.isEnabledFor(logging.DEBUG): - logger.debug(f"Forwarded metric: {json.dumps(metric)}") - - lambda_stats.distribution( - "{}.metrics_forwarded".format(DD_FORWARDER_TELEMETRY_NAMESPACE_PREFIX), - len(metrics), - tags=get_forwarder_telemetry_tags(), - ) - - -def forward_traces(trace_payloads): - if logger.isEnabledFor(logging.DEBUG): - logger.debug(f"Forwarding {len(trace_payloads)} traces") - - try: - trace_connection.send_traces(trace_payloads) - except Exception: - logger.exception( - f"Exception while forwarding traces {json.dumps(trace_payloads)}" - ) - else: - if logger.isEnabledFor(logging.DEBUG): - logger.debug(f"Forwarded traces: {json.dumps(trace_payloads)}") - - lambda_stats.distribution( - "{}.traces_forwarded".format(DD_FORWARDER_TELEMETRY_NAMESPACE_PREFIX), - len(trace_payloads), - tags=get_forwarder_telemetry_tags(), - ) +lambda_handler = datadog_lambda_wrapper(datadog_forwarder) diff --git a/aws/logs_monitoring/logs.py b/aws/logs_monitoring/logs.py index 44fbf7ab4..23f7492d1 100644 --- a/aws/logs_monitoring/logs.py +++ b/aws/logs_monitoring/logs.py @@ -3,135 +3,26 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2021 Datadog, Inc. -from settings import DD_FORWARDER_VERSION -import gzip -import json -import os -from concurrent.futures import as_completed -import re +import os import socket import ssl import logging import time +from concurrent.futures import as_completed from requests_futures.sessions import FuturesSession +from logs_helpers import compress_logs, compileRegex +from exceptions import RetriableException, ScrubbingException -from datadog_lambda.metric import lambda_stats -from telemetry import ( - DD_FORWARDER_TELEMETRY_NAMESPACE_PREFIX, - get_forwarder_telemetry_tags, -) from settings import ( - DD_API_KEY, - DD_USE_TCP, DD_USE_COMPRESSION, DD_COMPRESSION_LEVEL, - DD_NO_SSL, - DD_SKIP_SSL_VALIDATION, - DD_URL, - DD_PORT, - SCRUBBING_RULE_CONFIGS, - INCLUDE_AT_MATCH, - EXCLUDE_AT_MATCH, DD_MAX_WORKERS, + DD_FORWARDER_VERSION, ) logger = logging.getLogger() - - -class RetriableException(Exception): - pass - - -class ScrubbingException(Exception): - pass - - -def forward_logs(logs): - """Forward logs to Datadog""" - if logger.isEnabledFor(logging.DEBUG): - logger.debug(f"Forwarding {len(logs)} logs") - logs_to_forward = filter_logs( - [json.dumps(log, ensure_ascii=False) for log in logs], - include_pattern=INCLUDE_AT_MATCH, - exclude_pattern=EXCLUDE_AT_MATCH, - ) - scrubber = DatadogScrubber(SCRUBBING_RULE_CONFIGS) - if DD_USE_TCP: - batcher = DatadogBatcher(256 * 1000, 256 * 1000, 1) - cli = DatadogTCPClient(DD_URL, DD_PORT, DD_NO_SSL, DD_API_KEY, scrubber) - else: - batcher = DatadogBatcher(512 * 1000, 4 * 1000 * 1000, 400) - cli = DatadogHTTPClient( - DD_URL, DD_PORT, DD_NO_SSL, DD_SKIP_SSL_VALIDATION, DD_API_KEY, scrubber - ) - - with DatadogClient(cli) as client: - for batch in batcher.batch(logs_to_forward): - try: - client.send(batch) - except Exception: - logger.exception(f"Exception while forwarding log batch {batch}") - else: - if logger.isEnabledFor(logging.DEBUG): - logger.debug(f"Forwarded log batch: {json.dumps(batch)}") - - lambda_stats.distribution( - "{}.logs_forwarded".format(DD_FORWARDER_TELEMETRY_NAMESPACE_PREFIX), - len(logs_to_forward), - tags=get_forwarder_telemetry_tags(), - ) - - -def compileRegex(rule, pattern): - if pattern is not None: - if pattern == "": - # If pattern is an empty string, raise exception - raise Exception( - "No pattern provided:\nAdd pattern or remove {} environment variable".format( - rule - ) - ) - try: - return re.compile(pattern) - except Exception: - raise Exception( - "could not compile {} regex with pattern: {}".format(rule, pattern) - ) - - -def filter_logs(logs, include_pattern=None, exclude_pattern=None): - """ - Applies log filtering rules. - If no filtering rules exist, return all the logs. - """ - if include_pattern is None and exclude_pattern is None: - return logs - # Add logs that should be sent to logs_to_send - logs_to_send = [] - for log in logs: - if exclude_pattern is not None or include_pattern is not None: - logger.debug("Filtering log event:") - logger.debug(log) - try: - if exclude_pattern is not None: - # if an exclude match is found, do not add log to logs_to_send - logger.debug(f"Applying exclude pattern: {exclude_pattern}") - exclude_regex = compileRegex("EXCLUDE_AT_MATCH", exclude_pattern) - if re.search(exclude_regex, log): - logger.debug("Exclude pattern matched, excluding log event") - continue - if include_pattern is not None: - # if no include match is found, do not add log to logs_to_send - logger.debug(f"Applying include pattern: {include_pattern}") - include_regex = compileRegex("INCLUDE_AT_MATCH", include_pattern) - if not re.search(include_regex, log): - logger.debug("Include pattern did not match, excluding log event") - continue - logs_to_send.append(log) - except ScrubbingException: - raise Exception("could not filter the payload") - return logs_to_send +logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper())) class DatadogClient(object): @@ -163,17 +54,6 @@ def __exit__(self, ex_type, ex_value, traceback): self._client.__exit__(ex_type, ex_value, traceback) -def compress_logs(batch, level): - if level < 0: - compression_level = 0 - elif level > 9: - compression_level = 9 - else: - compression_level = level - - return gzip.compress(bytes(batch, "utf-8"), compression_level) - - class DatadogScrubber(object): def __init__(self, configs): rules = [] diff --git a/aws/logs_monitoring/logs_helpers.py b/aws/logs_monitoring/logs_helpers.py new file mode 100644 index 000000000..43044afbf --- /dev/null +++ b/aws/logs_monitoring/logs_helpers.py @@ -0,0 +1,71 @@ +import logging +import re +import gzip +import os +from exceptions import ScrubbingException + +logger = logging.getLogger() +logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper())) + + +def filter_logs(logs, include_pattern=None, exclude_pattern=None): + """ + Applies log filtering rules. + If no filtering rules exist, return all the logs. + """ + if include_pattern is None and exclude_pattern is None: + return logs + # Add logs that should be sent to logs_to_send + logs_to_send = [] + for log in logs: + if exclude_pattern is not None or include_pattern is not None: + logger.debug("Filtering log event:") + logger.debug(log) + try: + if exclude_pattern is not None: + # if an exclude match is found, do not add log to logs_to_send + logger.debug(f"Applying exclude pattern: {exclude_pattern}") + exclude_regex = compileRegex("EXCLUDE_AT_MATCH", exclude_pattern) + if re.search(exclude_regex, log): + logger.debug("Exclude pattern matched, excluding log event") + continue + if include_pattern is not None: + # if no include match is found, do not add log to logs_to_send + logger.debug(f"Applying include pattern: {include_pattern}") + include_regex = compileRegex("INCLUDE_AT_MATCH", include_pattern) + if not re.search(include_regex, log): + logger.debug("Include pattern did not match, excluding log event") + continue + logs_to_send.append(log) + except ScrubbingException: + raise Exception("could not filter the payload") + + return logs_to_send + + +def compress_logs(batch, level): + if level < 0: + compression_level = 0 + elif level > 9: + compression_level = 9 + else: + compression_level = level + + return gzip.compress(bytes(batch, "utf-8"), compression_level) + + +def compileRegex(rule, pattern): + if pattern is not None: + if pattern == "": + # If pattern is an empty string, raise exception + raise Exception( + "No pattern provided:\nAdd pattern or remove {} environment variable".format( + rule + ) + ) + try: + return re.compile(pattern) + except Exception: + raise Exception( + "could not compile {} regex with pattern: {}".format(rule, pattern) + ) diff --git a/aws/logs_monitoring/parsing.py b/aws/logs_monitoring/parsing.py index f2e4317df..b2724309c 100644 --- a/aws/logs_monitoring/parsing.py +++ b/aws/logs_monitoring/parsing.py @@ -7,8 +7,6 @@ import gzip import json import os -import copy - import boto3 import botocore import itertools @@ -16,9 +14,7 @@ import urllib import logging from io import BytesIO, BufferedReader - from datadog_lambda.metric import lambda_stats - from customized_log_group import ( get_lambda_function_name_from_logstream_name, is_lambda_customized_log_group, @@ -662,166 +658,6 @@ def cwevent_handler(event, metadata): yield data -def parse_aws_waf_logs(event): - """Parse out complex arrays of objects in AWS WAF logs - - Attributes to convert: - httpRequest.headers - nonTerminatingMatchingRules - rateBasedRuleList - ruleGroupList - - This prevents having an unparsable array of objects in the final log. - """ - if isinstance(event, str): - try: - event = json.loads(event) - except json.JSONDecodeError: - logger.debug("Argument provided for waf parser is not valid JSON") - return event - if event.get(DD_SOURCE) != "waf": - return event - - event_copy = copy.deepcopy(event) - - message = event_copy.get("message", {}) - if isinstance(message, str): - try: - message = json.loads(message) - except json.JSONDecodeError: - logger.debug("Failed to decode waf message") - return event - - headers = message.get("httpRequest", {}).get("headers") - if headers: - message["httpRequest"]["headers"] = convert_rule_to_nested_json(headers) - - # Iterate through rules in ruleGroupList and nest them under the group id - # ruleGroupList has three attributes that need to be handled separately - rule_groups = message.get("ruleGroupList", {}) - if rule_groups and isinstance(rule_groups, list): - message["ruleGroupList"] = {} - for rule_group in rule_groups: - group_id = None - if "ruleGroupId" in rule_group and rule_group["ruleGroupId"]: - group_id = rule_group.pop("ruleGroupId", None) - if group_id not in message["ruleGroupList"]: - message["ruleGroupList"][group_id] = {} - - # Extract the terminating rule and nest it under its own id - if "terminatingRule" in rule_group and rule_group["terminatingRule"]: - terminating_rule = rule_group.pop("terminatingRule", None) - if not "terminatingRule" in message["ruleGroupList"][group_id]: - message["ruleGroupList"][group_id]["terminatingRule"] = {} - message["ruleGroupList"][group_id]["terminatingRule"].update( - convert_rule_to_nested_json(terminating_rule) - ) - - # Iterate through array of non-terminating rules and nest each under its own id - if "nonTerminatingMatchingRules" in rule_group and isinstance( - rule_group["nonTerminatingMatchingRules"], list - ): - non_terminating_rules = rule_group.pop( - "nonTerminatingMatchingRules", None - ) - if ( - "nonTerminatingMatchingRules" - not in message["ruleGroupList"][group_id] - ): - message["ruleGroupList"][group_id][ - "nonTerminatingMatchingRules" - ] = {} - message["ruleGroupList"][group_id][ - "nonTerminatingMatchingRules" - ].update(convert_rule_to_nested_json(non_terminating_rules)) - - # Iterate through array of excluded rules and nest each under its own id - if "excludedRules" in rule_group and isinstance( - rule_group["excludedRules"], list - ): - excluded_rules = rule_group.pop("excludedRules", None) - if "excludedRules" not in message["ruleGroupList"][group_id]: - message["ruleGroupList"][group_id]["excludedRules"] = {} - message["ruleGroupList"][group_id]["excludedRules"].update( - convert_rule_to_nested_json(excluded_rules) - ) - - rate_based_rules = message.get("rateBasedRuleList", {}) - if rate_based_rules: - message["rateBasedRuleList"] = convert_rule_to_nested_json(rate_based_rules) - - non_terminating_rules = message.get("nonTerminatingMatchingRules", {}) - if non_terminating_rules: - message["nonTerminatingMatchingRules"] = convert_rule_to_nested_json( - non_terminating_rules - ) - - event_copy["message"] = message - return event_copy - - -def convert_rule_to_nested_json(rule): - key = None - result_obj = {} - if not isinstance(rule, list): - if "ruleId" in rule and rule["ruleId"]: - key = rule.pop("ruleId", None) - result_obj.update({key: rule}) - return result_obj - for entry in rule: - if "ruleId" in entry and entry["ruleId"]: - key = entry.pop("ruleId", None) - elif "rateBasedRuleName" in entry and entry["rateBasedRuleName"]: - key = entry.pop("rateBasedRuleName", None) - elif "name" in entry and "value" in entry: - key = entry["name"] - entry = entry["value"] - result_obj.update({key: entry}) - return result_obj - - -def separate_security_hub_findings(event): - """Replace Security Hub event with series of events based on findings - - Each event should contain one finding only. - This prevents having an unparsable array of objects in the final log. - """ - if event.get(DD_SOURCE) != "securityhub" or not event.get("detail", {}).get( - "findings" - ): - return None - events = [] - event_copy = copy.deepcopy(event) - # Copy findings before separating - findings = event_copy.get("detail", {}).get("findings") - if findings: - # Remove findings from the original event once we have a copy - del event_copy["detail"]["findings"] - # For each finding create a separate log event - for index, item in enumerate(findings): - # Copy the original event with source and other metadata - new_event = copy.deepcopy(event_copy) - current_finding = findings[index] - # Get the resources array from the current finding - resources = current_finding.get("Resources", {}) - new_event["detail"]["finding"] = current_finding - new_event["detail"]["finding"]["resources"] = {} - # Separate objects in resources array into distinct attributes - if resources: - # Remove from current finding once we have a copy - del current_finding["Resources"] - for item in resources: - current_resource = item - # Capture the type and use it as the distinguishing key - resource_type = current_resource.get("Type", {}) - del current_resource["Type"] - new_event["detail"]["finding"]["resources"][ - resource_type - ] = current_resource - events.append(new_event) - return events - - # Handle Sns events def sns_handler(event, metadata): data = event diff --git a/aws/logs_monitoring/splitting.py b/aws/logs_monitoring/splitting.py new file mode 100644 index 000000000..6923da82f --- /dev/null +++ b/aws/logs_monitoring/splitting.py @@ -0,0 +1,74 @@ +import logging +import json +import os +from settings import DD_CUSTOM_TAGS + +logger = logging.getLogger() +logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper())) + + +def split(events): + """Split events into metrics, logs, and trace payloads""" + metrics, logs, trace_payloads = [], [], [] + for event in events: + metric = extract_metric(event) + trace_payload = extract_trace_payload(event) + if metric: + metrics.append(metric) + elif trace_payload: + trace_payloads.append(trace_payload) + else: + logs.append(event) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + f"Extracted {len(metrics)} metrics, {len(trace_payloads)} traces, and {len(logs)} logs" + ) + + return metrics, logs, trace_payloads + + +def extract_metric(event): + """Extract metric from an event if possible""" + try: + metric = json.loads(event["message"]) + required_attrs = {"m", "v", "e", "t"} + if not all(attr in metric for attr in required_attrs): + return None + if not isinstance(metric["t"], list): + return None + if not (isinstance(metric["v"], int) or isinstance(metric["v"], float)): + return None + + lambda_log_metadata = event.get("lambda", {}) + lambda_log_arn = lambda_log_metadata.get("arn") + + if lambda_log_arn: + metric["t"] += [f"function_arn:{lambda_log_arn.lower()}"] + + metric["t"] += event[DD_CUSTOM_TAGS].split(",") + return metric + except Exception: + return None + + +def extract_trace_payload(event): + """Extract trace payload from an event if possible""" + try: + message = event["message"] + obj = json.loads(event["message"]) + + obj_has_traces = "traces" in obj + traces_is_a_list = isinstance(obj["traces"], list) + # check that the log is not containing a trace array unrelated to Datadog + trace_id_found = ( + len(obj["traces"]) > 0 + and len(obj["traces"][0]) > 0 + and obj["traces"][0][0]["trace_id"] is not None + ) + + if obj_has_traces and traces_is_a_list and trace_id_found: + return {"message": message, "tags": event[DD_CUSTOM_TAGS]} + return None + except Exception: + return None diff --git a/aws/logs_monitoring/tests/test_cloudtrail_s3.py b/aws/logs_monitoring/tests/test_cloudtrail_s3.py index 36d057a50..213a0d811 100644 --- a/aws/logs_monitoring/tests/test_cloudtrail_s3.py +++ b/aws/logs_monitoring/tests/test_cloudtrail_s3.py @@ -24,7 +24,7 @@ env_patch.start() import lambda_function -import parsing +from parsing import parse env_patch.stop() @@ -120,7 +120,7 @@ def test_s3_cloudtrail_pasing_and_enrichment( } } - result = parsing.parse({"Records": [payload]}, context) + result = parse({"Records": [payload]}, context) expected = copy.deepcopy([test_data["Records"][0]]) expected[0].update( diff --git a/aws/logs_monitoring/tests/test_enrichment.py b/aws/logs_monitoring/tests/test_enrichment.py new file mode 100644 index 000000000..783d4767c --- /dev/null +++ b/aws/logs_monitoring/tests/test_enrichment.py @@ -0,0 +1,142 @@ +import unittest +import json +from enrichment import ( + extract_host_from_cloudtrails, + extract_host_from_guardduty, + extract_host_from_route53, + extract_ddtags_from_message, +) + + +class TestMergeMessageTags(unittest.TestCase): + message_tags = '{"ddtags":"service:my_application_service,custom_tag_1:value1"}' + custom_tags = "custom_tag_2:value2,service:my_custom_service" + + def test_extract_ddtags_from_message_str(self): + event = { + "message": self.message_tags, + "ddtags": self.custom_tags, + "service": "my_service", + } + + extract_ddtags_from_message(event) + + self.assertEqual( + event["ddtags"], + "custom_tag_2:value2,service:my_application_service,custom_tag_1:value1", + ) + self.assertEqual( + event["service"], + "my_application_service", + ) + + def test_extract_ddtags_from_message_dict(self): + loaded_message_tags = json.loads(self.message_tags) + event = { + "message": loaded_message_tags, + "ddtags": self.custom_tags, + "service": "my_service", + } + + extract_ddtags_from_message(event) + + self.assertEqual( + event["ddtags"], + "custom_tag_2:value2,service:my_application_service,custom_tag_1:value1", + ) + self.assertEqual( + event["service"], + "my_application_service", + ) + + def test_extract_ddtags_from_message_service_tag_setting(self): + loaded_message_tags = json.loads(self.message_tags) + loaded_message_tags["ddtags"] = ",".join( + [ + tag + for tag in loaded_message_tags["ddtags"].split(",") + if not tag.startswith("service:") + ] + ) + event = { + "message": loaded_message_tags, + "ddtags": self.custom_tags, + "service": "my_custom_service", + } + + extract_ddtags_from_message(event) + + self.assertEqual( + event["ddtags"], + "custom_tag_2:value2,service:my_custom_service,custom_tag_1:value1", + ) + self.assertEqual( + event["service"], + "my_custom_service", + ) + + def test_extract_ddtags_from_message_multiple_service_tag_values(self): + custom_tags = self.custom_tags + ",service:my_custom_service_2" + event = {"message": self.message_tags, "ddtags": custom_tags} + + extract_ddtags_from_message(event) + + self.assertEqual( + event["ddtags"], + "custom_tag_2:value2,service:my_application_service,custom_tag_1:value1", + ) + self.assertEqual( + event["service"], + "my_application_service", + ) + + def test_extract_ddtags_from_message_multiple_values_tag(self): + loaded_message_tags = json.loads(self.message_tags) + loaded_message_tags["ddtags"] += ",custom_tag_3:value4" + custom_tags = self.custom_tags + ",custom_tag_3:value3" + event = {"message": loaded_message_tags, "ddtags": custom_tags} + + extract_ddtags_from_message(event) + + self.assertEqual( + event["ddtags"], + "custom_tag_2:value2,custom_tag_3:value3,service:my_application_service,custom_tag_1:value1,custom_tag_3:value4", + ) + self.assertEqual( + event["service"], + "my_application_service", + ) + + +class TestExtractHostFromLogEvents(unittest.TestCase): + def test_parse_source_cloudtrail(self): + event = { + "ddsource": "cloudtrail", + "message": { + "userIdentity": { + "arn": "arn:aws:sts::601427279990:assumed-role/gke-90725aa7-management/i-99999999" + } + }, + } + extract_host_from_cloudtrails(event) + self.assertEqual(event["host"], "i-99999999") + + def test_parse_source_guardduty(self): + event = { + "ddsource": "guardduty", + "detail": {"resource": {"instanceDetails": {"instanceId": "i-99999999"}}}, + } + extract_host_from_guardduty(event) + self.assertEqual(event["host"], "i-99999999") + + def test_parse_source_route53(self): + event = { + "ddsource": "route53", + "message": {"srcids": {"instance": "i-99999999"}}, + } + extract_host_from_route53(event) + self.assertEqual(event["host"], "i-99999999") + + +if __name__ == "__main__": + unittest.main() diff --git a/aws/logs_monitoring/tests/test_lambda_function.py b/aws/logs_monitoring/tests/test_lambda_function.py index 21872de0a..ebc24fab4 100644 --- a/aws/logs_monitoring/tests/test_lambda_function.py +++ b/aws/logs_monitoring/tests/test_lambda_function.py @@ -26,51 +26,20 @@ }, ) env_patch.start() -from lambda_function import ( - invoke_additional_target_lambdas, - extract_metric, - extract_host_from_cloudtrails, - extract_host_from_guardduty, - extract_host_from_route53, - extract_trace_payload, - enrich, - transform, - split, - extract_ddtags_from_message, -) +from lambda_function import invoke_additional_target_lambdas +from enrichment import enrich +from transformation import transform +from splitting import split from parsing import parse, parse_event_type env_patch.stop() -class TestExtractHostFromLogEvents(unittest.TestCase): - def test_parse_source_cloudtrail(self): - event = { - "ddsource": "cloudtrail", - "message": { - "userIdentity": { - "arn": "arn:aws:sts::601427279990:assumed-role/gke-90725aa7-management/i-99999999" - } - }, - } - extract_host_from_cloudtrails(event) - self.assertEqual(event["host"], "i-99999999") - - def test_parse_source_guardduty(self): - event = { - "ddsource": "guardduty", - "detail": {"resource": {"instanceDetails": {"instanceId": "i-99999999"}}}, - } - extract_host_from_guardduty(event) - self.assertEqual(event["host"], "i-99999999") - - def test_parse_source_route53(self): - event = { - "ddsource": "route53", - "message": {"srcids": {"instance": "i-99999999"}}, - } - extract_host_from_route53(event) - self.assertEqual(event["host"], "i-99999999") +class Context: + function_version = 0 + invoked_function_arn = "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + function_name = "inferred-spans-python-dev-initsender" + memory_limit_in_mb = "10" class TestInvokeAdditionalTargetLambdas(unittest.TestCase): @@ -100,38 +69,6 @@ def test_lambda_invocation_exception(self, boto3): ) -class TestExtractMetric(unittest.TestCase): - def test_empty_event(self): - self.assertEqual(extract_metric({}), None) - - def test_missing_keys(self): - self.assertEqual(extract_metric({"e": 0, "v": 1, "m": "foo"}), None) - - def test_tags_instance(self): - self.assertEqual(extract_metric({"e": 0, "v": 1, "m": "foo", "t": 666}), None) - - def test_value_instance(self): - self.assertEqual(extract_metric({"e": 0, "v": 1.1, "m": "foo", "t": []}), None) - - def test_value_instance_float(self): - self.assertEqual(extract_metric({"e": 0, "v": None, "m": "foo", "t": []}), None) - - -class Context: - function_version = 0 - invoked_function_arn = "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" - function_name = "inferred-spans-python-dev-initsender" - memory_limit_in_mb = "10" - - -def create_cloudwatch_log_event_from_data(data): - # CloudWatch log event data is a base64-encoded ZIP archive - # see https://docs.aws.amazon.com/lambda/latest/dg/services-cloudwatchlogs.html - gzipped_data = gzip.compress(bytes(data, encoding="utf-8")) - encoded_data = base64.b64encode(gzipped_data).decode("utf-8") - return encoded_data - - class TestLambdaFunctionEndToEnd(unittest.TestCase): @patch("enhanced_lambda_metrics.LambdaTagsCache.get_cache_from_s3") def test_datadog_forwarder(self, mock_get_s3_cache): @@ -149,7 +86,9 @@ def test_datadog_forwarder(self, mock_get_s3_cache): ) context = Context() input_data = self._get_input_data() - event = {"awslogs": {"data": create_cloudwatch_log_event_from_data(input_data)}} + event = { + "awslogs": {"data": self._create_cloudwatch_log_event_from_data(input_data)} + } os.environ["DD_FETCH_LAMBDA_TAGS"] = "True" event_type = parse_event_type(event) @@ -206,7 +145,9 @@ def test_setting_service_tag_from_log_group_cache(self, cw_logs_tags_get): cw_logs_tags_get.return_value = ["service:log_group_service"] context = Context() input_data = self._get_input_data() - event = {"awslogs": {"data": create_cloudwatch_log_event_from_data(input_data)}} + event = { + "awslogs": {"data": self._create_cloudwatch_log_event_from_data(input_data)} + } normalized_events = parse(event, context) enriched_events = enrich(normalized_events) @@ -225,7 +166,9 @@ def test_service_override_from_dd_tags(self, cw_logs_tags_get): cw_logs_tags_get.return_value = ["service:log_group_service"] context = Context() input_data = self._get_input_data() - event = {"awslogs": {"data": create_cloudwatch_log_event_from_data(input_data)}} + event = { + "awslogs": {"data": self._create_cloudwatch_log_event_from_data(input_data)} + } normalized_events = parse(event, context) enriched_events = enrich(normalized_events) @@ -246,7 +189,9 @@ def test_overrding_service_tag_from_lambda_cache( context = Context() input_data = self._get_input_data() - event = {"awslogs": {"data": create_cloudwatch_log_event_from_data(input_data)}} + event = { + "awslogs": {"data": self._create_cloudwatch_log_event_from_data(input_data)} + } normalized_events = parse(event, context) enriched_events = enrich(normalized_events) @@ -268,7 +213,9 @@ def test_overrding_service_tag_from_lambda_cache_when_dd_tags_is_set( context = Context() input_data = self._get_input_data() - event = {"awslogs": {"data": create_cloudwatch_log_event_from_data(input_data)}} + event = { + "awslogs": {"data": self._create_cloudwatch_log_event_from_data(input_data)} + } normalized_events = parse(event, context) enriched_events = enrich(normalized_events) @@ -291,134 +238,12 @@ def _get_input_data(self): return input_data - -class TestLambdaFunctionExtractTracePayload(unittest.TestCase): - def test_extract_trace_payload_none_no_trace(self): - message_json = """{ - "key": "value" - }""" - self.assertEqual(extract_trace_payload({"message": message_json}), None) - - def test_extract_trace_payload_none_exception(self): - message_json = """{ - "invalid_json" - }""" - self.assertEqual(extract_trace_payload({"message": message_json}), None) - - def test_extract_trace_payload_unrelated_datadog_trace(self): - message_json = """{"traces":["I am a trace"]}""" - self.assertEqual(extract_trace_payload({"message": message_json}), None) - - def test_extract_trace_payload_valid_trace(self): - message_json = """{"traces":[[{"trace_id":1234}]]}""" - tags_json = """["key0:value", "key1:value1"]""" - item = { - "message": '{"traces":[[{"trace_id":1234}]]}', - "tags": '["key0:value", "key1:value1"]', - } - self.assertEqual( - extract_trace_payload({"message": message_json, "ddtags": tags_json}), item - ) - - -class TestMergeMessageTags(unittest.TestCase): - message_tags = '{"ddtags":"service:my_application_service,custom_tag_1:value1"}' - custom_tags = "custom_tag_2:value2,service:my_custom_service" - - def test_extract_ddtags_from_message_str(self): - event = { - "message": self.message_tags, - "ddtags": self.custom_tags, - "service": "my_service", - } - - extract_ddtags_from_message(event) - - self.assertEqual( - event["ddtags"], - "custom_tag_2:value2,service:my_application_service,custom_tag_1:value1", - ) - self.assertEqual( - event["service"], - "my_application_service", - ) - - def test_extract_ddtags_from_message_dict(self): - loaded_message_tags = json.loads(self.message_tags) - event = { - "message": loaded_message_tags, - "ddtags": self.custom_tags, - "service": "my_service", - } - - extract_ddtags_from_message(event) - - self.assertEqual( - event["ddtags"], - "custom_tag_2:value2,service:my_application_service,custom_tag_1:value1", - ) - self.assertEqual( - event["service"], - "my_application_service", - ) - - def test_extract_ddtags_from_message_service_tag_setting(self): - loaded_message_tags = json.loads(self.message_tags) - loaded_message_tags["ddtags"] = ",".join( - [ - tag - for tag in loaded_message_tags["ddtags"].split(",") - if not tag.startswith("service:") - ] - ) - event = { - "message": loaded_message_tags, - "ddtags": self.custom_tags, - "service": "my_custom_service", - } - - extract_ddtags_from_message(event) - - self.assertEqual( - event["ddtags"], - "custom_tag_2:value2,service:my_custom_service,custom_tag_1:value1", - ) - self.assertEqual( - event["service"], - "my_custom_service", - ) - - def test_extract_ddtags_from_message_multiple_service_tag_values(self): - custom_tags = self.custom_tags + ",service:my_custom_service_2" - event = {"message": self.message_tags, "ddtags": custom_tags} - - extract_ddtags_from_message(event) - - self.assertEqual( - event["ddtags"], - "custom_tag_2:value2,service:my_application_service,custom_tag_1:value1", - ) - self.assertEqual( - event["service"], - "my_application_service", - ) - - def test_extract_ddtags_from_message_multiple_values_tag(self): - loaded_message_tags = json.loads(self.message_tags) - loaded_message_tags["ddtags"] += ",custom_tag_3:value4" - custom_tags = self.custom_tags + ",custom_tag_3:value3" - event = {"message": loaded_message_tags, "ddtags": custom_tags} - - extract_ddtags_from_message(event) - - self.assertEqual( - event["ddtags"], - "custom_tag_2:value2,custom_tag_3:value3,service:my_application_service,custom_tag_1:value1,custom_tag_3:value4", - ) - self.assertEqual( - event["service"], - "my_application_service", - ) + def _create_cloudwatch_log_event_from_data(self, data): + # CloudWatch log event data is a base64-encoded ZIP archive + # see https://docs.aws.amazon.com/lambda/latest/dg/services-cloudwatchlogs.html + gzipped_data = gzip.compress(bytes(data, encoding="utf-8")) + encoded_data = base64.b64encode(gzipped_data).decode("utf-8") + return encoded_data if __name__ == "__main__": diff --git a/aws/logs_monitoring/tests/test_logs.py b/aws/logs_monitoring/tests/test_logs.py index 5a08fb819..635c0ea8b 100644 --- a/aws/logs_monitoring/tests/test_logs.py +++ b/aws/logs_monitoring/tests/test_logs.py @@ -1,7 +1,8 @@ import unittest import os -from logs import DatadogScrubber, filter_logs +from logs import DatadogScrubber +from logs_helpers import filter_logs from settings import ScrubbingRuleConfig, SCRUBBING_RULE_CONFIGS, get_env_var @@ -80,3 +81,7 @@ def test_exclude_overrides_include(self): def test_no_filtering_rules(self): filtered_logs = filter_logs(self.example_logs) self.assertEqual(filtered_logs, self.example_logs) + + +if __name__ == "__main__": + unittest.main() diff --git a/aws/logs_monitoring/tests/test_parsing.py b/aws/logs_monitoring/tests/test_parsing.py index f0de20d43..8f4d224e8 100644 --- a/aws/logs_monitoring/tests/test_parsing.py +++ b/aws/logs_monitoring/tests/test_parsing.py @@ -27,8 +27,6 @@ awslogs_handler, parse_event_source, parse_service_arn, - separate_security_hub_findings, - parse_aws_waf_logs, get_service_from_tags_and_remove_duplicates, get_state_machine_arn, get_lower_cased_lambda_function_name, @@ -397,221 +395,6 @@ def test_elb_s3_key_multi_prefix_gov(self): ) -class TestParseAwsWafLogs(unittest.TestCase): - def test_waf_string_invalid_json(self): - event = "This is not valid JSON." - self.assertEqual(parse_aws_waf_logs(event), "This is not valid JSON.") - - def test_waf_string_json(self): - event = '{"ddsource":"waf","message":"This is a string of JSON"}' - self.assertEqual( - parse_aws_waf_logs(event), - {"ddsource": "waf", "message": "This is a string of JSON"}, - ) - - def test_waf_headers(self): - event = { - "ddsource": "waf", - "message": { - "httpRequest": { - "headers": [ - {"name": "header1", "value": "value1"}, - {"name": "header2", "value": "value2"}, - ] - } - }, - } - verify_as_json(parse_aws_waf_logs(event)) - - def test_waf_non_terminating_matching_rules(self): - event = { - "ddsource": "waf", - "message": { - "nonTerminatingMatchingRules": [ - {"ruleId": "nonterminating1", "action": "COUNT"}, - {"ruleId": "nonterminating2", "action": "COUNT"}, - ] - }, - } - verify_as_json(parse_aws_waf_logs(event)) - - def test_waf_rate_based_rules(self): - event = { - "ddsource": "waf", - "message": { - "rateBasedRuleList": [ - { - "limitValue": "195.154.122.189", - "rateBasedRuleName": "tf-rate-limit-5-min", - "rateBasedRuleId": "arn:aws:wafv2:ap-southeast-2:068133125972_MANAGED:regional/ipset/0f94bd8b-0fa5-4865-81ce-d11a60051fb4_fef50279-8b9a-4062-b733-88ecd1cfd889_IPV4/fef50279-8b9a-4062-b733-88ecd1cfd889", - "maxRateAllowed": 300, - "limitKey": "IP", - }, - { - "limitValue": "195.154.122.189", - "rateBasedRuleName": "no-rate-limit", - "rateBasedRuleId": "arn:aws:wafv2:ap-southeast-2:068133125972_MANAGED:regional/ipset/0f94bd8b-0fa5-4865-81ce-d11a60051fb4_fef50279-8b9a-4062-b733-88ecd1cfd889_IPV4/fef50279-8b9a-4062-b733-88ecd1cfd889", - "maxRateAllowed": 300, - "limitKey": "IP", - }, - ] - }, - } - verify_as_json(parse_aws_waf_logs(event)) - - def test_waf_rule_group_with_excluded_and_nonterminating_rules(self): - event = { - "ddsource": "waf", - "message": { - "ruleGroupList": [ - { - "ruleGroupId": "AWS#AWSManagedRulesSQLiRuleSet", - "terminatingRule": { - "ruleId": "SQLi_QUERYARGUMENTS", - "action": "BLOCK", - }, - "nonTerminatingMatchingRules": [ - { - "exclusionType": "REGULAR", - "ruleId": "first_nonterminating", - }, - { - "exclusionType": "REGULAR", - "ruleId": "second_nonterminating", - }, - ], - "excludedRules": [ - { - "exclusionType": "EXCLUDED_AS_COUNT", - "ruleId": "GenericRFI_BODY", - }, - { - "exclusionType": "EXCLUDED_AS_COUNT", - "ruleId": "second_exclude", - }, - ], - } - ] - }, - } - verify_as_json(parse_aws_waf_logs(event)) - - def test_waf_rule_group_two_rules_same_group_id(self): - event = { - "ddsource": "waf", - "message": { - "ruleGroupList": [ - { - "ruleGroupId": "AWS#AWSManagedRulesSQLiRuleSet", - "terminatingRule": { - "ruleId": "SQLi_QUERYARGUMENTS", - "action": "BLOCK", - }, - }, - { - "ruleGroupId": "AWS#AWSManagedRulesSQLiRuleSet", - "terminatingRule": {"ruleId": "secondRULE", "action": "BLOCK"}, - }, - ] - }, - } - verify_as_json(parse_aws_waf_logs(event)) - - def test_waf_rule_group_three_rules_two_group_ids(self): - event = { - "ddsource": "waf", - "message": { - "ruleGroupList": [ - { - "ruleGroupId": "AWS#AWSManagedRulesSQLiRuleSet", - "terminatingRule": { - "ruleId": "SQLi_QUERYARGUMENTS", - "action": "BLOCK", - }, - }, - { - "ruleGroupId": "AWS#AWSManagedRulesSQLiRuleSet", - "terminatingRule": {"ruleId": "secondRULE", "action": "BLOCK"}, - }, - { - "ruleGroupId": "A_DIFFERENT_ID", - "terminatingRule": {"ruleId": "thirdRULE", "action": "BLOCK"}, - }, - ] - }, - } - verify_as_json(parse_aws_waf_logs(event)) - - -class TestParseSecurityHubEvents(unittest.TestCase): - def test_security_hub_no_findings(self): - event = {"ddsource": "securityhub"} - self.assertEqual( - separate_security_hub_findings(event), - None, - ) - - def test_security_hub_one_finding_no_resources(self): - event = { - "ddsource": "securityhub", - "detail": {"findings": [{"myattribute": "somevalue"}]}, - } - verify_as_json(separate_security_hub_findings(event)) - - def test_security_hub_two_findings_one_resource_each(self): - event = { - "ddsource": "securityhub", - "detail": { - "findings": [ - { - "myattribute": "somevalue", - "Resources": [ - {"Region": "us-east-1", "Type": "AwsEc2SecurityGroup"} - ], - }, - { - "myattribute": "somevalue", - "Resources": [ - {"Region": "us-east-1", "Type": "AwsEc2SecurityGroup"} - ], - }, - ] - }, - } - verify_as_json(separate_security_hub_findings(event)) - - def test_security_hub_multiple_findings_multiple_resources(self): - event = { - "ddsource": "securityhub", - "detail": { - "findings": [ - { - "myattribute": "somevalue", - "Resources": [ - {"Region": "us-east-1", "Type": "AwsEc2SecurityGroup"} - ], - }, - { - "myattribute": "somevalue", - "Resources": [ - {"Region": "us-east-1", "Type": "AwsEc2SecurityGroup"}, - {"Region": "us-east-1", "Type": "AwsOtherSecurityGroup"}, - ], - }, - { - "myattribute": "somevalue", - "Resources": [ - {"Region": "us-east-1", "Type": "AwsEc2SecurityGroup"}, - {"Region": "us-east-1", "Type": "AwsOtherSecurityGroup"}, - {"Region": "us-east-1", "Type": "AwsAnotherSecurityGroup"}, - ], - }, - ] - }, - } - verify_as_json(separate_security_hub_findings(event)) - - class TestAWSLogsHandler(unittest.TestCase): @patch("parsing.CloudwatchLogGroupTagsCache.get") @patch("parsing.CloudwatchLogGroupTagsCache.release_s3_cache_lock") diff --git a/aws/logs_monitoring/tests/test_splitting.py b/aws/logs_monitoring/tests/test_splitting.py new file mode 100644 index 000000000..1fa85365d --- /dev/null +++ b/aws/logs_monitoring/tests/test_splitting.py @@ -0,0 +1,55 @@ +import unittest +from splitting import ( + extract_metric, + extract_trace_payload, +) + + +class TestExtractMetric(unittest.TestCase): + def test_empty_event(self): + self.assertEqual(extract_metric({}), None) + + def test_missing_keys(self): + self.assertEqual(extract_metric({"e": 0, "v": 1, "m": "foo"}), None) + + def test_tags_instance(self): + self.assertEqual(extract_metric({"e": 0, "v": 1, "m": "foo", "t": 666}), None) + + def test_value_instance(self): + self.assertEqual(extract_metric({"e": 0, "v": 1.1, "m": "foo", "t": []}), None) + + def test_value_instance_float(self): + self.assertEqual(extract_metric({"e": 0, "v": None, "m": "foo", "t": []}), None) + + +class TestLambdaFunctionExtractTracePayload(unittest.TestCase): + def test_extract_trace_payload_none_no_trace(self): + message_json = """{ + "key": "value" + }""" + self.assertEqual(extract_trace_payload({"message": message_json}), None) + + def test_extract_trace_payload_none_exception(self): + message_json = """{ + "invalid_json" + }""" + self.assertEqual(extract_trace_payload({"message": message_json}), None) + + def test_extract_trace_payload_unrelated_datadog_trace(self): + message_json = """{"traces":["I am a trace"]}""" + self.assertEqual(extract_trace_payload({"message": message_json}), None) + + def test_extract_trace_payload_valid_trace(self): + message_json = """{"traces":[[{"trace_id":1234}]]}""" + tags_json = """["key0:value", "key1:value1"]""" + item = { + "message": '{"traces":[[{"trace_id":1234}]]}', + "tags": '["key0:value", "key1:value1"]', + } + self.assertEqual( + extract_trace_payload({"message": message_json, "ddtags": tags_json}), item + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/aws/logs_monitoring/tests/test_transformation.py b/aws/logs_monitoring/tests/test_transformation.py new file mode 100644 index 000000000..0f16a8cd6 --- /dev/null +++ b/aws/logs_monitoring/tests/test_transformation.py @@ -0,0 +1,225 @@ +import unittest +from approvaltests.approvals import verify_as_json +from transformation import ( + separate_security_hub_findings, + parse_aws_waf_logs, +) + + +class TestParseAwsWafLogs(unittest.TestCase): + def test_waf_string_invalid_json(self): + event = "This is not valid JSON." + self.assertEqual(parse_aws_waf_logs(event), "This is not valid JSON.") + + def test_waf_string_json(self): + event = '{"ddsource":"waf","message":"This is a string of JSON"}' + self.assertEqual( + parse_aws_waf_logs(event), + {"ddsource": "waf", "message": "This is a string of JSON"}, + ) + + def test_waf_headers(self): + event = { + "ddsource": "waf", + "message": { + "httpRequest": { + "headers": [ + {"name": "header1", "value": "value1"}, + {"name": "header2", "value": "value2"}, + ] + } + }, + } + verify_as_json(parse_aws_waf_logs(event)) + + def test_waf_non_terminating_matching_rules(self): + event = { + "ddsource": "waf", + "message": { + "nonTerminatingMatchingRules": [ + {"ruleId": "nonterminating1", "action": "COUNT"}, + {"ruleId": "nonterminating2", "action": "COUNT"}, + ] + }, + } + verify_as_json(parse_aws_waf_logs(event)) + + def test_waf_rate_based_rules(self): + event = { + "ddsource": "waf", + "message": { + "rateBasedRuleList": [ + { + "limitValue": "195.154.122.189", + "rateBasedRuleName": "tf-rate-limit-5-min", + "rateBasedRuleId": "arn:aws:wafv2:ap-southeast-2:068133125972_MANAGED:regional/ipset/0f94bd8b-0fa5-4865-81ce-d11a60051fb4_fef50279-8b9a-4062-b733-88ecd1cfd889_IPV4/fef50279-8b9a-4062-b733-88ecd1cfd889", + "maxRateAllowed": 300, + "limitKey": "IP", + }, + { + "limitValue": "195.154.122.189", + "rateBasedRuleName": "no-rate-limit", + "rateBasedRuleId": "arn:aws:wafv2:ap-southeast-2:068133125972_MANAGED:regional/ipset/0f94bd8b-0fa5-4865-81ce-d11a60051fb4_fef50279-8b9a-4062-b733-88ecd1cfd889_IPV4/fef50279-8b9a-4062-b733-88ecd1cfd889", + "maxRateAllowed": 300, + "limitKey": "IP", + }, + ] + }, + } + verify_as_json(parse_aws_waf_logs(event)) + + def test_waf_rule_group_with_excluded_and_nonterminating_rules(self): + event = { + "ddsource": "waf", + "message": { + "ruleGroupList": [ + { + "ruleGroupId": "AWS#AWSManagedRulesSQLiRuleSet", + "terminatingRule": { + "ruleId": "SQLi_QUERYARGUMENTS", + "action": "BLOCK", + }, + "nonTerminatingMatchingRules": [ + { + "exclusionType": "REGULAR", + "ruleId": "first_nonterminating", + }, + { + "exclusionType": "REGULAR", + "ruleId": "second_nonterminating", + }, + ], + "excludedRules": [ + { + "exclusionType": "EXCLUDED_AS_COUNT", + "ruleId": "GenericRFI_BODY", + }, + { + "exclusionType": "EXCLUDED_AS_COUNT", + "ruleId": "second_exclude", + }, + ], + } + ] + }, + } + verify_as_json(parse_aws_waf_logs(event)) + + def test_waf_rule_group_two_rules_same_group_id(self): + event = { + "ddsource": "waf", + "message": { + "ruleGroupList": [ + { + "ruleGroupId": "AWS#AWSManagedRulesSQLiRuleSet", + "terminatingRule": { + "ruleId": "SQLi_QUERYARGUMENTS", + "action": "BLOCK", + }, + }, + { + "ruleGroupId": "AWS#AWSManagedRulesSQLiRuleSet", + "terminatingRule": {"ruleId": "secondRULE", "action": "BLOCK"}, + }, + ] + }, + } + verify_as_json(parse_aws_waf_logs(event)) + + def test_waf_rule_group_three_rules_two_group_ids(self): + event = { + "ddsource": "waf", + "message": { + "ruleGroupList": [ + { + "ruleGroupId": "AWS#AWSManagedRulesSQLiRuleSet", + "terminatingRule": { + "ruleId": "SQLi_QUERYARGUMENTS", + "action": "BLOCK", + }, + }, + { + "ruleGroupId": "AWS#AWSManagedRulesSQLiRuleSet", + "terminatingRule": {"ruleId": "secondRULE", "action": "BLOCK"}, + }, + { + "ruleGroupId": "A_DIFFERENT_ID", + "terminatingRule": {"ruleId": "thirdRULE", "action": "BLOCK"}, + }, + ] + }, + } + verify_as_json(parse_aws_waf_logs(event)) + + +class TestParseSecurityHubEvents(unittest.TestCase): + def test_security_hub_no_findings(self): + event = {"ddsource": "securityhub"} + self.assertEqual( + separate_security_hub_findings(event), + None, + ) + + def test_security_hub_one_finding_no_resources(self): + event = { + "ddsource": "securityhub", + "detail": {"findings": [{"myattribute": "somevalue"}]}, + } + verify_as_json(separate_security_hub_findings(event)) + + def test_security_hub_two_findings_one_resource_each(self): + event = { + "ddsource": "securityhub", + "detail": { + "findings": [ + { + "myattribute": "somevalue", + "Resources": [ + {"Region": "us-east-1", "Type": "AwsEc2SecurityGroup"} + ], + }, + { + "myattribute": "somevalue", + "Resources": [ + {"Region": "us-east-1", "Type": "AwsEc2SecurityGroup"} + ], + }, + ] + }, + } + verify_as_json(separate_security_hub_findings(event)) + + def test_security_hub_multiple_findings_multiple_resources(self): + event = { + "ddsource": "securityhub", + "detail": { + "findings": [ + { + "myattribute": "somevalue", + "Resources": [ + {"Region": "us-east-1", "Type": "AwsEc2SecurityGroup"} + ], + }, + { + "myattribute": "somevalue", + "Resources": [ + {"Region": "us-east-1", "Type": "AwsEc2SecurityGroup"}, + {"Region": "us-east-1", "Type": "AwsOtherSecurityGroup"}, + ], + }, + { + "myattribute": "somevalue", + "Resources": [ + {"Region": "us-east-1", "Type": "AwsEc2SecurityGroup"}, + {"Region": "us-east-1", "Type": "AwsOtherSecurityGroup"}, + {"Region": "us-east-1", "Type": "AwsAnotherSecurityGroup"}, + ], + }, + ] + }, + } + verify_as_json(separate_security_hub_findings(event)) + + +if __name__ == "__main__": + unittest.main() diff --git a/aws/logs_monitoring/transformation.py b/aws/logs_monitoring/transformation.py new file mode 100644 index 000000000..e02874c33 --- /dev/null +++ b/aws/logs_monitoring/transformation.py @@ -0,0 +1,188 @@ +import logging +import json +import copy +import os +from settings import DD_SOURCE + +logger = logging.getLogger() +logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper())) + + +def transform(events): + """Performs transformations on complex events + + Ex: handles special cases with nested arrays of JSON objects + Args: + events (dict[]): the list of event dicts we want to transform + """ + for event in reversed(events): + findings = separate_security_hub_findings(event) + if findings: + events.remove(event) + events.extend(findings) + + waf = parse_aws_waf_logs(event) + if waf != event: + events.remove(event) + events.append(waf) + return events + + +def separate_security_hub_findings(event): + """Replace Security Hub event with series of events based on findings + + Each event should contain one finding only. + This prevents having an unparsable array of objects in the final log. + """ + if event.get(DD_SOURCE) != "securityhub" or not event.get("detail", {}).get( + "findings" + ): + return None + events = [] + event_copy = copy.deepcopy(event) + # Copy findings before separating + findings = event_copy.get("detail", {}).get("findings") + if findings: + # Remove findings from the original event once we have a copy + del event_copy["detail"]["findings"] + # For each finding create a separate log event + for index, item in enumerate(findings): + # Copy the original event with source and other metadata + new_event = copy.deepcopy(event_copy) + current_finding = findings[index] + # Get the resources array from the current finding + resources = current_finding.get("Resources", {}) + new_event["detail"]["finding"] = current_finding + new_event["detail"]["finding"]["resources"] = {} + # Separate objects in resources array into distinct attributes + if resources: + # Remove from current finding once we have a copy + del current_finding["Resources"] + for item in resources: + current_resource = item + # Capture the type and use it as the distinguishing key + resource_type = current_resource.get("Type", {}) + del current_resource["Type"] + new_event["detail"]["finding"]["resources"][ + resource_type + ] = current_resource + events.append(new_event) + return events + + +def parse_aws_waf_logs(event): + """Parse out complex arrays of objects in AWS WAF logs + + Attributes to convert: + httpRequest.headers + nonTerminatingMatchingRules + rateBasedRuleList + ruleGroupList + + This prevents having an unparsable array of objects in the final log. + """ + if isinstance(event, str): + try: + event = json.loads(event) + except json.JSONDecodeError: + logger.debug("Argument provided for waf parser is not valid JSON") + return event + if event.get(DD_SOURCE) != "waf": + return event + + event_copy = copy.deepcopy(event) + + message = event_copy.get("message", {}) + if isinstance(message, str): + try: + message = json.loads(message) + except json.JSONDecodeError: + logger.debug("Failed to decode waf message") + return event + + headers = message.get("httpRequest", {}).get("headers") + if headers: + message["httpRequest"]["headers"] = convert_rule_to_nested_json(headers) + + # Iterate through rules in ruleGroupList and nest them under the group id + # ruleGroupList has three attributes that need to be handled separately + rule_groups = message.get("ruleGroupList", {}) + if rule_groups and isinstance(rule_groups, list): + message["ruleGroupList"] = {} + for rule_group in rule_groups: + group_id = None + if "ruleGroupId" in rule_group and rule_group["ruleGroupId"]: + group_id = rule_group.pop("ruleGroupId", None) + if group_id not in message["ruleGroupList"]: + message["ruleGroupList"][group_id] = {} + + # Extract the terminating rule and nest it under its own id + if "terminatingRule" in rule_group and rule_group["terminatingRule"]: + terminating_rule = rule_group.pop("terminatingRule", None) + if not "terminatingRule" in message["ruleGroupList"][group_id]: + message["ruleGroupList"][group_id]["terminatingRule"] = {} + message["ruleGroupList"][group_id]["terminatingRule"].update( + convert_rule_to_nested_json(terminating_rule) + ) + + # Iterate through array of non-terminating rules and nest each under its own id + if "nonTerminatingMatchingRules" in rule_group and isinstance( + rule_group["nonTerminatingMatchingRules"], list + ): + non_terminating_rules = rule_group.pop( + "nonTerminatingMatchingRules", None + ) + if ( + "nonTerminatingMatchingRules" + not in message["ruleGroupList"][group_id] + ): + message["ruleGroupList"][group_id][ + "nonTerminatingMatchingRules" + ] = {} + message["ruleGroupList"][group_id][ + "nonTerminatingMatchingRules" + ].update(convert_rule_to_nested_json(non_terminating_rules)) + + # Iterate through array of excluded rules and nest each under its own id + if "excludedRules" in rule_group and isinstance( + rule_group["excludedRules"], list + ): + excluded_rules = rule_group.pop("excludedRules", None) + if "excludedRules" not in message["ruleGroupList"][group_id]: + message["ruleGroupList"][group_id]["excludedRules"] = {} + message["ruleGroupList"][group_id]["excludedRules"].update( + convert_rule_to_nested_json(excluded_rules) + ) + + rate_based_rules = message.get("rateBasedRuleList", {}) + if rate_based_rules: + message["rateBasedRuleList"] = convert_rule_to_nested_json(rate_based_rules) + + non_terminating_rules = message.get("nonTerminatingMatchingRules", {}) + if non_terminating_rules: + message["nonTerminatingMatchingRules"] = convert_rule_to_nested_json( + non_terminating_rules + ) + + event_copy["message"] = message + return event_copy + + +def convert_rule_to_nested_json(rule): + key = None + result_obj = {} + if not isinstance(rule, list): + if "ruleId" in rule and rule["ruleId"]: + key = rule.pop("ruleId", None) + result_obj.update({key: rule}) + return result_obj + for entry in rule: + if "ruleId" in entry and entry["ruleId"]: + key = entry.pop("ruleId", None) + elif "rateBasedRuleName" in entry and entry["rateBasedRuleName"]: + key = entry.pop("rateBasedRuleName", None) + elif "name" in entry and "value" in entry: + key = entry["name"] + entry = entry["value"] + result_obj.update({key: entry}) + return result_obj From 6f9efe1cc9d19de5610f246c3afdfb036c27e2fd Mon Sep 17 00:00:00 2001 From: Georgi Date: Mon, 26 Feb 2024 15:26:04 +0100 Subject: [PATCH 2/8] Package caching --- aws/logs_monitoring/caching/__init__.py | 0 .../{ => caching}/base_tags_cache.py | 0 .../cloudwatch_log_group_cache.py | 2 +- .../{ => caching}/lambda_cache.py | 2 +- .../{ => caching}/step_functions_cache.py | 2 +- .../enhanced_lambda_metrics.py | 2 +- aws/logs_monitoring/parsing.py | 4 +- .../tests/test_cloudtrail_s3.py | 2 +- .../tests/test_enhanced_lambda_metrics.py | 54 +++++++++---------- .../tests/test_lambda_function.py | 12 ++--- aws/logs_monitoring/tests/test_parsing.py | 4 +- 11 files changed, 42 insertions(+), 42 deletions(-) create mode 100644 aws/logs_monitoring/caching/__init__.py rename aws/logs_monitoring/{ => caching}/base_tags_cache.py (100%) rename aws/logs_monitoring/{ => caching}/cloudwatch_log_group_cache.py (98%) rename aws/logs_monitoring/{ => caching}/lambda_cache.py (98%) rename aws/logs_monitoring/{ => caching}/step_functions_cache.py (99%) diff --git a/aws/logs_monitoring/caching/__init__.py b/aws/logs_monitoring/caching/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/aws/logs_monitoring/base_tags_cache.py b/aws/logs_monitoring/caching/base_tags_cache.py similarity index 100% rename from aws/logs_monitoring/base_tags_cache.py rename to aws/logs_monitoring/caching/base_tags_cache.py diff --git a/aws/logs_monitoring/cloudwatch_log_group_cache.py b/aws/logs_monitoring/caching/cloudwatch_log_group_cache.py similarity index 98% rename from aws/logs_monitoring/cloudwatch_log_group_cache.py rename to aws/logs_monitoring/caching/cloudwatch_log_group_cache.py index 5949f079f..c503bfac9 100644 --- a/aws/logs_monitoring/cloudwatch_log_group_cache.py +++ b/aws/logs_monitoring/caching/cloudwatch_log_group_cache.py @@ -1,6 +1,6 @@ import boto3 -from base_tags_cache import ( +from caching.base_tags_cache import ( BaseTagsCache, logger, sanitize_aws_tag_string, diff --git a/aws/logs_monitoring/lambda_cache.py b/aws/logs_monitoring/caching/lambda_cache.py similarity index 98% rename from aws/logs_monitoring/lambda_cache.py rename to aws/logs_monitoring/caching/lambda_cache.py index d0bbf7cc7..7f444658c 100644 --- a/aws/logs_monitoring/lambda_cache.py +++ b/aws/logs_monitoring/caching/lambda_cache.py @@ -1,6 +1,6 @@ from botocore.exceptions import ClientError -from base_tags_cache import ( +from caching.base_tags_cache import ( GET_RESOURCES_LAMBDA_FILTER, BaseTagsCache, logger, diff --git a/aws/logs_monitoring/step_functions_cache.py b/aws/logs_monitoring/caching/step_functions_cache.py similarity index 99% rename from aws/logs_monitoring/step_functions_cache.py rename to aws/logs_monitoring/caching/step_functions_cache.py index c8560625b..8160df9d6 100644 --- a/aws/logs_monitoring/step_functions_cache.py +++ b/aws/logs_monitoring/caching/step_functions_cache.py @@ -1,6 +1,6 @@ from botocore.exceptions import ClientError -from base_tags_cache import ( +from caching.base_tags_cache import ( BaseTagsCache, logger, parse_get_resources_response_for_tags_by_arn, diff --git a/aws/logs_monitoring/enhanced_lambda_metrics.py b/aws/logs_monitoring/enhanced_lambda_metrics.py index 2521d074c..6d6a9ca3c 100644 --- a/aws/logs_monitoring/enhanced_lambda_metrics.py +++ b/aws/logs_monitoring/enhanced_lambda_metrics.py @@ -7,7 +7,7 @@ import re import datetime from time import time -from lambda_cache import LambdaTagsCache +from caching.lambda_cache import LambdaTagsCache ENHANCED_METRICS_NAMESPACE_PREFIX = "aws.lambda.enhanced" diff --git a/aws/logs_monitoring/parsing.py b/aws/logs_monitoring/parsing.py index b2724309c..ec29a5218 100644 --- a/aws/logs_monitoring/parsing.py +++ b/aws/logs_monitoring/parsing.py @@ -19,8 +19,8 @@ get_lambda_function_name_from_logstream_name, is_lambda_customized_log_group, ) -from step_functions_cache import StepFunctionsTagsCache -from cloudwatch_log_group_cache import CloudwatchLogGroupTagsCache +from caching.step_functions_cache import StepFunctionsTagsCache +from caching.cloudwatch_log_group_cache import CloudwatchLogGroupTagsCache from telemetry import ( DD_FORWARDER_TELEMETRY_NAMESPACE_PREFIX, get_forwarder_telemetry_tags, diff --git a/aws/logs_monitoring/tests/test_cloudtrail_s3.py b/aws/logs_monitoring/tests/test_cloudtrail_s3.py index 213a0d811..12179d38d 100644 --- a/aws/logs_monitoring/tests/test_cloudtrail_s3.py +++ b/aws/logs_monitoring/tests/test_cloudtrail_s3.py @@ -98,7 +98,7 @@ class TestS3CloudwatchParsing(unittest.TestCase): def setUp(self): self.maxDiff = 9000 - @patch("base_tags_cache.boto3") + @patch("caching.base_tags_cache.boto3") @patch("parsing.boto3") @patch("lambda_function.boto3") def test_s3_cloudtrail_pasing_and_enrichment( diff --git a/aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py b/aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py index f47eb15f0..b762795c7 100644 --- a/aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py +++ b/aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py @@ -13,12 +13,12 @@ create_out_of_memory_enhanced_metric, ) -from base_tags_cache import ( +from caching.base_tags_cache import ( sanitize_aws_tag_string, parse_get_resources_response_for_tags_by_arn, get_dd_tag_string_from_aws_dict, ) -from lambda_cache import LambdaTagsCache +from caching.lambda_cache import LambdaTagsCache class TestEnhancedLambdaMetrics(unittest.TestCase): @@ -313,8 +313,8 @@ def test_create_out_of_memory_enhanced_metric(self): success_message = "Success!" self.assertEqual(len(create_out_of_memory_enhanced_metric(success_message)), 0) - @patch("base_tags_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics( self, mock_get_s3_cache, mock_forward_metrics ): @@ -411,8 +411,8 @@ def test_generate_enhanced_lambda_metrics( del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("base_tags_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics_with_tags( self, mock_get_s3_cache, mock_forward_metrics ): @@ -524,8 +524,8 @@ def test_generate_enhanced_lambda_metrics_with_tags( del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("base_tags_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics_once_with_missing_arn( self, mock_get_s3_cache, mock_forward_metrics ): @@ -560,8 +560,8 @@ def test_generate_enhanced_lambda_metrics_once_with_missing_arn( del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("base_tags_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics_refresh_on_new_arn( self, mock_get_s3_cache, mock_forward_metrics ): @@ -603,12 +603,12 @@ def test_generate_enhanced_lambda_metrics_refresh_on_new_arn( del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("lambda_cache.LambdaTagsCache.release_s3_cache_lock") - @patch("lambda_cache.LambdaTagsCache.acquire_s3_cache_lock") - @patch("lambda_cache.LambdaTagsCache.write_cache_to_s3") - @patch("lambda_cache.LambdaTagsCache.build_tags_cache") - @patch("lambda_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.lambda_cache.LambdaTagsCache.release_s3_cache_lock") + @patch("caching.lambda_cache.LambdaTagsCache.acquire_s3_cache_lock") + @patch("caching.lambda_cache.LambdaTagsCache.write_cache_to_s3") + @patch("caching.lambda_cache.LambdaTagsCache.build_tags_cache") + @patch("caching.lambda_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics_refresh_s3_cache( self, mock_get_s3_cache, @@ -664,13 +664,13 @@ def test_generate_enhanced_lambda_metrics_refresh_s3_cache( del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("lambda_cache.LambdaTagsCache.release_s3_cache_lock") - @patch("lambda_cache.LambdaTagsCache.acquire_s3_cache_lock") - @patch("lambda_cache.resource_tagging_client") - @patch("lambda_cache.LambdaTagsCache.write_cache_to_s3") - @patch("lambda_cache.parse_get_resources_response_for_tags_by_arn") - @patch("lambda_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.lambda_cache.LambdaTagsCache.release_s3_cache_lock") + @patch("caching.lambda_cache.LambdaTagsCache.acquire_s3_cache_lock") + @patch("caching.lambda_cache.resource_tagging_client") + @patch("caching.lambda_cache.LambdaTagsCache.write_cache_to_s3") + @patch("caching.lambda_cache.parse_get_resources_response_for_tags_by_arn") + @patch("caching.lambda_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics_client_error( self, mock_get_s3_cache, @@ -724,8 +724,8 @@ def test_generate_enhanced_lambda_metrics_client_error( del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("base_tags_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics_timeout( self, mock_get_s3_cache, mock_forward_metrics ): @@ -782,8 +782,8 @@ def test_generate_enhanced_lambda_metrics_timeout( ) del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("base_tags_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics_out_of_memory( self, mock_get_s3_cache, mock_forward_metrics ): diff --git a/aws/logs_monitoring/tests/test_lambda_function.py b/aws/logs_monitoring/tests/test_lambda_function.py index ebc24fab4..cf36eefdf 100644 --- a/aws/logs_monitoring/tests/test_lambda_function.py +++ b/aws/logs_monitoring/tests/test_lambda_function.py @@ -138,7 +138,7 @@ def test_datadog_forwarder(self, mock_get_s3_cache): del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") + @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") def test_setting_service_tag_from_log_group_cache(self, cw_logs_tags_get): reload(sys.modules["settings"]) reload(sys.modules["parsing"]) @@ -159,7 +159,7 @@ def test_setting_service_tag_from_log_group_cache(self, cw_logs_tags_get): self.assertEqual(log["service"], "log_group_service") @patch.dict(os.environ, {"DD_TAGS": "service:dd_tag_service"}, clear=True) - @patch("cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") + @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") def test_service_override_from_dd_tags(self, cw_logs_tags_get): reload(sys.modules["settings"]) reload(sys.modules["parsing"]) @@ -179,8 +179,8 @@ def test_service_override_from_dd_tags(self, cw_logs_tags_get): for log in logs: self.assertEqual(log["service"], "dd_tag_service") - @patch("lambda_cache.LambdaTagsCache.get") - @patch("cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") + @patch("caching.lambda_cache.LambdaTagsCache.get") + @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") def test_overrding_service_tag_from_lambda_cache( self, lambda_tags_get, cw_logs_tags_get ): @@ -203,8 +203,8 @@ def test_overrding_service_tag_from_lambda_cache( self.assertEqual(log["service"], "lambda_service") @patch.dict(os.environ, {"DD_TAGS": "service:dd_tag_service"}, clear=True) - @patch("lambda_cache.LambdaTagsCache.get") - @patch("cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") + @patch("caching.lambda_cache.LambdaTagsCache.get") + @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") def test_overrding_service_tag_from_lambda_cache_when_dd_tags_is_set( self, lambda_tags_get, cw_logs_tags_get ): diff --git a/aws/logs_monitoring/tests/test_parsing.py b/aws/logs_monitoring/tests/test_parsing.py index 8f4d224e8..3ceb1c102 100644 --- a/aws/logs_monitoring/tests/test_parsing.py +++ b/aws/logs_monitoring/tests/test_parsing.py @@ -400,7 +400,7 @@ class TestAWSLogsHandler(unittest.TestCase): @patch("parsing.CloudwatchLogGroupTagsCache.release_s3_cache_lock") @patch("parsing.CloudwatchLogGroupTagsCache.acquire_s3_cache_lock") @patch("parsing.CloudwatchLogGroupTagsCache.write_cache_to_s3") - @patch("base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") @patch("parsing.CloudwatchLogGroupTagsCache.get_cache_from_s3") def test_awslogs_handler_rds_postgresql( self, @@ -456,7 +456,7 @@ def test_awslogs_handler_rds_postgresql( @patch("parsing.StepFunctionsTagsCache.release_s3_cache_lock") @patch("parsing.StepFunctionsTagsCache.acquire_s3_cache_lock") @patch("parsing.StepFunctionsTagsCache.write_cache_to_s3") - @patch("base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") @patch("parsing.StepFunctionsTagsCache.get_cache_from_s3") def test_awslogs_handler_step_functions_tags_added_properly( self, From ebf2fa96d907d647d72a06851195fcfa0acf3978 Mon Sep 17 00:00:00 2001 From: Georgi Date: Mon, 26 Feb 2024 16:41:41 +0100 Subject: [PATCH 3/8] Package steps --- aws/logs_monitoring/caching/__init__.py | 0 aws/logs_monitoring/forwarder.py | 4 ++-- aws/logs_monitoring/lambda_function.py | 8 +++---- aws/logs_monitoring/{ => logs}/exceptions.py | 0 aws/logs_monitoring/{ => logs}/logs.py | 4 ++-- .../{ => logs}/logs_helpers.py | 2 +- aws/logs_monitoring/setup.py | 1 + aws/logs_monitoring/{ => steps}/enrichment.py | 0 aws/logs_monitoring/{ => steps}/parsing.py | 0 aws/logs_monitoring/{ => steps}/splitting.py | 0 .../{ => steps}/transformation.py | 0 .../tests/test_cloudtrail_s3.py | 4 ++-- aws/logs_monitoring/tests/test_enrichment.py | 2 +- .../tests/test_lambda_function.py | 12 +++++----- aws/logs_monitoring/tests/test_logs.py | 4 ++-- aws/logs_monitoring/tests/test_parsing.py | 24 +++++++++---------- aws/logs_monitoring/tests/test_splitting.py | 2 +- .../tests/test_transformation.py | 2 +- 18 files changed, 35 insertions(+), 34 deletions(-) delete mode 100644 aws/logs_monitoring/caching/__init__.py rename aws/logs_monitoring/{ => logs}/exceptions.py (100%) rename aws/logs_monitoring/{ => logs}/logs.py (98%) rename aws/logs_monitoring/{ => logs}/logs_helpers.py (98%) rename aws/logs_monitoring/{ => steps}/enrichment.py (100%) rename aws/logs_monitoring/{ => steps}/parsing.py (100%) rename aws/logs_monitoring/{ => steps}/splitting.py (100%) rename aws/logs_monitoring/{ => steps}/transformation.py (100%) diff --git a/aws/logs_monitoring/caching/__init__.py b/aws/logs_monitoring/caching/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/aws/logs_monitoring/forwarder.py b/aws/logs_monitoring/forwarder.py index f4f5ab681..dc5980876 100644 --- a/aws/logs_monitoring/forwarder.py +++ b/aws/logs_monitoring/forwarder.py @@ -8,14 +8,14 @@ ) from datadog_lambda.metric import lambda_stats from trace_forwarder.connection import TraceConnection -from logs import ( +from logs.logs import ( DatadogScrubber, DatadogBatcher, DatadogClient, DatadogHTTPClient, DatadogTCPClient, ) -from logs_helpers import filter_logs +from logs.logs_helpers import filter_logs from settings import ( DD_API_KEY, DD_USE_TCP, diff --git a/aws/logs_monitoring/lambda_function.py b/aws/logs_monitoring/lambda_function.py index da5530a9d..780715958 100644 --- a/aws/logs_monitoring/lambda_function.py +++ b/aws/logs_monitoring/lambda_function.py @@ -11,10 +11,10 @@ from datadog_lambda.wrapper import datadog_lambda_wrapper from datadog import api from enhanced_lambda_metrics import parse_and_submit_enhanced_metrics -from parsing import parse -from enrichment import enrich -from transformation import transform -from splitting import split +from steps.parsing import parse +from steps.enrichment import enrich +from steps.transformation import transform +from steps.splitting import split from forwarder import ( forward_metrics, forward_traces, diff --git a/aws/logs_monitoring/exceptions.py b/aws/logs_monitoring/logs/exceptions.py similarity index 100% rename from aws/logs_monitoring/exceptions.py rename to aws/logs_monitoring/logs/exceptions.py diff --git a/aws/logs_monitoring/logs.py b/aws/logs_monitoring/logs/logs.py similarity index 98% rename from aws/logs_monitoring/logs.py rename to aws/logs_monitoring/logs/logs.py index 23f7492d1..77e2d8cac 100644 --- a/aws/logs_monitoring/logs.py +++ b/aws/logs_monitoring/logs/logs.py @@ -11,8 +11,8 @@ import time from concurrent.futures import as_completed from requests_futures.sessions import FuturesSession -from logs_helpers import compress_logs, compileRegex -from exceptions import RetriableException, ScrubbingException +from logs.logs_helpers import compress_logs, compileRegex +from logs.exceptions import RetriableException, ScrubbingException from settings import ( DD_USE_COMPRESSION, diff --git a/aws/logs_monitoring/logs_helpers.py b/aws/logs_monitoring/logs/logs_helpers.py similarity index 98% rename from aws/logs_monitoring/logs_helpers.py rename to aws/logs_monitoring/logs/logs_helpers.py index 43044afbf..f3b5c24ec 100644 --- a/aws/logs_monitoring/logs_helpers.py +++ b/aws/logs_monitoring/logs/logs_helpers.py @@ -2,7 +2,7 @@ import re import gzip import os -from exceptions import ScrubbingException +from logs.exceptions import ScrubbingException logger = logging.getLogger() logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper())) diff --git a/aws/logs_monitoring/setup.py b/aws/logs_monitoring/setup.py index e6ce2f18b..9ccaba65f 100644 --- a/aws/logs_monitoring/setup.py +++ b/aws/logs_monitoring/setup.py @@ -24,4 +24,5 @@ extras_require={ "dev": ["nose2==0.9.1", "flake8==3.7.9", "requests==2.22.0", "boto3==1.10.33"] }, + py_modules=[], ) diff --git a/aws/logs_monitoring/enrichment.py b/aws/logs_monitoring/steps/enrichment.py similarity index 100% rename from aws/logs_monitoring/enrichment.py rename to aws/logs_monitoring/steps/enrichment.py diff --git a/aws/logs_monitoring/parsing.py b/aws/logs_monitoring/steps/parsing.py similarity index 100% rename from aws/logs_monitoring/parsing.py rename to aws/logs_monitoring/steps/parsing.py diff --git a/aws/logs_monitoring/splitting.py b/aws/logs_monitoring/steps/splitting.py similarity index 100% rename from aws/logs_monitoring/splitting.py rename to aws/logs_monitoring/steps/splitting.py diff --git a/aws/logs_monitoring/transformation.py b/aws/logs_monitoring/steps/transformation.py similarity index 100% rename from aws/logs_monitoring/transformation.py rename to aws/logs_monitoring/steps/transformation.py diff --git a/aws/logs_monitoring/tests/test_cloudtrail_s3.py b/aws/logs_monitoring/tests/test_cloudtrail_s3.py index 12179d38d..c90421ebf 100644 --- a/aws/logs_monitoring/tests/test_cloudtrail_s3.py +++ b/aws/logs_monitoring/tests/test_cloudtrail_s3.py @@ -24,7 +24,7 @@ env_patch.start() import lambda_function -from parsing import parse +from steps.parsing import parse env_patch.stop() @@ -99,7 +99,7 @@ def setUp(self): self.maxDiff = 9000 @patch("caching.base_tags_cache.boto3") - @patch("parsing.boto3") + @patch("steps.parsing.boto3") @patch("lambda_function.boto3") def test_s3_cloudtrail_pasing_and_enrichment( self, lambda_boto3, parsing_boto3, cache_boto3 diff --git a/aws/logs_monitoring/tests/test_enrichment.py b/aws/logs_monitoring/tests/test_enrichment.py index 783d4767c..8f73902e3 100644 --- a/aws/logs_monitoring/tests/test_enrichment.py +++ b/aws/logs_monitoring/tests/test_enrichment.py @@ -1,6 +1,6 @@ import unittest import json -from enrichment import ( +from steps.enrichment import ( extract_host_from_cloudtrails, extract_host_from_guardduty, extract_host_from_route53, diff --git a/aws/logs_monitoring/tests/test_lambda_function.py b/aws/logs_monitoring/tests/test_lambda_function.py index cf36eefdf..e8da0476c 100644 --- a/aws/logs_monitoring/tests/test_lambda_function.py +++ b/aws/logs_monitoring/tests/test_lambda_function.py @@ -27,10 +27,10 @@ ) env_patch.start() from lambda_function import invoke_additional_target_lambdas -from enrichment import enrich -from transformation import transform -from splitting import split -from parsing import parse, parse_event_type +from steps.enrichment import enrich +from steps.transformation import transform +from steps.splitting import split +from steps.parsing import parse, parse_event_type env_patch.stop() @@ -141,7 +141,7 @@ def test_datadog_forwarder(self, mock_get_s3_cache): @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") def test_setting_service_tag_from_log_group_cache(self, cw_logs_tags_get): reload(sys.modules["settings"]) - reload(sys.modules["parsing"]) + reload(sys.modules["steps.parsing"]) cw_logs_tags_get.return_value = ["service:log_group_service"] context = Context() input_data = self._get_input_data() @@ -162,7 +162,7 @@ def test_setting_service_tag_from_log_group_cache(self, cw_logs_tags_get): @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") def test_service_override_from_dd_tags(self, cw_logs_tags_get): reload(sys.modules["settings"]) - reload(sys.modules["parsing"]) + reload(sys.modules["steps.parsing"]) cw_logs_tags_get.return_value = ["service:log_group_service"] context = Context() input_data = self._get_input_data() diff --git a/aws/logs_monitoring/tests/test_logs.py b/aws/logs_monitoring/tests/test_logs.py index 635c0ea8b..2a6c4dd2e 100644 --- a/aws/logs_monitoring/tests/test_logs.py +++ b/aws/logs_monitoring/tests/test_logs.py @@ -1,8 +1,8 @@ import unittest import os -from logs import DatadogScrubber -from logs_helpers import filter_logs +from logs.logs import DatadogScrubber +from logs.logs_helpers import filter_logs from settings import ScrubbingRuleConfig, SCRUBBING_RULE_CONFIGS, get_env_var diff --git a/aws/logs_monitoring/tests/test_parsing.py b/aws/logs_monitoring/tests/test_parsing.py index 3ceb1c102..2b451e19b 100644 --- a/aws/logs_monitoring/tests/test_parsing.py +++ b/aws/logs_monitoring/tests/test_parsing.py @@ -23,7 +23,7 @@ }, ) env_patch.start() -from parsing import ( +from steps.parsing import ( awslogs_handler, parse_event_source, parse_service_arn, @@ -396,12 +396,12 @@ def test_elb_s3_key_multi_prefix_gov(self): class TestAWSLogsHandler(unittest.TestCase): - @patch("parsing.CloudwatchLogGroupTagsCache.get") - @patch("parsing.CloudwatchLogGroupTagsCache.release_s3_cache_lock") - @patch("parsing.CloudwatchLogGroupTagsCache.acquire_s3_cache_lock") - @patch("parsing.CloudwatchLogGroupTagsCache.write_cache_to_s3") + @patch("steps.parsing.CloudwatchLogGroupTagsCache.get") + @patch("steps.parsing.CloudwatchLogGroupTagsCache.release_s3_cache_lock") + @patch("steps.parsing.CloudwatchLogGroupTagsCache.acquire_s3_cache_lock") + @patch("steps.parsing.CloudwatchLogGroupTagsCache.write_cache_to_s3") @patch("caching.base_tags_cache.send_forwarder_internal_metrics") - @patch("parsing.CloudwatchLogGroupTagsCache.get_cache_from_s3") + @patch("steps.parsing.CloudwatchLogGroupTagsCache.get_cache_from_s3") def test_awslogs_handler_rds_postgresql( self, mock_get_s3_cache, @@ -451,13 +451,13 @@ def test_awslogs_handler_rds_postgresql( verify_as_json(list(awslogs_handler(event, context, metadata))) verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) - @patch("parsing.CloudwatchLogGroupTagsCache.get") - @patch("parsing.StepFunctionsTagsCache.get") - @patch("parsing.StepFunctionsTagsCache.release_s3_cache_lock") - @patch("parsing.StepFunctionsTagsCache.acquire_s3_cache_lock") - @patch("parsing.StepFunctionsTagsCache.write_cache_to_s3") + @patch("steps.parsing.CloudwatchLogGroupTagsCache.get") + @patch("steps.parsing.StepFunctionsTagsCache.get") + @patch("steps.parsing.StepFunctionsTagsCache.release_s3_cache_lock") + @patch("steps.parsing.StepFunctionsTagsCache.acquire_s3_cache_lock") + @patch("steps.parsing.StepFunctionsTagsCache.write_cache_to_s3") @patch("caching.base_tags_cache.send_forwarder_internal_metrics") - @patch("parsing.StepFunctionsTagsCache.get_cache_from_s3") + @patch("steps.parsing.StepFunctionsTagsCache.get_cache_from_s3") def test_awslogs_handler_step_functions_tags_added_properly( self, mock_get_s3_cache, diff --git a/aws/logs_monitoring/tests/test_splitting.py b/aws/logs_monitoring/tests/test_splitting.py index 1fa85365d..0ba747c3e 100644 --- a/aws/logs_monitoring/tests/test_splitting.py +++ b/aws/logs_monitoring/tests/test_splitting.py @@ -1,5 +1,5 @@ import unittest -from splitting import ( +from steps.splitting import ( extract_metric, extract_trace_payload, ) diff --git a/aws/logs_monitoring/tests/test_transformation.py b/aws/logs_monitoring/tests/test_transformation.py index 0f16a8cd6..1dc517aaa 100644 --- a/aws/logs_monitoring/tests/test_transformation.py +++ b/aws/logs_monitoring/tests/test_transformation.py @@ -1,6 +1,6 @@ import unittest from approvaltests.approvals import verify_as_json -from transformation import ( +from steps.transformation import ( separate_security_hub_findings, parse_aws_waf_logs, ) From 6746b17b4c8eccc1dc8f96d1211475a08657e935 Mon Sep 17 00:00:00 2001 From: Georgi Date: Tue, 27 Feb 2024 13:54:29 +0100 Subject: [PATCH 4/8] break down parsing --- .../{forwarder.py => forwarders.py} | 0 aws/logs_monitoring/lambda_function.py | 2 +- aws/logs_monitoring/steps/common.py | 188 ++++++ .../steps/handlers/awslogs_handler.py | 214 +++++++ .../steps/handlers/s3_handler.py | 219 +++++++ aws/logs_monitoring/steps/parsing.py | 585 +----------------- aws/logs_monitoring/tests/run_unit_tests.sh | 2 +- .../tests/test_awslogs_handler.py | 218 +++++++ .../tests/test_cloudtrail_s3.py | 2 +- .../tests/test_lambda_function.py | 12 +- aws/logs_monitoring/tests/test_parsing.py | 323 +--------- aws/logs_monitoring/tests/test_s3_handler.py | 118 ++++ 12 files changed, 976 insertions(+), 907 deletions(-) rename aws/logs_monitoring/{forwarder.py => forwarders.py} (100%) create mode 100644 aws/logs_monitoring/steps/common.py create mode 100644 aws/logs_monitoring/steps/handlers/awslogs_handler.py create mode 100644 aws/logs_monitoring/steps/handlers/s3_handler.py create mode 100644 aws/logs_monitoring/tests/test_awslogs_handler.py create mode 100644 aws/logs_monitoring/tests/test_s3_handler.py diff --git a/aws/logs_monitoring/forwarder.py b/aws/logs_monitoring/forwarders.py similarity index 100% rename from aws/logs_monitoring/forwarder.py rename to aws/logs_monitoring/forwarders.py diff --git a/aws/logs_monitoring/lambda_function.py b/aws/logs_monitoring/lambda_function.py index 780715958..b0c68ac18 100644 --- a/aws/logs_monitoring/lambda_function.py +++ b/aws/logs_monitoring/lambda_function.py @@ -15,7 +15,7 @@ from steps.enrichment import enrich from steps.transformation import transform from steps.splitting import split -from forwarder import ( +from forwarders import ( forward_metrics, forward_traces, forward_logs, diff --git a/aws/logs_monitoring/steps/common.py b/aws/logs_monitoring/steps/common.py new file mode 100644 index 000000000..b93dac19c --- /dev/null +++ b/aws/logs_monitoring/steps/common.py @@ -0,0 +1,188 @@ +import re + +from settings import ( + DD_SOURCE, + DD_CUSTOM_TAGS, +) + +CLOUDTRAIL_REGEX = re.compile( + "\d+_CloudTrail(|-Digest)_\w{2}(|-gov|-cn)-\w{4,9}-\d_(|.+)\d{8}T\d{4,6}Z(|.+).json.gz$", + re.I, +) + + +def parse_event_source(event, key): + """Parse out the source that will be assigned to the log in Datadog + Args: + event (dict): The AWS-formatted log event that the forwarder was triggered with + key (string): The S3 object key if the event is from S3 or the CW Log Group if the event is from CW Logs + """ + lowercase_key = str(key).lower() + + # Determines if the key matches any known sources for Cloudwatch logs + if "awslogs" in event: + return find_cloudwatch_source(lowercase_key) + + # Determines if the key matches any known sources for S3 logs + if "Records" in event and len(event["Records"]) > 0: + if "s3" in event["Records"][0]: + if is_cloudtrail(str(key)): + return "cloudtrail" + + return find_s3_source(lowercase_key) + + return "aws" + + +def find_cloudwatch_source(log_group): + # e.g. /aws/rds/instance/my-mariadb/error + if log_group.startswith("/aws/rds"): + for engine in ["mariadb", "mysql", "postgresql"]: + if engine in log_group: + return engine + return "rds" + + if log_group.startswith( + ( + # default location for rest api execution logs + "api-gateway", # e.g. Api-Gateway-Execution-Logs_xxxxxx/dev + # default location set by serverless framework for rest api access logs + "/aws/api-gateway", # e.g. /aws/api-gateway/my-project + # default location set by serverless framework for http api logs + "/aws/http-api", # e.g. /aws/http-api/my-project + ) + ): + return "apigateway" + + if log_group.startswith("/aws/vendedlogs/states"): + return "stepfunction" + + # e.g. dms-tasks-test-instance + if log_group.startswith("dms-tasks"): + return "dms" + + # e.g. sns/us-east-1/123456779121/SnsTopicX + if log_group.startswith("sns/"): + return "sns" + + # e.g. /aws/fsx/windows/xxx + if log_group.startswith("/aws/fsx/windows"): + return "aws.fsx" + + if log_group.startswith("/aws/appsync/"): + return "appsync" + + for source in [ + "/aws/lambda", # e.g. /aws/lambda/helloDatadog + "/aws/codebuild", # e.g. /aws/codebuild/my-project + "/aws/kinesis", # e.g. /aws/kinesisfirehose/dev + "/aws/docdb", # e.g. /aws/docdb/yourClusterName/profile + "/aws/eks", # e.g. /aws/eks/yourClusterName/profile + ]: + if log_group.startswith(source): + return source.replace("/aws/", "") + + # the below substrings must be in your log group to be detected + for source in [ + "network-firewall", + "route53", + "vpc", + "fargate", + "cloudtrail", + "msk", + "elasticsearch", + "transitgateway", + "verified-access", + "bedrock", + ]: + if source in log_group: + return source + + return "cloudwatch" + + +def find_s3_source(key): + # e.g. AWSLogs/123456779121/elasticloadbalancing/us-east-1/2020/10/02/123456779121_elasticloadbalancing_us-east-1_app.alb.xxxxx.xx.xxx.xxx_x.log.gz + if "elasticloadbalancing" in key: + return "elb" + + # e.g. AWSLogs/123456779121/vpcflowlogs/us-east-1/2020/10/02/123456779121_vpcflowlogs_us-east-1_fl-xxxxx.log.gz + if "vpcflowlogs" in key: + return "vpc" + + # e.g. AWSLogs/123456779121/vpcdnsquerylogs/vpc-********/2021/05/11/vpc-********_vpcdnsquerylogs_********_20210511T0910Z_71584702.log.gz + if "vpcdnsquerylogs" in key: + return "route53" + + # e.g. 2020/10/02/21/aws-waf-logs-testing-1-2020-10-02-21-25-30-x123x-x456x or AWSLogs/123456779121/WAFLogs/us-east-1/xxxxxx-waf/2022/10/11/14/10/123456779121_waflogs_us-east-1_xxxxx-waf_20221011T1410Z_12756524.log.gz + if "aws-waf-logs" in key or "waflogs" in key: + return "waf" + + # e.g. AWSLogs/123456779121/redshift/us-east-1/2020/10/21/123456779121_redshift_us-east-1_mycluster_userlog_2020-10-21T18:01.gz + if "_redshift_" in key: + return "redshift" + + # this substring must be in your target prefix to be detected + if "amazon_documentdb" in key: + return "docdb" + + # e.g. carbon-black-cloud-forwarder/alerts/org_key=*****/year=2021/month=7/day=19/hour=18/minute=15/second=41/8436e850-7e78-40e4-b3cd-6ebbc854d0a2.jsonl.gz + if "carbon-black" in key: + return "carbonblack" + + # the below substrings must be in your target prefix to be detected + for source in [ + "amazon_codebuild", + "amazon_kinesis", + "amazon_dms", + "amazon_msk", + "network-firewall", + "cloudfront", + "verified-access", + "bedrock", + ]: + if source in key: + return source.replace("amazon_", "") + + return "s3" + + +def get_service_from_tags_and_remove_duplicates(metadata): + service = "" + tagsplit = metadata[DD_CUSTOM_TAGS].split(",") + for i, tag in enumerate(tagsplit): + if tag.startswith("service:"): + if service: + # remove duplicate entry from the tags + del tagsplit[i] + else: + service = tag[8:] + + metadata[DD_CUSTOM_TAGS] = ",".join(tagsplit) + + # Default service to source value + return service if service else metadata[DD_SOURCE] + + +def is_cloudtrail(key): + match = CLOUDTRAIL_REGEX.search(key) + return bool(match) + + +def merge_dicts(a, b, path=None): + if path is None: + path = [] + for key in b: + if key in a: + if isinstance(a[key], dict) and isinstance(b[key], dict): + merge_dicts(a[key], b[key], path + [str(key)]) + elif a[key] == b[key]: + pass # same leaf value + else: + raise Exception( + "Conflict while merging metadatas and the log entry at %s" + % ".".join(path + [str(key)]) + ) + else: + a[key] = b[key] + return a diff --git a/aws/logs_monitoring/steps/handlers/awslogs_handler.py b/aws/logs_monitoring/steps/handlers/awslogs_handler.py new file mode 100644 index 000000000..e9342604b --- /dev/null +++ b/aws/logs_monitoring/steps/handlers/awslogs_handler.py @@ -0,0 +1,214 @@ +import base64 +import gzip +import json +import logging +import os +import re +from io import BufferedReader, BytesIO + +from steps.common import ( + merge_dicts, + parse_event_source, + get_service_from_tags_and_remove_duplicates, +) +from customized_log_group import ( + is_lambda_customized_log_group, + get_lambda_function_name_from_logstream_name, +) +from caching.cloudwatch_log_group_cache import CloudwatchLogGroupTagsCache +from caching.step_functions_cache import StepFunctionsTagsCache +from settings import ( + DD_SOURCE, + DD_SERVICE, + DD_HOST, + DD_CUSTOM_TAGS, +) + +RDS_REGEX = re.compile("/aws/rds/(instance|cluster)/(?P[^/]+)/(?P[^/]+)") + +# Store the cache in the global scope so that it will be reused as long as +# the log forwarder Lambda container is running +account_step_functions_tags_cache = StepFunctionsTagsCache() +account_cw_logs_tags_cache = CloudwatchLogGroupTagsCache() + +logger = logging.getLogger() +logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper())) + + +# Handle CloudWatch logs +def awslogs_handler(event, context, metadata): + # Get logs + with gzip.GzipFile( + fileobj=BytesIO(base64.b64decode(event["awslogs"]["data"])) + ) as decompress_stream: + # Reading line by line avoid a bug where gzip would take a very long + # time (>5min) for file around 60MB gzipped + data = b"".join(BufferedReader(decompress_stream)) + logs = json.loads(data) + + # Set the source on the logs + source = logs.get("logGroup", "cloudwatch") + + # Use the logStream to identify if this is a CloudTrail, TransitGateway, or Bedrock event + # i.e. 123456779121_CloudTrail_us-east-1 + if "_CloudTrail_" in logs["logStream"]: + source = "cloudtrail" + if "tgw-attach" in logs["logStream"]: + source = "transitgateway" + if logs["logStream"] == "aws/bedrock/modelinvocations": + source = "bedrock" + metadata[DD_SOURCE] = parse_event_source(event, source) + + # Special handling for customized log group of Lambda functions + # Multiple Lambda functions can share one single customized log group + # Need to parse logStream name to determine whether it is a Lambda function + if is_lambda_customized_log_group(logs["logStream"]): + metadata[DD_SOURCE] = "lambda" + + # Build aws attributes + aws_attributes = { + "aws": { + "awslogs": { + "logGroup": logs["logGroup"], + "logStream": logs["logStream"], + "owner": logs["owner"], + } + } + } + + formatted_tags = account_cw_logs_tags_cache.get(logs["logGroup"]) + if len(formatted_tags) > 0: + metadata[DD_CUSTOM_TAGS] = ( + ",".join(formatted_tags) + if not metadata[DD_CUSTOM_TAGS] + else metadata[DD_CUSTOM_TAGS] + "," + ",".join(formatted_tags) + ) + + # Set service from custom tags, which may include the tags set on the log group + # Returns DD_SOURCE by default + metadata[DD_SERVICE] = get_service_from_tags_and_remove_duplicates(metadata) + + # Set host as log group where cloudwatch is source + if metadata[DD_SOURCE] == "cloudwatch" or metadata.get(DD_HOST, None) == None: + metadata[DD_HOST] = aws_attributes["aws"]["awslogs"]["logGroup"] + + if metadata[DD_SOURCE] == "appsync": + metadata[DD_HOST] = aws_attributes["aws"]["awslogs"]["logGroup"].split("/")[-1] + + if metadata[DD_SOURCE] == "verified-access": + try: + message = json.loads(logs["logEvents"][0]["message"]) + metadata[DD_HOST] = message["http_request"]["url"]["hostname"] + except Exception as e: + logger.debug("Unable to set verified-access log host: %s" % e) + + if metadata[DD_SOURCE] == "stepfunction" and logs["logStream"].startswith( + "states/" + ): + state_machine_arn = "" + try: + state_machine_arn = get_state_machine_arn( + json.loads(logs["logEvents"][0]["message"]) + ) + if state_machine_arn: # not empty + metadata[DD_HOST] = state_machine_arn + except Exception as e: + logger.debug( + "Unable to set stepfunction host or get state_machine_arn: %s" % e + ) + + formatted_stepfunctions_tags = account_step_functions_tags_cache.get( + state_machine_arn + ) + if len(formatted_stepfunctions_tags) > 0: + metadata[DD_CUSTOM_TAGS] = ( + ",".join(formatted_stepfunctions_tags) + if not metadata[DD_CUSTOM_TAGS] + else metadata[DD_CUSTOM_TAGS] + + "," + + ",".join(formatted_stepfunctions_tags) + ) + + # When parsing rds logs, use the cloudwatch log group name to derive the + # rds instance name, and add the log name of the stream ingested + if metadata[DD_SOURCE] in ["rds", "mariadb", "mysql", "postgresql"]: + match = RDS_REGEX.match(logs["logGroup"]) + if match is not None: + metadata[DD_HOST] = match.group("host") + metadata[DD_CUSTOM_TAGS] = ( + metadata[DD_CUSTOM_TAGS] + ",logname:" + match.group("name") + ) + + # For Lambda logs we want to extract the function name, + # then rebuild the arn of the monitored lambda using that name. + if metadata[DD_SOURCE] == "lambda": + process_lambda_logs(logs, aws_attributes, context, metadata) + + # The EKS log group contains various sources from the K8S control plane. + # In order to have these automatically trigger the correct pipelines they + # need to send their events with the correct log source. + if metadata[DD_SOURCE] == "eks": + if logs["logStream"].startswith("kube-apiserver-audit-"): + metadata[DD_SOURCE] = "kubernetes.audit" + elif logs["logStream"].startswith("kube-scheduler-"): + metadata[DD_SOURCE] = "kube_scheduler" + elif logs["logStream"].startswith("kube-apiserver-"): + metadata[DD_SOURCE] = "kube-apiserver" + elif logs["logStream"].startswith("kube-controller-manager-"): + metadata[DD_SOURCE] = "kube-controller-manager" + elif logs["logStream"].startswith("authenticator-"): + metadata[DD_SOURCE] = "aws-iam-authenticator" + # In case the conditions above don't match we maintain eks as the source + + # Create and send structured logs to Datadog + for log in logs["logEvents"]: + yield merge_dicts(log, aws_attributes) + + +def get_state_machine_arn(message): + if message.get("execution_arn") is not None: + execution_arn = message["execution_arn"] + arn_tokens = re.split(r"[:/\\]", execution_arn) + arn_tokens[5] = "stateMachine" + return ":".join(arn_tokens[:7]) + return "" + + +# Lambda logs can be from either default or customized log group +def process_lambda_logs(logs, aws_attributes, context, metadata): + lower_cased_lambda_function_name = get_lower_cased_lambda_function_name(logs) + if lower_cased_lambda_function_name is None: + return + # Split the arn of the forwarder to extract the prefix + arn_parts = context.invoked_function_arn.split("function:") + if len(arn_parts) > 0: + arn_prefix = arn_parts[0] + # Rebuild the arn with the lowercased function name + lower_cased_lambda__arn = ( + arn_prefix + "function:" + lower_cased_lambda_function_name + ) + # Add the lowe_rcased arn as a log attribute + arn_attributes = {"lambda": {"arn": lower_cased_lambda__arn}} + aws_attributes = merge_dicts(aws_attributes, arn_attributes) + env_tag_exists = ( + metadata[DD_CUSTOM_TAGS].startswith("env:") + or ",env:" in metadata[DD_CUSTOM_TAGS] + ) + # If there is no env specified, default to env:none + if not env_tag_exists: + metadata[DD_CUSTOM_TAGS] += ",env:none" + + + +# The lambda function name can be inferred from either a customized logstream name, or a loggroup name +def get_lower_cased_lambda_function_name(logs): + logstream_name = logs["logStream"] + # function name parsed from logstream is preferred for handling some edge cases + function_name = get_lambda_function_name_from_logstream_name(logstream_name) + if function_name is None: + log_group_parts = logs["logGroup"].split("/lambda/") + if len(log_group_parts) > 1: + function_name = log_group_parts[1] + else: + return None + return function_name.lower() diff --git a/aws/logs_monitoring/steps/handlers/s3_handler.py b/aws/logs_monitoring/steps/handlers/s3_handler.py new file mode 100644 index 000000000..fb621efce --- /dev/null +++ b/aws/logs_monitoring/steps/handlers/s3_handler.py @@ -0,0 +1,219 @@ +import logging +import gzip +import json + +import os +import re +import urllib.parse +from io import BufferedReader, BytesIO +import boto3 +import botocore + +from steps.common import ( + merge_dicts, + is_cloudtrail, + parse_event_source, + get_service_from_tags_and_remove_duplicates, +) +from settings import ( + DD_SOURCE, + DD_SERVICE, + DD_USE_VPC, + DD_MULTILINE_LOG_REGEX_PATTERN, + DD_HOST, +) + +GOV, CN = "gov", "cn" +if DD_MULTILINE_LOG_REGEX_PATTERN: + try: + MULTILINE_REGEX = re.compile( + "[\n\r\f]+(?={})".format(DD_MULTILINE_LOG_REGEX_PATTERN) + ) + except Exception: + raise Exception( + "could not compile multiline regex with pattern: {}".format( + DD_MULTILINE_LOG_REGEX_PATTERN + ) + ) + MULTILINE_REGEX_START_PATTERN = re.compile( + "^{}".format(DD_MULTILINE_LOG_REGEX_PATTERN) + ) + +logger = logging.getLogger() +logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper())) + +# Handle S3 events +def s3_handler(event, context, metadata): + # Need to use path style to access s3 via VPC Endpoints + # https://github.com/gford1000-aws/lambda_s3_access_using_vpc_endpoint#boto3-specific-notes + if DD_USE_VPC: + s3 = boto3.client( + "s3", + os.environ["AWS_REGION"], + config=botocore.config.Config(s3={"addressing_style": "path"}), + ) + else: + s3 = boto3.client("s3") + # if this is a S3 event carried in a SNS message, extract it and override the event + if "Sns" in event["Records"][0]: + event = json.loads(event["Records"][0]["Sns"]["Message"]) + + # Get the object from the event and show its content type + bucket = event["Records"][0]["s3"]["bucket"]["name"] + key = urllib.parse.unquote_plus(event["Records"][0]["s3"]["object"]["key"]) + + source = parse_event_source(event, key) + if "transit-gateway" in bucket: + source = "transitgateway" + metadata[DD_SOURCE] = source + + metadata[DD_SERVICE] = get_service_from_tags_and_remove_duplicates(metadata) + + ##Get the ARN of the service and set it as the hostname + hostname = parse_service_arn(source, key, bucket, context) + if hostname: + metadata[DD_HOST] = hostname + + # Extract the S3 object + response = s3.get_object(Bucket=bucket, Key=key) + body = response["Body"] + data = body.read() + + yield from get_structured_lines_for_s3_handler(data, bucket, key, source) + + +def parse_service_arn(source, key, bucket, context): + if source == "elb": + # For ELB logs we parse the filename to extract parameters in order to rebuild the ARN + # 1. We extract the region from the filename + # 2. We extract the loadbalancer name and replace the "." by "/" to match the ARN format + # 3. We extract the id of the loadbalancer + # 4. We build the arn + idsplit = key.split("/") + if not idsplit: + logger.debug("Invalid service ARN, unable to parse ELB ARN") + return + # If there is a prefix on the S3 bucket, remove the prefix before splitting the key + if idsplit[0] != "AWSLogs": + try: + idsplit = idsplit[idsplit.index("AWSLogs") :] + keysplit = "/".join(idsplit).split("_") + except ValueError: + logger.debug("Invalid S3 key, doesn't contain AWSLogs") + return + # If no prefix, split the key + else: + keysplit = key.split("_") + if len(keysplit) > 3: + region = keysplit[2].lower() + name = keysplit[3] + elbname = name.replace(".", "/") + if len(idsplit) > 1: + idvalue = idsplit[1] + partition = get_partition_from_region(region) + return "arn:{}:elasticloadbalancing:{}:{}:loadbalancer/{}".format( + partition, region, idvalue, elbname + ) + if source == "s3": + # For S3 access logs we use the bucket name to rebuild the arn + if bucket: + return "arn:aws:s3:::{}".format(bucket) + if source == "cloudfront": + # For Cloudfront logs we need to get the account and distribution id from the lambda arn and the filename + # 1. We extract the cloudfront id from the filename + # 2. We extract the AWS account id from the lambda arn + # 3. We build the arn + namesplit = key.split("/") + if len(namesplit) > 0: + filename = namesplit[len(namesplit) - 1] + # (distribution-ID.YYYY-MM-DD-HH.unique-ID.gz) + filenamesplit = filename.split(".") + if len(filenamesplit) > 3: + distributionID = filenamesplit[len(filenamesplit) - 4].lower() + arn = context.invoked_function_arn + arnsplit = arn.split(":") + if len(arnsplit) == 7: + awsaccountID = arnsplit[4].lower() + return "arn:aws:cloudfront::{}:distribution/{}".format( + awsaccountID, distributionID + ) + if source == "redshift": + # For redshift logs we leverage the filename to extract the relevant information + # 1. We extract the region from the filename + # 2. We extract the account-id from the filename + # 3. We extract the name of the cluster + # 4. We build the arn: arn:aws:redshift:region:account-id:cluster:cluster-name + namesplit = key.split("/") + if len(namesplit) == 8: + region = namesplit[3].lower() + accountID = namesplit[1].lower() + filename = namesplit[7] + filesplit = filename.split("_") + if len(filesplit) == 6: + clustername = filesplit[3] + return "arn:{}:redshift:{}:{}:cluster:{}:".format( + get_partition_from_region(region), region, accountID, clustername + ) + return + + +def get_partition_from_region(region): + partition = "aws" + if region: + if GOV in region: + partition = "aws-us-gov" + elif CN in region: + partition = "aws-cn" + return partition + + +def get_structured_lines_for_s3_handler(data, bucket, key, source): + # Decompress data that has a .gz extension or magic header http://www.onicos.com/staff/iz/formats/gzip.html + if key[-3:] == ".gz" or data[:2] == b"\x1f\x8b": + with gzip.GzipFile(fileobj=BytesIO(data)) as decompress_stream: + # Reading line by line avoid a bug where gzip would take a very long time (>5min) for + # file around 60MB gzipped + data = b"".join(BufferedReader(decompress_stream)) + + is_cloudtrail_bucket = False + if is_cloudtrail(str(key)): + try: + cloud_trail = json.loads(data) + if cloud_trail.get("Records") is not None: + # only parse as a cloudtrail bucket if we have a Records field to parse + is_cloudtrail_bucket = True + for event in cloud_trail["Records"]: + # Create structured object and send it + structured_line = merge_dicts( + event, {"aws": {"s3": {"bucket": bucket, "key": key}}} + ) + yield structured_line + except Exception as e: + logger.debug("Unable to parse cloudtrail log: %s" % e) + + if not is_cloudtrail_bucket: + # Check if using multiline log regex pattern + # and determine whether line or pattern separated logs + data = data.decode("utf-8", errors="ignore") + if DD_MULTILINE_LOG_REGEX_PATTERN and multiline_regex_start_pattern.match(data): + split_data = multiline_regex.split(data) + else: + if DD_MULTILINE_LOG_REGEX_PATTERN: + logger.debug( + "DD_MULTILINE_LOG_REGEX_PATTERN %s did not match start of file, splitting by line", + DD_MULTILINE_LOG_REGEX_PATTERN, + ) + if source == "waf": + # WAF logs are \n separated + split_data = [d for d in data.split("\n") if d != ""] + else: + split_data = data.splitlines() + + # Send lines to Datadog + for line in split_data: + # Create structured object and send it + structured_line = { + "aws": {"s3": {"bucket": bucket, "key": key}}, + "message": line, + } + yield structured_line diff --git a/aws/logs_monitoring/steps/parsing.py b/aws/logs_monitoring/steps/parsing.py index ec29a5218..57d2c7af7 100644 --- a/aws/logs_monitoring/steps/parsing.py +++ b/aws/logs_monitoring/steps/parsing.py @@ -3,70 +3,32 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2021 Datadog, Inc. -import base64 -import gzip import json import os -import boto3 -import botocore import itertools -import re -import urllib import logging -from io import BytesIO, BufferedReader from datadog_lambda.metric import lambda_stats -from customized_log_group import ( - get_lambda_function_name_from_logstream_name, - is_lambda_customized_log_group, -) -from caching.step_functions_cache import StepFunctionsTagsCache -from caching.cloudwatch_log_group_cache import CloudwatchLogGroupTagsCache from telemetry import ( DD_FORWARDER_TELEMETRY_NAMESPACE_PREFIX, get_forwarder_telemetry_tags, set_forwarder_telemetry_tags, ) +from steps.handlers.awslogs_handler import awslogs_handler +from steps.handlers.s3_handler import s3_handler +from steps.common import ( + merge_dicts, + get_service_from_tags_and_remove_duplicates, +) from settings import ( DD_TAGS, - DD_MULTILINE_LOG_REGEX_PATTERN, DD_SOURCE, DD_CUSTOM_TAGS, DD_SERVICE, - DD_HOST, DD_FORWARDER_VERSION, - DD_USE_VPC, ) -GOV, CN = "gov", "cn" - logger = logging.getLogger() - -if DD_MULTILINE_LOG_REGEX_PATTERN: - try: - multiline_regex = re.compile( - "[\n\r\f]+(?={})".format(DD_MULTILINE_LOG_REGEX_PATTERN) - ) - except Exception: - raise Exception( - "could not compile multiline regex with pattern: {}".format( - DD_MULTILINE_LOG_REGEX_PATTERN - ) - ) - multiline_regex_start_pattern = re.compile( - "^{}".format(DD_MULTILINE_LOG_REGEX_PATTERN) - ) - -rds_regex = re.compile("/aws/rds/(instance|cluster)/(?P[^/]+)/(?P[^/]+)") - -cloudtrail_regex = re.compile( - "\d+_CloudTrail(|-Digest)_\w{2}(|-gov|-cn)-\w{4,9}-\d_(|.+)\d{8}T\d{4,6}Z(|.+).json.gz$", - re.I, -) - -# Store the cache in the global scope so that it will be reused as long as -# the log forwarder Lambda container is running -account_cw_logs_tags_cache = CloudwatchLogGroupTagsCache() -account_step_functions_tags_cache = StepFunctionsTagsCache() +logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper())) def parse(event, context): @@ -114,7 +76,6 @@ def generate_metadata(context): "forwarder_memorysize": context.memory_limit_in_mb, "forwarder_version": DD_FORWARDER_VERSION, } - metadata[DD_CUSTOM_TAGS] = ",".join( filter( None, @@ -157,490 +118,6 @@ def parse_event_type(event): raise Exception("Event type not supported (see #Event supported section)") -# Handle S3 events -def s3_handler(event, context, metadata): - # Need to use path style to access s3 via VPC Endpoints - # https://github.com/gford1000-aws/lambda_s3_access_using_vpc_endpoint#boto3-specific-notes - if DD_USE_VPC: - s3 = boto3.client( - "s3", - os.environ["AWS_REGION"], - config=botocore.config.Config(s3={"addressing_style": "path"}), - ) - else: - s3 = boto3.client("s3") - # if this is a S3 event carried in a SNS message, extract it and override the event - if "Sns" in event["Records"][0]: - event = json.loads(event["Records"][0]["Sns"]["Message"]) - - # Get the object from the event and show its content type - bucket = event["Records"][0]["s3"]["bucket"]["name"] - key = urllib.parse.unquote_plus(event["Records"][0]["s3"]["object"]["key"]) - - source = parse_event_source(event, key) - if "transit-gateway" in bucket: - source = "transitgateway" - metadata[DD_SOURCE] = source - - metadata[DD_SERVICE] = get_service_from_tags_and_remove_duplicates(metadata) - - ##Get the ARN of the service and set it as the hostname - hostname = parse_service_arn(source, key, bucket, context) - if hostname: - metadata[DD_HOST] = hostname - - # Extract the S3 object - response = s3.get_object(Bucket=bucket, Key=key) - body = response["Body"] - data = body.read() - - yield from get_structured_lines_for_s3_handler(data, bucket, key, source) - - -def get_structured_lines_for_s3_handler(data, bucket, key, source): - # Decompress data that has a .gz extension or magic header http://www.onicos.com/staff/iz/formats/gzip.html - if key[-3:] == ".gz" or data[:2] == b"\x1f\x8b": - with gzip.GzipFile(fileobj=BytesIO(data)) as decompress_stream: - # Reading line by line avoid a bug where gzip would take a very long time (>5min) for - # file around 60MB gzipped - data = b"".join(BufferedReader(decompress_stream)) - - is_cloudtrail_bucket = False - if is_cloudtrail(str(key)): - try: - cloud_trail = json.loads(data) - if cloud_trail.get("Records") is not None: - # only parse as a cloudtrail bucket if we have a Records field to parse - is_cloudtrail_bucket = True - for event in cloud_trail["Records"]: - # Create structured object and send it - structured_line = merge_dicts( - event, {"aws": {"s3": {"bucket": bucket, "key": key}}} - ) - yield structured_line - except Exception as e: - logger.debug("Unable to parse cloudtrail log: %s" % e) - - if not is_cloudtrail_bucket: - # Check if using multiline log regex pattern - # and determine whether line or pattern separated logs - data = data.decode("utf-8", errors="ignore") - if DD_MULTILINE_LOG_REGEX_PATTERN and multiline_regex_start_pattern.match(data): - split_data = multiline_regex.split(data) - else: - if DD_MULTILINE_LOG_REGEX_PATTERN: - logger.debug( - "DD_MULTILINE_LOG_REGEX_PATTERN %s did not match start of file, splitting by line", - DD_MULTILINE_LOG_REGEX_PATTERN, - ) - if source == "waf": - # WAF logs are \n separated - split_data = [d for d in data.split("\n") if d != ""] - else: - split_data = data.splitlines() - - # Send lines to Datadog - for line in split_data: - # Create structured object and send it - structured_line = { - "aws": {"s3": {"bucket": bucket, "key": key}}, - "message": line, - } - yield structured_line - - -def get_service_from_tags_and_remove_duplicates(metadata): - service = "" - tagsplit = metadata[DD_CUSTOM_TAGS].split(",") - for i, tag in enumerate(tagsplit): - if tag.startswith("service:"): - if service: - # remove duplicate entry from the tags - del tagsplit[i] - else: - service = tag[8:] - - metadata[DD_CUSTOM_TAGS] = ",".join(tagsplit) - - # Default service to source value - return service if service else metadata[DD_SOURCE] - - -def parse_event_source(event, key): - """Parse out the source that will be assigned to the log in Datadog - Args: - event (dict): The AWS-formatted log event that the forwarder was triggered with - key (string): The S3 object key if the event is from S3 or the CW Log Group if the event is from CW Logs - """ - lowercase_key = str(key).lower() - - # Determines if the key matches any known sources for Cloudwatch logs - if "awslogs" in event: - return find_cloudwatch_source(lowercase_key) - - # Determines if the key matches any known sources for S3 logs - if "Records" in event and len(event["Records"]) > 0: - if "s3" in event["Records"][0]: - if is_cloudtrail(str(key)): - return "cloudtrail" - - return find_s3_source(lowercase_key) - - return "aws" - - -def find_cloudwatch_source(log_group): - # e.g. /aws/rds/instance/my-mariadb/error - if log_group.startswith("/aws/rds"): - for engine in ["mariadb", "mysql", "postgresql"]: - if engine in log_group: - return engine - return "rds" - - if log_group.startswith( - ( - # default location for rest api execution logs - "api-gateway", # e.g. Api-Gateway-Execution-Logs_xxxxxx/dev - # default location set by serverless framework for rest api access logs - "/aws/api-gateway", # e.g. /aws/api-gateway/my-project - # default location set by serverless framework for http api logs - "/aws/http-api", # e.g. /aws/http-api/my-project - ) - ): - return "apigateway" - - if log_group.startswith("/aws/vendedlogs/states"): - return "stepfunction" - - # e.g. dms-tasks-test-instance - if log_group.startswith("dms-tasks"): - return "dms" - - # e.g. sns/us-east-1/123456779121/SnsTopicX - if log_group.startswith("sns/"): - return "sns" - - # e.g. /aws/fsx/windows/xxx - if log_group.startswith("/aws/fsx/windows"): - return "aws.fsx" - - if log_group.startswith("/aws/appsync/"): - return "appsync" - - for source in [ - "/aws/lambda", # e.g. /aws/lambda/helloDatadog - "/aws/codebuild", # e.g. /aws/codebuild/my-project - "/aws/kinesis", # e.g. /aws/kinesisfirehose/dev - "/aws/docdb", # e.g. /aws/docdb/yourClusterName/profile - "/aws/eks", # e.g. /aws/eks/yourClusterName/profile - ]: - if log_group.startswith(source): - return source.replace("/aws/", "") - - # the below substrings must be in your log group to be detected - for source in [ - "network-firewall", - "route53", - "vpc", - "fargate", - "cloudtrail", - "msk", - "elasticsearch", - "transitgateway", - "verified-access", - "bedrock", - ]: - if source in log_group: - return source - - return "cloudwatch" - - -def is_cloudtrail(key): - match = cloudtrail_regex.search(key) - return bool(match) - - -def find_s3_source(key): - # e.g. AWSLogs/123456779121/elasticloadbalancing/us-east-1/2020/10/02/123456779121_elasticloadbalancing_us-east-1_app.alb.xxxxx.xx.xxx.xxx_x.log.gz - if "elasticloadbalancing" in key: - return "elb" - - # e.g. AWSLogs/123456779121/vpcflowlogs/us-east-1/2020/10/02/123456779121_vpcflowlogs_us-east-1_fl-xxxxx.log.gz - if "vpcflowlogs" in key: - return "vpc" - - # e.g. AWSLogs/123456779121/vpcdnsquerylogs/vpc-********/2021/05/11/vpc-********_vpcdnsquerylogs_********_20210511T0910Z_71584702.log.gz - if "vpcdnsquerylogs" in key: - return "route53" - - # e.g. 2020/10/02/21/aws-waf-logs-testing-1-2020-10-02-21-25-30-x123x-x456x or AWSLogs/123456779121/WAFLogs/us-east-1/xxxxxx-waf/2022/10/11/14/10/123456779121_waflogs_us-east-1_xxxxx-waf_20221011T1410Z_12756524.log.gz - if "aws-waf-logs" in key or "waflogs" in key: - return "waf" - - # e.g. AWSLogs/123456779121/redshift/us-east-1/2020/10/21/123456779121_redshift_us-east-1_mycluster_userlog_2020-10-21T18:01.gz - if "_redshift_" in key: - return "redshift" - - # this substring must be in your target prefix to be detected - if "amazon_documentdb" in key: - return "docdb" - - # e.g. carbon-black-cloud-forwarder/alerts/org_key=*****/year=2021/month=7/day=19/hour=18/minute=15/second=41/8436e850-7e78-40e4-b3cd-6ebbc854d0a2.jsonl.gz - if "carbon-black" in key: - return "carbonblack" - - # the below substrings must be in your target prefix to be detected - for source in [ - "amazon_codebuild", - "amazon_kinesis", - "amazon_dms", - "amazon_msk", - "network-firewall", - "cloudfront", - "verified-access", - "bedrock", - ]: - if source in key: - return source.replace("amazon_", "") - - return "s3" - - -def get_partition_from_region(region): - partition = "aws" - if region: - if GOV in region: - partition = "aws-us-gov" - elif CN in region: - partition = "aws-cn" - return partition - - -def parse_service_arn(source, key, bucket, context): - if source == "elb": - # For ELB logs we parse the filename to extract parameters in order to rebuild the ARN - # 1. We extract the region from the filename - # 2. We extract the loadbalancer name and replace the "." by "/" to match the ARN format - # 3. We extract the id of the loadbalancer - # 4. We build the arn - idsplit = key.split("/") - if not idsplit: - logger.debug("Invalid service ARN, unable to parse ELB ARN") - return - # If there is a prefix on the S3 bucket, remove the prefix before splitting the key - if idsplit[0] != "AWSLogs": - try: - idsplit = idsplit[idsplit.index("AWSLogs") :] - keysplit = "/".join(idsplit).split("_") - except ValueError: - logger.debug("Invalid S3 key, doesn't contain AWSLogs") - return - # If no prefix, split the key - else: - keysplit = key.split("_") - if len(keysplit) > 3: - region = keysplit[2].lower() - name = keysplit[3] - elbname = name.replace(".", "/") - if len(idsplit) > 1: - idvalue = idsplit[1] - partition = get_partition_from_region(region) - return "arn:{}:elasticloadbalancing:{}:{}:loadbalancer/{}".format( - partition, region, idvalue, elbname - ) - if source == "s3": - # For S3 access logs we use the bucket name to rebuild the arn - if bucket: - return "arn:aws:s3:::{}".format(bucket) - if source == "cloudfront": - # For Cloudfront logs we need to get the account and distribution id from the lambda arn and the filename - # 1. We extract the cloudfront id from the filename - # 2. We extract the AWS account id from the lambda arn - # 3. We build the arn - namesplit = key.split("/") - if len(namesplit) > 0: - filename = namesplit[len(namesplit) - 1] - # (distribution-ID.YYYY-MM-DD-HH.unique-ID.gz) - filenamesplit = filename.split(".") - if len(filenamesplit) > 3: - distributionID = filenamesplit[len(filenamesplit) - 4].lower() - arn = context.invoked_function_arn - arnsplit = arn.split(":") - if len(arnsplit) == 7: - awsaccountID = arnsplit[4].lower() - return "arn:aws:cloudfront::{}:distribution/{}".format( - awsaccountID, distributionID - ) - if source == "redshift": - # For redshift logs we leverage the filename to extract the relevant information - # 1. We extract the region from the filename - # 2. We extract the account-id from the filename - # 3. We extract the name of the cluster - # 4. We build the arn: arn:aws:redshift:region:account-id:cluster:cluster-name - namesplit = key.split("/") - if len(namesplit) == 8: - region = namesplit[3].lower() - accountID = namesplit[1].lower() - filename = namesplit[7] - filesplit = filename.split("_") - if len(filesplit) == 6: - clustername = filesplit[3] - return "arn:{}:redshift:{}:{}:cluster:{}:".format( - get_partition_from_region(region), region, accountID, clustername - ) - return - - -# Handle CloudWatch logs -def awslogs_handler(event, context, metadata): - # Get logs - with gzip.GzipFile( - fileobj=BytesIO(base64.b64decode(event["awslogs"]["data"])) - ) as decompress_stream: - # Reading line by line avoid a bug where gzip would take a very long - # time (>5min) for file around 60MB gzipped - data = b"".join(BufferedReader(decompress_stream)) - logs = json.loads(data) - - # Set the source on the logs - source = logs.get("logGroup", "cloudwatch") - - # Use the logStream to identify if this is a CloudTrail, TransitGateway, or Bedrock event - # i.e. 123456779121_CloudTrail_us-east-1 - if "_CloudTrail_" in logs["logStream"]: - source = "cloudtrail" - if "tgw-attach" in logs["logStream"]: - source = "transitgateway" - if logs["logStream"] == "aws/bedrock/modelinvocations": - source = "bedrock" - metadata[DD_SOURCE] = parse_event_source(event, source) - - # Special handling for customized log group of Lambda functions - # Multiple Lambda functions can share one single customized log group - # Need to parse logStream name to determine whether it is a Lambda function - if is_lambda_customized_log_group(logs["logStream"]): - metadata[DD_SOURCE] = "lambda" - - # Build aws attributes - aws_attributes = { - "aws": { - "awslogs": { - "logGroup": logs["logGroup"], - "logStream": logs["logStream"], - "owner": logs["owner"], - } - } - } - - formatted_tags = account_cw_logs_tags_cache.get(logs["logGroup"]) - if len(formatted_tags) > 0: - metadata[DD_CUSTOM_TAGS] = ( - ",".join(formatted_tags) - if not metadata[DD_CUSTOM_TAGS] - else metadata[DD_CUSTOM_TAGS] + "," + ",".join(formatted_tags) - ) - - # Set service from custom tags, which may include the tags set on the log group - # Returns DD_SOURCE by default - metadata[DD_SERVICE] = get_service_from_tags_and_remove_duplicates(metadata) - - # Set host as log group where cloudwatch is source - if metadata[DD_SOURCE] == "cloudwatch" or metadata.get(DD_HOST, None) == None: - metadata[DD_HOST] = aws_attributes["aws"]["awslogs"]["logGroup"] - - if metadata[DD_SOURCE] == "appsync": - metadata[DD_HOST] = aws_attributes["aws"]["awslogs"]["logGroup"].split("/")[-1] - - if metadata[DD_SOURCE] == "verified-access": - try: - message = json.loads(logs["logEvents"][0]["message"]) - metadata[DD_HOST] = message["http_request"]["url"]["hostname"] - except Exception as e: - logger.debug("Unable to set verified-access log host: %s" % e) - - if metadata[DD_SOURCE] == "stepfunction" and logs["logStream"].startswith( - "states/" - ): - state_machine_arn = "" - try: - state_machine_arn = get_state_machine_arn( - json.loads(logs["logEvents"][0]["message"]) - ) - if state_machine_arn: # not empty - metadata[DD_HOST] = state_machine_arn - except Exception as e: - logger.debug( - "Unable to set stepfunction host or get state_machine_arn: %s" % e - ) - - formatted_stepfunctions_tags = account_step_functions_tags_cache.get( - state_machine_arn - ) - if len(formatted_stepfunctions_tags) > 0: - metadata[DD_CUSTOM_TAGS] = ( - ",".join(formatted_stepfunctions_tags) - if not metadata[DD_CUSTOM_TAGS] - else metadata[DD_CUSTOM_TAGS] - + "," - + ",".join(formatted_stepfunctions_tags) - ) - - # When parsing rds logs, use the cloudwatch log group name to derive the - # rds instance name, and add the log name of the stream ingested - if metadata[DD_SOURCE] in ["rds", "mariadb", "mysql", "postgresql"]: - match = rds_regex.match(logs["logGroup"]) - if match is not None: - metadata[DD_HOST] = match.group("host") - metadata[DD_CUSTOM_TAGS] = ( - metadata[DD_CUSTOM_TAGS] + ",logname:" + match.group("name") - ) - - # For Lambda logs we want to extract the function name, - # then rebuild the arn of the monitored lambda using that name. - if metadata[DD_SOURCE] == "lambda": - process_lambda_logs(logs, aws_attributes, context, metadata) - - # The EKS log group contains various sources from the K8S control plane. - # In order to have these automatically trigger the correct pipelines they - # need to send their events with the correct log source. - if metadata[DD_SOURCE] == "eks": - if logs["logStream"].startswith("kube-apiserver-audit-"): - metadata[DD_SOURCE] = "kubernetes.audit" - elif logs["logStream"].startswith("kube-scheduler-"): - metadata[DD_SOURCE] = "kube_scheduler" - elif logs["logStream"].startswith("kube-apiserver-"): - metadata[DD_SOURCE] = "kube-apiserver" - elif logs["logStream"].startswith("kube-controller-manager-"): - metadata[DD_SOURCE] = "kube-controller-manager" - elif logs["logStream"].startswith("authenticator-"): - metadata[DD_SOURCE] = "aws-iam-authenticator" - # In case the conditions above don't match we maintain eks as the source - - # Create and send structured logs to Datadog - for log in logs["logEvents"]: - yield merge_dicts(log, aws_attributes) - - -def merge_dicts(a, b, path=None): - if path is None: - path = [] - for key in b: - if key in a: - if isinstance(a[key], dict) and isinstance(b[key], dict): - merge_dicts(a[key], b[key], path + [str(key)]) - elif a[key] == b[key]: - pass # same leaf value - else: - raise Exception( - "Conflict while merging metadatas and the log entry at %s" - % ".".join(path + [str(key)]) - ) - else: - a[key] = b[key] - return a - - # Handle Cloudwatch Events def cwevent_handler(event, metadata): data = event @@ -702,51 +179,3 @@ def normalize_events(events, metadata): ) return normalized - - -def get_state_machine_arn(message): - if message.get("execution_arn") is not None: - execution_arn = message["execution_arn"] - arn_tokens = re.split(r"[:/\\]", execution_arn) - arn_tokens[5] = "stateMachine" - return ":".join(arn_tokens[:7]) - return "" - - -# Lambda logs can be from either default or customized log group -def process_lambda_logs(logs, aws_attributes, context, metadata): - lower_cased_lambda_function_name = get_lower_cased_lambda_function_name(logs) - if lower_cased_lambda_function_name is None: - return - # Split the arn of the forwarder to extract the prefix - arn_parts = context.invoked_function_arn.split("function:") - if len(arn_parts) > 0: - arn_prefix = arn_parts[0] - # Rebuild the arn with the lowercased function name - lower_cased_lambda__arn = ( - arn_prefix + "function:" + lower_cased_lambda_function_name - ) - # Add the lowe_rcased arn as a log attribute - arn_attributes = {"lambda": {"arn": lower_cased_lambda__arn}} - aws_attributes = merge_dicts(aws_attributes, arn_attributes) - env_tag_exists = ( - metadata[DD_CUSTOM_TAGS].startswith("env:") - or ",env:" in metadata[DD_CUSTOM_TAGS] - ) - # If there is no env specified, default to env:none - if not env_tag_exists: - metadata[DD_CUSTOM_TAGS] += ",env:none" - - -# The lambda function name can be inferred from either a customized logstream name, or a loggroup name -def get_lower_cased_lambda_function_name(logs): - logstream_name = logs["logStream"] - # function name parsed from logstream is preferred for handling some edge cases - function_name = get_lambda_function_name_from_logstream_name(logstream_name) - if function_name is None: - log_group_parts = logs["logGroup"].split("/lambda/") - if len(log_group_parts) > 1: - function_name = log_group_parts[1] - else: - return None - return function_name.lower() diff --git a/aws/logs_monitoring/tests/run_unit_tests.sh b/aws/logs_monitoring/tests/run_unit_tests.sh index 65a7107a3..46496d248 100755 --- a/aws/logs_monitoring/tests/run_unit_tests.sh +++ b/aws/logs_monitoring/tests/run_unit_tests.sh @@ -2,4 +2,4 @@ export DD_API_KEY=11111111111111111111111111111111 export DD_ADDITIONAL_TARGET_LAMBDAS=ironmaiden,megadeth -python3 -m unittest discover . \ No newline at end of file +python3 -m unittest discover . diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py new file mode 100644 index 000000000..0b282c62c --- /dev/null +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -0,0 +1,218 @@ +import base64 +import gzip +import json +import os +import unittest +import sys +from unittest.mock import patch, MagicMock +from approvaltests.approvals import verify_as_json +from approvaltests.namer import NamerFactory + +sys.modules["trace_forwarder.connection"] = MagicMock() +sys.modules["datadog_lambda.wrapper"] = MagicMock() +sys.modules["datadog_lambda.metric"] = MagicMock() +sys.modules["datadog"] = MagicMock() +sys.modules["requests"] = MagicMock() +sys.modules["requests_futures.sessions"] = MagicMock() + +env_patch = patch.dict( + os.environ, + { + "DD_API_KEY": "11111111111111111111111111111111", + }, +) +env_patch.start() +from steps.handlers.awslogs_handler import ( + awslogs_handler, + get_state_machine_arn, + get_lower_cased_lambda_function_name, +) +env_patch.stop() + + + +class TestAWSLogsHandler(unittest.TestCase): + @patch("steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.get") + @patch("steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.release_s3_cache_lock") + @patch("steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.acquire_s3_cache_lock") + @patch("steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.write_cache_to_s3") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") + @patch("steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.get_cache_from_s3") + def test_awslogs_handler_rds_postgresql( + self, + mock_get_s3_cache, + mock_forward_metrics, + mock_write_cache, + mock_acquire_lock, + mock_release_lock, + mock_cache_get, + ): + os.environ["DD_FETCH_LAMBDA_TAGS"] = "True" + os.environ["DD_FETCH_LOG_GROUP_TAGS"] = "True" + mock_acquire_lock.return_value = True + mock_cache_get.return_value = ["test_tag_key:test_tag_value"] + mock_get_s3_cache.return_value = ( + {}, + 1000, + ) + + event = { + "awslogs": { + "data": base64.b64encode( + gzip.compress( + bytes( + json.dumps( + { + "owner": "123456789012", + "logGroup": "/aws/rds/instance/datadog/postgresql", + "logStream": "datadog.0", + "logEvents": [ + { + "id": "31953106606966983378809025079804211143289615424298221568", + "timestamp": 1609556645000, + "message": "2021-01-02 03:04:05 UTC::@:[5306]:LOG: database system is ready to accept connections", + } + ], + } + ), + "utf-8", + ) + ) + ) + } + } + context = None + metadata = {"ddsource": "postgresql", "ddtags": "env:dev"} + + verify_as_json(list(awslogs_handler(event, context, metadata))) + verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) + + @patch("steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.get") + @patch("steps.handlers.awslogs_handler.StepFunctionsTagsCache.get") + @patch("steps.handlers.awslogs_handler.StepFunctionsTagsCache.release_s3_cache_lock") + @patch("steps.handlers.awslogs_handler.StepFunctionsTagsCache.acquire_s3_cache_lock") + @patch("steps.handlers.awslogs_handler.StepFunctionsTagsCache.write_cache_to_s3") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") + @patch("steps.handlers.awslogs_handler.StepFunctionsTagsCache.get_cache_from_s3") + def test_awslogs_handler_step_functions_tags_added_properly( + self, + mock_get_s3_cache, + mock_forward_metrics, + mock_write_cache, + mock_acquire_lock, + mock_release_lock, + mock_step_functions_cache_get, + mock_cw_log_group_cache_get, + ): + os.environ["DD_FETCH_LAMBDA_TAGS"] = "True" + os.environ["DD_FETCH_LOG_GROUP_TAGS"] = "True" + os.environ["DD_FETCH_STEP_FUNCTIONS_TAGS"] = "True" + mock_acquire_lock.return_value = True + mock_step_functions_cache_get.return_value = ["test_tag_key:test_tag_value"] + mock_cw_log_group_cache_get.return_value = [] + mock_get_s3_cache.return_value = ( + {}, + 1000, + ) + + event = { + "awslogs": { + "data": base64.b64encode( + gzip.compress( + bytes( + json.dumps( + { + "messageType": "DATA_MESSAGE", + "owner": "425362996713", + "logGroup": "/aws/vendedlogs/states/logs-to-traces-sequential-Logs", + "logStream": "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9", + "subscriptionFilters": ["testFilter"], + "logEvents": [ + { + "id": "37199773595581154154810589279545129148442535997644275712", + "timestamp": 1668095539607, + "message": '{"id":"1","type":"ExecutionStarted","details":{"input":"{"Comment": "Insert your JSON here"}","inputDetails":{"truncated":false},"roleArn":"arn:aws:iam::425362996713:role/service-role/StepFunctions-logs-to-traces-sequential-role-ccd69c03"},",previous_event_id":"0","event_timestamp":"1668095539607","execution_arn":"arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential:d0dbefd8-a0f6-b402-da4c-f4863def7456:7fa0cfbe-be28-4a20-9875-73c37f5dc39e"}', + } + ], + } + ), + "utf-8", + ) + ) + ) + } + } + context = None + metadata = {"ddsource": "postgresql", "ddtags": "env:dev"} + + verify_as_json(list(awslogs_handler(event, context, metadata))) + verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) + + +class TestLambdaCustomizedLogGroup(unittest.TestCase): + def test_get_lower_cased_lambda_function_name(self): + self.assertEqual(True, True) + # Non Lambda log + stepfunction_loggroup = { + "messageType": "DATA_MESSAGE", + "logGroup": "/aws/vendedlogs/states/logs-to-traces-sequential-Logs", + "logStream": "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9", + "logEvents": [], + } + self.assertEqual( + get_lower_cased_lambda_function_name(stepfunction_loggroup), None + ) + lambda_default_loggroup = { + "messageType": "DATA_MESSAGE", + "logGroup": "/aws/lambda/test-lambda-default-log-group", + "logStream": "2023/11/06/[$LATEST]b25b1f977b3e416faa45a00f427e7acb", + "logEvents": [], + } + self.assertEqual( + get_lower_cased_lambda_function_name(lambda_default_loggroup), + "test-lambda-default-log-group", + ) + lambda_customized_loggroup = { + "messageType": "DATA_MESSAGE", + "logGroup": "customizeLambdaGrop", + "logStream": "2023/11/06/test-customized-log-group1[$LATEST]13e304cba4b9446eb7ef082a00038990", + "logEvents": [], + } + self.assertEqual( + get_lower_cased_lambda_function_name(lambda_customized_loggroup), + "test-customized-log-group1", + ) + + +class TestParsingStepFunctionLogs(unittest.TestCase): + def test_get_state_machine_arn(self): + invalid_sf_log_message = {"no_execution_arn": "xxxx/yyy"} + self.assertEqual(get_state_machine_arn(invalid_sf_log_message), "") + + normal_sf_log_message = { + "execution_arn": "arn:aws:states:sa-east-1:425362996713:express:my-Various-States:7f653fda-c79a-430b-91e2-3f97eb87cabb:862e5d40-a457-4ca2-a3c1-78485bd94d3f" + } + self.assertEqual( + get_state_machine_arn(normal_sf_log_message), + "arn:aws:states:sa-east-1:425362996713:stateMachine:my-Various-States", + ) + + forward_slash_sf_log_message = { + "execution_arn": "arn:aws:states:sa-east-1:425362996713:express:my-Various-States/7f653fda-c79a-430b-91e2-3f97eb87cabb:862e5d40-a457-4ca2-a3c1-78485bd94d3f" + } + self.assertEqual( + get_state_machine_arn(forward_slash_sf_log_message), + "arn:aws:states:sa-east-1:425362996713:stateMachine:my-Various-States", + ) + + back_slash_sf_log_message = { + "execution_arn": "arn:aws:states:sa-east-1:425362996713:express:my-Various-States\\7f653fda-c79a-430b-91e2-3f97eb87cabb:862e5d40-a457-4ca2-a3c1-78485bd94d3f" + } + self.assertEqual( + get_state_machine_arn(back_slash_sf_log_message), + "arn:aws:states:sa-east-1:425362996713:stateMachine:my-Various-States", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/aws/logs_monitoring/tests/test_cloudtrail_s3.py b/aws/logs_monitoring/tests/test_cloudtrail_s3.py index c90421ebf..abe644364 100644 --- a/aws/logs_monitoring/tests/test_cloudtrail_s3.py +++ b/aws/logs_monitoring/tests/test_cloudtrail_s3.py @@ -99,7 +99,7 @@ def setUp(self): self.maxDiff = 9000 @patch("caching.base_tags_cache.boto3") - @patch("steps.parsing.boto3") + @patch("steps.handlers.s3_handler.boto3") @patch("lambda_function.boto3") def test_s3_cloudtrail_pasing_and_enrichment( self, lambda_boto3, parsing_boto3, cache_boto3 diff --git a/aws/logs_monitoring/tests/test_lambda_function.py b/aws/logs_monitoring/tests/test_lambda_function.py index e8da0476c..39207b25a 100644 --- a/aws/logs_monitoring/tests/test_lambda_function.py +++ b/aws/logs_monitoring/tests/test_lambda_function.py @@ -31,7 +31,6 @@ from steps.transformation import transform from steps.splitting import split from steps.parsing import parse, parse_event_type - env_patch.stop() @@ -70,9 +69,10 @@ def test_lambda_invocation_exception(self, boto3): class TestLambdaFunctionEndToEnd(unittest.TestCase): - @patch("enhanced_lambda_metrics.LambdaTagsCache.get_cache_from_s3") - def test_datadog_forwarder(self, mock_get_s3_cache): - mock_get_s3_cache.return_value = ( + @patch("enhanced_lambda_metrics.LambdaTagsCache.get") + @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get_cache_from_s3") + def test_datadog_forwarder(self, mock_get_lambda_tags, mock_get_s3_cache): + mock_get_lambda_tags.return_value = ( { "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender": [ "team:metrics", @@ -84,6 +84,7 @@ def test_datadog_forwarder(self, mock_get_s3_cache): }, time(), ) + mock_get_s3_cache.return_value = [] context = Context() input_data = self._get_input_data() event = { @@ -139,7 +140,8 @@ def test_datadog_forwarder(self, mock_get_s3_cache): del os.environ["DD_FETCH_LAMBDA_TAGS"] @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") - def test_setting_service_tag_from_log_group_cache(self, cw_logs_tags_get): + @patch("caching.lambda_cache.LambdaTagsCache.get") + def test_setting_service_tag_from_log_group_cache(self, cw_logs_tags_get, lambda_tags_get): reload(sys.modules["settings"]) reload(sys.modules["steps.parsing"]) cw_logs_tags_get.return_value = ["service:log_group_service"] diff --git a/aws/logs_monitoring/tests/test_parsing.py b/aws/logs_monitoring/tests/test_parsing.py index 2b451e19b..4727d8a40 100644 --- a/aws/logs_monitoring/tests/test_parsing.py +++ b/aws/logs_monitoring/tests/test_parsing.py @@ -1,44 +1,14 @@ -import base64 -import gzip -import json -from unittest.mock import MagicMock, patch -import os -import sys import unittest -from approvaltests.approvals import verify_as_json -from approvaltests.combination_approvals import verify_all_combinations -from approvaltests.namer import NamerFactory - -sys.modules["trace_forwarder.connection"] = MagicMock() -sys.modules["datadog_lambda.wrapper"] = MagicMock() -sys.modules["datadog_lambda.metric"] = MagicMock() -sys.modules["datadog"] = MagicMock() -sys.modules["requests"] = MagicMock() -sys.modules["requests_futures.sessions"] = MagicMock() - -env_patch = patch.dict( - os.environ, - { - "DD_API_KEY": "11111111111111111111111111111111", - }, -) -env_patch.start() -from steps.parsing import ( - awslogs_handler, + +from steps.common import ( parse_event_source, - parse_service_arn, get_service_from_tags_and_remove_duplicates, - get_state_machine_arn, - get_lower_cased_lambda_function_name, - get_structured_lines_for_s3_handler, ) from settings import ( DD_CUSTOM_TAGS, DD_SOURCE, ) -env_patch.stop() - class TestParseEventSource(unittest.TestCase): def test_aws_source_if_none_found(self): @@ -335,184 +305,6 @@ def test_s3_source_if_none_found(self): self.assertEqual(parse_event_source({"Records": ["logs-from-s3"]}, ""), "s3") -class TestParseServiceArn(unittest.TestCase): - def test_elb_s3_key_invalid(self): - self.assertEqual( - parse_service_arn( - "elb", - "123456789123/elasticloadbalancing/us-east-1/2022/02/08/123456789123_elasticloadbalancing_us-east-1_app.my-alb-name.123456789aabcdef_20220208T1127Z_10.0.0.2_1abcdef2.log.gz", - None, - None, - ), - None, - ) - - def test_elb_s3_key_no_prefix(self): - self.assertEqual( - parse_service_arn( - "elb", - "AWSLogs/123456789123/elasticloadbalancing/us-east-1/2022/02/08/123456789123_elasticloadbalancing_us-east-1_app.my-alb-name.123456789aabcdef_20220208T1127Z_10.0.0.2_1abcdef2.log.gz", - None, - None, - ), - "arn:aws:elasticloadbalancing:us-east-1:123456789123:loadbalancer/app/my-alb-name/123456789aabcdef", - ) - - def test_elb_s3_key_single_prefix(self): - self.assertEqual( - parse_service_arn( - "elb", - "elasticloadbalancing/AWSLogs/123456789123/elasticloadbalancing/us-east-1/2022/02/08/123456789123_elasticloadbalancing_us-east-1_app.my-alb-name.123456789aabcdef_20220208T1127Z_10.0.0.2_1abcdef2.log.gz", - None, - None, - ), - "arn:aws:elasticloadbalancing:us-east-1:123456789123:loadbalancer/app/my-alb-name/123456789aabcdef", - ) - - def test_elb_s3_key_multi_prefix(self): - self.assertEqual( - parse_service_arn( - "elb", - "elasticloadbalancing/my-alb-name/AWSLogs/123456789123/elasticloadbalancing/us-east-1/2022/02/08/123456789123_elasticloadbalancing_us-east-1_app.my-alb-name.123456789aabcdef_20220208T1127Z_10.0.0.2_1abcdef2.log.gz", - None, - None, - ), - "arn:aws:elasticloadbalancing:us-east-1:123456789123:loadbalancer/app/my-alb-name/123456789aabcdef", - ) - - def test_elb_s3_key_multi_prefix_gov(self): - self.assertEqual( - parse_service_arn( - "elb", - "elasticloadbalancing/my-alb-name/AWSLogs/123456789123/elasticloadbalancing/us-gov-east-1/2022/02/08" - "/123456789123_elasticloadbalancing_us-gov-east-1_app.my-alb-name.123456789aabcdef_20220208T1127Z_10" - ".0.0.2_1abcdef2.log.gz", - None, - None, - ), - "arn:aws-us-gov:elasticloadbalancing:us-gov-east-1:123456789123:loadbalancer/app/my-alb-name" - "/123456789aabcdef", - ) - - -class TestAWSLogsHandler(unittest.TestCase): - @patch("steps.parsing.CloudwatchLogGroupTagsCache.get") - @patch("steps.parsing.CloudwatchLogGroupTagsCache.release_s3_cache_lock") - @patch("steps.parsing.CloudwatchLogGroupTagsCache.acquire_s3_cache_lock") - @patch("steps.parsing.CloudwatchLogGroupTagsCache.write_cache_to_s3") - @patch("caching.base_tags_cache.send_forwarder_internal_metrics") - @patch("steps.parsing.CloudwatchLogGroupTagsCache.get_cache_from_s3") - def test_awslogs_handler_rds_postgresql( - self, - mock_get_s3_cache, - mock_forward_metrics, - mock_write_cache, - mock_acquire_lock, - mock_release_lock, - mock_cache_get, - ): - os.environ["DD_FETCH_LAMBDA_TAGS"] = "True" - os.environ["DD_FETCH_LOG_GROUP_TAGS"] = "True" - mock_acquire_lock.return_value = True - mock_cache_get.return_value = ["test_tag_key:test_tag_value"] - mock_get_s3_cache.return_value = ( - {}, - 1000, - ) - - event = { - "awslogs": { - "data": base64.b64encode( - gzip.compress( - bytes( - json.dumps( - { - "owner": "123456789012", - "logGroup": "/aws/rds/instance/datadog/postgresql", - "logStream": "datadog.0", - "logEvents": [ - { - "id": "31953106606966983378809025079804211143289615424298221568", - "timestamp": 1609556645000, - "message": "2021-01-02 03:04:05 UTC::@:[5306]:LOG: database system is ready to accept connections", - } - ], - } - ), - "utf-8", - ) - ) - ) - } - } - context = None - metadata = {"ddsource": "postgresql", "ddtags": "env:dev"} - - verify_as_json(list(awslogs_handler(event, context, metadata))) - verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) - - @patch("steps.parsing.CloudwatchLogGroupTagsCache.get") - @patch("steps.parsing.StepFunctionsTagsCache.get") - @patch("steps.parsing.StepFunctionsTagsCache.release_s3_cache_lock") - @patch("steps.parsing.StepFunctionsTagsCache.acquire_s3_cache_lock") - @patch("steps.parsing.StepFunctionsTagsCache.write_cache_to_s3") - @patch("caching.base_tags_cache.send_forwarder_internal_metrics") - @patch("steps.parsing.StepFunctionsTagsCache.get_cache_from_s3") - def test_awslogs_handler_step_functions_tags_added_properly( - self, - mock_get_s3_cache, - mock_forward_metrics, - mock_write_cache, - mock_acquire_lock, - mock_release_lock, - mock_step_functions_cache_get, - mock_cw_log_group_cache_get, - ): - os.environ["DD_FETCH_LAMBDA_TAGS"] = "True" - os.environ["DD_FETCH_LOG_GROUP_TAGS"] = "True" - os.environ["DD_FETCH_STEP_FUNCTIONS_TAGS"] = "True" - mock_acquire_lock.return_value = True - mock_step_functions_cache_get.return_value = ["test_tag_key:test_tag_value"] - mock_cw_log_group_cache_get.return_value = [] - mock_get_s3_cache.return_value = ( - {}, - 1000, - ) - - event = { - "awslogs": { - "data": base64.b64encode( - gzip.compress( - bytes( - json.dumps( - { - "messageType": "DATA_MESSAGE", - "owner": "425362996713", - "logGroup": "/aws/vendedlogs/states/logs-to-traces-sequential-Logs", - "logStream": "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9", - "subscriptionFilters": ["testFilter"], - "logEvents": [ - { - "id": "37199773595581154154810589279545129148442535997644275712", - "timestamp": 1668095539607, - "message": '{"id":"1","type":"ExecutionStarted","details":{"input":"{"Comment": "Insert your JSON here"}","inputDetails":{"truncated":false},"roleArn":"arn:aws:iam::425362996713:role/service-role/StepFunctions-logs-to-traces-sequential-role-ccd69c03"},",previous_event_id":"0","event_timestamp":"1668095539607","execution_arn":"arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential:d0dbefd8-a0f6-b402-da4c-f4863def7456:7fa0cfbe-be28-4a20-9875-73c37f5dc39e"}', - } - ], - } - ), - "utf-8", - ) - ) - ) - } - } - context = None - metadata = {"ddsource": "postgresql", "ddtags": "env:dev"} - - verify_as_json(list(awslogs_handler(event, context, metadata))) - verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) - - class TestGetServiceFromTags(unittest.TestCase): def test_get_service_from_tags(self): metadata = { @@ -539,116 +331,5 @@ def test_get_service_from_tags_removing_duplicates(self): ) -class TestParsingStepFunctionLogs(unittest.TestCase): - def test_get_state_machine_arn(self): - invalid_sf_log_message = {"no_execution_arn": "xxxx/yyy"} - self.assertEqual(get_state_machine_arn(invalid_sf_log_message), "") - - normal_sf_log_message = { - "execution_arn": "arn:aws:states:sa-east-1:425362996713:express:my-Various-States:7f653fda-c79a-430b-91e2-3f97eb87cabb:862e5d40-a457-4ca2-a3c1-78485bd94d3f" - } - self.assertEqual( - get_state_machine_arn(normal_sf_log_message), - "arn:aws:states:sa-east-1:425362996713:stateMachine:my-Various-States", - ) - - forward_slash_sf_log_message = { - "execution_arn": "arn:aws:states:sa-east-1:425362996713:express:my-Various-States/7f653fda-c79a-430b-91e2-3f97eb87cabb:862e5d40-a457-4ca2-a3c1-78485bd94d3f" - } - self.assertEqual( - get_state_machine_arn(forward_slash_sf_log_message), - "arn:aws:states:sa-east-1:425362996713:stateMachine:my-Various-States", - ) - - back_slash_sf_log_message = { - "execution_arn": "arn:aws:states:sa-east-1:425362996713:express:my-Various-States\\7f653fda-c79a-430b-91e2-3f97eb87cabb:862e5d40-a457-4ca2-a3c1-78485bd94d3f" - } - self.assertEqual( - get_state_machine_arn(back_slash_sf_log_message), - "arn:aws:states:sa-east-1:425362996713:stateMachine:my-Various-States", - ) - - -class TestLambdaCustomizedLogGroup(unittest.TestCase): - def test_get_lower_cased_lambda_function_name(self): - self.assertEqual(True, True) - # Non Lambda log - stepfunction_loggroup = { - "messageType": "DATA_MESSAGE", - "logGroup": "/aws/vendedlogs/states/logs-to-traces-sequential-Logs", - "logStream": "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9", - "logEvents": [], - } - self.assertEqual( - get_lower_cased_lambda_function_name(stepfunction_loggroup), None - ) - lambda_default_loggroup = { - "messageType": "DATA_MESSAGE", - "logGroup": "/aws/lambda/test-lambda-default-log-group", - "logStream": "2023/11/06/[$LATEST]b25b1f977b3e416faa45a00f427e7acb", - "logEvents": [], - } - self.assertEqual( - get_lower_cased_lambda_function_name(lambda_default_loggroup), - "test-lambda-default-log-group", - ) - lambda_customized_loggroup = { - "messageType": "DATA_MESSAGE", - "logGroup": "customizeLambdaGrop", - "logStream": "2023/11/06/test-customized-log-group1[$LATEST]13e304cba4b9446eb7ef082a00038990", - "logEvents": [], - } - self.assertEqual( - get_lower_cased_lambda_function_name(lambda_customized_loggroup), - "test-customized-log-group1", - ) - - -class TestS3EventsHandler(unittest.TestCase): - def parse_lines(self, data, key, source): - bucket = "my-bucket" - gzip_data = gzip.compress(bytes(data, "utf-8")) - - return [ - l - for l in get_structured_lines_for_s3_handler(gzip_data, bucket, key, source) - ] - - def test_get_structured_lines_waf(self): - key = "mykey" - source = "waf" - verify_all_combinations( - lambda d: self.parse_lines(d, key, source), - [ - [ - '{"timestamp": 12345, "key1": "value1", "key2":"value2"}\n', - '{"timestamp": 12345, "key1": "value1", "key2":"value2"}\n{"timestamp": 789760, "key1": "value1", "key3":"value4"}\n', - '{"timestamp": 12345, "key1": "value1", "key2":"value2" "key3": {"key5" : "value5"}}\r{"timestamp": 12345, "key1": "value1"}\n', - '{"timestamp": 12345, "key1": "value1", "key2":"value2" "key3": {"key5" : "value5"}}\f{"timestamp": 12345, "key1": "value1"}\n', - '{"timestamp": 12345, "key1": "value1", "key2":"value2" "key3": {"key5" : "value5"}}\u00A0{"timestamp": 12345, "key1": "value1"}\n', - "", - "\n", - ] - ], - ) - - def test_get_structured_lines_cloudtrail(self): - key = ( - "123456779121_CloudTrail_eu-west-3_20180707T1735Z_abcdefghi0MCRL2O.json.gz" - ) - source = "cloudtrail" - verify_all_combinations( - lambda d: self.parse_lines(d, key, source), - [ - [ - '{"Records": [{"event_key" : "logs-from-s3"}]}', - '{"Records": [{"event_key" : "logs-from-s3"}, {"key1" : "data1", "key2" : "data2"}]}', - '{"Records": {}}', - "", - ] - ], - ) - - if __name__ == "__main__": unittest.main() diff --git a/aws/logs_monitoring/tests/test_s3_handler.py b/aws/logs_monitoring/tests/test_s3_handler.py new file mode 100644 index 000000000..907a46616 --- /dev/null +++ b/aws/logs_monitoring/tests/test_s3_handler.py @@ -0,0 +1,118 @@ + +import gzip +import unittest +from approvaltests.combination_approvals import verify_all_combinations +from steps.handlers.s3_handler import ( + parse_service_arn, + get_structured_lines_for_s3_handler, +) + + +class TestS3EventsHandler(unittest.TestCase): + def parse_lines(self, data, key, source): + bucket = "my-bucket" + gzip_data = gzip.compress(bytes(data, "utf-8")) + + return [ + l + for l in get_structured_lines_for_s3_handler(gzip_data, bucket, key, source) + ] + + def test_get_structured_lines_waf(self): + key = "mykey" + source = "waf" + verify_all_combinations( + lambda d: self.parse_lines(d, key, source), + [ + [ + '{"timestamp": 12345, "key1": "value1", "key2":"value2"}\n', + '{"timestamp": 12345, "key1": "value1", "key2":"value2"}\n{"timestamp": 789760, "key1": "value1", "key3":"value4"}\n', + '{"timestamp": 12345, "key1": "value1", "key2":"value2" "key3": {"key5" : "value5"}}\r{"timestamp": 12345, "key1": "value1"}\n', + '{"timestamp": 12345, "key1": "value1", "key2":"value2" "key3": {"key5" : "value5"}}\f{"timestamp": 12345, "key1": "value1"}\n', + '{"timestamp": 12345, "key1": "value1", "key2":"value2" "key3": {"key5" : "value5"}}\u00A0{"timestamp": 12345, "key1": "value1"}\n', + "", + "\n", + ] + ], + ) + + def test_get_structured_lines_cloudtrail(self): + key = ( + "123456779121_CloudTrail_eu-west-3_20180707T1735Z_abcdefghi0MCRL2O.json.gz" + ) + source = "cloudtrail" + verify_all_combinations( + lambda d: self.parse_lines(d, key, source), + [ + [ + '{"Records": [{"event_key" : "logs-from-s3"}]}', + '{"Records": [{"event_key" : "logs-from-s3"}, {"key1" : "data1", "key2" : "data2"}]}', + '{"Records": {}}', + "", + ] + ], + ) + + +class TestParseServiceArn(unittest.TestCase): + def test_elb_s3_key_invalid(self): + self.assertEqual( + parse_service_arn( + "elb", + "123456789123/elasticloadbalancing/us-east-1/2022/02/08/123456789123_elasticloadbalancing_us-east-1_app.my-alb-name.123456789aabcdef_20220208T1127Z_10.0.0.2_1abcdef2.log.gz", + None, + None, + ), + None, + ) + + def test_elb_s3_key_no_prefix(self): + self.assertEqual( + parse_service_arn( + "elb", + "AWSLogs/123456789123/elasticloadbalancing/us-east-1/2022/02/08/123456789123_elasticloadbalancing_us-east-1_app.my-alb-name.123456789aabcdef_20220208T1127Z_10.0.0.2_1abcdef2.log.gz", + None, + None, + ), + "arn:aws:elasticloadbalancing:us-east-1:123456789123:loadbalancer/app/my-alb-name/123456789aabcdef", + ) + + def test_elb_s3_key_single_prefix(self): + self.assertEqual( + parse_service_arn( + "elb", + "elasticloadbalancing/AWSLogs/123456789123/elasticloadbalancing/us-east-1/2022/02/08/123456789123_elasticloadbalancing_us-east-1_app.my-alb-name.123456789aabcdef_20220208T1127Z_10.0.0.2_1abcdef2.log.gz", + None, + None, + ), + "arn:aws:elasticloadbalancing:us-east-1:123456789123:loadbalancer/app/my-alb-name/123456789aabcdef", + ) + + def test_elb_s3_key_multi_prefix(self): + self.assertEqual( + parse_service_arn( + "elb", + "elasticloadbalancing/my-alb-name/AWSLogs/123456789123/elasticloadbalancing/us-east-1/2022/02/08/123456789123_elasticloadbalancing_us-east-1_app.my-alb-name.123456789aabcdef_20220208T1127Z_10.0.0.2_1abcdef2.log.gz", + None, + None, + ), + "arn:aws:elasticloadbalancing:us-east-1:123456789123:loadbalancer/app/my-alb-name/123456789aabcdef", + ) + + def test_elb_s3_key_multi_prefix_gov(self): + self.assertEqual( + parse_service_arn( + "elb", + "elasticloadbalancing/my-alb-name/AWSLogs/123456789123/elasticloadbalancing/us-gov-east-1/2022/02/08" + "/123456789123_elasticloadbalancing_us-gov-east-1_app.my-alb-name.123456789aabcdef_20220208T1127Z_10" + ".0.0.2_1abcdef2.log.gz", + None, + None, + ), + "arn:aws-us-gov:elasticloadbalancing:us-gov-east-1:123456789123:loadbalancer/app/my-alb-name" + "/123456789aabcdef", + ) + + +if __name__ == "__main__": + unittest.main() From 7a82d83e0d2428d389963373e667c1c368b53062 Mon Sep 17 00:00:00 2001 From: Georgi Date: Tue, 27 Feb 2024 14:39:45 +0100 Subject: [PATCH 5/8] refactor some unittests --- .../steps/handlers/awslogs_handler.py | 1 - .../steps/handlers/s3_handler.py | 1 + .../tests/test_awslogs_handler.py | 80 +++++++++++++++++-- .../tests/test_cloudtrail_s3.py | 14 ++-- .../tests/test_lambda_function.py | 9 ++- aws/logs_monitoring/tests/test_logs.py | 24 +++++- aws/logs_monitoring/tests/test_s3_handler.py | 8 +- 7 files changed, 116 insertions(+), 21 deletions(-) diff --git a/aws/logs_monitoring/steps/handlers/awslogs_handler.py b/aws/logs_monitoring/steps/handlers/awslogs_handler.py index e9342604b..941a14cd4 100644 --- a/aws/logs_monitoring/steps/handlers/awslogs_handler.py +++ b/aws/logs_monitoring/steps/handlers/awslogs_handler.py @@ -199,7 +199,6 @@ def process_lambda_logs(logs, aws_attributes, context, metadata): metadata[DD_CUSTOM_TAGS] += ",env:none" - # The lambda function name can be inferred from either a customized logstream name, or a loggroup name def get_lower_cased_lambda_function_name(logs): logstream_name = logs["logStream"] diff --git a/aws/logs_monitoring/steps/handlers/s3_handler.py b/aws/logs_monitoring/steps/handlers/s3_handler.py index fb621efce..3a5d3a9b1 100644 --- a/aws/logs_monitoring/steps/handlers/s3_handler.py +++ b/aws/logs_monitoring/steps/handlers/s3_handler.py @@ -42,6 +42,7 @@ logger = logging.getLogger() logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper())) + # Handle S3 events def s3_handler(event, context, metadata): # Need to use path style to access s3 via VPC Endpoints diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 0b282c62c..4c9048c38 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -24,20 +24,29 @@ env_patch.start() from steps.handlers.awslogs_handler import ( awslogs_handler, + process_lambda_logs, get_state_machine_arn, get_lower_cased_lambda_function_name, ) -env_patch.stop() +env_patch.stop() class TestAWSLogsHandler(unittest.TestCase): @patch("steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.get") - @patch("steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.release_s3_cache_lock") - @patch("steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.acquire_s3_cache_lock") - @patch("steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.write_cache_to_s3") + @patch( + "steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.release_s3_cache_lock" + ) + @patch( + "steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.acquire_s3_cache_lock" + ) + @patch( + "steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.write_cache_to_s3" + ) @patch("caching.base_tags_cache.send_forwarder_internal_metrics") - @patch("steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.get_cache_from_s3") + @patch( + "steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.get_cache_from_s3" + ) def test_awslogs_handler_rds_postgresql( self, mock_get_s3_cache, @@ -89,8 +98,12 @@ def test_awslogs_handler_rds_postgresql( @patch("steps.handlers.awslogs_handler.CloudwatchLogGroupTagsCache.get") @patch("steps.handlers.awslogs_handler.StepFunctionsTagsCache.get") - @patch("steps.handlers.awslogs_handler.StepFunctionsTagsCache.release_s3_cache_lock") - @patch("steps.handlers.awslogs_handler.StepFunctionsTagsCache.acquire_s3_cache_lock") + @patch( + "steps.handlers.awslogs_handler.StepFunctionsTagsCache.release_s3_cache_lock" + ) + @patch( + "steps.handlers.awslogs_handler.StepFunctionsTagsCache.acquire_s3_cache_lock" + ) @patch("steps.handlers.awslogs_handler.StepFunctionsTagsCache.write_cache_to_s3") @patch("caching.base_tags_cache.send_forwarder_internal_metrics") @patch("steps.handlers.awslogs_handler.StepFunctionsTagsCache.get_cache_from_s3") @@ -148,6 +161,59 @@ def test_awslogs_handler_step_functions_tags_added_properly( verify_as_json(list(awslogs_handler(event, context, metadata))) verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) + def test_process_lambda_logs(self): + # Non Lambda log + stepfunction_loggroup = { + "messageType": "DATA_MESSAGE", + "logGroup": "/aws/vendedlogs/states/logs-to-traces-sequential-Logs", + "logStream": "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9", + "logEvents": [], + } + metadata = {"ddsource": "postgresql", "ddtags": ""} + aws_attributes = {} + context = None + process_lambda_logs(stepfunction_loggroup, aws_attributes, context, metadata) + self.assertEqual(metadata, {"ddsource": "postgresql", "ddtags": ""}) + + # Lambda log + lambda_default_loggroup = { + "messageType": "DATA_MESSAGE", + "logGroup": "/aws/lambda/test-lambda-default-log-group", + "logStream": "2023/11/06/[$LATEST]b25b1f977b3e416faa45a00f427e7acb", + "logEvents": [], + } + metadata = {"ddsource": "postgresql", "ddtags": "env:dev"} + aws_attributes = {} + context = MagicMock() + context.invoked_function_arn = "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + process_lambda_logs(lambda_default_loggroup, aws_attributes, context, metadata) + self.assertEqual( + metadata, + { + "ddsource": "postgresql", + "ddtags": "env:dev", + }, + ) + self.assertEqual( + aws_attributes, + { + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:test-lambda-default-log-group" + } + }, + ) + + # env not set + metadata = {"ddsource": "postgresql", "ddtags": ""} + process_lambda_logs(lambda_default_loggroup, aws_attributes, context, metadata) + self.assertEqual( + metadata, + { + "ddsource": "postgresql", + "ddtags": ",env:none", + }, + ) + class TestLambdaCustomizedLogGroup(unittest.TestCase): def test_get_lower_cased_lambda_function_name(self): diff --git a/aws/logs_monitoring/tests/test_cloudtrail_s3.py b/aws/logs_monitoring/tests/test_cloudtrail_s3.py index abe644364..db0bf7b27 100644 --- a/aws/logs_monitoring/tests/test_cloudtrail_s3.py +++ b/aws/logs_monitoring/tests/test_cloudtrail_s3.py @@ -22,7 +22,6 @@ }, ) env_patch.start() - import lambda_function from steps.parsing import parse @@ -88,16 +87,15 @@ class Context: } -def test_data_gzipped() -> io.BytesIO: - return io.BytesIO( - gzip.compress(json.dumps(copy.deepcopy(test_data)).encode("utf-8")) - ) - - class TestS3CloudwatchParsing(unittest.TestCase): def setUp(self): self.maxDiff = 9000 + def get_test_data_gzipped(self) -> io.BytesIO: + return io.BytesIO( + gzip.compress(json.dumps(copy.deepcopy(test_data)).encode("utf-8")) + ) + @patch("caching.base_tags_cache.boto3") @patch("steps.handlers.s3_handler.boto3") @patch("lambda_function.boto3") @@ -107,7 +105,7 @@ def test_s3_cloudtrail_pasing_and_enrichment( context = Context() boto3 = parsing_boto3.client() - boto3.get_object.return_value = {"Body": test_data_gzipped()} + boto3.get_object.return_value = {"Body": self.get_test_data_gzipped()} payload = { "s3": { diff --git a/aws/logs_monitoring/tests/test_lambda_function.py b/aws/logs_monitoring/tests/test_lambda_function.py index 39207b25a..e9d07e5aa 100644 --- a/aws/logs_monitoring/tests/test_lambda_function.py +++ b/aws/logs_monitoring/tests/test_lambda_function.py @@ -31,6 +31,7 @@ from steps.transformation import transform from steps.splitting import split from steps.parsing import parse, parse_event_type + env_patch.stop() @@ -70,7 +71,9 @@ def test_lambda_invocation_exception(self, boto3): class TestLambdaFunctionEndToEnd(unittest.TestCase): @patch("enhanced_lambda_metrics.LambdaTagsCache.get") - @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get_cache_from_s3") + @patch( + "caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get_cache_from_s3" + ) def test_datadog_forwarder(self, mock_get_lambda_tags, mock_get_s3_cache): mock_get_lambda_tags.return_value = ( { @@ -141,7 +144,9 @@ def test_datadog_forwarder(self, mock_get_lambda_tags, mock_get_s3_cache): @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") @patch("caching.lambda_cache.LambdaTagsCache.get") - def test_setting_service_tag_from_log_group_cache(self, cw_logs_tags_get, lambda_tags_get): + def test_setting_service_tag_from_log_group_cache( + self, cw_logs_tags_get, lambda_tags_get + ): reload(sys.modules["settings"]) reload(sys.modules["steps.parsing"]) cw_logs_tags_get.return_value = ["service:log_group_service"] diff --git a/aws/logs_monitoring/tests/test_logs.py b/aws/logs_monitoring/tests/test_logs.py index 2a6c4dd2e..16abecf35 100644 --- a/aws/logs_monitoring/tests/test_logs.py +++ b/aws/logs_monitoring/tests/test_logs.py @@ -1,8 +1,11 @@ import unittest import os -from logs.logs import DatadogScrubber -from logs.logs_helpers import filter_logs +from logs.logs import DatadogScrubber, DatadogBatcher +from logs.logs_helpers import ( + filter_logs, + compress_logs, +) from settings import ScrubbingRuleConfig, SCRUBBING_RULE_CONFIGS, get_env_var @@ -36,6 +39,23 @@ def test_non_ascii(self): os.environ.pop("DD_SCRUBBING_RULE", None) +class TestDatadogBatcher(unittest.TestCase): + def test_batch(self): + batcher = DatadogBatcher(256, 512, 1) + logs = [ + "a" * 100, + "b" * 100, + "c" * 100, + "d" * 100, + ] + batches = list(batcher.batch(logs)) + self.assertEqual(len(batches), 4) + + batcher = DatadogBatcher(256, 512, 2) + batches = list(batcher.batch(logs)) + self.assertEqual(len(batches), 2) + + class TestFilterLogs(unittest.TestCase): example_logs = [ "START RequestId: ...", diff --git a/aws/logs_monitoring/tests/test_s3_handler.py b/aws/logs_monitoring/tests/test_s3_handler.py index 907a46616..d5ff0f487 100644 --- a/aws/logs_monitoring/tests/test_s3_handler.py +++ b/aws/logs_monitoring/tests/test_s3_handler.py @@ -1,9 +1,9 @@ - import gzip import unittest from approvaltests.combination_approvals import verify_all_combinations from steps.handlers.s3_handler import ( parse_service_arn, + get_partition_from_region, get_structured_lines_for_s3_handler, ) @@ -53,6 +53,12 @@ def test_get_structured_lines_cloudtrail(self): ], ) + def test_get_partition_from_region(self): + self.assertEqual(get_partition_from_region("us-east-1"), "aws") + self.assertEqual(get_partition_from_region("us-gov-west-1"), "aws-us-gov") + self.assertEqual(get_partition_from_region("cn-north-1"), "aws-cn") + self.assertEqual(get_partition_from_region(None), "aws") + class TestParseServiceArn(unittest.TestCase): def test_elb_s3_key_invalid(self): From 02fd1ff8098ea3147716de0ddabdb2689bdb5dc9 Mon Sep 17 00:00:00 2001 From: Georgi Date: Tue, 27 Feb 2024 14:43:33 +0100 Subject: [PATCH 6/8] Fix Lint --- aws/logs_monitoring/steps/handlers/s3_handler.py | 4 ++-- aws/logs_monitoring/tests/test_logs.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/aws/logs_monitoring/steps/handlers/s3_handler.py b/aws/logs_monitoring/steps/handlers/s3_handler.py index 3a5d3a9b1..7ca7dbfd2 100644 --- a/aws/logs_monitoring/steps/handlers/s3_handler.py +++ b/aws/logs_monitoring/steps/handlers/s3_handler.py @@ -196,8 +196,8 @@ def get_structured_lines_for_s3_handler(data, bucket, key, source): # Check if using multiline log regex pattern # and determine whether line or pattern separated logs data = data.decode("utf-8", errors="ignore") - if DD_MULTILINE_LOG_REGEX_PATTERN and multiline_regex_start_pattern.match(data): - split_data = multiline_regex.split(data) + if DD_MULTILINE_LOG_REGEX_PATTERN and MULTILINE_REGEX_START_PATTERN.match(data): + split_data = MULTILINE_REGEX.split(data) else: if DD_MULTILINE_LOG_REGEX_PATTERN: logger.debug( diff --git a/aws/logs_monitoring/tests/test_logs.py b/aws/logs_monitoring/tests/test_logs.py index 16abecf35..8d12caea3 100644 --- a/aws/logs_monitoring/tests/test_logs.py +++ b/aws/logs_monitoring/tests/test_logs.py @@ -2,10 +2,7 @@ import os from logs.logs import DatadogScrubber, DatadogBatcher -from logs.logs_helpers import ( - filter_logs, - compress_logs, -) +from logs.logs_helpers import filter_logs from settings import ScrubbingRuleConfig, SCRUBBING_RULE_CONFIGS, get_env_var From 343f2656e7b1d9d5437de4f5540136aadd5b2901 Mon Sep 17 00:00:00 2001 From: Georgi Date: Tue, 27 Feb 2024 18:20:34 +0100 Subject: [PATCH 7/8] Refactor parsing/handlers --- aws/logs_monitoring/lambda_function.py | 5 +- aws/logs_monitoring/settings.py | 11 ++ aws/logs_monitoring/steps/common.py | 6 +- .../steps/handlers/awslogs_handler.py | 83 ++++++---- .../steps/handlers/s3_handler.py | 151 ++++++++++-------- aws/logs_monitoring/steps/parsing.py | 80 +++++----- 6 files changed, 194 insertions(+), 142 deletions(-) diff --git a/aws/logs_monitoring/lambda_function.py b/aws/logs_monitoring/lambda_function.py index b0c68ac18..e0c6142f2 100644 --- a/aws/logs_monitoring/lambda_function.py +++ b/aws/logs_monitoring/lambda_function.py @@ -66,7 +66,10 @@ def datadog_forwarder(event, context): if DD_ADDITIONAL_TARGET_LAMBDAS: invoke_additional_target_lambdas(event) - metrics, logs, trace_payloads = split(transform(enrich(parse(event, context)))) + parsed = parse(event, context) + enriched = enrich(parsed) + transformed = transform(enriched) + metrics, logs, trace_payloads = split(transformed) if DD_FORWARD_LOG: forward_logs(logs) diff --git a/aws/logs_monitoring/settings.py b/aws/logs_monitoring/settings.py index 2e2820fa3..2d8ad7da3 100644 --- a/aws/logs_monitoring/settings.py +++ b/aws/logs_monitoring/settings.py @@ -239,6 +239,17 @@ def __init__(self, name, pattern, placeholder): DD_HOST = "host" DD_FORWARDER_VERSION = "3.103.0" +# CONST STRINGS +AWS_STRING = "aws" +FUNCTIONVERSION_STRING = "function_version" +INVOKEDFUNCTIONARN_STRING = "invoked_function_arn" +SOURCECATEGORY_STRING = "ddsourcecategory" +FORWARDERNAME_STRING = "forwardername" +FORWARDERMEMSIZE_STRING = "forwarder_memorysize" +FORWARDERVERSION_STRING = "forwarder_version" +GOV_STRING = "gov" +CN_STRING = "cn" + # Additional target lambda invoked async with event data DD_ADDITIONAL_TARGET_LAMBDAS = get_env_var("DD_ADDITIONAL_TARGET_LAMBDAS", default=None) diff --git a/aws/logs_monitoring/steps/common.py b/aws/logs_monitoring/steps/common.py index b93dac19c..6d6516734 100644 --- a/aws/logs_monitoring/steps/common.py +++ b/aws/logs_monitoring/steps/common.py @@ -2,6 +2,7 @@ from settings import ( DD_SOURCE, + DD_SERVICE, DD_CUSTOM_TAGS, ) @@ -157,13 +158,16 @@ def get_service_from_tags_and_remove_duplicates(metadata): del tagsplit[i] else: service = tag[8:] - metadata[DD_CUSTOM_TAGS] = ",".join(tagsplit) # Default service to source value return service if service else metadata[DD_SOURCE] +def add_service_tag(metadata): + metadata[DD_SERVICE] = get_service_from_tags_and_remove_duplicates(metadata) + + def is_cloudtrail(key): match = CLOUDTRAIL_REGEX.search(key) return bool(match) diff --git a/aws/logs_monitoring/steps/handlers/awslogs_handler.py b/aws/logs_monitoring/steps/handlers/awslogs_handler.py index 941a14cd4..8ad99853c 100644 --- a/aws/logs_monitoring/steps/handlers/awslogs_handler.py +++ b/aws/logs_monitoring/steps/handlers/awslogs_handler.py @@ -7,9 +7,9 @@ from io import BufferedReader, BytesIO from steps.common import ( + add_service_tag, merge_dicts, parse_event_source, - get_service_from_tags_and_remove_duplicates, ) from customized_log_group import ( is_lambda_customized_log_group, @@ -19,7 +19,6 @@ from caching.step_functions_cache import StepFunctionsTagsCache from settings import ( DD_SOURCE, - DD_SERVICE, DD_HOST, DD_CUSTOM_TAGS, ) @@ -38,17 +37,44 @@ # Handle CloudWatch logs def awslogs_handler(event, context, metadata): # Get logs + logs = extract_logs(event) + # Set the source on the logs + set_source(event, metadata, logs) + # Build aws attributes + aws_attributes = init_attributes(logs) + add_cloudwatch_tags_from_cache(metadata, logs) + # Set service from custom tags, which may include the tags set on the log group + # Returns DD_SOURCE by default + add_service_tag(metadata) + # Set host as log group where cloudwatch is source + set_host(metadata, logs, aws_attributes) + # For Lambda logs we want to extract the function name, + # then rebuild the arn of the monitored lambda using that name. + if metadata[DD_SOURCE] == "lambda": + process_lambda_logs(logs, aws_attributes, context, metadata) + # The EKS log group contains various sources from the K8S control plane. + # In order to have these automatically trigger the correct pipelines they + # need to send their events with the correct log source. + if metadata[DD_SOURCE] == "eks": + process_eks_logs(logs, metadata) + + # Create and send structured logs to Datadog + for log in logs["logEvents"]: + yield merge_dicts(log, aws_attributes) + + +def extract_logs(event): with gzip.GzipFile( fileobj=BytesIO(base64.b64decode(event["awslogs"]["data"])) ) as decompress_stream: # Reading line by line avoid a bug where gzip would take a very long # time (>5min) for file around 60MB gzipped data = b"".join(BufferedReader(decompress_stream)) - logs = json.loads(data) + return json.loads(data) - # Set the source on the logs - source = logs.get("logGroup", "cloudwatch") +def set_source(event, metadata, logs): + source = logs.get("logGroup", "cloudwatch") # Use the logStream to identify if this is a CloudTrail, TransitGateway, or Bedrock event # i.e. 123456779121_CloudTrail_us-east-1 if "_CloudTrail_" in logs["logStream"]: @@ -65,8 +91,9 @@ def awslogs_handler(event, context, metadata): if is_lambda_customized_log_group(logs["logStream"]): metadata[DD_SOURCE] = "lambda" - # Build aws attributes - aws_attributes = { + +def init_attributes(logs): + return { "aws": { "awslogs": { "logGroup": logs["logGroup"], @@ -76,6 +103,8 @@ def awslogs_handler(event, context, metadata): } } + +def add_cloudwatch_tags_from_cache(metadata, logs): formatted_tags = account_cw_logs_tags_cache.get(logs["logGroup"]) if len(formatted_tags) > 0: metadata[DD_CUSTOM_TAGS] = ( @@ -84,11 +113,8 @@ def awslogs_handler(event, context, metadata): else metadata[DD_CUSTOM_TAGS] + "," + ",".join(formatted_tags) ) - # Set service from custom tags, which may include the tags set on the log group - # Returns DD_SOURCE by default - metadata[DD_SERVICE] = get_service_from_tags_and_remove_duplicates(metadata) - # Set host as log group where cloudwatch is source +def set_host(metadata, logs, aws_attributes): if metadata[DD_SOURCE] == "cloudwatch" or metadata.get(DD_HOST, None) == None: metadata[DD_HOST] = aws_attributes["aws"]["awslogs"]["logGroup"] @@ -139,30 +165,19 @@ def awslogs_handler(event, context, metadata): metadata[DD_CUSTOM_TAGS] + ",logname:" + match.group("name") ) - # For Lambda logs we want to extract the function name, - # then rebuild the arn of the monitored lambda using that name. - if metadata[DD_SOURCE] == "lambda": - process_lambda_logs(logs, aws_attributes, context, metadata) - - # The EKS log group contains various sources from the K8S control plane. - # In order to have these automatically trigger the correct pipelines they - # need to send their events with the correct log source. - if metadata[DD_SOURCE] == "eks": - if logs["logStream"].startswith("kube-apiserver-audit-"): - metadata[DD_SOURCE] = "kubernetes.audit" - elif logs["logStream"].startswith("kube-scheduler-"): - metadata[DD_SOURCE] = "kube_scheduler" - elif logs["logStream"].startswith("kube-apiserver-"): - metadata[DD_SOURCE] = "kube-apiserver" - elif logs["logStream"].startswith("kube-controller-manager-"): - metadata[DD_SOURCE] = "kube-controller-manager" - elif logs["logStream"].startswith("authenticator-"): - metadata[DD_SOURCE] = "aws-iam-authenticator" - # In case the conditions above don't match we maintain eks as the source - # Create and send structured logs to Datadog - for log in logs["logEvents"]: - yield merge_dicts(log, aws_attributes) +def process_eks_logs(logs, metadata): + if logs["logStream"].startswith("kube-apiserver-audit-"): + metadata[DD_SOURCE] = "kubernetes.audit" + elif logs["logStream"].startswith("kube-scheduler-"): + metadata[DD_SOURCE] = "kube_scheduler" + elif logs["logStream"].startswith("kube-apiserver-"): + metadata[DD_SOURCE] = "kube-apiserver" + elif logs["logStream"].startswith("kube-controller-manager-"): + metadata[DD_SOURCE] = "kube-controller-manager" + elif logs["logStream"].startswith("authenticator-"): + metadata[DD_SOURCE] = "aws-iam-authenticator" + # In case the conditions above don't match we maintain eks as the source def get_state_machine_arn(message): diff --git a/aws/logs_monitoring/steps/handlers/s3_handler.py b/aws/logs_monitoring/steps/handlers/s3_handler.py index 7ca7dbfd2..be7f6da91 100644 --- a/aws/logs_monitoring/steps/handlers/s3_handler.py +++ b/aws/logs_monitoring/steps/handlers/s3_handler.py @@ -10,20 +10,20 @@ import botocore from steps.common import ( + add_service_tag, merge_dicts, is_cloudtrail, parse_event_source, - get_service_from_tags_and_remove_duplicates, ) from settings import ( + GOV_STRING, + CN_STRING, DD_SOURCE, - DD_SERVICE, DD_USE_VPC, DD_MULTILINE_LOG_REGEX_PATTERN, DD_HOST, ) -GOV, CN = "gov", "cn" if DD_MULTILINE_LOG_REGEX_PATTERN: try: MULTILINE_REGEX = re.compile( @@ -45,6 +45,27 @@ # Handle S3 events def s3_handler(event, context, metadata): + s3 = get_s3_client() + # if this is a S3 event carried in a SNS message, extract it and override the event + first_record = event["Records"][0] + if "Sns" in first_record: + event = json.loads(first_record["Sns"]["Message"]) + # Get the object from the event and show its content type + bucket = first_record["s3"]["bucket"]["name"] + key = urllib.parse.unquote_plus(first_record["s3"]["object"]["key"]) + source = set_source(event, metadata, bucket, key) + add_service_tag(metadata) + ##Get the ARN of the service and set it as the hostname + set_host(context, metadata, bucket, key, source) + # Extract the S3 object + response = s3.get_object(Bucket=bucket, Key=key) + body = response["Body"] + data = body.read() + + yield from get_structured_lines_for_s3_handler(data, bucket, key, source) + + +def get_s3_client(): # Need to use path style to access s3 via VPC Endpoints # https://github.com/gford1000-aws/lambda_s3_access_using_vpc_endpoint#boto3-specific-notes if DD_USE_VPC: @@ -55,32 +76,74 @@ def s3_handler(event, context, metadata): ) else: s3 = boto3.client("s3") - # if this is a S3 event carried in a SNS message, extract it and override the event - if "Sns" in event["Records"][0]: - event = json.loads(event["Records"][0]["Sns"]["Message"]) + return s3 - # Get the object from the event and show its content type - bucket = event["Records"][0]["s3"]["bucket"]["name"] - key = urllib.parse.unquote_plus(event["Records"][0]["s3"]["object"]["key"]) +def set_source(event, metadata, bucket, key): source = parse_event_source(event, key) if "transit-gateway" in bucket: source = "transitgateway" metadata[DD_SOURCE] = source - metadata[DD_SERVICE] = get_service_from_tags_and_remove_duplicates(metadata) + return source - ##Get the ARN of the service and set it as the hostname + +def set_host(context, metadata, bucket, key, source): hostname = parse_service_arn(source, key, bucket, context) if hostname: metadata[DD_HOST] = hostname - # Extract the S3 object - response = s3.get_object(Bucket=bucket, Key=key) - body = response["Body"] - data = body.read() - yield from get_structured_lines_for_s3_handler(data, bucket, key, source) +def get_structured_lines_for_s3_handler(data, bucket, key, source): + # Decompress data that has a .gz extension or magic header http://www.onicos.com/staff/iz/formats/gzip.html + if key[-3:] == ".gz" or data[:2] == b"\x1f\x8b": + with gzip.GzipFile(fileobj=BytesIO(data)) as decompress_stream: + # Reading line by line avoid a bug where gzip would take a very long time (>5min) for + # file around 60MB gzipped + data = b"".join(BufferedReader(decompress_stream)) + + is_cloudtrail_bucket = False + if is_cloudtrail(str(key)): + try: + cloud_trail = json.loads(data) + if cloud_trail.get("Records") is not None: + # only parse as a cloudtrail bucket if we have a Records field to parse + is_cloudtrail_bucket = True + for event in cloud_trail["Records"]: + # Create structured object and send it + structured_line = merge_dicts( + event, {"aws": {"s3": {"bucket": bucket, "key": key}}} + ) + yield structured_line + except Exception as e: + logger.debug("Unable to parse cloudtrail log: %s" % e) + + if not is_cloudtrail_bucket: + # Check if using multiline log regex pattern + # and determine whether line or pattern separated logs + data = data.decode("utf-8", errors="ignore") + if DD_MULTILINE_LOG_REGEX_PATTERN and MULTILINE_REGEX_START_PATTERN.match(data): + split_data = MULTILINE_REGEX_START_PATTERN.split(data) + else: + if DD_MULTILINE_LOG_REGEX_PATTERN: + logger.debug( + "DD_MULTILINE_LOG_REGEX_PATTERN %s did not match start of file, splitting by line", + DD_MULTILINE_LOG_REGEX_PATTERN, + ) + if source == "waf": + # WAF logs are \n separated + split_data = [d for d in data.split("\n") if d != ""] + else: + split_data = data.splitlines() + + # Send lines to Datadog + for line in split_data: + # Create structured object and send it + structured_line = { + "aws": {"s3": {"bucket": bucket, "key": key}}, + "message": line, + } + yield structured_line def parse_service_arn(source, key, bucket, context): @@ -161,60 +224,8 @@ def parse_service_arn(source, key, bucket, context): def get_partition_from_region(region): partition = "aws" if region: - if GOV in region: + if GOV_STRING in region: partition = "aws-us-gov" - elif CN in region: + elif CN_STRING in region: partition = "aws-cn" return partition - - -def get_structured_lines_for_s3_handler(data, bucket, key, source): - # Decompress data that has a .gz extension or magic header http://www.onicos.com/staff/iz/formats/gzip.html - if key[-3:] == ".gz" or data[:2] == b"\x1f\x8b": - with gzip.GzipFile(fileobj=BytesIO(data)) as decompress_stream: - # Reading line by line avoid a bug where gzip would take a very long time (>5min) for - # file around 60MB gzipped - data = b"".join(BufferedReader(decompress_stream)) - - is_cloudtrail_bucket = False - if is_cloudtrail(str(key)): - try: - cloud_trail = json.loads(data) - if cloud_trail.get("Records") is not None: - # only parse as a cloudtrail bucket if we have a Records field to parse - is_cloudtrail_bucket = True - for event in cloud_trail["Records"]: - # Create structured object and send it - structured_line = merge_dicts( - event, {"aws": {"s3": {"bucket": bucket, "key": key}}} - ) - yield structured_line - except Exception as e: - logger.debug("Unable to parse cloudtrail log: %s" % e) - - if not is_cloudtrail_bucket: - # Check if using multiline log regex pattern - # and determine whether line or pattern separated logs - data = data.decode("utf-8", errors="ignore") - if DD_MULTILINE_LOG_REGEX_PATTERN and MULTILINE_REGEX_START_PATTERN.match(data): - split_data = MULTILINE_REGEX.split(data) - else: - if DD_MULTILINE_LOG_REGEX_PATTERN: - logger.debug( - "DD_MULTILINE_LOG_REGEX_PATTERN %s did not match start of file, splitting by line", - DD_MULTILINE_LOG_REGEX_PATTERN, - ) - if source == "waf": - # WAF logs are \n separated - split_data = [d for d in data.split("\n") if d != ""] - else: - split_data = data.splitlines() - - # Send lines to Datadog - for line in split_data: - # Create structured object and send it - structured_line = { - "aws": {"s3": {"bucket": bucket, "key": key}}, - "message": line, - } - yield structured_line diff --git a/aws/logs_monitoring/steps/parsing.py b/aws/logs_monitoring/steps/parsing.py index 57d2c7af7..3c1fabff1 100644 --- a/aws/logs_monitoring/steps/parsing.py +++ b/aws/logs_monitoring/steps/parsing.py @@ -20,6 +20,13 @@ get_service_from_tags_and_remove_duplicates, ) from settings import ( + AWS_STRING, + FUNCTIONVERSION_STRING, + INVOKEDFUNCTIONARN_STRING, + SOURCECATEGORY_STRING, + FORWARDERNAME_STRING, + FORWARDERMEMSIZE_STRING, + FORWARDERVERSION_STRING, DD_TAGS, DD_SOURCE, DD_CUSTOM_TAGS, @@ -40,16 +47,19 @@ def parse(event, context): event_type = parse_event_type(event) if logger.isEnabledFor(logging.DEBUG): logger.debug(f"Parsed event type: {event_type}") - if event_type == "s3": - events = s3_handler(event, context, metadata) - elif event_type == "awslogs": - events = awslogs_handler(event, context, metadata) - elif event_type == "events": - events = cwevent_handler(event, metadata) - elif event_type == "sns": - events = sns_handler(event, metadata) - elif event_type == "kinesis": - events = kinesis_awslogs_handler(event, context, metadata) + match event_type: + case "s3": + events = s3_handler(event, context, metadata) + case "awslogs": + events = awslogs_handler(event, context, metadata) + case "events": + events = cwevent_handler(event, metadata) + case "sns": + events = sns_handler(event, metadata) + case "kinesis": + events = kinesis_awslogs_handler(event, context, metadata) + case _: + events = ["Parsing: Unsupported event type"] except Exception as e: # Logs through the socket the error err_message = "Error parsing the object. Exception: {} for event {}".format( @@ -64,18 +74,14 @@ def parse(event, context): def generate_metadata(context): metadata = { - "ddsourcecategory": "aws", - "aws": { - "function_version": context.function_version, - "invoked_function_arn": context.invoked_function_arn, + SOURCECATEGORY_STRING: AWS_STRING, + AWS_STRING: { + FUNCTIONVERSION_STRING: context.function_version, + INVOKEDFUNCTIONARN_STRING: context.invoked_function_arn, }, } # Add custom tags here by adding new value with the following format "key1:value1, key2:value2" - might be subject to modifications - dd_custom_tags_data = { - "forwardername": context.function_name.lower(), - "forwarder_memorysize": context.memory_limit_in_mb, - "forwarder_version": DD_FORWARDER_VERSION, - } + dd_custom_tags_data = generate_custom_tags(context) metadata[DD_CUSTOM_TAGS] = ",".join( filter( None, @@ -91,14 +97,23 @@ def generate_metadata(context): return metadata +def generate_custom_tags(context): + dd_custom_tags_data = { + FORWARDERNAME_STRING: context.function_name.lower(), + FORWARDERMEMSIZE_STRING: context.memory_limit_in_mb, + FORWARDERVERSION_STRING: DD_FORWARDER_VERSION, + } + + return dd_custom_tags_data + + def parse_event_type(event): - if "Records" in event and len(event["Records"]) > 0: - if "s3" in event["Records"][0]: + if "Records" in event and event["Records"]: + record = event["Records"][0] + if "s3" in record: return "s3" - elif "Sns" in event["Records"][0]: - # it's not uncommon to fan out s3 notifications through SNS, - # should treat it as an s3 event rather than sns event. - sns_msg = event["Records"][0]["Sns"]["Message"] + elif "Sns" in record: + sns_msg = record["Sns"]["Message"] try: sns_msg_dict = json.loads(sns_msg) if "Records" in sns_msg_dict and "s3" in sns_msg_dict["Records"][0]: @@ -107,12 +122,10 @@ def parse_event_type(event): if logger.isEnabledFor(logging.DEBUG): logger.debug(f"No s3 event detected from SNS message: {sns_msg}") return "sns" - elif "kinesis" in event["Records"][0]: + elif "kinesis" in record: return "kinesis" - elif "awslogs" in event: return "awslogs" - elif "detail" in event: return "events" raise Exception("Event type not supported (see #Event supported section)") @@ -120,28 +133,23 @@ def parse_event_type(event): # Handle Cloudwatch Events def cwevent_handler(event, metadata): - data = event - # Set the source on the log - source = data.get("source", "cloudwatch") + source = event.get("source", "cloudwatch") service = source.split(".") if len(service) > 1: metadata[DD_SOURCE] = service[1] else: metadata[DD_SOURCE] = "cloudwatch" - metadata[DD_SERVICE] = get_service_from_tags_and_remove_duplicates(metadata) - yield data + yield event # Handle Sns events def sns_handler(event, metadata): - data = event # Set the source on the log metadata[DD_SOURCE] = "sns" - - for ev in data["Records"]: + for ev in event["Records"]: # Create structured object and send it structured_line = ev yield structured_line From c87a5d2fd1374ff54de9ed1037ae694207d30764 Mon Sep 17 00:00:00 2001 From: Georgi Date: Wed, 28 Feb 2024 10:06:32 +0100 Subject: [PATCH 8/8] Update codeowners file --- .github/CODEOWNERS | 2 ++ aws/logs_monitoring/steps/handlers/awslogs_handler.py | 5 +++-- aws/logs_monitoring/steps/handlers/s3_handler.py | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e2bb9e5b4..db3f6d6ee 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,4 @@ # Azure Integrations azure/ @DataDog/azure-integrations +# AWS Integrations +aws/ @DataDog/aws-integrations diff --git a/aws/logs_monitoring/steps/handlers/awslogs_handler.py b/aws/logs_monitoring/steps/handlers/awslogs_handler.py index 8ad99853c..6768cba4b 100644 --- a/aws/logs_monitoring/steps/handlers/awslogs_handler.py +++ b/aws/logs_monitoring/steps/handlers/awslogs_handler.py @@ -38,10 +38,11 @@ def awslogs_handler(event, context, metadata): # Get logs logs = extract_logs(event) - # Set the source on the logs - set_source(event, metadata, logs) # Build aws attributes aws_attributes = init_attributes(logs) + # Set the source on the logs + set_source(event, metadata, logs) + # Add custom tags from cache add_cloudwatch_tags_from_cache(metadata, logs) # Set service from custom tags, which may include the tags set on the log group # Returns DD_SOURCE by default diff --git a/aws/logs_monitoring/steps/handlers/s3_handler.py b/aws/logs_monitoring/steps/handlers/s3_handler.py index be7f6da91..3a5864925 100644 --- a/aws/logs_monitoring/steps/handlers/s3_handler.py +++ b/aws/logs_monitoring/steps/handlers/s3_handler.py @@ -45,6 +45,7 @@ # Handle S3 events def s3_handler(event, context, metadata): + # Get the S3 client s3 = get_s3_client() # if this is a S3 event carried in a SNS message, extract it and override the event first_record = event["Records"][0]