From 6f9efe1cc9d19de5610f246c3afdfb036c27e2fd Mon Sep 17 00:00:00 2001 From: Georgi Date: Mon, 26 Feb 2024 15:26:04 +0100 Subject: [PATCH] Package caching --- aws/logs_monitoring/caching/__init__.py | 0 .../{ => caching}/base_tags_cache.py | 0 .../cloudwatch_log_group_cache.py | 2 +- .../{ => caching}/lambda_cache.py | 2 +- .../{ => caching}/step_functions_cache.py | 2 +- .../enhanced_lambda_metrics.py | 2 +- aws/logs_monitoring/parsing.py | 4 +- .../tests/test_cloudtrail_s3.py | 2 +- .../tests/test_enhanced_lambda_metrics.py | 54 +++++++++---------- .../tests/test_lambda_function.py | 12 ++--- aws/logs_monitoring/tests/test_parsing.py | 4 +- 11 files changed, 42 insertions(+), 42 deletions(-) create mode 100644 aws/logs_monitoring/caching/__init__.py rename aws/logs_monitoring/{ => caching}/base_tags_cache.py (100%) rename aws/logs_monitoring/{ => caching}/cloudwatch_log_group_cache.py (98%) rename aws/logs_monitoring/{ => caching}/lambda_cache.py (98%) rename aws/logs_monitoring/{ => caching}/step_functions_cache.py (99%) diff --git a/aws/logs_monitoring/caching/__init__.py b/aws/logs_monitoring/caching/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/aws/logs_monitoring/base_tags_cache.py b/aws/logs_monitoring/caching/base_tags_cache.py similarity index 100% rename from aws/logs_monitoring/base_tags_cache.py rename to aws/logs_monitoring/caching/base_tags_cache.py diff --git a/aws/logs_monitoring/cloudwatch_log_group_cache.py b/aws/logs_monitoring/caching/cloudwatch_log_group_cache.py similarity index 98% rename from aws/logs_monitoring/cloudwatch_log_group_cache.py rename to aws/logs_monitoring/caching/cloudwatch_log_group_cache.py index 5949f079f..c503bfac9 100644 --- a/aws/logs_monitoring/cloudwatch_log_group_cache.py +++ b/aws/logs_monitoring/caching/cloudwatch_log_group_cache.py @@ -1,6 +1,6 @@ import boto3 -from base_tags_cache import ( +from caching.base_tags_cache import ( BaseTagsCache, logger, sanitize_aws_tag_string, diff --git a/aws/logs_monitoring/lambda_cache.py b/aws/logs_monitoring/caching/lambda_cache.py similarity index 98% rename from aws/logs_monitoring/lambda_cache.py rename to aws/logs_monitoring/caching/lambda_cache.py index d0bbf7cc7..7f444658c 100644 --- a/aws/logs_monitoring/lambda_cache.py +++ b/aws/logs_monitoring/caching/lambda_cache.py @@ -1,6 +1,6 @@ from botocore.exceptions import ClientError -from base_tags_cache import ( +from caching.base_tags_cache import ( GET_RESOURCES_LAMBDA_FILTER, BaseTagsCache, logger, diff --git a/aws/logs_monitoring/step_functions_cache.py b/aws/logs_monitoring/caching/step_functions_cache.py similarity index 99% rename from aws/logs_monitoring/step_functions_cache.py rename to aws/logs_monitoring/caching/step_functions_cache.py index c8560625b..8160df9d6 100644 --- a/aws/logs_monitoring/step_functions_cache.py +++ b/aws/logs_monitoring/caching/step_functions_cache.py @@ -1,6 +1,6 @@ from botocore.exceptions import ClientError -from base_tags_cache import ( +from caching.base_tags_cache import ( BaseTagsCache, logger, parse_get_resources_response_for_tags_by_arn, diff --git a/aws/logs_monitoring/enhanced_lambda_metrics.py b/aws/logs_monitoring/enhanced_lambda_metrics.py index 2521d074c..6d6a9ca3c 100644 --- a/aws/logs_monitoring/enhanced_lambda_metrics.py +++ b/aws/logs_monitoring/enhanced_lambda_metrics.py @@ -7,7 +7,7 @@ import re import datetime from time import time -from lambda_cache import LambdaTagsCache +from caching.lambda_cache import LambdaTagsCache ENHANCED_METRICS_NAMESPACE_PREFIX = "aws.lambda.enhanced" diff --git a/aws/logs_monitoring/parsing.py b/aws/logs_monitoring/parsing.py index b2724309c..ec29a5218 100644 --- a/aws/logs_monitoring/parsing.py +++ b/aws/logs_monitoring/parsing.py @@ -19,8 +19,8 @@ get_lambda_function_name_from_logstream_name, is_lambda_customized_log_group, ) -from step_functions_cache import StepFunctionsTagsCache -from cloudwatch_log_group_cache import CloudwatchLogGroupTagsCache +from caching.step_functions_cache import StepFunctionsTagsCache +from caching.cloudwatch_log_group_cache import CloudwatchLogGroupTagsCache from telemetry import ( DD_FORWARDER_TELEMETRY_NAMESPACE_PREFIX, get_forwarder_telemetry_tags, diff --git a/aws/logs_monitoring/tests/test_cloudtrail_s3.py b/aws/logs_monitoring/tests/test_cloudtrail_s3.py index 213a0d811..12179d38d 100644 --- a/aws/logs_monitoring/tests/test_cloudtrail_s3.py +++ b/aws/logs_monitoring/tests/test_cloudtrail_s3.py @@ -98,7 +98,7 @@ class TestS3CloudwatchParsing(unittest.TestCase): def setUp(self): self.maxDiff = 9000 - @patch("base_tags_cache.boto3") + @patch("caching.base_tags_cache.boto3") @patch("parsing.boto3") @patch("lambda_function.boto3") def test_s3_cloudtrail_pasing_and_enrichment( diff --git a/aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py b/aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py index f47eb15f0..b762795c7 100644 --- a/aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py +++ b/aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py @@ -13,12 +13,12 @@ create_out_of_memory_enhanced_metric, ) -from base_tags_cache import ( +from caching.base_tags_cache import ( sanitize_aws_tag_string, parse_get_resources_response_for_tags_by_arn, get_dd_tag_string_from_aws_dict, ) -from lambda_cache import LambdaTagsCache +from caching.lambda_cache import LambdaTagsCache class TestEnhancedLambdaMetrics(unittest.TestCase): @@ -313,8 +313,8 @@ def test_create_out_of_memory_enhanced_metric(self): success_message = "Success!" self.assertEqual(len(create_out_of_memory_enhanced_metric(success_message)), 0) - @patch("base_tags_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics( self, mock_get_s3_cache, mock_forward_metrics ): @@ -411,8 +411,8 @@ def test_generate_enhanced_lambda_metrics( del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("base_tags_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics_with_tags( self, mock_get_s3_cache, mock_forward_metrics ): @@ -524,8 +524,8 @@ def test_generate_enhanced_lambda_metrics_with_tags( del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("base_tags_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics_once_with_missing_arn( self, mock_get_s3_cache, mock_forward_metrics ): @@ -560,8 +560,8 @@ def test_generate_enhanced_lambda_metrics_once_with_missing_arn( del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("base_tags_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics_refresh_on_new_arn( self, mock_get_s3_cache, mock_forward_metrics ): @@ -603,12 +603,12 @@ def test_generate_enhanced_lambda_metrics_refresh_on_new_arn( del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("lambda_cache.LambdaTagsCache.release_s3_cache_lock") - @patch("lambda_cache.LambdaTagsCache.acquire_s3_cache_lock") - @patch("lambda_cache.LambdaTagsCache.write_cache_to_s3") - @patch("lambda_cache.LambdaTagsCache.build_tags_cache") - @patch("lambda_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.lambda_cache.LambdaTagsCache.release_s3_cache_lock") + @patch("caching.lambda_cache.LambdaTagsCache.acquire_s3_cache_lock") + @patch("caching.lambda_cache.LambdaTagsCache.write_cache_to_s3") + @patch("caching.lambda_cache.LambdaTagsCache.build_tags_cache") + @patch("caching.lambda_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics_refresh_s3_cache( self, mock_get_s3_cache, @@ -664,13 +664,13 @@ def test_generate_enhanced_lambda_metrics_refresh_s3_cache( del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("lambda_cache.LambdaTagsCache.release_s3_cache_lock") - @patch("lambda_cache.LambdaTagsCache.acquire_s3_cache_lock") - @patch("lambda_cache.resource_tagging_client") - @patch("lambda_cache.LambdaTagsCache.write_cache_to_s3") - @patch("lambda_cache.parse_get_resources_response_for_tags_by_arn") - @patch("lambda_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.lambda_cache.LambdaTagsCache.release_s3_cache_lock") + @patch("caching.lambda_cache.LambdaTagsCache.acquire_s3_cache_lock") + @patch("caching.lambda_cache.resource_tagging_client") + @patch("caching.lambda_cache.LambdaTagsCache.write_cache_to_s3") + @patch("caching.lambda_cache.parse_get_resources_response_for_tags_by_arn") + @patch("caching.lambda_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics_client_error( self, mock_get_s3_cache, @@ -724,8 +724,8 @@ def test_generate_enhanced_lambda_metrics_client_error( del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("base_tags_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics_timeout( self, mock_get_s3_cache, mock_forward_metrics ): @@ -782,8 +782,8 @@ def test_generate_enhanced_lambda_metrics_timeout( ) del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("base_tags_cache.send_forwarder_internal_metrics") - @patch("lambda_cache.LambdaTagsCache.get_cache_from_s3") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3") def test_generate_enhanced_lambda_metrics_out_of_memory( self, mock_get_s3_cache, mock_forward_metrics ): diff --git a/aws/logs_monitoring/tests/test_lambda_function.py b/aws/logs_monitoring/tests/test_lambda_function.py index ebc24fab4..cf36eefdf 100644 --- a/aws/logs_monitoring/tests/test_lambda_function.py +++ b/aws/logs_monitoring/tests/test_lambda_function.py @@ -138,7 +138,7 @@ def test_datadog_forwarder(self, mock_get_s3_cache): del os.environ["DD_FETCH_LAMBDA_TAGS"] - @patch("cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") + @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") def test_setting_service_tag_from_log_group_cache(self, cw_logs_tags_get): reload(sys.modules["settings"]) reload(sys.modules["parsing"]) @@ -159,7 +159,7 @@ def test_setting_service_tag_from_log_group_cache(self, cw_logs_tags_get): self.assertEqual(log["service"], "log_group_service") @patch.dict(os.environ, {"DD_TAGS": "service:dd_tag_service"}, clear=True) - @patch("cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") + @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") def test_service_override_from_dd_tags(self, cw_logs_tags_get): reload(sys.modules["settings"]) reload(sys.modules["parsing"]) @@ -179,8 +179,8 @@ def test_service_override_from_dd_tags(self, cw_logs_tags_get): for log in logs: self.assertEqual(log["service"], "dd_tag_service") - @patch("lambda_cache.LambdaTagsCache.get") - @patch("cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") + @patch("caching.lambda_cache.LambdaTagsCache.get") + @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") def test_overrding_service_tag_from_lambda_cache( self, lambda_tags_get, cw_logs_tags_get ): @@ -203,8 +203,8 @@ def test_overrding_service_tag_from_lambda_cache( self.assertEqual(log["service"], "lambda_service") @patch.dict(os.environ, {"DD_TAGS": "service:dd_tag_service"}, clear=True) - @patch("lambda_cache.LambdaTagsCache.get") - @patch("cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") + @patch("caching.lambda_cache.LambdaTagsCache.get") + @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.get") def test_overrding_service_tag_from_lambda_cache_when_dd_tags_is_set( self, lambda_tags_get, cw_logs_tags_get ): diff --git a/aws/logs_monitoring/tests/test_parsing.py b/aws/logs_monitoring/tests/test_parsing.py index 8f4d224e8..3ceb1c102 100644 --- a/aws/logs_monitoring/tests/test_parsing.py +++ b/aws/logs_monitoring/tests/test_parsing.py @@ -400,7 +400,7 @@ class TestAWSLogsHandler(unittest.TestCase): @patch("parsing.CloudwatchLogGroupTagsCache.release_s3_cache_lock") @patch("parsing.CloudwatchLogGroupTagsCache.acquire_s3_cache_lock") @patch("parsing.CloudwatchLogGroupTagsCache.write_cache_to_s3") - @patch("base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") @patch("parsing.CloudwatchLogGroupTagsCache.get_cache_from_s3") def test_awslogs_handler_rds_postgresql( self, @@ -456,7 +456,7 @@ def test_awslogs_handler_rds_postgresql( @patch("parsing.StepFunctionsTagsCache.release_s3_cache_lock") @patch("parsing.StepFunctionsTagsCache.acquire_s3_cache_lock") @patch("parsing.StepFunctionsTagsCache.write_cache_to_s3") - @patch("base_tags_cache.send_forwarder_internal_metrics") + @patch("caching.base_tags_cache.send_forwarder_internal_metrics") @patch("parsing.StepFunctionsTagsCache.get_cache_from_s3") def test_awslogs_handler_step_functions_tags_added_properly( self,