fix some parsing issues + add more tests

aws/logs_monitoring/parsing.py

@@ -236,8 +236,7 @@ def get_structured_lines_for_s3_handler(data, bucket, key, source):
                 )
             if source == "waf":
                 # WAF logs are \n separated
-                split_data = data.split("\n")
-                split_data.remove("")
+                split_data = [d for d in data.split("\n") if d != ""]
             else:
                 split_data = data.splitlines()
 

aws/logs_monitoring/tests/approved_files/TestS3EventsHandler.test_get_structured_lines_waf.approved.txt

@@ -0,0 +1,5 @@
+args: ('123\n456\n789\n',) => [{'aws': {'s3': {'bucket': 'my-bucket', 'key': 'mykey'}}, 'message': '123'}, {'aws': {'s3': {'bucket': 'my-bucket', 'key': 'mykey'}}, 'message': '456'}, {'aws': {'s3': {'bucket': 'my-bucket', 'key': 'mykey'}}, 'message': '789'}]
+args: ('123\n456\n789',) => [{'aws': {'s3': {'bucket': 'my-bucket', 'key': 'mykey'}}, 'message': '123'}, {'aws': {'s3': {'bucket': 'my-bucket', 'key': 'mykey'}}, 'message': '456'}, {'aws': {'s3': {'bucket': 'my-bucket', 'key': 'mykey'}}, 'message': '789'}]
+args: ('Hello\rWorld!\x0c',) => [{'aws': {'s3': {'bucket': 'my-bucket', 'key': 'mykey'}}, 'message': 'Hello\rWorld!\x0c'}]
+args: ('Hello\n\rWorld!\x0c',) => [{'aws': {'s3': {'bucket': 'my-bucket', 'key': 'mykey'}}, 'message': 'Hello'}, {'aws': {'s3': {'bucket': 'my-bucket', 'key': 'mykey'}}, 'message': '\rWorld!\x0c'}]
+args: ('',) => []

aws/logs_monitoring/tests/test_parsing.py

@@ -5,7 +5,8 @@
 import os
 import sys
 import unittest
-from approvaltests.approvals import verify_as_json
+from approvaltests.approvals import verify_as_json, verify
+from approvaltests.combination_approvals import verify_all_combinations
 from approvaltests.namer import NamerFactory
 
 sys.modules["trace_forwarder.connection"] = MagicMock()
@@ -820,56 +821,27 @@ def test_get_lower_cased_lambda_function_name(self):
         )
 
 
-class TestS3EventsHandler(unittest.TestCase):
-    def test_get_structured_lines_waf(self):
-        key = "mykey"
-        source = "waf"
-        bucket = "my-bucket"
-        data = gzip.compress(bytes("123\n456\n789\n", "utf-8"))
-        expected_lines = [
-            {"aws": {"s3": {"bucket": bucket, "key": key}}, "message": "123"},
-            {"aws": {"s3": {"bucket": bucket, "key": key}}, "message": "456"},
-            {"aws": {"s3": {"bucket": bucket, "key": key}}, "message": "789"},
-        ]
-
-        lines = yield from get_structured_lines_for_s3_handler(
-            data, bucket, key, source
-        )
-        lines = list(lines)
 
-        self.assertEqual(len(lines), 3)
-        self.assertEqual(lines, expected_lines)
+class TestS3EventsHandler(unittest.TestCase):
 
-    def test_get_structured_lines_waf_single_line(self):
+    def parse_lines(self,data):
         key = "mykey"
         source = "waf"
         bucket = "my-bucket"
-        message = "Hello\rWorld!\f"
-        data = gzip.compress(bytes(message, "utf-8"))
-        expected_lines = [
-            {"aws": {"s3": {"bucket": bucket, "key": key}}, "message": message},
-        ]
 
-        lines = yield from get_structured_lines_for_s3_handler(
-            data, bucket, key, source
-        )
-        lines = list(lines)
-
-        self.assertEqual(len(lines), 1)
-        self.assertEqual(lines, expected_lines)
-
-    def test_get_structured_lines_empty_waf(self):
-        key = "mykey"
-        source = "waf"
-        bucket = "my-bucket"
-        data = gzip.compress(bytes("", "utf-8"))
+        return [l for l in get_structured_lines_for_s3_handler(
+            gzip.compress(bytes(data, "utf-8")), bucket, key, source
+        )]
 
-        lines = yield from get_structured_lines_for_s3_handler(
-            data, bucket, key, source
-        )
-        lines = list(lines)
+    def test_get_structured_lines_waf(self):
+        verify_all_combinations(lambda d: self.parse_lines(d), [[
+            "123\n456\n789\n",
+            "123\n456\n789",
+            "Hello\rWorld!\f",
+            "Hello\n\rWorld!\f",
+            ""
+        ]])
 
-        self.assertEqual(len(lines), 0)
 
     def test_get_structured_lines_cloudtrail(self):
         key = "23456789123_CloudTrail_us-east-1"