Address issues with serialization and test runner

botocore/serialize.py

@@ -398,9 +398,6 @@ def _serialize_type_structure(self, serialized, value, shape, key):
                 serialized = new_serialized
             members = shape.members
             for member_key, member_value in value.items():
-                if member_value is None:
-                    # Don't serialize any parameter with a None value.
-                    continue
                 member_shape = members[member_key]
                 if 'name' in member_shape.serialization:
                     member_key = member_shape.serialization['name']
@@ -642,6 +639,9 @@ def _partition_parameters(
                 partitioned['query_string_kwargs'][key_name] = new_param
         elif location == 'header':
             shape = shape_members[param_name]
+            if not param_value and shape.type_name == 'list':
+                # Empty lists should not be set on the headers
+                return
             value = self._convert_header_value(shape, param_value)
             partitioned['headers'][key_name] = value
         elif location == 'headers':
@@ -704,7 +704,7 @@ def _convert_header_value(self, shape, value):
                     for v in value
                     if v is not None
                 ]
-            return ", ".join(converted_value)
+            return ",".join(converted_value)
         elif is_json_value_header(shape):
             # Serialize with no spaces after separators to save space in
             # the header.
@@ -735,8 +735,7 @@ def _requires_empty_body(self, shape):
 
     def _serialize_content_type(self, serialized, shape, shape_members):
         """Set Content-Type to application/json for all structured bodies."""
-        has_content_type = has_header('Content-Type', serialized['headers'])
-        if has_content_type:
+        if has_header('Content-Type', serialized['headers']):
             return
         payload = shape.serialization.get('payload')
         if self._has_streaming_payload(payload, shape_members):
@@ -746,9 +745,8 @@ def _serialize_content_type(self, serialized, shape, shape_members):
                 serialized['headers']['Content-Type'] = (
                     'application/octet-stream'
                 )
-        else:
-            if serialized['body'] != b'':
-                serialized['headers']['Content-Type'] = 'application/json'
+        elif serialized['body'] != b'':
+            serialized['headers']['Content-Type'] = 'application/json'
 
     def _serialize_body_params(self, params, shape):
         serialized_body = self.MAP_TYPE()
@@ -790,9 +788,9 @@ def _serialize_type_structure(self, xmlnode, params, shape, name):
             # xmlAttribute.  Rather than serializing into an XML child node,
             # we instead serialize the shape to an XML attribute of the
             # *current* node.
-            # if value is None:
-            #     # Don't serialize any param whose value is None.
-            #     return
+            if value is None:
+                # Don't serialize any param whose value is None.
+                return
             if member_shape.serialization.get('xmlAttribute'):
                 # xmlAttributes must have a serialization name.
                 xml_attribute_name = member_shape.serialization['name']
@@ -871,8 +869,7 @@ def _default_serialize(self, xmlnode, params, shape, name):
 
     def _serialize_content_type(self, serialized, shape, shape_members):
         """Set Content-Type to application/xml for all structured bodies."""
-        has_content_type = has_header('Content-Type', serialized['headers'])
-        if has_content_type:
+        if has_header('Content-Type', serialized['headers']):
             return
         payload = shape.serialization.get('payload')
         if self._has_streaming_payload(payload, shape_members):

tests/unit/test_protocols.py

@@ -127,7 +127,7 @@ def _compliance_tests(test_type=None):
                     protocol,
                     "input" if inp else "output",
                     model['description'],
-                    case['id'],
+                    case.get('id'),
                 ):
                     continue
                 if 'params' in case and inp:
@@ -158,7 +158,9 @@ def test_input_compliance(json_description, case, basename):
     client_endpoint = service_description.get('clientEndpoint')
     try:
         _assert_request_body_is_bytes(request['body'])
-        _assert_requests_equal(request, case['serialized'], protocol_type)
+        _assert_requests_equal(
+            request, case['serialized'], protocol_type, operation_model
+        )
         _assert_endpoints_equal(request, case['serialized'], client_endpoint)
     except AssertionError as e:
         _input_failure_message(protocol_type, case, request, e)
@@ -436,7 +438,7 @@ def _serialize_request_description(request_dict):
                 request_dict['url_path'] += f'&{encoded}'
 
 
-def _assert_requests_equal(actual, expected, protocol_type):
+def _assert_requests_equal(actual, expected, protocol_type, operation_model):
     if 'body' in expected:
         expected_body = expected['body'].encode('utf-8')
         actual_body = actual['body']
@@ -446,7 +448,11 @@ def _assert_requests_equal(actual, expected, protocol_type):
     expected_headers = HeadersDict(expected.get('headers', {}))
     excluded_headers = expected.get('forbidHeaders', [])
     _assert_expected_headers_in_request(
-        actual_headers, expected_headers, excluded_headers, protocol_type
+        actual_headers,
+        expected_headers,
+        excluded_headers,
+        protocol_type,
+        operation_model,
     )
     assert_equal(actual['url_path'], expected.get('uri', ''), "URI")
     if 'method' in expected:
@@ -489,22 +495,71 @@ def _assert_xml_bodies(actual, expected):
 
 
 def _assert_expected_headers_in_request(
-    actual, expected, excluded_headers, protocol_type
+    actual, expected, excluded_headers, protocol_type, operation_model
 ):
+    _clean_list_header_values(actual, expected, operation_model)
     if protocol_type in ['query', 'ec2']:
         # Botocore sets the Content-Type header to the following for query and ec2:
         # Content-Type: application/x-www-form-urlencoded; charset=utf-8
         # The protocol test files do not include "; charset=utf-8".
         # We'll add this to the expected header value before asserting equivalence.
-        if expected.get('Content-Type'):
-            expected['Content-Type'] += '; charset=utf-8'
+        content_type = expected.get('Content-Type', '')
+        if 'charset=utf-8' not in content_type:
+            expected['Content-Type'] = content_type + '; charset=utf-8'
     for header, value in expected.items():
         assert header in actual
         assert actual[header] == value
     for header in excluded_headers:
         assert header not in actual
 
 
+def _clean_list_header_values(
+    actual_headers, expected_headers, operation_model
+):
+    """
+    Standardizes list-type header values in HTTP request headers based on an AWS operation model.
+    Ensures consistency between expected and actual header values, particularly for lists and timestamps.
+
+    Expected list header values in Smithy protocol tests are joined by ", ". Example: "foo, bar, baz".
+    Actual list headers values generated in botocore are  joined by ",". Example "foo,bar,baz".
+    We need to standardize these header values to assert equivalence appropriately.
+    """
+    input_shape = operation_model.input_shape
+
+    if not (
+        input_shape
+        and input_shape.type_name == "structure"
+        and input_shape.members
+    ):
+        return
+
+    for member, shape in input_shape.members.items():
+        if (
+            shape.serialization.get("location") != "header"
+            or shape.type_name != "list"
+        ):
+            continue
+
+        header_name = shape.serialization.get("name")
+        if not header_name:
+            continue
+
+        # Standardize expected header values by removing spaces after commas
+        if header_name in expected_headers:
+            expected_headers[header_name] = expected_headers[
+                header_name
+            ].replace(", ", ",")
+
+        # Standardize actual header values only if they exist and the list contains timestamps
+        if (
+            shape.member.type_name == "timestamp"
+            and header_name in actual_headers
+        ):
+            actual_headers[header_name] = actual_headers[header_name].replace(
+                ", ", ","
+            )
+
+
 def _walk_files():
     # Check for a shortcut when running the tests interactively.
     # If a BOTOCORE_TEST env var is defined, that file is used as the