From d58d2c3d0f8755dcd982071e3dc12e9f14db55f6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 5 Dec 2024 16:11:44 -0600 Subject: [PATCH] Disable zero copy writes in the ``StreamWriter`` (#10125) --- CHANGES/10125.bugfix.rst | 1 + aiohttp/http_writer.py | 2 +- tests/test_http_writer.py | 27 +++++++++++++-------------- 3 files changed, 15 insertions(+), 15 deletions(-) create mode 100644 CHANGES/10125.bugfix.rst diff --git a/CHANGES/10125.bugfix.rst b/CHANGES/10125.bugfix.rst new file mode 100644 index 00000000000..4ece1e68d96 --- /dev/null +++ b/CHANGES/10125.bugfix.rst @@ -0,0 +1 @@ +Disabled zero copy writes in the ``StreamWriter`` -- by :user:`bdraco`. diff --git a/aiohttp/http_writer.py b/aiohttp/http_writer.py index c66fda3d8d0..edd19ed65da 100644 --- a/aiohttp/http_writer.py +++ b/aiohttp/http_writer.py @@ -90,7 +90,7 @@ def _writelines(self, chunks: Iterable[bytes]) -> None: transport = self._protocol.transport if transport is None or transport.is_closing(): raise ClientConnectionResetError("Cannot write to closing transport") - transport.writelines(chunks) + transport.write(b"".join(chunks)) async def write( self, chunk: bytes, *, drain: bool = True, LIMIT: int = 0x10000 diff --git a/tests/test_http_writer.py b/tests/test_http_writer.py index 69e89bb34c2..65afc05ae10 100644 --- a/tests/test_http_writer.py +++ b/tests/test_http_writer.py @@ -123,16 +123,15 @@ async def test_write_large_payload_deflate_compression_data_in_eof( assert transport.write.called # type: ignore[attr-defined] chunks = [c[1][0] for c in list(transport.write.mock_calls)] # type: ignore[attr-defined] transport.write.reset_mock() # type: ignore[attr-defined] - assert not transport.writelines.called # type: ignore[attr-defined] # This payload compresses to 20447 bytes payload = b"".join( [bytes((*range(0, i), *range(i, 0, -1))) for i in range(255) for _ in range(64)] ) await msg.write_eof(payload) - assert not transport.write.called # type: ignore[attr-defined] - assert transport.writelines.called # type: ignore[attr-defined] - chunks.extend(transport.writelines.mock_calls[0][1][0]) # type: ignore[attr-defined] + chunks.extend([c[1][0] for c in list(transport.write.mock_calls)]) # type: ignore[attr-defined] + + assert all(chunks) content = b"".join(chunks) assert zlib.decompress(content) == (b"data" * 4096) + payload @@ -202,7 +201,7 @@ async def test_write_payload_deflate_compression_chunked( await msg.write(b"data") await msg.write_eof() - chunks = [b"".join(c[1][0]) for c in list(transport.writelines.mock_calls)] # type: ignore[attr-defined] + chunks = [c[1][0] for c in list(transport.write.mock_calls)] # type: ignore[attr-defined] assert all(chunks) content = b"".join(chunks) assert content == expected @@ -238,7 +237,7 @@ async def test_write_payload_deflate_compression_chunked_data_in_eof( await msg.write(b"data") await msg.write_eof(b"end") - chunks = [b"".join(c[1][0]) for c in list(transport.writelines.mock_calls)] # type: ignore[attr-defined] + chunks = [c[1][0] for c in list(transport.write.mock_calls)] # type: ignore[attr-defined] assert all(chunks) content = b"".join(chunks) assert content == expected @@ -257,16 +256,16 @@ async def test_write_large_payload_deflate_compression_chunked_data_in_eof( # This payload compresses to 1111 bytes payload = b"".join([bytes((*range(0, i), *range(i, 0, -1))) for i in range(255)]) await msg.write_eof(payload) - assert not transport.write.called # type: ignore[attr-defined] - chunks = [] - for write_lines_call in transport.writelines.mock_calls: # type: ignore[attr-defined] - chunked_payload = list(write_lines_call[1][0])[1:] - chunked_payload.pop() - chunks.extend(chunked_payload) + compressed = [] + chunks = [c[1][0] for c in list(transport.write.mock_calls)] # type: ignore[attr-defined] + chunked_body = b"".join(chunks) + split_body = chunked_body.split(b"\r\n") + while split_body: + if split_body.pop(0): + compressed.append(split_body.pop(0)) - assert all(chunks) - content = b"".join(chunks) + content = b"".join(compressed) assert zlib.decompress(content) == (b"data" * 4096) + payload