From 59687074bbf7f3cace7a9eb77e6ebaf3b3d2fdf9 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 15 Jan 2026 16:12:15 +0000 Subject: [PATCH 001/199] Test chunk splits after pause --- tests/test_http_parser.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index bfd84aae0d2..59c1ebd5bd0 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -827,6 +827,24 @@ def test_max_header_value_size_under_limit(parser: HttpRequestParser) -> None: assert msg.url == URL("/test") +async def test_chunk_split_after_pause(parser: HttpRequestParser) -> None: + value = b"t" * size + text = ( + b"GET /test HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n" + + b"1\r\nb\r\n" * 50000 + + b"0\r\n\r\n" + ) + + payload = None + messages, upgrade, tail = parser.feed_data(text) + payload = messages[0][-1] + # Payload should have paused reading and stopped receiving new chunks after 16k. + assert len(payload._http_chunk_splits) == 160001 + # We should still get the full result after read(), as it will continue processing. + result = await payload.read() + assert result == b"b" * 50000 + + @pytest.mark.parametrize("size", [40965, 8191]) def test_max_header_value_size_continuation( response: HttpResponseParser, size: int From 2eeeb5e1d00788c303c58970e36e52d4d4afd1a5 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 15 Jan 2026 16:13:03 +0000 Subject: [PATCH 002/199] Update tests/test_http_parser.py --- tests/test_http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 59c1ebd5bd0..cc2708ca7d9 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -827,7 +827,7 @@ def test_max_header_value_size_under_limit(parser: HttpRequestParser) -> None: assert msg.url == URL("/test") -async def test_chunk_split_after_pause(parser: HttpRequestParser) -> None: +async def test_chunk_splits_after_pause(parser: HttpRequestParser) -> None: value = b"t" * size text = ( b"GET /test HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n" From 8b0cfee646556a845dd56ebac3216f0d5e14bcbe Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 15 Jan 2026 16:13:38 +0000 Subject: [PATCH 003/199] Apply suggestions from code review --- tests/test_http_parser.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index cc2708ca7d9..e61db01b5f0 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -828,14 +828,12 @@ def test_max_header_value_size_under_limit(parser: HttpRequestParser) -> None: async def test_chunk_splits_after_pause(parser: HttpRequestParser) -> None: - value = b"t" * size text = ( b"GET /test HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n" + b"1\r\nb\r\n" * 50000 + b"0\r\n\r\n" ) - payload = None messages, upgrade, tail = parser.feed_data(text) payload = messages[0][-1] # Payload should have paused reading and stopped receiving new chunks after 16k. From 9cd91549432112dcd496e30942b133e29ded54ba Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 15 Jan 2026 16:15:34 +0000 Subject: [PATCH 004/199] Update tests/test_http_parser.py --- tests/test_http_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index e61db01b5f0..a787c5860b6 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -837,6 +837,7 @@ async def test_chunk_splits_after_pause(parser: HttpRequestParser) -> None: messages, upgrade, tail = parser.feed_data(text) payload = messages[0][-1] # Payload should have paused reading and stopped receiving new chunks after 16k. + assert payload._http_chunk_splits is not None assert len(payload._http_chunk_splits) == 160001 # We should still get the full result after read(), as it will continue processing. result = await payload.read() From 83fc87c2b1b0818c9dda7d905cdda96e9bdb3864 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 15 Jan 2026 16:29:26 +0000 Subject: [PATCH 005/199] Update tests/test_http_parser.py --- tests/test_http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index a787c5860b6..c421e3f49e9 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -838,7 +838,7 @@ async def test_chunk_splits_after_pause(parser: HttpRequestParser) -> None: payload = messages[0][-1] # Payload should have paused reading and stopped receiving new chunks after 16k. assert payload._http_chunk_splits is not None - assert len(payload._http_chunk_splits) == 160001 + assert len(payload._http_chunk_splits) == 16001 # We should still get the full result after read(), as it will continue processing. result = await payload.read() assert result == b"b" * 50000 From efe2ca9cf0ab600eb794d8e2c3dfbf611a259032 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 15 Jan 2026 16:51:04 +0000 Subject: [PATCH 006/199] Read small chunks from decompressors --- aiohttp/base_protocol.py | 19 ++++- aiohttp/client_proto.py | 4 - aiohttp/compression_utils.py | 38 +++++++++- aiohttp/http_parser.py | 130 ++++++++++++++++++++++---------- aiohttp/multipart.py | 2 +- aiohttp/streams.py | 3 +- aiohttp/web_protocol.py | 13 ++-- tests/test_client_functional.py | 6 +- tests/test_http_parser.py | 1 + 9 files changed, 153 insertions(+), 63 deletions(-) diff --git a/aiohttp/base_protocol.py b/aiohttp/base_protocol.py index 7f01830f4e9..f7e1f6532d8 100644 --- a/aiohttp/base_protocol.py +++ b/aiohttp/base_protocol.py @@ -3,6 +3,7 @@ from .client_exceptions import ClientConnectionResetError from .helpers import set_exception +from .http_parser import HttpParser from .tcp_helpers import tcp_nodelay @@ -10,17 +11,19 @@ class BaseProtocol(asyncio.Protocol): __slots__ = ( "_loop", "_paused", + "_parser", "_drain_waiter", "_connection_lost", "_reading_paused", "transport", ) - def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + def __init__(self, loop: asyncio.AbstractEventLoop, parser: HttpParser) -> None: self._loop: asyncio.AbstractEventLoop = loop self._paused = False self._drain_waiter: asyncio.Future[None] | None = None self._reading_paused = False + self._parser = parser self.transport: asyncio.Transport | None = None @@ -48,15 +51,23 @@ def resume_writing(self) -> None: waiter.set_result(None) def pause_reading(self) -> None: - if not self._reading_paused and self.transport is not None: + self._reading_paused = True + self._parser.pause_reading() + if self.transport is not None: try: self.transport.pause_reading() except (AttributeError, NotImplementedError, RuntimeError): pass - self._reading_paused = True def resume_reading(self) -> None: - if self._reading_paused and self.transport is not None: + self._reading_paused = False + + # This will resume parsing any unprocessed data from the last pause. + self.data_received(b"") + + # Reading may have been paused again in the above call if there was a lot of + # compressed data still pending. + if not self._reading_paused and self.transport is not None: try: self.transport.resume_reading() except (AttributeError, NotImplementedError, RuntimeError): diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index 601b545c82a..54570471071 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -39,7 +39,6 @@ def __init__(self, loop: asyncio.AbstractEventLoop) -> None: self._tail = b"" self._upgraded = False - self._parser: HttpResponseParser | None = None self._read_timeout: float | None = None self._read_timeout_handle: asyncio.TimerHandle | None = None @@ -293,9 +292,6 @@ def _on_read_timeout(self) -> None: def data_received(self, data: bytes) -> None: self._reschedule_timeout() - if not data: - return - # custom payload parser - currently always WebSocketReader if self._payload_parser is not None: eof, tail = self._payload_parser.feed_data(data) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index 0bc4a30d8ed..ff7a50327c5 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -34,7 +34,7 @@ MAX_SYNC_CHUNK_SIZE = 4096 -DEFAULT_MAX_DECOMPRESS_SIZE = 2**25 # 32MiB +DEFAULT_MAX_DECOMPRESS_SIZE = 256 * 1024 # Unlimited decompression constants - different libraries use different conventions ZLIB_MAX_LENGTH_UNLIMITED = 0 # zlib uses 0 to mean unlimited @@ -179,6 +179,16 @@ async def decompress( ) return self.decompress_sync(data, max_length) + @abstractmethod + @property + def unconsumed_tail(self) -> bytes: + """Unused input that must be fed back to decompress() after using max_length.""" + + @abstractmethod + @property + def data_available(self) -> bool: + """Return True if more output is available using only .unconsumed_tail.""" + class ZLibCompressor: def __init__( @@ -271,7 +281,7 @@ def __init__( def decompress_sync( self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED ) -> bytes: - return self._decompressor.decompress(data, max_length) + return self._decompressor.decompress(self._decompressor._unconsumed_tail + data, max_length) def flush(self, length: int = 0) -> bytes: return ( @@ -280,6 +290,14 @@ def flush(self, length: int = 0) -> bytes: else self._decompressor.flush() ) + @property + def unconsumed_tail(self) -> bytes: + return self._decompressor.unconsumed_tail + + @property + def data_available(self) -> bool: + return bool(self._decompressor.unconsumed_tail) + @property def eof(self) -> bool: return self._decompressor.eof @@ -317,6 +335,14 @@ def flush(self) -> bytes: return cast(bytes, self._obj.flush()) return b"" + @property + def unconsumed_tail(self) -> bytes: + pass # TODO + + @property + def data_available(self) -> bool: + pass # TODO + class ZSTDDecompressor(DecompressionBaseHandler): def __init__( @@ -346,3 +372,11 @@ def decompress_sync( def flush(self) -> bytes: return b"" + + @property + def unconsumed_tail(self) -> bytes: + return b"" + + @property + def data_available(self) -> bool: + return not self._obj.needs_input diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index c5560b7a5ac..c720092c467 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -100,6 +100,12 @@ class RawResponseMessage(NamedTuple): _MsgT = TypeVar("_MsgT", RawRequestMessage, RawResponseMessage) +class PayloadState(IntEnum): + PAYLOAD_COMPLETE = 0 + PAYLOAD_NEEDS_INPUT = 1 + PAYLOAD_HAS_PENDING_INPUT = 2 + + class ParseState(IntEnum): PARSE_NONE = 0 PARSE_LENGTH = 1 @@ -246,6 +252,10 @@ def parse_message(self, lines: list[bytes]) -> _MsgT: ... @abc.abstractmethod def _is_chunked_te(self, te: str) -> bool: ... + def pause_reading(self) -> None: + assert self._payload_parser is not None + self._payload_parser.pause_reading() + def feed_eof(self) -> _MsgT | None: if self._payload_parser is not None: self._payload_parser.feed_eof() @@ -445,7 +455,7 @@ def get_content_length() -> int | None: assert not self._lines assert self._payload_parser is not None try: - eof, data = self._payload_parser.feed_data(data[start_pos:], SEP) + payload_state, data = self._payload_parser.feed_data(data[start_pos:], SEP) except BaseException as underlying_exc: reraised_exc = underlying_exc if self.payload_exception is not None: @@ -457,18 +467,21 @@ def get_content_length() -> int | None: underlying_exc, ) - eof = True + payload_state = PayloadState.PAYLOAD_COMPLETE data = b"" if isinstance( underlying_exc, (InvalidHeader, TransferEncodingError) ): raise - if eof: - start_pos = 0 - data_len = len(data) - self._payload_parser = None - continue + if payload_state is not PayloadState.PAYLOAD_COMPLETE: + # We've either consumed all available data, or we're pausing + # until the reader buffer is freed up. + break + + start_pos = 0 + data_len = len(data) + self._payload_parser = None else: break @@ -751,6 +764,7 @@ def __init__( max_trailers: int = 128, ) -> None: self._length = 0 + self._paused = False self._type = ParseState.PARSE_UNTIL_EOF self._chunk = ChunkState.PARSE_CHUNKED_SIZE self._chunk_size = 0 @@ -789,6 +803,9 @@ def __init__( self.payload = real_payload + def pause_reading(self) -> None: + self._paused = True + def feed_eof(self) -> None: if self._type == ParseState.PARSE_UNTIL_EOF: self.payload.feed_eof() @@ -803,27 +820,50 @@ def feed_eof(self) -> None: def feed_data( self, chunk: bytes, SEP: _SEP = b"\r\n", CHUNK_EXT: bytes = b";" - ) -> tuple[bool, bytes]: + ) -> tuple[PayloadState, bytes]: + """Receive a chunk of data to process. + + Return: + PayloadState - The current state of payload processing. + This function may be called with empty bytes after returning + PAYLOAD_HAS_PENDING_INPUT to continue processing after a pause. + bytes - If payload is complete, this is the unconsumed bytes intended for the + next message/payload, b"" otherwise. + """ # Read specified amount of bytes if self._type == ParseState.PARSE_LENGTH: - required = self._length - self._length = max(required - len(chunk), 0) - self.payload.feed_data(chunk[:required]) - if self._length == 0: - self.payload.feed_eof() - return True, chunk[required:] + if self._chunk_tail: + chunk = self._chunk_tail + chunk + self._chunk_tail = b"" + + while chunk: + required = self._length + self._length = max(required - len(chunk), 0) + tail = self.payload.feed_data(chunk[:required]) + + if tail is not None: + self._length += len(tail) + chunk = tail + chunk[required:] + if self._paused: + self._paused = False + self._chunk_tail = chunk + return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" + if self._length == 0: + self.payload.feed_eof() + return PayloadState.PAYLOAD_COMPLETE, chunk[required:] + if tail is None: + break # Chunked transfer encoding parser elif self._type == ParseState.PARSE_CHUNKED: if self._chunk_tail: - # We should never have a tail if we're inside the payload body. - assert self._chunk != ChunkState.PARSE_CHUNKED_CHUNK - # We should check the length is sane. - max_line_length = self._max_line_size - if self._chunk == ChunkState.PARSE_TRAILERS: - max_line_length = self._max_field_size - if len(self._chunk_tail) > max_line_length: - raise LineTooLong(self._chunk_tail[:100] + b"...", max_line_length) + # We should check the length is sane when not processing payload body. + if self._chunk != ChunkState.PARSE_CHUNKED_CHUNK: + max_line_length = self._max_line_size + if self._chunk == ChunkState.PARSE_TRAILERS: + max_line_length = self._max_field_size + if len(self._chunk_tail) > max_line_length: + raise LineTooLong(self._chunk_tail[:100] + b"...", max_line_length) chunk = self._chunk_tail + chunk self._chunk_tail = b"" @@ -868,16 +908,26 @@ def feed_data( self.payload.begin_http_chunk_receiving() else: self._chunk_tail = chunk - return False, b"" + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" # read chunk and feed buffer if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK: + if self._paused: + self._paused = False + self._chunk_tail = chunk + return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" + required = self._chunk_size self._chunk_size = max(required - len(chunk), 0) - self.payload.feed_data(chunk[:required]) + tail = self.payload.feed_data(chunk[:required]) + + if tail is not None: + self._chunk_size += len(tail) + chunk = tail + chunk[required:] + continue if self._chunk_size: - return False, b"" + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" chunk = chunk[required:] self._chunk = ChunkState.PARSE_CHUNKED_CHUNK_EOF self.payload.end_http_chunk_receiving() @@ -891,13 +941,13 @@ def feed_data( self._chunk = ChunkState.PARSE_CHUNKED_SIZE else: self._chunk_tail = chunk - return False, b"" + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" if self._chunk == ChunkState.PARSE_TRAILERS: pos = chunk.find(SEP) if pos < 0: # No line found self._chunk_tail = chunk - return False, b"" + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" line = chunk[:pos] chunk = chunk[pos + len(SEP) :] @@ -923,13 +973,18 @@ def feed_data( finally: self._trailer_lines.clear() self.payload.feed_eof() - return True, chunk + return PayloadState.PAYLOAD_COMPLETE, chunk # Read all bytes until eof elif self._type == ParseState.PARSE_UNTIL_EOF: - self.payload.feed_data(chunk) + while chunk is not None: + if self._paused: + self._paused = False + self._chunk_tail = chunk + return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" + chunk = self.payload.feed_data(chunk) - return False, b"" + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" class DeflateBuffer: @@ -974,9 +1029,9 @@ def set_exception( ) -> None: set_exception(self.out, exc, exc_cause) - def feed_data(self, chunk: bytes) -> None: + def feed_data(self, chunk: bytes) -> bytes | None: if not chunk: - return + return None self.size += len(chunk) self.out.total_compressed_bytes = self.size @@ -996,9 +1051,8 @@ def feed_data(self, chunk: bytes) -> None: ) try: - # Decompress with limit + 1 so we can detect if output exceeds limit chunk = self.decompressor.decompress_sync( - chunk, max_length=self._max_decompress_size + 1 + chunk, max_length=self._max_decompress_size ) except Exception: raise ContentEncodingError( @@ -1007,15 +1061,9 @@ def feed_data(self, chunk: bytes) -> None: self._started_decoding = True - # Check if decompression limit was exceeded - if len(chunk) > self._max_decompress_size: - raise DecompressSizeError( - "Decompressed data exceeds the configured limit of %d bytes" - % self._max_decompress_size - ) - if chunk: self.out.feed_data(chunk) + return self.decompressor.unconsumed_tail if self.decompressor.data_available else None def feed_eof(self) -> None: chunk = self.decompressor.flush() diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 21697b8c175..51a813ed7fe 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -551,7 +551,7 @@ async def _decode_content_async(self, data: bytes) -> bytes: return await ZLibDecompressor( encoding=encoding, suppress_deflate_header=True, - ).decompress(data, max_length=self._max_decompress_size) + ).decompress(data, max_length=self._max_decompress_size) # TODO raise RuntimeError(f"unknown content encoding: {encoding}") diff --git a/aiohttp/streams.py b/aiohttp/streams.py index 034fcc540c0..4420769dced 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -277,7 +277,7 @@ def feed_data(self, data: bytes) -> None: assert not self._eof, "feed_data after feed_eof" if not data: - return + return None data_len = len(data) self._size += data_len @@ -291,6 +291,7 @@ def feed_data(self, data: bytes) -> None: if self._size > self._high_water and not self._protocol._reading_paused: self._protocol.pause_reading() + return None def begin_http_chunk_receiving(self) -> None: if self._http_chunk_splits is None: diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index bd39c48050d..31bb9a71136 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -170,7 +170,6 @@ class RequestHandler(BaseProtocol, Generic[_Request]): "_task_handler", "_upgrade", "_payload_parser", - "_request_parser", "logger", "access_log", "access_logger", @@ -383,7 +382,7 @@ def connection_lost(self, exc: BaseException | None) -> None: self._manager = None self._request_factory = None self._request_handler = None - self._request_parser = None + self._parser = None if self._keepalive_handle is not None: self._keepalive_handle.cancel() @@ -421,9 +420,9 @@ def data_received(self, data: bytes) -> None: # parse http messages messages: Sequence[_MsgType] if self._payload_parser is None and not self._upgrade: - assert self._request_parser is not None + assert self._parser is not None try: - messages, upgraded, tail = self._request_parser.feed_data(data) + messages, upgraded, tail = self._parser.feed_data(data) except HttpProcessingError as exc: messages = [ (_ErrInfo(status=400, exc=exc, message=exc.message), EMPTY_PAYLOAD) @@ -705,11 +704,11 @@ async def finish_response( prematurely. """ request._finish() - if self._request_parser is not None: - self._request_parser.set_upgraded(False) + if self._parser is not None: + self._parser.set_upgraded(False) self._upgrade = False if self._message_tail: - self._request_parser.feed_data(self._message_tail) + self._parser.feed_data(self._message_tail) self._message_tail = b"" try: prepare_meth = resp.prepare diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index fec3d3c6c3e..97d189d4b02 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -2406,7 +2406,7 @@ async def handler(request: web.Request) -> web.Response: async with client.get("/") as resp: assert resp.status == 200 - with pytest.raises(aiohttp.ClientPayloadError) as exc_info: + with pytest.raises(aiohttp.ClientPayloadError) as exc_info: # TODO await resp.read() assert isinstance(exc_info.value.__cause__, DecompressSizeError) @@ -2436,7 +2436,7 @@ async def handler(request: web.Request) -> web.Response: async with client.get("/") as resp: assert resp.status == 200 - with pytest.raises(aiohttp.ClientPayloadError) as exc_info: + with pytest.raises(aiohttp.ClientPayloadError) as exc_info: # TODO await resp.read() assert isinstance(exc_info.value.__cause__, DecompressSizeError) @@ -2467,7 +2467,7 @@ async def handler(request: web.Request) -> web.Response: async with client.get("/") as resp: assert resp.status == 200 - with pytest.raises(aiohttp.ClientPayloadError) as exc_info: + with pytest.raises(aiohttp.ClientPayloadError) as exc_info: # TODO await resp.read() assert isinstance(exc_info.value.__cause__, DecompressSizeError) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index c421e3f49e9..86fd6dcdffd 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -837,6 +837,7 @@ async def test_chunk_splits_after_pause(parser: HttpRequestParser) -> None: messages, upgrade, tail = parser.feed_data(text) payload = messages[0][-1] # Payload should have paused reading and stopped receiving new chunks after 16k. + assert parser._payload_parser._paused assert payload._http_chunk_splits is not None assert len(payload._http_chunk_splits) == 16001 # We should still get the full result after read(), as it will continue processing. From 57bf4a29641dd271e980e76af2b81047df61c6e0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Jan 2026 16:52:01 +0000 Subject: [PATCH 007/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/compression_utils.py | 8 +++++--- aiohttp/http_parser.py | 14 +++++++++++--- aiohttp/multipart.py | 4 +++- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index ff7a50327c5..c1666ee7b5f 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -281,7 +281,9 @@ def __init__( def decompress_sync( self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED ) -> bytes: - return self._decompressor.decompress(self._decompressor._unconsumed_tail + data, max_length) + return self._decompressor.decompress( + self._decompressor._unconsumed_tail + data, max_length + ) def flush(self, length: int = 0) -> bytes: return ( @@ -337,11 +339,11 @@ def flush(self) -> bytes: @property def unconsumed_tail(self) -> bytes: - pass # TODO + pass # TODO @property def data_available(self) -> bool: - pass # TODO + pass # TODO class ZSTDDecompressor(DecompressionBaseHandler): diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index c720092c467..6ef78a53b43 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -455,7 +455,9 @@ def get_content_length() -> int | None: assert not self._lines assert self._payload_parser is not None try: - payload_state, data = self._payload_parser.feed_data(data[start_pos:], SEP) + payload_state, data = self._payload_parser.feed_data( + data[start_pos:], SEP + ) except BaseException as underlying_exc: reraised_exc = underlying_exc if self.payload_exception is not None: @@ -863,7 +865,9 @@ def feed_data( if self._chunk == ChunkState.PARSE_TRAILERS: max_line_length = self._max_field_size if len(self._chunk_tail) > max_line_length: - raise LineTooLong(self._chunk_tail[:100] + b"...", max_line_length) + raise LineTooLong( + self._chunk_tail[:100] + b"...", max_line_length + ) chunk = self._chunk_tail + chunk self._chunk_tail = b"" @@ -1063,7 +1067,11 @@ def feed_data(self, chunk: bytes) -> bytes | None: if chunk: self.out.feed_data(chunk) - return self.decompressor.unconsumed_tail if self.decompressor.data_available else None + return ( + self.decompressor.unconsumed_tail + if self.decompressor.data_available + else None + ) def feed_eof(self) -> None: chunk = self.decompressor.flush() diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 51a813ed7fe..f78693a12a8 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -551,7 +551,9 @@ async def _decode_content_async(self, data: bytes) -> bytes: return await ZLibDecompressor( encoding=encoding, suppress_deflate_header=True, - ).decompress(data, max_length=self._max_decompress_size) # TODO + ).decompress( + data, max_length=self._max_decompress_size + ) # TODO raise RuntimeError(f"unknown content encoding: {encoding}") From 82a4dfb18142579d2f76814d12366a7c82273e13 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 15 Jan 2026 17:24:30 +0000 Subject: [PATCH 008/199] Fix --- aiohttp/base_protocol.py | 8 +++++--- aiohttp/client_proto.py | 2 +- aiohttp/compression_utils.py | 6 +++++- aiohttp/http_parser.py | 7 ++++--- aiohttp/web_protocol.py | 22 +++++++++++----------- tests/test_http_parser.py | 1 + 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/aiohttp/base_protocol.py b/aiohttp/base_protocol.py index f7e1f6532d8..636105a38ea 100644 --- a/aiohttp/base_protocol.py +++ b/aiohttp/base_protocol.py @@ -1,11 +1,13 @@ import asyncio -from typing import cast +from typing import TYPE_CHECKING, cast from .client_exceptions import ClientConnectionResetError from .helpers import set_exception -from .http_parser import HttpParser from .tcp_helpers import tcp_nodelay +if TYPE_CHECKING: + from .http_parser import HttpParser + class BaseProtocol(asyncio.Protocol): __slots__ = ( @@ -18,7 +20,7 @@ class BaseProtocol(asyncio.Protocol): "transport", ) - def __init__(self, loop: asyncio.AbstractEventLoop, parser: HttpParser) -> None: + def __init__(self, loop: asyncio.AbstractEventLoop, parser: "HttpParser | None") -> None: self._loop: asyncio.AbstractEventLoop = loop self._paused = False self._drain_waiter: asyncio.Future[None] | None = None diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index 54570471071..7ede8a5766c 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -26,7 +26,7 @@ class ResponseHandler(BaseProtocol, DataQueue[tuple[RawResponseMessage, StreamRe """Helper class to adapt between Protocol and StreamReader.""" def __init__(self, loop: asyncio.AbstractEventLoop) -> None: - BaseProtocol.__init__(self, loop=loop) + BaseProtocol.__init__(self, loop=loop, parser=None) DataQueue.__init__(self, loop) self._should_close = False diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index ff7a50327c5..07a2b3cb396 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -96,6 +96,10 @@ def __init__(self, _zlib_backend: ZLibBackendProtocol): def name(self) -> str: return getattr(self._zlib_backend, "__name__", "undefined") + @property + def unconsumed_tail(self) -> bytes: + return self._zlib_backend.unconsumed_tail + @property def MAX_WBITS(self) -> int: return self._zlib_backend.MAX_WBITS @@ -281,7 +285,7 @@ def __init__( def decompress_sync( self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED ) -> bytes: - return self._decompressor.decompress(self._decompressor._unconsumed_tail + data, max_length) + return self._decompressor.decompress(self._decompressor.unconsumed_tail + data, max_length) def flush(self, length: int = 0) -> bytes: return ( diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index c720092c467..8023e5cfd23 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -977,12 +977,13 @@ def feed_data( # Read all bytes until eof elif self._type == ParseState.PARSE_UNTIL_EOF: - while chunk is not None: + tail = chunk + while tail is not None: if self._paused: self._paused = False - self._chunk_tail = chunk + self._chunk_tail = tail return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" - chunk = self.payload.feed_data(chunk) + tail = self.payload.feed_data(tail) return PayloadState.PAYLOAD_NEEDS_INPUT, b"" diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index 31bb9a71136..a716c1d3b5a 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -202,7 +202,17 @@ def __init__( auto_decompress: bool = True, timeout_ceil_threshold: float = 5, ): - super().__init__(loop) + parser = HttpRequestParser( + self, + loop, + read_bufsize, + max_line_size=max_line_size, + max_field_size=max_field_size, + max_headers=max_headers, + payload_exception=RequestPayloadError, + auto_decompress=auto_decompress, + ) + super().__init__(loop, parser) # _request_count is the number of requests processed with the same connection. self._request_count = 0 @@ -232,16 +242,6 @@ def __init__( self._upgrade = False self._payload_parser: Any = None - self._request_parser: HttpRequestParser | None = HttpRequestParser( - self, - loop, - read_bufsize, - max_line_size=max_line_size, - max_field_size=max_field_size, - max_headers=max_headers, - payload_exception=RequestPayloadError, - auto_decompress=auto_decompress, - ) self._timeout_ceil_threshold: float = 5 try: diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 86fd6dcdffd..33450db9754 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -837,6 +837,7 @@ async def test_chunk_splits_after_pause(parser: HttpRequestParser) -> None: messages, upgrade, tail = parser.feed_data(text) payload = messages[0][-1] # Payload should have paused reading and stopped receiving new chunks after 16k. + assert parser._payload_parser is not None assert parser._payload_parser._paused assert payload._http_chunk_splits is not None assert len(payload._http_chunk_splits) == 16001 From a85f38baadcee3a9c86975142e9f1b1df42e4e49 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Jan 2026 17:27:09 +0000 Subject: [PATCH 009/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/base_protocol.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aiohttp/base_protocol.py b/aiohttp/base_protocol.py index 636105a38ea..df629a0627c 100644 --- a/aiohttp/base_protocol.py +++ b/aiohttp/base_protocol.py @@ -20,7 +20,9 @@ class BaseProtocol(asyncio.Protocol): "transport", ) - def __init__(self, loop: asyncio.AbstractEventLoop, parser: "HttpParser | None") -> None: + def __init__( + self, loop: asyncio.AbstractEventLoop, parser: "HttpParser | None" + ) -> None: self._loop: asyncio.AbstractEventLoop = loop self._paused = False self._drain_waiter: asyncio.Future[None] | None = None From 602eb510d95b7ff17730fac4c04dbdbbf8e274d5 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 15 Jan 2026 23:58:07 +0000 Subject: [PATCH 010/199] Update compression_utils.py --- aiohttp/compression_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index 111f669e278..5925041d879 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -183,13 +183,13 @@ async def decompress( ) return self.decompress_sync(data, max_length) - @abstractmethod @property + @abstractmethod def unconsumed_tail(self) -> bytes: """Unused input that must be fed back to decompress() after using max_length.""" - @abstractmethod @property + @abstractmethod def data_available(self) -> bool: """Return True if more output is available using only .unconsumed_tail.""" From 32f0a84639b76630caa66af92ab1b0dbd1493780 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 00:36:45 +0000 Subject: [PATCH 011/199] Update test_http_parser.py --- tests/test_http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 33450db9754..2aee6928c5c 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -58,7 +58,7 @@ @pytest.fixture def protocol() -> Any: - return mock.create_autospec(BaseProtocol, spec_set=True, instance=True) + return mock.create_autospec(BaseProtocol, transport=None, pause_reading=BaseProtocol.pause_reading, resume_reading=BaseProtocol.resume_reading, spec_set=True, instance=True) def _gen_ids(parsers: Iterable[type[HttpParser[Any]]]) -> list[str]: From 94c70a9296e03f512bf5b160ea055b6afa0a45f1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jan 2026 00:38:53 +0000 Subject: [PATCH 012/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_http_parser.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 2aee6928c5c..41e2ed36b00 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -58,7 +58,14 @@ @pytest.fixture def protocol() -> Any: - return mock.create_autospec(BaseProtocol, transport=None, pause_reading=BaseProtocol.pause_reading, resume_reading=BaseProtocol.resume_reading, spec_set=True, instance=True) + return mock.create_autospec( + BaseProtocol, + transport=None, + pause_reading=BaseProtocol.pause_reading, + resume_reading=BaseProtocol.resume_reading, + spec_set=True, + instance=True, + ) def _gen_ids(parsers: Iterable[type[HttpParser[Any]]]) -> list[str]: From bd34cea729308026192d19453254181f5526ac67 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 00:55:09 +0000 Subject: [PATCH 013/199] Update test_http_parser.py --- tests/test_http_parser.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 41e2ed36b00..59e0f4d666c 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -58,14 +58,15 @@ @pytest.fixture def protocol() -> Any: - return mock.create_autospec( + m = mock.create_autospec( BaseProtocol, transport=None, - pause_reading=BaseProtocol.pause_reading, - resume_reading=BaseProtocol.resume_reading, spec_set=True, instance=True, ) + m.pause_reading = BaseProtocol.pause_reading.__get__(m, BaseProtocol) + m.resume_reading = BaseProtocol.resume_reading.__get__(m, BaseProtocol) + return m def _gen_ids(parsers: Iterable[type[HttpParser[Any]]]) -> list[str]: From 0d4683c341a0b390add91d109ca80b3e8dfcfcac Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 01:07:39 +0000 Subject: [PATCH 014/199] Update test_http_parser.py --- tests/test_http_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 59e0f4d666c..242cabe5fdb 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -845,8 +845,8 @@ async def test_chunk_splits_after_pause(parser: HttpRequestParser) -> None: messages, upgrade, tail = parser.feed_data(text) payload = messages[0][-1] # Payload should have paused reading and stopped receiving new chunks after 16k. - assert parser._payload_parser is not None - assert parser._payload_parser._paused + #assert parser._payload_parser is not None + #assert parser._payload_parser._paused assert payload._http_chunk_splits is not None assert len(payload._http_chunk_splits) == 16001 # We should still get the full result after read(), as it will continue processing. From d3df80158f7c88609c383b29b85588c94e1ff788 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jan 2026 01:08:19 +0000 Subject: [PATCH 015/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_http_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 242cabe5fdb..deffb295c1c 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -845,8 +845,8 @@ async def test_chunk_splits_after_pause(parser: HttpRequestParser) -> None: messages, upgrade, tail = parser.feed_data(text) payload = messages[0][-1] # Payload should have paused reading and stopped receiving new chunks after 16k. - #assert parser._payload_parser is not None - #assert parser._payload_parser._paused + # assert parser._payload_parser is not None + # assert parser._payload_parser._paused assert payload._http_chunk_splits is not None assert len(payload._http_chunk_splits) == 16001 # We should still get the full result after read(), as it will continue processing. From b81f901891cb81ed48de792948e3cd24ee0b7b2b Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 01:16:58 +0000 Subject: [PATCH 016/199] Update test_http_parser.py --- tests/test_http_parser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index deffb295c1c..9e45d07d559 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -6,6 +6,7 @@ import zlib from collections.abc import Iterable from contextlib import suppress +from types import MethodType from typing import Any from unittest import mock from urllib.parse import quote @@ -64,8 +65,8 @@ def protocol() -> Any: spec_set=True, instance=True, ) - m.pause_reading = BaseProtocol.pause_reading.__get__(m, BaseProtocol) - m.resume_reading = BaseProtocol.resume_reading.__get__(m, BaseProtocol) + m.pause_reading = MethodType(BaseProtocol.pause_reading, m) + m.resume_reading = MethodType(BaseProtocol.resume_reading, m) return m From 4dd22795e180622e48996f9daa1dddf6934eea00 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 01:30:57 +0000 Subject: [PATCH 017/199] Fix --- tests/test_http_parser.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 9e45d07d559..772d29ac062 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -84,7 +84,7 @@ def parser( request: pytest.FixtureRequest, ) -> HttpRequestParser: # Parser implementations - return request.param( # type: ignore[no-any-return] + parser = request.param( protocol, loop, 2**16, @@ -92,6 +92,8 @@ def parser( max_headers=128, max_field_size=8190, ) + protocol._parser = parser + return parser # type: ignore[no-any-return] @pytest.fixture(params=REQUEST_PARSERS, ids=_gen_ids(REQUEST_PARSERS)) @@ -107,7 +109,7 @@ def response( request: pytest.FixtureRequest, ) -> HttpResponseParser: # Parser implementations - return request.param( # type: ignore[no-any-return] + parser = request.param( protocol, loop, 2**16, @@ -116,6 +118,8 @@ def response( max_field_size=8190, read_until_eof=True, ) + protocol._parser = parser + return parser # type: ignore[no-any-return] @pytest.fixture(params=RESPONSE_PARSERS, ids=_gen_ids(RESPONSE_PARSERS)) @@ -173,6 +177,7 @@ def test_invalid_character( max_line_size=8190, max_field_size=8190, ) + protocol._parser = parser text = b"POST / HTTP/1.1\r\nHost: localhost:8080\r\nSet-Cookie: abc\x01def\r\n\r\n" error_detail = re.escape( r""": @@ -197,6 +202,7 @@ def test_invalid_linebreak( max_line_size=8190, max_field_size=8190, ) + protocol._parser = parser text = b"GET /world HTTP/1.1\r\nHost: 127.0.0.1\n\r\n" error_detail = re.escape( r""": @@ -262,6 +268,7 @@ def test_unpaired_surrogate_in_header_py( max_line_size=8190, max_field_size=8190, ) + protocol._parser = parser text = b"POST / HTTP/1.1\r\n\xff\r\n\r\n" message = None try: @@ -1283,6 +1290,7 @@ async def test_http_response_parser_bad_chunked_strict_py( max_line_size=8190, max_field_size=8190, ) + protocol._parser = parser text = ( b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5 \r\nabcde\r\n0\r\n\r\n" ) @@ -1305,6 +1313,7 @@ async def test_http_response_parser_bad_chunked_strict_c( max_line_size=8190, max_field_size=8190, ) + protocol._parser = parser text = ( b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5 \r\nabcde\r\n0\r\n\r\n" ) @@ -1459,6 +1468,7 @@ def test_parse_no_length_or_te_on_post( request_cls: type[HttpRequestParser], ) -> None: parser = request_cls(protocol, loop, limit=2**16) + protocol._parser = parser text = b"POST /test HTTP/1.1\r\n\r\n" msg, payload = parser.feed_data(text)[0][0] @@ -1471,6 +1481,7 @@ def test_parse_payload_response_without_body( response_cls: type[HttpResponseParser], ) -> None: parser = response_cls(protocol, loop, 2**16, response_with_body=False) + protocol._parser = parser text = b"HTTP/1.1 200 Ok\r\ncontent-length: 10\r\n\r\n" msg, payload = parser.feed_data(text)[0][0] @@ -1742,6 +1753,7 @@ def test_parse_bad_method_for_c_parser_raises( max_headers=128, max_field_size=8190, ) + protocol._parser = parser with pytest.raises(aiohttp.http_exceptions.BadStatusLine): messages, upgrade, tail = parser.feed_data(payload) From 3a604a46ae1bb5612c467f0a21870720bbd78767 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 01:40:58 +0000 Subject: [PATCH 018/199] Update test_http_parser.py --- tests/test_http_parser.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 772d29ac062..b490981f187 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -18,6 +18,7 @@ import aiohttp from aiohttp import http_exceptions, streams from aiohttp.base_protocol import BaseProtocol +from aiohttp.client_proto import ResponseHandler from aiohttp.helpers import NO_EXTENSIONS from aiohttp.http_parser import ( DeflateBuffer, @@ -30,6 +31,7 @@ HttpResponseParserPy, ) from aiohttp.http_writer import HttpVersion +from aiohttp.web_protocol import RequestHandler try: try: @@ -93,6 +95,7 @@ def parser( max_field_size=8190, ) protocol._parser = parser + protocol.data_received = MethodType(RequestHandler.data_received, protocol) return parser # type: ignore[no-any-return] @@ -119,6 +122,7 @@ def response( read_until_eof=True, ) protocol._parser = parser + protocol.data_received = MethodType(ResponseHandler.data_received, protocol) return parser # type: ignore[no-any-return] From 10feb8bbb0509be8fb6946fa2c72def83010a83c Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 03:18:33 +0000 Subject: [PATCH 019/199] Fix --- aiohttp/streams.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/aiohttp/streams.py b/aiohttp/streams.py index 4420769dced..c1e2d9159b3 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -328,10 +328,7 @@ def end_http_chunk_receiving(self) -> None: # If we get too many small chunks before self._high_water is reached, then any # .read() call becomes computationally expensive, and could block the event loop # for too long, hence an additional self._high_water_chunks here. - if ( - len(self._http_chunk_splits) > self._high_water_chunks - and not self._protocol._reading_paused - ): + if (len(self._http_chunk_splits) > self._high_water_chunks): self._protocol.pause_reading() # wake up readchunk when end of http chunk received From d02bbf5c4843ffc75c919deccfbb08d8dce04fde Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jan 2026 03:19:21 +0000 Subject: [PATCH 020/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/streams.py b/aiohttp/streams.py index c1e2d9159b3..98275565074 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -328,7 +328,7 @@ def end_http_chunk_receiving(self) -> None: # If we get too many small chunks before self._high_water is reached, then any # .read() call becomes computationally expensive, and could block the event loop # for too long, hence an additional self._high_water_chunks here. - if (len(self._http_chunk_splits) > self._high_water_chunks): + if len(self._http_chunk_splits) > self._high_water_chunks: self._protocol.pause_reading() # wake up readchunk when end of http chunk received From 67bd57d54faaf0a6d071e234f8de451c9ae9f4db Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 03:28:09 +0000 Subject: [PATCH 021/199] Update test_http_parser.py --- tests/test_http_parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index b490981f187..222cf01150c 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -857,10 +857,10 @@ async def test_chunk_splits_after_pause(parser: HttpRequestParser) -> None: messages, upgrade, tail = parser.feed_data(text) payload = messages[0][-1] # Payload should have paused reading and stopped receiving new chunks after 16k. - # assert parser._payload_parser is not None - # assert parser._payload_parser._paused assert payload._http_chunk_splits is not None - assert len(payload._http_chunk_splits) == 16001 + assert len(payload._http_chunk_splits) == 16385 + assert parser._payload_parser is not None + assert parser._payload_parser._paused # We should still get the full result after read(), as it will continue processing. result = await payload.read() assert result == b"b" * 50000 From 04b717f1f8c0360661224f97b7e8b24ed9585ff4 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 03:51:42 +0000 Subject: [PATCH 022/199] Fix --- tests/test_http_parser.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 222cf01150c..911ca07790c 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -859,8 +859,6 @@ async def test_chunk_splits_after_pause(parser: HttpRequestParser) -> None: # Payload should have paused reading and stopped receiving new chunks after 16k. assert payload._http_chunk_splits is not None assert len(payload._http_chunk_splits) == 16385 - assert parser._payload_parser is not None - assert parser._payload_parser._paused # We should still get the full result after read(), as it will continue processing. result = await payload.read() assert result == b"b" * 50000 From 53ac968658e9c6228ef9fb6d627eb25838c93367 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 04:04:51 +0000 Subject: [PATCH 023/199] Update test_http_parser.py --- tests/test_http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 911ca07790c..95029851502 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -64,7 +64,6 @@ def protocol() -> Any: m = mock.create_autospec( BaseProtocol, transport=None, - spec_set=True, instance=True, ) m.pause_reading = MethodType(BaseProtocol.pause_reading, m) @@ -94,6 +93,7 @@ def parser( max_headers=128, max_field_size=8190, ) + protocol._force_close = False protocol._parser = parser protocol.data_received = MethodType(RequestHandler.data_received, protocol) return parser # type: ignore[no-any-return] From 44c12a1517da82b3edf083e6191006ea0fda415b Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 14:29:19 +0000 Subject: [PATCH 024/199] Fix --- tests/test_http_parser.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 911ca07790c..a26073cc031 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -63,12 +63,9 @@ def protocol() -> Any: m = mock.create_autospec( BaseProtocol, - transport=None, spec_set=True, instance=True, ) - m.pause_reading = MethodType(BaseProtocol.pause_reading, m) - m.resume_reading = MethodType(BaseProtocol.resume_reading, m) return m @@ -82,9 +79,10 @@ def _gen_ids(parsers: Iterable[type[HttpParser[Any]]]) -> list[str]: @pytest.fixture(params=REQUEST_PARSERS, ids=_gen_ids(REQUEST_PARSERS)) def parser( loop: asyncio.AbstractEventLoop, - protocol: BaseProtocol, request: pytest.FixtureRequest, ) -> HttpRequestParser: + protocol = RequestHandler(mock.create_autospec(Server, spec_set=True, instance=True), loop=loop) + # Parser implementations parser = request.param( protocol, @@ -95,7 +93,6 @@ def parser( max_field_size=8190, ) protocol._parser = parser - protocol.data_received = MethodType(RequestHandler.data_received, protocol) return parser # type: ignore[no-any-return] @@ -108,9 +105,10 @@ def request_cls(request: pytest.FixtureRequest) -> type[HttpRequestParser]: @pytest.fixture(params=RESPONSE_PARSERS, ids=_gen_ids(RESPONSE_PARSERS)) def response( loop: asyncio.AbstractEventLoop, - protocol: BaseProtocol, request: pytest.FixtureRequest, ) -> HttpResponseParser: + protocol = ResponseHandler(loop) + # Parser implementations parser = request.param( protocol, @@ -122,7 +120,6 @@ def response( read_until_eof=True, ) protocol._parser = parser - protocol.data_received = MethodType(ResponseHandler.data_received, protocol) return parser # type: ignore[no-any-return] @@ -171,9 +168,10 @@ def test_reject_obsolete_line_folding(parser: HttpRequestParser) -> None: @pytest.mark.skipif(NO_EXTENSIONS, reason="Only tests C parser.") def test_invalid_character( loop: asyncio.AbstractEventLoop, - protocol: BaseProtocol, request: pytest.FixtureRequest, ) -> None: + protocol = RequestHandler(mock.create_autospec(Server, spec_set=True, instance=True), loop) + parser = HttpRequestParserC( protocol, loop, @@ -196,9 +194,10 @@ def test_invalid_character( @pytest.mark.skipif(NO_EXTENSIONS, reason="Only tests C parser.") def test_invalid_linebreak( loop: asyncio.AbstractEventLoop, - protocol: BaseProtocol, request: pytest.FixtureRequest, ) -> None: + protocol = RequestHandler(mock.create_autospec(Server, spec_set=True, instance=True), loop) + parser = HttpRequestParserC( protocol, loop, @@ -263,8 +262,10 @@ def test_bad_headers(parser: HttpRequestParser, hdr: str) -> None: def test_unpaired_surrogate_in_header_py( - loop: asyncio.AbstractEventLoop, protocol: BaseProtocol + loop: asyncio.AbstractEventLoop ) -> None: + protocol = RequestHandler(mock.create_autospec(Server, spec_set=True, instance=True), loop) + parser = HttpRequestParserPy( protocol, loop, @@ -1283,8 +1284,10 @@ async def test_http_response_parser_bad_chunked_lax( @pytest.mark.dev_mode async def test_http_response_parser_bad_chunked_strict_py( - loop: asyncio.AbstractEventLoop, protocol: BaseProtocol + loop: asyncio.AbstractEventLoop ) -> None: + protocol = ResponseHandler(loop) + response = HttpResponseParserPy( protocol, loop, @@ -1306,8 +1309,10 @@ async def test_http_response_parser_bad_chunked_strict_py( reason="C based HTTP parser not available", ) async def test_http_response_parser_bad_chunked_strict_c( - loop: asyncio.AbstractEventLoop, protocol: BaseProtocol + loop: asyncio.AbstractEventLoop ) -> None: + protocol = ResponseHandler(loop) + response = HttpResponseParserC( protocol, loop, @@ -1466,9 +1471,9 @@ async def test_request_chunked_reject_bad_trailer(parser: HttpRequestParser) -> def test_parse_no_length_or_te_on_post( loop: asyncio.AbstractEventLoop, - protocol: BaseProtocol, request_cls: type[HttpRequestParser], ) -> None: + protocol = RequestHandler(mock.create_autospec(Server, spec_set=True, instance=True), loop) parser = request_cls(protocol, loop, limit=2**16) protocol._parser = parser text = b"POST /test HTTP/1.1\r\n\r\n" @@ -1479,9 +1484,9 @@ def test_parse_no_length_or_te_on_post( def test_parse_payload_response_without_body( loop: asyncio.AbstractEventLoop, - protocol: BaseProtocol, response_cls: type[HttpResponseParser], ) -> None: + protocol = ResponseHandler(loop) parser = response_cls(protocol, loop, 2**16, response_with_body=False) protocol._parser = parser text = b"HTTP/1.1 200 Ok\r\ncontent-length: 10\r\n\r\n" @@ -1744,8 +1749,10 @@ def test_parse_uri_utf8_percent_encoded(parser: HttpRequestParser) -> None: reason="C based HTTP parser not available", ) def test_parse_bad_method_for_c_parser_raises( - loop: asyncio.AbstractEventLoop, protocol: BaseProtocol + loop: asyncio.AbstractEventLoop ) -> None: + protocol = RequestHandler(mock.create_autospec(Server, spec_set=True, instance=True), loop) + payload = b"GET1 /test HTTP/1.1\r\n\r\n" parser = HttpRequestParserC( protocol, From 75794c237f37a91d5afa519925233c89835012c1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jan 2026 14:30:53 +0000 Subject: [PATCH 025/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_http_parser.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index cc35c135c00..d660bb6648a 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -81,7 +81,9 @@ def parser( loop: asyncio.AbstractEventLoop, request: pytest.FixtureRequest, ) -> HttpRequestParser: - protocol = RequestHandler(mock.create_autospec(Server, spec_set=True, instance=True), loop=loop) + protocol = RequestHandler( + mock.create_autospec(Server, spec_set=True, instance=True), loop=loop + ) # Parser implementations parser = request.param( @@ -171,7 +173,9 @@ def test_invalid_character( loop: asyncio.AbstractEventLoop, request: pytest.FixtureRequest, ) -> None: - protocol = RequestHandler(mock.create_autospec(Server, spec_set=True, instance=True), loop) + protocol = RequestHandler( + mock.create_autospec(Server, spec_set=True, instance=True), loop + ) parser = HttpRequestParserC( protocol, @@ -197,7 +201,9 @@ def test_invalid_linebreak( loop: asyncio.AbstractEventLoop, request: pytest.FixtureRequest, ) -> None: - protocol = RequestHandler(mock.create_autospec(Server, spec_set=True, instance=True), loop) + protocol = RequestHandler( + mock.create_autospec(Server, spec_set=True, instance=True), loop + ) parser = HttpRequestParserC( protocol, @@ -262,10 +268,10 @@ def test_bad_headers(parser: HttpRequestParser, hdr: str) -> None: parser.feed_data(text) -def test_unpaired_surrogate_in_header_py( - loop: asyncio.AbstractEventLoop -) -> None: - protocol = RequestHandler(mock.create_autospec(Server, spec_set=True, instance=True), loop) +def test_unpaired_surrogate_in_header_py(loop: asyncio.AbstractEventLoop) -> None: + protocol = RequestHandler( + mock.create_autospec(Server, spec_set=True, instance=True), loop + ) parser = HttpRequestParserPy( protocol, @@ -1285,7 +1291,7 @@ async def test_http_response_parser_bad_chunked_lax( @pytest.mark.dev_mode async def test_http_response_parser_bad_chunked_strict_py( - loop: asyncio.AbstractEventLoop + loop: asyncio.AbstractEventLoop, ) -> None: protocol = ResponseHandler(loop) @@ -1310,7 +1316,7 @@ async def test_http_response_parser_bad_chunked_strict_py( reason="C based HTTP parser not available", ) async def test_http_response_parser_bad_chunked_strict_c( - loop: asyncio.AbstractEventLoop + loop: asyncio.AbstractEventLoop, ) -> None: protocol = ResponseHandler(loop) @@ -1474,7 +1480,9 @@ def test_parse_no_length_or_te_on_post( loop: asyncio.AbstractEventLoop, request_cls: type[HttpRequestParser], ) -> None: - protocol = RequestHandler(mock.create_autospec(Server, spec_set=True, instance=True), loop) + protocol = RequestHandler( + mock.create_autospec(Server, spec_set=True, instance=True), loop + ) parser = request_cls(protocol, loop, limit=2**16) protocol._parser = parser text = b"POST /test HTTP/1.1\r\n\r\n" @@ -1749,10 +1757,10 @@ def test_parse_uri_utf8_percent_encoded(parser: HttpRequestParser) -> None: "HttpRequestParserC" not in dir(aiohttp.http_parser), reason="C based HTTP parser not available", ) -def test_parse_bad_method_for_c_parser_raises( - loop: asyncio.AbstractEventLoop -) -> None: - protocol = RequestHandler(mock.create_autospec(Server, spec_set=True, instance=True), loop) +def test_parse_bad_method_for_c_parser_raises(loop: asyncio.AbstractEventLoop) -> None: + protocol = RequestHandler( + mock.create_autospec(Server, spec_set=True, instance=True), loop + ) payload = b"GET1 /test HTTP/1.1\r\n\r\n" parser = HttpRequestParserC( From ab736261709e696b8c90eed0a34daf3fc037b35d Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 14:42:43 +0000 Subject: [PATCH 026/199] Update test_http_parser.py --- tests/test_http_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index d660bb6648a..36d1df200de 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -32,6 +32,7 @@ ) from aiohttp.http_writer import HttpVersion from aiohttp.web_protocol import RequestHandler +from aiohttp.web_server import Server try: try: From ae46ee1e94262e39f4f139a4dda03391c1be4c87 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 15:02:22 +0000 Subject: [PATCH 027/199] Update test_http_parser.py --- tests/test_http_parser.py | 42 ++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 36d1df200de..15f859ad9a3 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -60,6 +60,16 @@ RESPONSE_PARSERS.append(HttpResponseParserC) +@pytest.fixture +def server() -> Any: + m = mock.create_autospec( + Server, + request_handler=mock.AsyncMock() + instance=True, + ) + return m + + @pytest.fixture def protocol() -> Any: m = mock.create_autospec( @@ -80,11 +90,10 @@ def _gen_ids(parsers: Iterable[type[HttpParser[Any]]]) -> list[str]: @pytest.fixture(params=REQUEST_PARSERS, ids=_gen_ids(REQUEST_PARSERS)) def parser( loop: asyncio.AbstractEventLoop, + server: Server, request: pytest.FixtureRequest, ) -> HttpRequestParser: - protocol = RequestHandler( - mock.create_autospec(Server, spec_set=True, instance=True), loop=loop - ) + protocol = RequestHandler(server, loop=loop) # Parser implementations parser = request.param( @@ -172,11 +181,10 @@ def test_reject_obsolete_line_folding(parser: HttpRequestParser) -> None: @pytest.mark.skipif(NO_EXTENSIONS, reason="Only tests C parser.") def test_invalid_character( loop: asyncio.AbstractEventLoop, + server: Server, request: pytest.FixtureRequest, ) -> None: - protocol = RequestHandler( - mock.create_autospec(Server, spec_set=True, instance=True), loop - ) + protocol = RequestHandler(server, loop) parser = HttpRequestParserC( protocol, @@ -200,11 +208,10 @@ def test_invalid_character( @pytest.mark.skipif(NO_EXTENSIONS, reason="Only tests C parser.") def test_invalid_linebreak( loop: asyncio.AbstractEventLoop, + server: Server, request: pytest.FixtureRequest, ) -> None: - protocol = RequestHandler( - mock.create_autospec(Server, spec_set=True, instance=True), loop - ) + protocol = RequestHandler(server, loop) parser = HttpRequestParserC( protocol, @@ -269,10 +276,8 @@ def test_bad_headers(parser: HttpRequestParser, hdr: str) -> None: parser.feed_data(text) -def test_unpaired_surrogate_in_header_py(loop: asyncio.AbstractEventLoop) -> None: - protocol = RequestHandler( - mock.create_autospec(Server, spec_set=True, instance=True), loop - ) +def test_unpaired_surrogate_in_header_py(loop: asyncio.AbstractEventLoop, server: Server) -> None: + protocol = RequestHandler(server, loop) parser = HttpRequestParserPy( protocol, @@ -1479,11 +1484,10 @@ async def test_request_chunked_reject_bad_trailer(parser: HttpRequestParser) -> def test_parse_no_length_or_te_on_post( loop: asyncio.AbstractEventLoop, + server: Server, request_cls: type[HttpRequestParser], ) -> None: - protocol = RequestHandler( - mock.create_autospec(Server, spec_set=True, instance=True), loop - ) + protocol = RequestHandler(server, loop) parser = request_cls(protocol, loop, limit=2**16) protocol._parser = parser text = b"POST /test HTTP/1.1\r\n\r\n" @@ -1758,10 +1762,8 @@ def test_parse_uri_utf8_percent_encoded(parser: HttpRequestParser) -> None: "HttpRequestParserC" not in dir(aiohttp.http_parser), reason="C based HTTP parser not available", ) -def test_parse_bad_method_for_c_parser_raises(loop: asyncio.AbstractEventLoop) -> None: - protocol = RequestHandler( - mock.create_autospec(Server, spec_set=True, instance=True), loop - ) +def test_parse_bad_method_for_c_parser_raises(loop: asyncio.AbstractEventLoop, server: Server) -> None: + protocol = RequestHandler(server, loop) payload = b"GET1 /test HTTP/1.1\r\n\r\n" parser = HttpRequestParserC( From d7380311a456719f9028b0847b91a8df7a11383a Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 15:07:44 +0000 Subject: [PATCH 028/199] Update test_http_parser.py --- tests/test_http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 15f859ad9a3..9c31b4a9c90 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -64,7 +64,7 @@ def server() -> Any: m = mock.create_autospec( Server, - request_handler=mock.AsyncMock() + request_handler=mock.AsyncMock(), instance=True, ) return m From 1d5fc0e963fe68914a9d88106fdeeda9a9b8a26d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jan 2026 15:08:24 +0000 Subject: [PATCH 029/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_http_parser.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 9c31b4a9c90..82a0701af3c 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -276,7 +276,9 @@ def test_bad_headers(parser: HttpRequestParser, hdr: str) -> None: parser.feed_data(text) -def test_unpaired_surrogate_in_header_py(loop: asyncio.AbstractEventLoop, server: Server) -> None: +def test_unpaired_surrogate_in_header_py( + loop: asyncio.AbstractEventLoop, server: Server +) -> None: protocol = RequestHandler(server, loop) parser = HttpRequestParserPy( @@ -1762,7 +1764,9 @@ def test_parse_uri_utf8_percent_encoded(parser: HttpRequestParser) -> None: "HttpRequestParserC" not in dir(aiohttp.http_parser), reason="C based HTTP parser not available", ) -def test_parse_bad_method_for_c_parser_raises(loop: asyncio.AbstractEventLoop, server: Server) -> None: +def test_parse_bad_method_for_c_parser_raises( + loop: asyncio.AbstractEventLoop, server: Server +) -> None: protocol = RequestHandler(server, loop) payload = b"GET1 /test HTTP/1.1\r\n\r\n" From 28dffda3991a3444d6f9394b39ba8f7364a898ab Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 15:23:39 +0000 Subject: [PATCH 030/199] Update test_http_parser.py --- tests/test_http_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 82a0701af3c..0e472414807 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -64,6 +64,7 @@ def server() -> Any: m = mock.create_autospec( Server, + request_factory=mock.Mock(), request_handler=mock.AsyncMock(), instance=True, ) From 2eb92be6fecb181c56dca53cae59f8debf38e6c3 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 15:37:26 +0000 Subject: [PATCH 031/199] Update test_http_parser.py --- tests/test_http_parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 0e472414807..3b771e5ef89 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -107,7 +107,8 @@ def parser( ) protocol._force_close = False protocol._parser = parser - return parser # type: ignore[no-any-return] + with mock.patch.object(protocol, "connected", True): + yield parser # type: ignore[no-any-return] @pytest.fixture(params=REQUEST_PARSERS, ids=_gen_ids(REQUEST_PARSERS)) From 3dbf9c8a761c0b9775e4d65f24fdecc59389ce50 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 15:54:06 +0000 Subject: [PATCH 032/199] Update test_http_parser.py --- tests/test_http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 3b771e5ef89..109e69d92b0 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -107,7 +107,7 @@ def parser( ) protocol._force_close = False protocol._parser = parser - with mock.patch.object(protocol, "connected", True): + with mock.patch.object(protocol, "transport", True): yield parser # type: ignore[no-any-return] From 3ea25ca5c41a307a925a7569ec5e585931041324 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 16:13:16 +0000 Subject: [PATCH 033/199] Fix --- aiohttp/streams.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/aiohttp/streams.py b/aiohttp/streams.py index 98275565074..e07108eda13 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -218,8 +218,7 @@ def feed_eof(self) -> None: self._eof_waiter = None set_result(waiter, None) - if self._protocol._reading_paused: - self._protocol.resume_reading() + self._protocol.resume_reading() for cb in self._eof_callbacks: try: @@ -289,7 +288,7 @@ def feed_data(self, data: bytes) -> None: self._waiter = None set_result(waiter, None) - if self._size > self._high_water and not self._protocol._reading_paused: + if self._size > self._high_water: self._protocol.pause_reading() return None @@ -526,7 +525,6 @@ def _read_nowait_chunk(self, n: int) -> bytes: chunk_splits.popleft() if ( - self._protocol._reading_paused and self._size < self._low_water and ( self._http_chunk_splits is None From d99fc3470ea8e13d1ccccc3a628d96779d4a7ad6 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 16:17:27 +0000 Subject: [PATCH 034/199] Update streams.py --- aiohttp/streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/streams.py b/aiohttp/streams.py index e07108eda13..2fd82f3c205 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -525,7 +525,7 @@ def _read_nowait_chunk(self, n: int) -> bytes: chunk_splits.popleft() if ( - and self._size < self._low_water + self._size < self._low_water and ( self._http_chunk_splits is None or len(self._http_chunk_splits) < self._low_water_chunks From 11cf432276da3bda3414a0b69de739e223d0218a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jan 2026 16:18:08 +0000 Subject: [PATCH 035/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/streams.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/aiohttp/streams.py b/aiohttp/streams.py index 2fd82f3c205..398b7174a75 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -524,12 +524,9 @@ def _read_nowait_chunk(self, n: int) -> bytes: while chunk_splits and chunk_splits[0] < self._cursor: chunk_splits.popleft() - if ( - self._size < self._low_water - and ( - self._http_chunk_splits is None - or len(self._http_chunk_splits) < self._low_water_chunks - ) + if self._size < self._low_water and ( + self._http_chunk_splits is None + or len(self._http_chunk_splits) < self._low_water_chunks ): self._protocol.resume_reading() return data From de6fca2b51dfc7015fe276566d06475231993194 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 17:35:09 +0000 Subject: [PATCH 036/199] Update http_parser.py --- aiohttp/http_parser.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index b1b7e610fba..6ae44c87f01 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -242,6 +242,7 @@ def __init__( self._upgraded = False self._payload = None self._payload_parser: HttpPayloadParser | None = None + self._payload_has_more_data = False self._auto_decompress = auto_decompress self._limit = limit self._headers_parser = HeadersParser(max_field_size, self.lax) @@ -292,7 +293,7 @@ def feed_data( max_line_length = self.max_line_size should_close = False - while start_pos < data_len: + while start_pos < data_len or self._payload_has_more_data: # read HTTP message (request/response line + headers), \r\n\r\n # and split by lines if self._payload_parser is None and not self._upgraded: @@ -451,7 +452,7 @@ def get_content_length() -> int | None: break # feed payload - elif data and start_pos < data_len: + elif self._payload_has_more_data or (data and start_pos < data_len): assert not self._lines assert self._payload_parser is not None try: @@ -476,6 +477,8 @@ def get_content_length() -> int | None: ): raise + self._payload_has_more_data = payload_state == PayloadState.PAYLOAD_HAS_PENDING_DATA + if payload_state is not PayloadState.PAYLOAD_COMPLETE: # We've either consumed all available data, or we're pausing # until the reader buffer is freed up. From 3a804560b411a6b2d676a24bfa93ed9a84cbcaf5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jan 2026 17:35:47 +0000 Subject: [PATCH 037/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/http_parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 6ae44c87f01..331f2d93664 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -477,7 +477,9 @@ def get_content_length() -> int | None: ): raise - self._payload_has_more_data = payload_state == PayloadState.PAYLOAD_HAS_PENDING_DATA + self._payload_has_more_data = ( + payload_state == PayloadState.PAYLOAD_HAS_PENDING_DATA + ) if payload_state is not PayloadState.PAYLOAD_COMPLETE: # We've either consumed all available data, or we're pausing From 7da23c165d3e91810c94a8e598176c29e7799701 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 17:47:48 +0000 Subject: [PATCH 038/199] Update http_parser.py --- aiohttp/http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 331f2d93664..1d5eec0e4d6 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -478,7 +478,7 @@ def get_content_length() -> int | None: raise self._payload_has_more_data = ( - payload_state == PayloadState.PAYLOAD_HAS_PENDING_DATA + payload_state == PayloadState.PAYLOAD_HAS_PENDING_INPUT ) if payload_state is not PayloadState.PAYLOAD_COMPLETE: From a4fc7c1bc5d2f8f43739619bf674074d1985011b Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 18:48:57 +0000 Subject: [PATCH 039/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 51 +++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index f7c393ed42a..254ad856fa5 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -291,6 +291,7 @@ cdef class HttpParser: bint _response_with_body bint _read_until_eof + bytes _tail bint _started object _url bytearray _buf @@ -300,6 +301,7 @@ cdef class HttpParser: list _raw_headers bint _upgraded list _messages + bint _paused object _payload bint _payload_error object _payload_exception @@ -345,6 +347,7 @@ cdef class HttpParser: self._timer = timer self._buf = bytearray() + self._paused = False self._payload = None self._payload_error = 0 self._payload_exception = payload_exception @@ -355,6 +358,7 @@ cdef class HttpParser: self._has_value = False self._header_name_size = 0 + self._tail = EMPTY_BYTES self._max_line_size = max_line_size self._max_headers = max_headers self._max_field_size = max_field_size @@ -503,6 +507,10 @@ cdef class HttpParser: ### Public API ### + def pause_reading(self): + assert pyparser._payload is not None + self._paused = True + def feed_eof(self): cdef bytes desc @@ -529,6 +537,16 @@ cdef class HttpParser: size_t nb cdef cparser.llhttp_errno_t errno + if self._tail: + tail = self._tail + self._tail = b"" + result = cb_on_body(self, tail, -1) + + if result == cparser.HPE_PAUSED: + assert data == b"" + return (), False, EMPTY_BYTES + # TODO: Do we need to handle error case (-1)? + PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE) data_len = self.py_buf.len @@ -569,7 +587,7 @@ cdef class HttpParser: if self._upgraded: return messages, True, data[nb:] else: - return messages, False, b"" + return messages, False, EMPTY_BYTES def set_upgraded(self, val): self._upgraded = val @@ -761,20 +779,25 @@ cdef int cb_on_headers_complete(cparser.llhttp_t* parser) except -1: cdef int cb_on_body(cparser.llhttp_t* parser, const char *at, size_t length) except -1: cdef HttpParser pyparser = parser.data - cdef bytes body = at[:length] - try: - pyparser._payload.feed_data(body) - except BaseException as underlying_exc: - reraised_exc = underlying_exc - if pyparser._payload_exception is not None: - reraised_exc = pyparser._payload_exception(str(underlying_exc)) + body = at[:length] + while body is not None: + if pyparser._paused: + pyparser._paused = False + pyparser._tail = body + return cparser.HPE_PAUSED - set_exception(pyparser._payload, reraised_exc, underlying_exc) - - pyparser._payload_error = 1 - return -1 - else: - return 0 + try: + body = pyparser._payload.feed_data(body) + except BaseException as underlying_exc: + reraised_exc = underlying_exc + if pyparser._payload_exception is not None: + reraised_exc = pyparser._payload_exception(str(underlying_exc)) + + set_exception(pyparser._payload, reraised_exc, underlying_exc) + + pyparser._payload_error = 1 + return -1 + return 0 cdef int cb_on_message_complete(cparser.llhttp_t* parser) except -1: From b3c77d7c2ed3f5d3b5b205a398e1971f76b19f80 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jan 2026 18:49:37 +0000 Subject: [PATCH 040/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/_http_parser.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 254ad856fa5..7e8b14c5572 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -546,7 +546,7 @@ cdef class HttpParser: assert data == b"" return (), False, EMPTY_BYTES # TODO: Do we need to handle error case (-1)? - + PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE) data_len = self.py_buf.len @@ -792,9 +792,9 @@ cdef int cb_on_body(cparser.llhttp_t* parser, reraised_exc = underlying_exc if pyparser._payload_exception is not None: reraised_exc = pyparser._payload_exception(str(underlying_exc)) - + set_exception(pyparser._payload, reraised_exc, underlying_exc) - + pyparser._payload_error = 1 return -1 return 0 From f26edc06a86796bdeee27f4d138d3d1bbd4d05a7 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 19:04:22 +0000 Subject: [PATCH 041/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 7e8b14c5572..c7c56a1f776 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -508,7 +508,7 @@ cdef class HttpParser: ### Public API ### def pause_reading(self): - assert pyparser._payload is not None + assert self._payload is not None self._paused = True def feed_eof(self): @@ -540,7 +540,7 @@ cdef class HttpParser: if self._tail: tail = self._tail self._tail = b"" - result = cb_on_body(self, tail, -1) + result = cb_on_body(self._cparser, tail, -1) if result == cparser.HPE_PAUSED: assert data == b"" From 1997c8b8b57167ecf65b4f58d7101efc34d4e1dd Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 19:40:15 +0000 Subject: [PATCH 042/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index c7c56a1f776..78a9086078e 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -557,12 +557,14 @@ cdef class HttpParser: if errno is cparser.HPE_PAUSED_UPGRADE: cparser.llhttp_resume_after_upgrade(self._cparser) - + nb = cparser.llhttp_get_error_pos(self._cparser) - self.py_buf.buf + elif errno is cparser.HPE_PAUSED: + cparser.llhttp_resume(self._cparser) nb = cparser.llhttp_get_error_pos(self._cparser) - self.py_buf.buf PyBuffer_Release(&self.py_buf) - if errno not in (cparser.HPE_OK, cparser.HPE_PAUSED_UPGRADE): + if errno not in (cparser.HPE_OK, cparser.HPE_PAUSED, cparser.HPE_PAUSED_UPGRADE): if self._payload_error == 0: if self._last_error is not None: ex = self._last_error @@ -586,6 +588,8 @@ cdef class HttpParser: if self._upgraded: return messages, True, data[nb:] + elif errno is cparser.HPE_PAUSED: + return messages, False, data[nb:] else: return messages, False, EMPTY_BYTES From b4443b7978994620ab64b1f571affde333e71754 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 19:40:48 +0000 Subject: [PATCH 043/199] Apply suggestions from code review --- aiohttp/_http_parser.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 78a9086078e..a10a422af6d 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -542,7 +542,7 @@ cdef class HttpParser: self._tail = b"" result = cb_on_body(self._cparser, tail, -1) - if result == cparser.HPE_PAUSED: + if result is cparser.HPE_PAUSED: assert data == b"" return (), False, EMPTY_BYTES # TODO: Do we need to handle error case (-1)? From b17c01384c43b77fcc4bc390acb4fac68727ffa5 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 19:58:24 +0000 Subject: [PATCH 044/199] Fix --- aiohttp/_cparser.pxd | 1 + 1 file changed, 1 insertion(+) diff --git a/aiohttp/_cparser.pxd b/aiohttp/_cparser.pxd index 1b3be6d4efb..cc7ef58d664 100644 --- a/aiohttp/_cparser.pxd +++ b/aiohttp/_cparser.pxd @@ -145,6 +145,7 @@ cdef extern from "llhttp.h": int llhttp_should_keep_alive(const llhttp_t* parser) + void llhttp_resume(llhttp_t* parser) void llhttp_resume_after_upgrade(llhttp_t* parser) llhttp_errno_t llhttp_get_errno(const llhttp_t* parser) From d628cee8a1bfc502be248fc4b21ba4f8b9d96522 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 16 Jan 2026 20:14:17 +0000 Subject: [PATCH 045/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index a10a422af6d..a1a3e7b63db 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -540,7 +540,7 @@ cdef class HttpParser: if self._tail: tail = self._tail self._tail = b"" - result = cb_on_body(self._cparser, tail, -1) + result = cb_on_body(self._cparser, tail, len(tail)) if result is cparser.HPE_PAUSED: assert data == b"" From cd2e07ac3788998c4cc83c6092943234f021a1d6 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 17 Jan 2026 16:06:48 +0000 Subject: [PATCH 046/199] Update http_parser.py --- aiohttp/http_parser.py | 58 ++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 36 deletions(-) diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 1d5eec0e4d6..b9ad3214a6d 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -782,6 +782,7 @@ def __init__( self._max_line_size = max_line_size self._max_field_size = max_field_size self._max_trailers = max_trailers + self._more_data_available = False self._trailer_lines: list[bytes] = [] self.done = False @@ -843,24 +844,19 @@ def feed_data( chunk = self._chunk_tail + chunk self._chunk_tail = b"" - while chunk: - required = self._length - self._length = max(required - len(chunk), 0) - tail = self.payload.feed_data(chunk[:required]) - - if tail is not None: - self._length += len(tail) - chunk = tail + chunk[required:] - if self._paused: - self._paused = False - self._chunk_tail = chunk - return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" + required = self._length + self._length = max(required - len(chunk), 0) + self._more_data_available = self.payload.feed_data(chunk[:required]) + while self._more_data_available: + if self._paused: + self._paused = False + self._chunk_tail = chunk[required:] + return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" + self._more_data_available = self.payload.feed_data(b"") - if self._length == 0: - self.payload.feed_eof() - return PayloadState.PAYLOAD_COMPLETE, chunk[required:] - if tail is None: - break + if self._length == 0: + self.payload.feed_eof() + return PayloadState.PAYLOAD_COMPLETE, chunk[required:] # Chunked transfer encoding parser elif self._type == ParseState.PARSE_CHUNKED: if self._chunk_tail: @@ -877,7 +873,7 @@ def feed_data( chunk = self._chunk_tail + chunk self._chunk_tail = b"" - while chunk: + while chunk or self._more_data_available: # read next chunk size if self._chunk == ChunkState.PARSE_CHUNKED_SIZE: pos = chunk.find(SEP) @@ -928,16 +924,14 @@ def feed_data( required = self._chunk_size self._chunk_size = max(required - len(chunk), 0) - tail = self.payload.feed_data(chunk[:required]) + self._more_data_available = self.payload.feed_data(chunk[:required]) + chunk = chunk[required:] - if tail is not None: - self._chunk_size += len(tail) - chunk = tail + chunk[required:] + if self._more_data_available: continue if self._chunk_size: return PayloadState.PAYLOAD_NEEDS_INPUT, b"" - chunk = chunk[required:] self._chunk = ChunkState.PARSE_CHUNKED_CHUNK_EOF self.payload.end_http_chunk_receiving() @@ -986,13 +980,11 @@ def feed_data( # Read all bytes until eof elif self._type == ParseState.PARSE_UNTIL_EOF: - tail = chunk - while tail is not None: + while self._more_data_available := self.payload.feed_data(chunk): + chunk = b"" if self._paused: self._paused = False - self._chunk_tail = tail return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" - tail = self.payload.feed_data(tail) return PayloadState.PAYLOAD_NEEDS_INPUT, b"" @@ -1039,10 +1031,8 @@ def set_exception( ) -> None: set_exception(self.out, exc, exc_cause) - def feed_data(self, chunk: bytes) -> bytes | None: - if not chunk: - return None - + def feed_data(self, chunk: bytes) -> bool: + """Return True if more data is available and this method should be called again with b"".""" self.size += len(chunk) self.out.total_compressed_bytes = self.size @@ -1073,11 +1063,7 @@ def feed_data(self, chunk: bytes) -> bytes | None: if chunk: self.out.feed_data(chunk) - return ( - self.decompressor.unconsumed_tail - if self.decompressor.data_available - else None - ) + return self.decompressor.data_available def feed_eof(self) -> None: chunk = self.decompressor.flush() From fd74c03ea60630e22d58f886f2294b8402afff36 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 17 Jan 2026 16:09:46 +0000 Subject: [PATCH 047/199] Update compression_utils.py --- aiohttp/compression_utils.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index 5925041d879..313ca7032cc 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -183,11 +183,6 @@ async def decompress( ) return self.decompress_sync(data, max_length) - @property - @abstractmethod - def unconsumed_tail(self) -> bytes: - """Unused input that must be fed back to decompress() after using max_length.""" - @property @abstractmethod def data_available(self) -> bool: @@ -296,10 +291,6 @@ def flush(self, length: int = 0) -> bytes: else self._decompressor.flush() ) - @property - def unconsumed_tail(self) -> bytes: - return self._decompressor.unconsumed_tail - @property def data_available(self) -> bool: return bool(self._decompressor.unconsumed_tail) @@ -341,10 +332,6 @@ def flush(self) -> bytes: return cast(bytes, self._obj.flush()) return b"" - @property - def unconsumed_tail(self) -> bytes: - pass # TODO - @property def data_available(self) -> bool: pass # TODO @@ -379,10 +366,6 @@ def decompress_sync( def flush(self) -> bytes: return b"" - @property - def unconsumed_tail(self) -> bytes: - return b"" - @property def data_available(self) -> bool: return not self._obj.needs_input From 087de363759d077ca24dbcd99d09fe176f9232b1 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 17 Jan 2026 16:10:23 +0000 Subject: [PATCH 048/199] Update streams.py --- aiohttp/streams.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aiohttp/streams.py b/aiohttp/streams.py index 398b7174a75..cc64e5e0906 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -272,11 +272,11 @@ def unread_data(self, data: bytes) -> None: self._buffer.appendleft(data) self._eof_counter = 0 - def feed_data(self, data: bytes) -> None: + def feed_data(self, data: bytes) -> bool: assert not self._eof, "feed_data after feed_eof" if not data: - return None + return False data_len = len(data) self._size += data_len @@ -290,7 +290,7 @@ def feed_data(self, data: bytes) -> None: if self._size > self._high_water: self._protocol.pause_reading() - return None + return False def begin_http_chunk_receiving(self) -> None: if self._http_chunk_splits is None: From 103c4a61dadf01d2d4e8608bb24bcfa33161b499 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 17 Jan 2026 16:47:06 +0000 Subject: [PATCH 049/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index a1a3e7b63db..a3bedfb307d 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -291,7 +291,7 @@ cdef class HttpParser: bint _response_with_body bint _read_until_eof - bytes _tail + bint _more_data_available bint _started object _url bytearray _buf @@ -347,6 +347,7 @@ cdef class HttpParser: self._timer = timer self._buf = bytearray() + self._more_data_available = False self._paused = False self._payload = None self._payload_error = 0 @@ -358,7 +359,6 @@ cdef class HttpParser: self._has_value = False self._header_name_size = 0 - self._tail = EMPTY_BYTES self._max_line_size = max_line_size self._max_headers = max_headers self._max_field_size = max_field_size @@ -537,11 +537,8 @@ cdef class HttpParser: size_t nb cdef cparser.llhttp_errno_t errno - if self._tail: - tail = self._tail - self._tail = b"" - result = cb_on_body(self._cparser, tail, len(tail)) - + if self._more_data_available: + result = cb_on_body(self._cparser, EMPTY_BYTES, 0) if result is cparser.HPE_PAUSED: assert data == b"" return (), False, EMPTY_BYTES @@ -784,14 +781,14 @@ cdef int cb_on_body(cparser.llhttp_t* parser, const char *at, size_t length) except -1: cdef HttpParser pyparser = parser.data body = at[:length] - while body is not None: + while more_data_available: if pyparser._paused: pyparser._paused = False - pyparser._tail = body + pyparser._more_data_available = True return cparser.HPE_PAUSED try: - body = pyparser._payload.feed_data(body) + more_data_available = pyparser._payload.feed_data(body) except BaseException as underlying_exc: reraised_exc = underlying_exc if pyparser._payload_exception is not None: @@ -801,6 +798,8 @@ cdef int cb_on_body(cparser.llhttp_t* parser, pyparser._payload_error = 1 return -1 + body = EMPTY_BYTES + pyparser._more_data_available = False return 0 From a249c040242cccbcaf5d6658261d76a1a179320e Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 17 Jan 2026 16:49:40 +0000 Subject: [PATCH 050/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index a3bedfb307d..b52d23f7d63 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -291,7 +291,6 @@ cdef class HttpParser: bint _response_with_body bint _read_until_eof - bint _more_data_available bint _started object _url bytearray _buf @@ -301,6 +300,7 @@ cdef class HttpParser: list _raw_headers bint _upgraded list _messages + bint _more_data_available bint _paused object _payload bint _payload_error From f20ddc7145d100efc2fca495b3f14e53655583ea Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 17 Jan 2026 16:55:10 +0000 Subject: [PATCH 051/199] Update http_parser.py --- aiohttp/http_parser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index b9ad3214a6d..3844a0acf4c 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -980,11 +980,12 @@ def feed_data( # Read all bytes until eof elif self._type == ParseState.PARSE_UNTIL_EOF: - while self._more_data_available := self.payload.feed_data(chunk): - chunk = b"" + self._more_data_available = self.payload.feed_data(chunk) + while self._more_data_available: if self._paused: self._paused = False return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" + self._more_data_available = self.payload.feed_data(b"") return PayloadState.PAYLOAD_NEEDS_INPUT, b"" From 0fe70001490503d735311a32c5e550e0af389d56 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 17 Jan 2026 17:46:58 +0000 Subject: [PATCH 052/199] Update web_protocol.py --- aiohttp/web_protocol.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index a716c1d3b5a..0fc928e5594 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -401,8 +401,7 @@ def connection_lost(self, exc: BaseException | None) -> None: self._payload_parser.feed_eof() self._payload_parser = None - def set_parser(self, parser: Any) -> None: - # Actual type is WebReader + def set_parser(self, parser: WebSocketReader) -> None: assert self._payload_parser is None self._payload_parser = parser From 513026674a0a25328d151462b69f267c3c43db6f Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 17 Jan 2026 17:58:24 +0000 Subject: [PATCH 053/199] Fix --- aiohttp/_http_parser.pyx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index b52d23f7d63..41e47ff935a 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -291,6 +291,7 @@ cdef class HttpParser: bint _response_with_body bint _read_until_eof + bytes _tail bint _started object _url bytearray _buf @@ -356,6 +357,7 @@ cdef class HttpParser: self._raw_name = EMPTY_BYTES self._raw_value = EMPTY_BYTES + self._tail = b"" self._has_value = False self._header_name_size = 0 @@ -537,10 +539,13 @@ cdef class HttpParser: size_t nb cdef cparser.llhttp_errno_t errno + if self._tail: + data, self._tail = self._tail + data, b"" + if self._more_data_available: result = cb_on_body(self._cparser, EMPTY_BYTES, 0) if result is cparser.HPE_PAUSED: - assert data == b"" + self._tail = data return (), False, EMPTY_BYTES # TODO: Do we need to handle error case (-1)? @@ -558,6 +563,7 @@ cdef class HttpParser: elif errno is cparser.HPE_PAUSED: cparser.llhttp_resume(self._cparser) nb = cparser.llhttp_get_error_pos(self._cparser) - self.py_buf.buf + self._tail = data[nb:] PyBuffer_Release(&self.py_buf) @@ -585,8 +591,6 @@ cdef class HttpParser: if self._upgraded: return messages, True, data[nb:] - elif errno is cparser.HPE_PAUSED: - return messages, False, data[nb:] else: return messages, False, EMPTY_BYTES @@ -781,7 +785,7 @@ cdef int cb_on_body(cparser.llhttp_t* parser, const char *at, size_t length) except -1: cdef HttpParser pyparser = parser.data body = at[:length] - while more_data_available: + while body or pyparser._more_data_available: if pyparser._paused: pyparser._paused = False pyparser._more_data_available = True From 7db01206e985e2cffa7ba0bdba9d5f2a58801f08 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 17 Jan 2026 18:07:36 +0000 Subject: [PATCH 054/199] Update web_protocol.py --- aiohttp/web_protocol.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index 0fc928e5594..8f2eff20200 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -22,6 +22,7 @@ HttpVersion10, RawRequestMessage, StreamWriter, + WebSocketReader, ) from .http_exceptions import BadHttpMethod from .log import access_logger, server_logger From d113ccb356d5ac71ad2e78e47e6bc88ac20f0235 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 17 Jan 2026 20:41:53 +0000 Subject: [PATCH 055/199] Update http_parser.py --- aiohttp/http_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 3844a0acf4c..6a3795784a2 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -931,6 +931,7 @@ def feed_data( continue if self._chunk_size: + self._paused = False return PayloadState.PAYLOAD_NEEDS_INPUT, b"" self._chunk = ChunkState.PARSE_CHUNKED_CHUNK_EOF self.payload.end_http_chunk_receiving() From 185db0b9dd2fb4bfd8e15a450c06daf7db1969ac Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 17 Jan 2026 21:40:28 +0000 Subject: [PATCH 056/199] Update aiohttp/_http_parser.pyx --- aiohttp/_http_parser.pyx | 1 + 1 file changed, 1 insertion(+) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 41e47ff935a..59c3a4e6354 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -804,6 +804,7 @@ cdef int cb_on_body(cparser.llhttp_t* parser, return -1 body = EMPTY_BYTES pyparser._more_data_available = False + pyparser._paused = False return 0 From 3be7ac6ad0b08455a3fd0ceb39a154b7c4468bf3 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 17 Jan 2026 22:43:46 +0000 Subject: [PATCH 057/199] Update base_protocol.py --- aiohttp/base_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/base_protocol.py b/aiohttp/base_protocol.py index df629a0627c..ed9d3396dc2 100644 --- a/aiohttp/base_protocol.py +++ b/aiohttp/base_protocol.py @@ -21,7 +21,7 @@ class BaseProtocol(asyncio.Protocol): ) def __init__( - self, loop: asyncio.AbstractEventLoop, parser: "HttpParser | None" + self, loop: asyncio.AbstractEventLoop, parser: "HttpParser | None" = None ) -> None: self._loop: asyncio.AbstractEventLoop = loop self._paused = False From 80eb8c7fd955c36fe7ab1104902f40cde38cba56 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 19 Jan 2026 20:18:29 +0000 Subject: [PATCH 058/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 59c3a4e6354..7c48ef922d0 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -562,8 +562,8 @@ cdef class HttpParser: nb = cparser.llhttp_get_error_pos(self._cparser) - self.py_buf.buf elif errno is cparser.HPE_PAUSED: cparser.llhttp_resume(self._cparser) - nb = cparser.llhttp_get_error_pos(self._cparser) - self.py_buf.buf - self._tail = data[nb:] + pos = cparser.llhttp_get_error_pos(self._cparser) - self.py_buf.buf + self._tail = data[pos:] PyBuffer_Release(&self.py_buf) From 518344183f878d448e2479eb8790f03618e381dc Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 19 Jan 2026 20:22:32 +0000 Subject: [PATCH 059/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 7c48ef922d0..83589438fc6 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -792,7 +792,7 @@ cdef int cb_on_body(cparser.llhttp_t* parser, return cparser.HPE_PAUSED try: - more_data_available = pyparser._payload.feed_data(body) + pyparser._more_data_available = pyparser._payload.feed_data(body) except BaseException as underlying_exc: reraised_exc = underlying_exc if pyparser._payload_exception is not None: @@ -803,7 +803,6 @@ cdef int cb_on_body(cparser.llhttp_t* parser, pyparser._payload_error = 1 return -1 body = EMPTY_BYTES - pyparser._more_data_available = False pyparser._paused = False return 0 From 60c61e389f1f6eb68717aa0aecdd0ddc729d076c Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 19 Jan 2026 20:30:24 +0000 Subject: [PATCH 060/199] Update test_http_parser.py --- tests/test_http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 109e69d92b0..b7043b7f8cf 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -6,7 +6,6 @@ import zlib from collections.abc import Iterable from contextlib import suppress -from types import MethodType from typing import Any from unittest import mock from urllib.parse import quote @@ -879,6 +878,7 @@ async def test_chunk_splits_after_pause(parser: HttpRequestParser) -> None: assert len(payload._http_chunk_splits) == 16385 # We should still get the full result after read(), as it will continue processing. result = await payload.read() + assert len(result) == 50000 # Compare len first, as it's easier to debug in diff. assert result == b"b" * 50000 From 2a6eff82dbac1e4d23e7a2d81cf7560e5f724455 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 19 Jan 2026 20:52:36 +0000 Subject: [PATCH 061/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 83589438fc6..fefaa52073e 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -786,11 +786,6 @@ cdef int cb_on_body(cparser.llhttp_t* parser, cdef HttpParser pyparser = parser.data body = at[:length] while body or pyparser._more_data_available: - if pyparser._paused: - pyparser._paused = False - pyparser._more_data_available = True - return cparser.HPE_PAUSED - try: pyparser._more_data_available = pyparser._payload.feed_data(body) except BaseException as underlying_exc: @@ -803,6 +798,11 @@ cdef int cb_on_body(cparser.llhttp_t* parser, pyparser._payload_error = 1 return -1 body = EMPTY_BYTES + + if pyparser._paused and (body or pyparser._more_data_available): + pyparser._paused = False + pyparser._more_data_available = True + return cparser.HPE_PAUSED pyparser._paused = False return 0 From dbfc0c886ad4fa35b82e0274885027e7bca0ae87 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 19 Jan 2026 21:07:33 +0000 Subject: [PATCH 062/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index fefaa52073e..0ebdb3639a2 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -799,9 +799,8 @@ cdef int cb_on_body(cparser.llhttp_t* parser, return -1 body = EMPTY_BYTES - if pyparser._paused and (body or pyparser._more_data_available): + if pyparser._paused and pyparser._more_data_available: pyparser._paused = False - pyparser._more_data_available = True return cparser.HPE_PAUSED pyparser._paused = False return 0 From eac561ef530fc736bc1c21cb3a9712b43fb98c97 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 21 Jan 2026 18:20:10 +0000 Subject: [PATCH 063/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 0ebdb3639a2..2e93b78eadd 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -799,7 +799,7 @@ cdef int cb_on_body(cparser.llhttp_t* parser, return -1 body = EMPTY_BYTES - if pyparser._paused and pyparser._more_data_available: + if pyparser._paused: pyparser._paused = False return cparser.HPE_PAUSED pyparser._paused = False From 1a407814194bb839831034811653fd47bd2640b3 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 21 Jan 2026 18:43:39 +0000 Subject: [PATCH 064/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 2e93b78eadd..06353fba761 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -784,7 +784,7 @@ cdef int cb_on_headers_complete(cparser.llhttp_t* parser) except -1: cdef int cb_on_body(cparser.llhttp_t* parser, const char *at, size_t length) except -1: cdef HttpParser pyparser = parser.data - body = at[:length] + cdef bytes body = at[:length] while body or pyparser._more_data_available: try: pyparser._more_data_available = pyparser._payload.feed_data(body) From 520b64a09ea0eedf6b681ec6036736be5d6be84c Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 22 Jan 2026 15:07:37 +0000 Subject: [PATCH 065/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 1 + 1 file changed, 1 insertion(+) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 06353fba761..2236c62ac70 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -796,6 +796,7 @@ cdef int cb_on_body(cparser.llhttp_t* parser, set_exception(pyparser._payload, reraised_exc, underlying_exc) pyparser._payload_error = 1 + pyparser._paused = False return -1 body = EMPTY_BYTES From 9c2987bc1f60933b646f4f3dbd247b2b8b946146 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sun, 25 Jan 2026 18:35:57 +0000 Subject: [PATCH 066/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 2236c62ac70..7374fad88b2 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -548,6 +548,8 @@ cdef class HttpParser: self._tail = data return (), False, EMPTY_BYTES # TODO: Do we need to handle error case (-1)? + elif not data: + return (), False, EMPTY_BYTES PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE) data_len = self.py_buf.len From c4b058d29e4e572f890c294aee40809d9ff39d58 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sun, 25 Jan 2026 18:36:27 +0000 Subject: [PATCH 067/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 7374fad88b2..a68dae6f6fa 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -540,7 +540,7 @@ cdef class HttpParser: cdef cparser.llhttp_errno_t errno if self._tail: - data, self._tail = self._tail + data, b"" + data, self._tail = self._tail + data, EMPTY_BYTES if self._more_data_available: result = cb_on_body(self._cparser, EMPTY_BYTES, 0) From fa644c74ebd52207996ff9cf8427185cdacaee8d Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sun, 25 Jan 2026 20:41:15 +0000 Subject: [PATCH 068/199] Update test_client_functional.py --- tests/test_client_functional.py | 41 +++++++++++++++++---------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 97d189d4b02..68854f3af45 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -2386,10 +2386,9 @@ async def test_payload_decompress_size_limit(aiohttp_client: AiohttpClient) -> N When a compressed payload expands beyond the configured limit, we raise DecompressSizeError. """ - # Create a highly compressible payload that exceeds the decompression limit. - # 64MiB of repeated bytes compresses to ~32KB but expands beyond the - # 32MiB per-call limit. - original = b"A" * (64 * 2**20) + # Create a highly compressible payload. + payload_size = 64 * 2**20 + original = b"A" * payload_size compressed = zlib.compress(original) assert len(original) > DEFAULT_MAX_DECOMPRESS_SIZE @@ -2406,11 +2405,11 @@ async def handler(request: web.Request) -> web.Response: async with client.get("/") as resp: assert resp.status == 200 - with pytest.raises(aiohttp.ClientPayloadError) as exc_info: # TODO - await resp.read() + received = 0 + async for chunk in resp.content.iter_chunked(1024): + received += len(chunk) - assert isinstance(exc_info.value.__cause__, DecompressSizeError) - assert "Decompressed data exceeds" in str(exc_info.value.__cause__) + assert received == payload_size @pytest.mark.skipif(brotli is None, reason="brotli is not installed") @@ -2419,8 +2418,9 @@ async def test_payload_decompress_size_limit_brotli( ) -> None: """Test that brotli decompression size limit triggers DecompressSizeError.""" assert brotli is not None - # Create a highly compressible payload that exceeds the decompression limit. - original = b"A" * (64 * 2**20) + # Create a highly compressible payload + payload_size = 64 * 2**20 + original = b"A" * payload_size compressed = brotli.compress(original) assert len(original) > DEFAULT_MAX_DECOMPRESS_SIZE @@ -2436,11 +2436,11 @@ async def handler(request: web.Request) -> web.Response: async with client.get("/") as resp: assert resp.status == 200 - with pytest.raises(aiohttp.ClientPayloadError) as exc_info: # TODO - await resp.read() + received = 0 + async for chunk in resp.content.iter_chunked(1024): + received += len(chunk) - assert isinstance(exc_info.value.__cause__, DecompressSizeError) - assert "Decompressed data exceeds" in str(exc_info.value.__cause__) + assert received == payload_size @pytest.mark.skipif(ZstdCompressor is None, reason="backports.zstd is not installed") @@ -2449,8 +2449,9 @@ async def test_payload_decompress_size_limit_zstd( ) -> None: """Test that zstd decompression size limit triggers DecompressSizeError.""" assert ZstdCompressor is not None - # Create a highly compressible payload that exceeds the decompression limit. - original = b"A" * (64 * 2**20) + # Create a highly compressible payload. + payload_size = 64 * 2**20 + original = b"A" * payload_size compressor = ZstdCompressor() compressed = compressor.compress(original) + compressor.flush() assert len(original) > DEFAULT_MAX_DECOMPRESS_SIZE @@ -2467,11 +2468,11 @@ async def handler(request: web.Request) -> web.Response: async with client.get("/") as resp: assert resp.status == 200 - with pytest.raises(aiohttp.ClientPayloadError) as exc_info: # TODO - await resp.read() + received = 0 + async for chunk in resp.content.iter_chunked(1024): + received += len(chunk) - assert isinstance(exc_info.value.__cause__, DecompressSizeError) - assert "Decompressed data exceeds" in str(exc_info.value.__cause__) + assert received == payload_size async def test_bad_payload_chunked_encoding(aiohttp_client: AiohttpClient) -> None: From dd82b9f290cee58e5508e792cb7ad559cf6efeae Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sun, 25 Jan 2026 20:48:33 +0000 Subject: [PATCH 069/199] Update test_base_protocol.py --- tests/test_base_protocol.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_base_protocol.py b/tests/test_base_protocol.py index 713dba2d0c2..d53e0cfb303 100644 --- a/tests/test_base_protocol.py +++ b/tests/test_base_protocol.py @@ -5,6 +5,7 @@ import pytest from aiohttp.base_protocol import BaseProtocol +from aiohttp.http_parser import HttpParser async def test_loop() -> None: @@ -26,25 +27,30 @@ async def test_pause_writing() -> None: async def test_pause_reading_no_transport() -> None: loop = asyncio.get_event_loop() - pr = BaseProtocol(loop) + parser = mock.create_autospec(HttpParser, spec_set=True, instance=True) + pr = BaseProtocol(loop, parser=parser) assert not pr._reading_paused pr.pause_reading() assert not pr._reading_paused + parser.pause_reading.assert_called_once() async def test_pause_reading_stub_transport() -> None: loop = asyncio.get_event_loop() - pr = BaseProtocol(loop) + parser = mock.create_autospec(HttpParser, spec_set=True, instance=True) + pr = BaseProtocol(loop, parser=parser) tr = asyncio.Transport() pr.transport = tr assert not pr._reading_paused pr.pause_reading() assert pr._reading_paused + parser.pause_reading.assert_called_once() async def test_resume_reading_no_transport() -> None: loop = asyncio.get_event_loop() - pr = BaseProtocol(loop) + parser = mock.create_autospec(HttpParser, spec_set=True, instance=True) + pr = BaseProtocol(loop, parser=parser) pr._reading_paused = True pr.resume_reading() assert pr._reading_paused @@ -52,7 +58,8 @@ async def test_resume_reading_no_transport() -> None: async def test_resume_reading_stub_transport() -> None: loop = asyncio.get_event_loop() - pr = BaseProtocol(loop) + parser = mock.create_autospec(HttpParser, spec_set=True, instance=True) + pr = BaseProtocol(loop, parser=parser) tr = asyncio.Transport() pr.transport = tr pr._reading_paused = True From 3099a4099a1de56fa6e81d1c83cf2b59610fe9fc Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 14:29:57 +0000 Subject: [PATCH 070/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index a68dae6f6fa..941bacfe28b 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -548,7 +548,7 @@ cdef class HttpParser: self._tail = data return (), False, EMPTY_BYTES # TODO: Do we need to handle error case (-1)? - elif not data: + if not data: return (), False, EMPTY_BYTES PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE) From 777579309c6e5de1cd15d6ac81f7f984459d6c23 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 15:12:14 +0000 Subject: [PATCH 071/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 941bacfe28b..83afac5d3b5 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -298,6 +298,7 @@ cdef class HttpParser: str _path str _reason list _headers + bint _last_had_more_data list _raw_headers bint _upgraded list _messages @@ -348,6 +349,7 @@ cdef class HttpParser: self._timer = timer self._buf = bytearray() + self._last_had_more_data = False self._more_data_available = False self._paused = False self._payload = None @@ -542,14 +544,19 @@ cdef class HttpParser: if self._tail: data, self._tail = self._tail + data, EMPTY_BYTES + had_more_data = self._more_data_available if self._more_data_available: result = cb_on_body(self._cparser, EMPTY_BYTES, 0) if result is cparser.HPE_PAUSED: + self._last_had_more_data = had_more_data self._tail = data return (), False, EMPTY_BYTES # TODO: Do we need to handle error case (-1)? - if not data: + # If the last pause had more data, then we probably paused at the + # end of the body. Therefore we need to continue with empty bytes. + if not data and not self._last_had_more_data: return (), False, EMPTY_BYTES + self._last_had_more_data = False PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE) data_len = self.py_buf.len From 9525459e4a8eca7ecfa6d19cfbeb7d4d3fd3a29b Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 16:19:42 +0000 Subject: [PATCH 072/199] Update base_protocol.py --- aiohttp/base_protocol.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/aiohttp/base_protocol.py b/aiohttp/base_protocol.py index ed9d3396dc2..2e29a6467a5 100644 --- a/aiohttp/base_protocol.py +++ b/aiohttp/base_protocol.py @@ -17,6 +17,7 @@ class BaseProtocol(asyncio.Protocol): "_drain_waiter", "_connection_lost", "_reading_paused", + "_upgraded", "transport", ) @@ -28,6 +29,7 @@ def __init__( self._drain_waiter: asyncio.Future[None] | None = None self._reading_paused = False self._parser = parser + self._upgraded = False self.transport: asyncio.Transport | None = None @@ -56,7 +58,9 @@ def resume_writing(self) -> None: def pause_reading(self) -> None: self._reading_paused = True - self._parser.pause_reading() + # Parser shouldn't be paused on websockets. + if not self._upgraded: + self._parser.pause_reading() if self.transport is not None: try: self.transport.pause_reading() @@ -67,7 +71,8 @@ def resume_reading(self) -> None: self._reading_paused = False # This will resume parsing any unprocessed data from the last pause. - self.data_received(b"") + if not self._upgraded: + self.data_received(b"") # Reading may have been paused again in the above call if there was a lot of # compressed data still pending. From f4985a2a5cc89ecfbc423036dd793753725bb523 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 16:20:12 +0000 Subject: [PATCH 073/199] Update client_proto.py --- aiohttp/client_proto.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index 7ede8a5766c..befad60ada6 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -36,9 +36,7 @@ def __init__(self, loop: asyncio.AbstractEventLoop) -> None: self._payload_parser: WebSocketReader | None = None self._timer = None - self._tail = b"" - self._upgraded = False self._read_timeout: float | None = None self._read_timeout_handle: asyncio.TimerHandle | None = None From 32d5c5fa0aef473b2710515f26d935b0bce3b835 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 16:21:33 +0000 Subject: [PATCH 074/199] Update web_protocol.py --- aiohttp/web_protocol.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index 8f2eff20200..973d6ba298a 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -169,7 +169,6 @@ class RequestHandler(BaseProtocol, Generic[_Request]): "_handler_waiter", "_waiter", "_task_handler", - "_upgrade", "_payload_parser", "logger", "access_log", @@ -240,8 +239,6 @@ def __init__( self._waiter: asyncio.Future[None] | None = None self._handler_waiter: asyncio.Future[None] | None = None self._task_handler: asyncio.Task[None] | None = None - - self._upgrade = False self._payload_parser: Any = None self._timeout_ceil_threshold: float = 5 @@ -419,7 +416,7 @@ def data_received(self, data: bytes) -> None: return # parse http messages messages: Sequence[_MsgType] - if self._payload_parser is None and not self._upgrade: + if self._payload_parser is None and not self._upgraded: assert self._parser is not None try: messages, upgraded, tail = self._parser.feed_data(data) @@ -439,12 +436,12 @@ def data_received(self, data: bytes) -> None: # don't set result twice waiter.set_result(None) - self._upgrade = upgraded + self._upgraded = upgraded if upgraded and tail: self._message_tail = tail # no parser, just store - elif self._payload_parser is None and self._upgrade and data: + elif self._payload_parser is None and self._upgraded and data: self._message_tail += data # feed payload @@ -706,7 +703,7 @@ async def finish_response( request._finish() if self._parser is not None: self._parser.set_upgraded(False) - self._upgrade = False + self._upgraded = False if self._message_tail: self._parser.feed_data(self._message_tail) self._message_tail = b"" From afa2b557a93c6bb3b1704eefc9161b9bfecaebfc Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 18:44:01 +0000 Subject: [PATCH 075/199] Update test_base_protocol.py --- tests/test_base_protocol.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_base_protocol.py b/tests/test_base_protocol.py index d53e0cfb303..9f871362b11 100644 --- a/tests/test_base_protocol.py +++ b/tests/test_base_protocol.py @@ -29,9 +29,7 @@ async def test_pause_reading_no_transport() -> None: loop = asyncio.get_event_loop() parser = mock.create_autospec(HttpParser, spec_set=True, instance=True) pr = BaseProtocol(loop, parser=parser) - assert not pr._reading_paused pr.pause_reading() - assert not pr._reading_paused parser.pause_reading.assert_called_once() From 30c23d40f47e5229d2d3f2796aafd2b81f10efc7 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 18:45:59 +0000 Subject: [PATCH 076/199] Update test_client_proto.py --- tests/test_client_proto.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_client_proto.py b/tests/test_client_proto.py index 49a81c8dbb3..f0df9c3bc53 100644 --- a/tests/test_client_proto.py +++ b/tests/test_client_proto.py @@ -35,7 +35,8 @@ async def test_oserror(loop: asyncio.AbstractEventLoop) -> None: async def test_pause_resume_on_error(loop: asyncio.AbstractEventLoop) -> None: - proto = ResponseHandler(loop=loop) + parser = mock.create_autospec(HttpParser, spec_set=True, instance=True) + proto = ResponseHandler(loop=loop, parser=parser) transport = mock.Mock() proto.connection_made(transport) From 6a5a2c7f8f9ebdd28ebb4fad814d66182d4049c9 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 18:47:45 +0000 Subject: [PATCH 077/199] Update test_http_parser.py --- tests/test_http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index b7043b7f8cf..662c6ea1c68 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -185,7 +185,7 @@ def test_invalid_character( server: Server, request: pytest.FixtureRequest, ) -> None: - protocol = RequestHandler(server, loop) + protocol = RequestHandler(server, loop=loop) parser = HttpRequestParserC( protocol, From 45f66d12c0dddd751239d18c9ec0fdf5e3c5456c Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 18:48:46 +0000 Subject: [PATCH 078/199] Update test_http_parser.py --- tests/test_http_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 662c6ea1c68..4be13704ab3 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -212,7 +212,7 @@ def test_invalid_linebreak( server: Server, request: pytest.FixtureRequest, ) -> None: - protocol = RequestHandler(server, loop) + protocol = RequestHandler(server, loop=loop) parser = HttpRequestParserC( protocol, @@ -280,7 +280,7 @@ def test_bad_headers(parser: HttpRequestParser, hdr: str) -> None: def test_unpaired_surrogate_in_header_py( loop: asyncio.AbstractEventLoop, server: Server ) -> None: - protocol = RequestHandler(server, loop) + protocol = RequestHandler(server, loop=loop) parser = HttpRequestParserPy( protocol, From 80d955ffe0b3ddb99b4bba20180fd3f4473ccd57 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 18:54:24 +0000 Subject: [PATCH 079/199] Update test_flowcontrol_streams.py --- tests/test_flowcontrol_streams.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_flowcontrol_streams.py b/tests/test_flowcontrol_streams.py index 9e21f786610..3654ba4aad2 100644 --- a/tests/test_flowcontrol_streams.py +++ b/tests/test_flowcontrol_streams.py @@ -5,6 +5,7 @@ from aiohttp import streams from aiohttp.base_protocol import BaseProtocol +from aiohttp.http_parser import HttpParser @pytest.fixture @@ -38,7 +39,6 @@ async def test_readline(self, stream: streams.StreamReader) -> None: stream.feed_data(b"d\n") res = await stream.readline() assert res == b"d\n" - assert not stream._protocol.resume_reading.called # type: ignore[attr-defined] async def test_readline_resume_paused(self, stream: streams.StreamReader) -> None: stream._protocol._reading_paused = True @@ -51,7 +51,6 @@ async def test_readany(self, stream: streams.StreamReader) -> None: stream.feed_data(b"data") res = await stream.readany() assert res == b"data" - assert not stream._protocol.resume_reading.called # type: ignore[attr-defined] async def test_readany_resume_paused(self, stream: streams.StreamReader) -> None: stream._protocol._reading_paused = True @@ -65,7 +64,6 @@ async def test_readchunk(self, stream: streams.StreamReader) -> None: res, end_of_http_chunk = await stream.readchunk() assert res == b"data" assert not end_of_http_chunk - assert not stream._protocol.resume_reading.called # type: ignore[attr-defined] async def test_readchunk_resume_paused(self, stream: streams.StreamReader) -> None: stream._protocol._reading_paused = True @@ -120,7 +118,8 @@ async def test_resumed_on_eof(self, stream: streams.StreamReader) -> None: async def test_stream_reader_eof_when_full() -> None: loop = asyncio.get_event_loop() - protocol = BaseProtocol(loop=loop) + parser = mock.create_autospec(HttpParser, spec_set=True, instance=True) + protocol = BaseProtocol(loop=loop, parser=parser) protocol.transport = asyncio.Transport() stream = streams.StreamReader(protocol, 1024, loop=loop) From 0cc627591bf31e5c25378d028e1540d2de563bc1 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 18:54:55 +0000 Subject: [PATCH 080/199] Update test_client_proto.py --- tests/test_client_proto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_client_proto.py b/tests/test_client_proto.py index f0df9c3bc53..3fa4ce7e336 100644 --- a/tests/test_client_proto.py +++ b/tests/test_client_proto.py @@ -10,7 +10,7 @@ from aiohttp.client_proto import ResponseHandler from aiohttp.client_reqrep import ClientResponse from aiohttp.helpers import TimerNoop -from aiohttp.http_parser import RawResponseMessage +from aiohttp.http_parser import HttpParser, RawResponseMessage async def test_force_close(loop: asyncio.AbstractEventLoop) -> None: From 69e3bbdff4e0611061c5cabb72d829f4edc70fb3 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 18:56:14 +0000 Subject: [PATCH 081/199] Update test_http_parser.py --- tests/test_http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 4be13704ab3..3e7684b0a30 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1491,7 +1491,7 @@ def test_parse_no_length_or_te_on_post( server: Server, request_cls: type[HttpRequestParser], ) -> None: - protocol = RequestHandler(server, loop) + protocol = RequestHandler(server, loop=loop) parser = request_cls(protocol, loop, limit=2**16) protocol._parser = parser text = b"POST /test HTTP/1.1\r\n\r\n" From 397e905c32e16229621f39ebc09fcaea6553c9b6 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 19:00:42 +0000 Subject: [PATCH 082/199] Update test_http_parser.py --- tests/test_http_parser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 3e7684b0a30..c6e3145219e 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -28,6 +28,7 @@ HttpRequestParserPy, HttpResponseParser, HttpResponseParserPy, + PayloadState, ) from aiohttp.http_writer import HttpVersion from aiohttp.web_protocol import RequestHandler @@ -1885,8 +1886,8 @@ async def test_parse_chunked_payload_split_end_trailers4( async def test_http_payload_parser_length(self, protocol: BaseProtocol) -> None: out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) p = HttpPayloadParser(out, length=2, headers_parser=HeadersParser()) - eof, tail = p.feed_data(b"1245") - assert eof + state, tail = p.feed_data(b"1245") + assert state is PayloadState.PAYLOAD_COMPLETE assert b"12" == out._buffer[0] assert b"45" == tail From aed6863ecd675f0b75dfbe048b205935f0b1e4e9 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 19:21:43 +0000 Subject: [PATCH 083/199] Update test_websocket_parser.py --- tests/test_websocket_parser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_websocket_parser.py b/tests/test_websocket_parser.py index 6de09a2cb00..c965a9b4395 100644 --- a/tests/test_websocket_parser.py +++ b/tests/test_websocket_parser.py @@ -19,7 +19,7 @@ from aiohttp._websocket.reader import WebSocketDataQueue from aiohttp.base_protocol import BaseProtocol from aiohttp.compression_utils import ZLibBackend, ZLibBackendWrapper -from aiohttp.http import WebSocketError, WSCloseCode, WSMsgType +from aiohttp.http import HttpParser, WebSocketError, WSCloseCode, WSMsgType from aiohttp.http_websocket import ( WebSocketReader, WSMessageBinary, @@ -113,8 +113,9 @@ def build_close_frame( @pytest.fixture() def protocol(loop: asyncio.AbstractEventLoop) -> BaseProtocol: + parser = mock.create_autospec(HttpParser, spec_set=True, instance=True) transport = mock.Mock(spec_set=asyncio.Transport) - protocol = BaseProtocol(loop) + protocol = BaseProtocol(loop, parser=parser) protocol.connection_made(transport) return protocol From a0fb83b1d54a221ba785699ff3185f2eb5d93dda Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 19:26:07 +0000 Subject: [PATCH 084/199] Update test_base_protocol.py --- tests/test_base_protocol.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/test_base_protocol.py b/tests/test_base_protocol.py index 9f871362b11..8cac3ff398e 100644 --- a/tests/test_base_protocol.py +++ b/tests/test_base_protocol.py @@ -45,15 +45,6 @@ async def test_pause_reading_stub_transport() -> None: parser.pause_reading.assert_called_once() -async def test_resume_reading_no_transport() -> None: - loop = asyncio.get_event_loop() - parser = mock.create_autospec(HttpParser, spec_set=True, instance=True) - pr = BaseProtocol(loop, parser=parser) - pr._reading_paused = True - pr.resume_reading() - assert pr._reading_paused - - async def test_resume_reading_stub_transport() -> None: loop = asyncio.get_event_loop() parser = mock.create_autospec(HttpParser, spec_set=True, instance=True) From 360f6de75cbb25a732c568307f0f158a30d5c30c Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 19:28:13 +0000 Subject: [PATCH 085/199] Update test_client_proto.py --- tests/test_client_proto.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_client_proto.py b/tests/test_client_proto.py index 3fa4ce7e336..0a26a211453 100644 --- a/tests/test_client_proto.py +++ b/tests/test_client_proto.py @@ -36,7 +36,8 @@ async def test_oserror(loop: asyncio.AbstractEventLoop) -> None: async def test_pause_resume_on_error(loop: asyncio.AbstractEventLoop) -> None: parser = mock.create_autospec(HttpParser, spec_set=True, instance=True) - proto = ResponseHandler(loop=loop, parser=parser) + proto = ResponseHandler(loop=loop) + proto._parser = parser transport = mock.Mock() proto.connection_made(transport) From 6e04d89f1b60b93b1f9c6d83df6f760066676063 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 26 Jan 2026 19:44:11 +0000 Subject: [PATCH 086/199] Update test_http_parser.py --- tests/test_http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index c6e3145219e..5514a470ceb 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1770,7 +1770,7 @@ def test_parse_uri_utf8_percent_encoded(parser: HttpRequestParser) -> None: def test_parse_bad_method_for_c_parser_raises( loop: asyncio.AbstractEventLoop, server: Server ) -> None: - protocol = RequestHandler(server, loop) + protocol = RequestHandler(server, loop=loop) payload = b"GET1 /test HTTP/1.1\r\n\r\n" parser = HttpRequestParserC( From 698e0cc196c8051ac95f594f29a6fecc005cbe40 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 27 Jan 2026 14:25:51 +0000 Subject: [PATCH 087/199] Update compression_utils.py --- aiohttp/compression_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index 313ca7032cc..c47b08d7d52 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -368,4 +368,4 @@ def flush(self) -> bytes: @property def data_available(self) -> bool: - return not self._obj.needs_input + return not self._obj.needs_input and not self._obj.eof From 48d411907e288575f957bf100516ae3aa3a5354d Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 28 Jan 2026 13:39:46 +0000 Subject: [PATCH 088/199] Update streams.py --- aiohttp/streams.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aiohttp/streams.py b/aiohttp/streams.py index cc64e5e0906..7a199c81de6 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -218,7 +218,8 @@ def feed_eof(self) -> None: self._eof_waiter = None set_result(waiter, None) - self._protocol.resume_reading() + # At EOF the parser is done, there won't be unprocessed data. + self._protocol.resume_reading(resume_parser=False) for cb in self._eof_callbacks: try: From 6598ff657a49e3fe3a5e09da094935b1845f35aa Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 28 Jan 2026 13:40:46 +0000 Subject: [PATCH 089/199] Update base_protocol.py --- aiohttp/base_protocol.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiohttp/base_protocol.py b/aiohttp/base_protocol.py index 2e29a6467a5..4acab2532e3 100644 --- a/aiohttp/base_protocol.py +++ b/aiohttp/base_protocol.py @@ -67,11 +67,11 @@ def pause_reading(self) -> None: except (AttributeError, NotImplementedError, RuntimeError): pass - def resume_reading(self) -> None: + def resume_reading(self, resume_parsing: bool = True) -> None: self._reading_paused = False # This will resume parsing any unprocessed data from the last pause. - if not self._upgraded: + if not self._upgraded and resume_parsing: self.data_received(b"") # Reading may have been paused again in the above call if there was a lot of From 3f76e2ca581cf0ee2b118f391e8145ce6cb82745 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 28 Jan 2026 13:56:41 +0000 Subject: [PATCH 090/199] Update base_protocol.py --- aiohttp/base_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/base_protocol.py b/aiohttp/base_protocol.py index 4acab2532e3..3f5fcac6e2d 100644 --- a/aiohttp/base_protocol.py +++ b/aiohttp/base_protocol.py @@ -67,7 +67,7 @@ def pause_reading(self) -> None: except (AttributeError, NotImplementedError, RuntimeError): pass - def resume_reading(self, resume_parsing: bool = True) -> None: + def resume_reading(self, resume_parser: bool = True) -> None: self._reading_paused = False # This will resume parsing any unprocessed data from the last pause. From faf6e404fdbc9bf6402a29a43d77e659b1913833 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 28 Jan 2026 14:14:46 +0000 Subject: [PATCH 091/199] Update client_proto.py --- aiohttp/client_proto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index befad60ada6..fd1fc2a0ad1 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -187,8 +187,8 @@ def pause_reading(self) -> None: super().pause_reading() self._drop_timeout() - def resume_reading(self) -> None: - super().resume_reading() + def resume_reading(self, resume_parser: bool = True) -> None: + super().resume_reading(resume_parser) self._reschedule_timeout() def set_exception( From add2b7082541f93c3739354945d0bf32ffa0352b Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 28 Jan 2026 14:26:03 +0000 Subject: [PATCH 092/199] Update base_protocol.py --- aiohttp/base_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/base_protocol.py b/aiohttp/base_protocol.py index 3f5fcac6e2d..9f0b75d9d00 100644 --- a/aiohttp/base_protocol.py +++ b/aiohttp/base_protocol.py @@ -71,7 +71,7 @@ def resume_reading(self, resume_parser: bool = True) -> None: self._reading_paused = False # This will resume parsing any unprocessed data from the last pause. - if not self._upgraded and resume_parsing: + if not self._upgraded and resume_parser: self.data_received(b"") # Reading may have been paused again in the above call if there was a lot of From 3396079d867808083214dbc50adb2d77d0531222 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 28 Jan 2026 16:27:07 +0000 Subject: [PATCH 093/199] Update compression_utils.py --- aiohttp/compression_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index c47b08d7d52..1deddd654c2 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -334,7 +334,7 @@ def flush(self) -> bytes: @property def data_available(self) -> bool: - pass # TODO + return not self._obj.can_accept_more_data() class ZSTDDecompressor(DecompressionBaseHandler): From 69a59a840bd0c24c185d59688250783040ac3c5d Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 28 Jan 2026 18:23:44 +0000 Subject: [PATCH 094/199] Update compression_utils.py --- aiohttp/compression_utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index 1deddd654c2..3bfebefaef2 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -316,6 +316,7 @@ def __init__( "Please install `Brotli` module" ) self._obj = brotli.Decompressor() + self._last_empty = False super().__init__(executor=executor, max_sync_chunk_size=max_sync_chunk_size) def decompress_sync( @@ -323,8 +324,11 @@ def decompress_sync( ) -> bytes: """Decompress the given data.""" if hasattr(self._obj, "decompress"): - return cast(bytes, self._obj.decompress(data, max_length)) - return cast(bytes, self._obj.process(data, max_length)) + result = cast(bytes, self._obj.decompress(data, max_length)) + result = cast(bytes, self._obj.process(data, max_length)) + # Only way to know that brotli has no further data is checking we get no output + self._last_empty = result == b"" + return result def flush(self) -> bytes: """Flush the decompressor.""" @@ -334,7 +338,7 @@ def flush(self) -> bytes: @property def data_available(self) -> bool: - return not self._obj.can_accept_more_data() + return not self._obj.is_finished() and not self._last_empty class ZSTDDecompressor(DecompressionBaseHandler): From c5f6e6a2d6558a6a22774f5d6801c29ea3a23c60 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 29 Jan 2026 17:58:08 +0000 Subject: [PATCH 095/199] Update compression_utils.py --- aiohttp/compression_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index 3bfebefaef2..850bffc4ffa 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -325,7 +325,8 @@ def decompress_sync( """Decompress the given data.""" if hasattr(self._obj, "decompress"): result = cast(bytes, self._obj.decompress(data, max_length)) - result = cast(bytes, self._obj.process(data, max_length)) + else: + result = cast(bytes, self._obj.process(data, max_length)) # Only way to know that brotli has no further data is checking we get no output self._last_empty = result == b"" return result From 51eda1be597420776d4e936b97e3414c022be8cb Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 29 Jan 2026 18:25:04 +0000 Subject: [PATCH 096/199] Update compression_utils.py --- aiohttp/compression_utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index 850bffc4ffa..bb2d0fe043e 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -53,6 +53,9 @@ def flush(self, length: int = ..., /) -> bytes: ... @property def eof(self) -> bool: ... + @property + def unconsumed_tail(self) -> bytes: ... + class ZLibBackendProtocol(Protocol): MAX_WBITS: int @@ -96,10 +99,6 @@ def __init__(self, _zlib_backend: ZLibBackendProtocol): def name(self) -> str: return getattr(self._zlib_backend, "__name__", "undefined") - @property - def unconsumed_tail(self) -> bytes: - return self._zlib_backend.unconsumed_tail - @property def MAX_WBITS(self) -> int: return self._zlib_backend.MAX_WBITS From 156fb3cc9a9ec48d5020da55ecdbbd8131062bb4 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 29 Jan 2026 18:28:59 +0000 Subject: [PATCH 097/199] Update base_protocol.py --- aiohttp/base_protocol.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aiohttp/base_protocol.py b/aiohttp/base_protocol.py index 9f0b75d9d00..179391cd3a0 100644 --- a/aiohttp/base_protocol.py +++ b/aiohttp/base_protocol.py @@ -1,5 +1,5 @@ import asyncio -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Any, cast from .client_exceptions import ClientConnectionResetError from .helpers import set_exception @@ -22,7 +22,7 @@ class BaseProtocol(asyncio.Protocol): ) def __init__( - self, loop: asyncio.AbstractEventLoop, parser: "HttpParser | None" = None + self, loop: asyncio.AbstractEventLoop, parser: "HttpParser[Any] | None" = None ) -> None: self._loop: asyncio.AbstractEventLoop = loop self._paused = False @@ -60,6 +60,7 @@ def pause_reading(self) -> None: self._reading_paused = True # Parser shouldn't be paused on websockets. if not self._upgraded: + assert self._parser is not None self._parser.pause_reading() if self.transport is not None: try: From 8b05bfc47573a064184e2a1d300432f8ab41fbfc Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 29 Jan 2026 18:34:57 +0000 Subject: [PATCH 098/199] Update streams.py --- aiohttp/streams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiohttp/streams.py b/aiohttp/streams.py index 7a199c81de6..123c974970d 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -587,8 +587,8 @@ def at_eof(self) -> bool: async def wait_eof(self) -> None: return - def feed_data(self, data: bytes) -> None: - pass + def feed_data(self, data: bytes) -> bool: + return False async def readline(self) -> bytes: return b"" From f1bfba5f3b690c99bf5848ab99d97c3d12ccd060 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 29 Jan 2026 18:46:08 +0000 Subject: [PATCH 099/199] Update test_http_parser.py --- tests/test_http_parser.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 5514a470ceb..aa8e6c5eef7 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -4,7 +4,7 @@ import re import sys import zlib -from collections.abc import Iterable +from collections.abc import Iterable, Iterator from contextlib import suppress from typing import Any from unittest import mock @@ -32,6 +32,7 @@ ) from aiohttp.http_writer import HttpVersion from aiohttp.web_protocol import RequestHandler +from aiohttp.web_request import Request from aiohttp.web_server import Server try: @@ -91,9 +92,9 @@ def _gen_ids(parsers: Iterable[type[HttpParser[Any]]]) -> list[str]: @pytest.fixture(params=REQUEST_PARSERS, ids=_gen_ids(REQUEST_PARSERS)) def parser( loop: asyncio.AbstractEventLoop, - server: Server, + server: Server[Request], request: pytest.FixtureRequest, -) -> HttpRequestParser: +) -> Iterator[HttpRequestParser]: protocol = RequestHandler(server, loop=loop) # Parser implementations @@ -108,7 +109,7 @@ def parser( protocol._force_close = False protocol._parser = parser with mock.patch.object(protocol, "transport", True): - yield parser # type: ignore[no-any-return] + yield parser @pytest.fixture(params=REQUEST_PARSERS, ids=_gen_ids(REQUEST_PARSERS)) @@ -181,9 +182,9 @@ def test_reject_obsolete_line_folding(parser: HttpRequestParser) -> None: @pytest.mark.skipif(NO_EXTENSIONS, reason="Only tests C parser.") -def test_invalid_character( +def test_invalid_character( # type: ignore[misc] loop: asyncio.AbstractEventLoop, - server: Server, + server: Server[Request], request: pytest.FixtureRequest, ) -> None: protocol = RequestHandler(server, loop=loop) @@ -208,9 +209,9 @@ def test_invalid_character( @pytest.mark.skipif(NO_EXTENSIONS, reason="Only tests C parser.") -def test_invalid_linebreak( +def test_invalid_linebreak( # type: ignore[misc] loop: asyncio.AbstractEventLoop, - server: Server, + server: Server[Request], request: pytest.FixtureRequest, ) -> None: protocol = RequestHandler(server, loop=loop) @@ -279,7 +280,7 @@ def test_bad_headers(parser: HttpRequestParser, hdr: str) -> None: def test_unpaired_surrogate_in_header_py( - loop: asyncio.AbstractEventLoop, server: Server + loop: asyncio.AbstractEventLoop, server: Server[Request] ) -> None: protocol = RequestHandler(server, loop=loop) @@ -1303,6 +1304,7 @@ async def test_http_response_parser_bad_chunked_lax( @pytest.mark.dev_mode async def test_http_response_parser_bad_chunked_strict_py( loop: asyncio.AbstractEventLoop, + parser: HttpParser, ) -> None: protocol = ResponseHandler(loop) @@ -1328,6 +1330,7 @@ async def test_http_response_parser_bad_chunked_strict_py( ) async def test_http_response_parser_bad_chunked_strict_c( loop: asyncio.AbstractEventLoop, + parser: HttpParser, ) -> None: protocol = ResponseHandler(loop) @@ -1489,7 +1492,7 @@ async def test_request_chunked_reject_bad_trailer(parser: HttpRequestParser) -> def test_parse_no_length_or_te_on_post( loop: asyncio.AbstractEventLoop, - server: Server, + server: Server[Request], request_cls: type[HttpRequestParser], ) -> None: protocol = RequestHandler(server, loop=loop) @@ -1767,8 +1770,8 @@ def test_parse_uri_utf8_percent_encoded(parser: HttpRequestParser) -> None: "HttpRequestParserC" not in dir(aiohttp.http_parser), reason="C based HTTP parser not available", ) -def test_parse_bad_method_for_c_parser_raises( - loop: asyncio.AbstractEventLoop, server: Server +def test_parse_bad_method_for_c_parser_raises( # type: ignore[misc] + loop: asyncio.AbstractEventLoop, server: Server[Request] ) -> None: protocol = RequestHandler(server, loop=loop) From 9c1ffa98f235ee5ad0b19489cda797e343f13aa0 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 29 Jan 2026 18:46:57 +0000 Subject: [PATCH 100/199] Update test_base_protocol.py --- tests/test_base_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_base_protocol.py b/tests/test_base_protocol.py index 8cac3ff398e..234e9927c02 100644 --- a/tests/test_base_protocol.py +++ b/tests/test_base_protocol.py @@ -42,7 +42,7 @@ async def test_pause_reading_stub_transport() -> None: assert not pr._reading_paused pr.pause_reading() assert pr._reading_paused - parser.pause_reading.assert_called_once() + parser.pause_reading.assert_called_once() # type: ignore[unreachable] async def test_resume_reading_stub_transport() -> None: From 79c9da6fdc5ddd540d247176cd491208140e1050 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 29 Jan 2026 18:50:05 +0000 Subject: [PATCH 101/199] Update test_streams.py --- tests/test_streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_streams.py b/tests/test_streams.py index e2fd1659191..3e2242a8768 100644 --- a/tests/test_streams.py +++ b/tests/test_streams.py @@ -1109,7 +1109,7 @@ async def test_empty_stream_reader() -> None: assert s.set_exception(ValueError()) is None # type: ignore[func-returns-value] assert s.exception() is None assert s.feed_eof() is None # type: ignore[func-returns-value] - assert s.feed_data(b"data") is None # type: ignore[func-returns-value] + assert s.feed_data(b"data") is False assert s.at_eof() await s.wait_eof() assert await s.read() == b"" From 9467ad2b263d711d9c721d2fe47354398aa61b40 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 29 Jan 2026 18:52:33 +0000 Subject: [PATCH 102/199] Update test_http_parser.py --- tests/test_http_parser.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index aa8e6c5eef7..9711412d742 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -182,7 +182,7 @@ def test_reject_obsolete_line_folding(parser: HttpRequestParser) -> None: @pytest.mark.skipif(NO_EXTENSIONS, reason="Only tests C parser.") -def test_invalid_character( # type: ignore[misc] +def test_invalid_character( loop: asyncio.AbstractEventLoop, server: Server[Request], request: pytest.FixtureRequest, @@ -209,7 +209,7 @@ def test_invalid_character( # type: ignore[misc] @pytest.mark.skipif(NO_EXTENSIONS, reason="Only tests C parser.") -def test_invalid_linebreak( # type: ignore[misc] +def test_invalid_linebreak( loop: asyncio.AbstractEventLoop, server: Server[Request], request: pytest.FixtureRequest, @@ -1304,7 +1304,6 @@ async def test_http_response_parser_bad_chunked_lax( @pytest.mark.dev_mode async def test_http_response_parser_bad_chunked_strict_py( loop: asyncio.AbstractEventLoop, - parser: HttpParser, ) -> None: protocol = ResponseHandler(loop) @@ -1315,7 +1314,7 @@ async def test_http_response_parser_bad_chunked_strict_py( max_line_size=8190, max_field_size=8190, ) - protocol._parser = parser + protocol._parser = response text = ( b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5 \r\nabcde\r\n0\r\n\r\n" ) @@ -1330,7 +1329,6 @@ async def test_http_response_parser_bad_chunked_strict_py( ) async def test_http_response_parser_bad_chunked_strict_c( loop: asyncio.AbstractEventLoop, - parser: HttpParser, ) -> None: protocol = ResponseHandler(loop) @@ -1341,7 +1339,7 @@ async def test_http_response_parser_bad_chunked_strict_c( max_line_size=8190, max_field_size=8190, ) - protocol._parser = parser + protocol._parser = response text = ( b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5 \r\nabcde\r\n0\r\n\r\n" ) @@ -1770,7 +1768,7 @@ def test_parse_uri_utf8_percent_encoded(parser: HttpRequestParser) -> None: "HttpRequestParserC" not in dir(aiohttp.http_parser), reason="C based HTTP parser not available", ) -def test_parse_bad_method_for_c_parser_raises( # type: ignore[misc] +def test_parse_bad_method_for_c_parser_raises( loop: asyncio.AbstractEventLoop, server: Server[Request] ) -> None: protocol = RequestHandler(server, loop=loop) From 5e8068f48f5624c921242358d0663773fdfb6706 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 30 Jan 2026 15:55:20 +0000 Subject: [PATCH 103/199] Apply suggestions from code review --- aiohttp/compression_utils.py | 4 +++- tests/test_http_parser.py | 6 ++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index bb2d0fe043e..0e3aa99fd71 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -34,6 +34,8 @@ MAX_SYNC_CHUNK_SIZE = 4096 +# Matches the max size we receive from sockets: +# https://github.com/python/cpython/blob/1857a40807daeae3a1bf5efb682de9c9ae6df845/Lib/asyncio/selector_events.py#L766 DEFAULT_MAX_DECOMPRESS_SIZE = 256 * 1024 # Unlimited decompression constants - different libraries use different conventions @@ -185,7 +187,7 @@ async def decompress( @property @abstractmethod def data_available(self) -> bool: - """Return True if more output is available using only .unconsumed_tail.""" + """Return True if more output is available by passing b"".""" class ZLibCompressor: diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 9711412d742..191a9a4e8a5 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -63,23 +63,21 @@ @pytest.fixture def server() -> Any: - m = mock.create_autospec( + return mock.create_autospec( Server, request_factory=mock.Mock(), request_handler=mock.AsyncMock(), instance=True, ) - return m @pytest.fixture def protocol() -> Any: - m = mock.create_autospec( + return mock.create_autospec( BaseProtocol, spec_set=True, instance=True, ) - return m def _gen_ids(parsers: Iterable[type[HttpParser[Any]]]) -> list[str]: From e1de72234e6b256bab1a64b0be82e2a9c49b2886 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 30 Jan 2026 16:27:13 +0000 Subject: [PATCH 104/199] Update http_parser.py --- aiohttp/http_parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 6a3795784a2..864a877b6fc 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -35,7 +35,6 @@ BadStatusLine, ContentEncodingError, ContentLengthError, - DecompressSizeError, InvalidHeader, InvalidURLError, LineTooLong, From 46713f5ed626359057c0a3f6002c31f23521ffe4 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 30 Jan 2026 16:27:52 +0000 Subject: [PATCH 105/199] Update test_client_functional.py --- tests/test_client_functional.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 68854f3af45..efded32169f 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -53,7 +53,6 @@ ) from aiohttp.client_reqrep import ClientRequest from aiohttp.compression_utils import DEFAULT_MAX_DECOMPRESS_SIZE -from aiohttp.http_exceptions import DecompressSizeError from aiohttp.payload import ( AsyncIterablePayload, BufferedReaderPayload, From 67c210da3ed8ef88355a3e776db36bc9bbd4103d Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 30 Jan 2026 16:29:02 +0000 Subject: [PATCH 106/199] Update http_exceptions.py --- aiohttp/http_exceptions.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/aiohttp/http_exceptions.py b/aiohttp/http_exceptions.py index cf3c05434c5..95d0d6373ae 100644 --- a/aiohttp/http_exceptions.py +++ b/aiohttp/http_exceptions.py @@ -73,10 +73,6 @@ class ContentLengthError(PayloadEncodingError): """Not enough data to satisfy content length header.""" -class DecompressSizeError(PayloadEncodingError): - """Decompressed size exceeds the configured limit.""" - - class LineTooLong(BadHttpMessage): def __init__(self, line: bytes, limit: int) -> None: super().__init__(f"Got more than {limit} bytes when reading: {line!r}.") From 2bc5d177d4f1c5902a0ecbbb4c4519aa207b509d Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 14:49:24 +0000 Subject: [PATCH 107/199] Update test_multipart.py --- tests/test_multipart.py | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 25672c9005a..2e380c5e935 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -354,12 +354,29 @@ async def test_read_with_content_encoding_gzip(self) -> None: result = await obj.read(decode=True) assert b"Time to Relax!" == result + @pytest.mark.skipif(sys.version_info < (3, 11), reason="wbits not available") async def test_read_with_content_encoding_deflate(self) -> None: + content = b"A" * 1_000_000 # Large enough to exceed max_length. + compressed = ZLibBackend.compress(content, wbits=-ZLibBackend.MAX_WBITS) + h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) - with Stream(b"\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00\r\n--:--") as stream: + with Stream(compressed + b"\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) result = await obj.read(decode=True) - assert b"Time to Relax!" == result + assert result == content + + @pytest.mark.skipif(sys.version_info < (3, 11), reason="wbits not available") + async def test_read_chunk_with_content_encoding_deflate(self) -> None: + content = b"A" * 1_000_000 # Large enough to exceed max_length. + compressed = ZLibBackend.compress(content, wbits=-ZLibBackend.MAX_WBITS) + + h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) + with Stream(compressed + b"\r\n--:--") as stream: + obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) + result = b"" + while chunk := await obj.read_chunk(decode=True): + result += chunk + assert result == content async def test_read_with_content_encoding_identity(self) -> None: thing = ( @@ -1720,6 +1737,24 @@ async def test_body_part_reader_payload_as_bytes() -> None: payload.decode() +async def test_body_part_reader_payload_write() -> None: + content = b"A" * 1_000_000 # Large enough to exceed max_length. + compressed = ZLibBackend.compress(content, wbits=-ZLibBackend.MAX_WBITS) + output = b"" + + async def write(inp: bytes) -> None: + output += inp + + h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) + writer = mock.create_autospec(AbstractStreamWriter, write=write, spec_set=True, instance=True) + with Stream(compressed + b"\r\n--:--") as stream: + body_part = aiohttp.BodyPartReader(BOUNDARY, h, stream) + payload = BodyPartReaderPayload(body_part) + await payload.write(writer) + + assert output == content + + async def test_multipart_writer_close_with_exceptions() -> None: """Test that MultipartWriter.close() continues closing all parts even if one raises.""" writer = aiohttp.MultipartWriter() From 46eeeda2e3fad0ba0cf0e1407c06640b33267e7b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Feb 2026 14:50:33 +0000 Subject: [PATCH 108/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_multipart.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 2e380c5e935..d103b7347f6 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -358,7 +358,7 @@ async def test_read_with_content_encoding_gzip(self) -> None: async def test_read_with_content_encoding_deflate(self) -> None: content = b"A" * 1_000_000 # Large enough to exceed max_length. compressed = ZLibBackend.compress(content, wbits=-ZLibBackend.MAX_WBITS) - + h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) with Stream(compressed + b"\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) @@ -369,7 +369,7 @@ async def test_read_with_content_encoding_deflate(self) -> None: async def test_read_chunk_with_content_encoding_deflate(self) -> None: content = b"A" * 1_000_000 # Large enough to exceed max_length. compressed = ZLibBackend.compress(content, wbits=-ZLibBackend.MAX_WBITS) - + h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) with Stream(compressed + b"\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) @@ -1746,7 +1746,9 @@ async def write(inp: bytes) -> None: output += inp h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) - writer = mock.create_autospec(AbstractStreamWriter, write=write, spec_set=True, instance=True) + writer = mock.create_autospec( + AbstractStreamWriter, write=write, spec_set=True, instance=True + ) with Stream(compressed + b"\r\n--:--") as stream: body_part = aiohttp.BodyPartReader(BOUNDARY, h, stream) payload = BodyPartReaderPayload(body_part) From 618309f844a1a83517c7271f700087126a616caa Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 15:01:01 +0000 Subject: [PATCH 109/199] Update test_web_functional.py --- tests/test_web_functional.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 71dc53b500e..dae505cb6d0 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -328,6 +328,27 @@ async def handler(request: web.Request) -> web.Response: resp.release() +async def test_multipart_client_max_size(aiohttp_client: AiohttpClient) -> None: + with multipart.MultipartWriter() as writer: + writer.append("A" * 1020) + + async def handler(request: web.Request) -> web.Response: + reader = await request.multipart() + assert isinstance(reader, multipart.MultipartReader) + + part = await reader.next() + assert isinstance(part, multipart.BodyPartReader) + await part.text() # Should raise HttpRequestEntityTooLarge + assert False + + app = web.Application(client_max_size=1000) + app.router.add_post("/", handler) + client = await aiohttp_client(app) + + async with client.post("/", data=writer) as resp: + assert resp.status == 413 + + async def test_multipart_empty(aiohttp_client: AiohttpClient) -> None: with multipart.MultipartWriter() as writer: pass From 2cc75675e23315fe664fd53929a717593c71c54c Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 15:05:06 +0000 Subject: [PATCH 110/199] Apply suggestions from code review --- tests/test_multipart.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_multipart.py b/tests/test_multipart.py index d103b7347f6..ccbd1421016 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -1743,6 +1743,7 @@ async def test_body_part_reader_payload_write() -> None: output = b"" async def write(inp: bytes) -> None: + nonlocal output output += inp h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) From 1f256dfe2d541d2856955496f9853b1581c1ebb3 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 15:22:21 +0000 Subject: [PATCH 111/199] Update test_multipart.py --- tests/test_multipart.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_multipart.py b/tests/test_multipart.py index ccbd1421016..ea184821733 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -363,6 +363,7 @@ async def test_read_with_content_encoding_deflate(self) -> None: with Stream(compressed + b"\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) result = await obj.read(decode=True) + assert len(result) == len(content) # Simplifies diff on failure assert result == content @pytest.mark.skipif(sys.version_info < (3, 11), reason="wbits not available") @@ -1755,6 +1756,7 @@ async def write(inp: bytes) -> None: payload = BodyPartReaderPayload(body_part) await payload.write(writer) + assert len(output) == len(content) # Simplifies diff on failure assert output == content From 70a51f50fcb8abe44b41841f859abab06b8736a7 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 15:52:46 +0000 Subject: [PATCH 112/199] Update multipart.py --- aiohttp/multipart.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index f78693a12a8..5b92a5fe0a1 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -314,10 +314,10 @@ async def read(self, *, decode: bool = False) -> bytes: data.extend(await self.read_chunk(self.chunk_size)) # https://github.com/python/mypy/issues/17537 if decode: # type: ignore[unreachable] - return await self.decode_async(data) + return b"".join(d async for d in self.decode_iter(data)) return data - async def read_chunk(self, size: int = chunk_size) -> bytes: + async def read_chunk(self, size: int = chunk_size, decode: bool = False) -> bytes: """Reads body part content chunk of the specified size. size: chunk size @@ -509,7 +509,7 @@ def decode(self, data: bytes) -> bytes: Decodes data according the specified Content-Encoding or Content-Transfer-Encoding headers value. - Note: For large payloads, consider using decode_async() instead + Note: For large payloads, consider using decode_iter() instead to avoid blocking the event loop during decompression. """ data = self._apply_content_transfer_decoding(data) @@ -517,7 +517,7 @@ def decode(self, data: bytes) -> bytes: return self._decode_content(data) return data - async def decode_async(self, data: bytes) -> bytes: + async def decode_iter(self, data: bytes) -> AsyncIterator[bytes]: """Decodes data asynchronously. Decodes data according the specified Content-Encoding @@ -543,17 +543,18 @@ def _decode_content(self, data: bytes) -> bytes: raise RuntimeError(f"unknown content encoding: {encoding}") - async def _decode_content_async(self, data: bytes) -> bytes: + async def _decode_content_async(self, data: bytes) -> AsyncIterator[bytes]: encoding = self.headers.get(CONTENT_ENCODING, "").lower() if encoding == "identity": - return data + yield data if encoding in {"deflate", "gzip"}: - return await ZLibDecompressor( + d = ZLibDecompressor( encoding=encoding, suppress_deflate_header=True, - ).decompress( - data, max_length=self._max_decompress_size - ) # TODO + ) + yield d.decompress(data, max_length=self._max_decompress_size) + while d.data_available: + yield d.decompress(b"", max_length=self._max_decompress_size) raise RuntimeError(f"unknown content encoding: {encoding}") @@ -626,10 +627,9 @@ async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> byt async def write(self, writer: AbstractStreamWriter) -> None: field = self._value - chunk = await field.read_chunk(size=2**16) - while chunk: - await writer.write(await field.decode_async(chunk)) - chunk = await field.read_chunk(size=2**16) + while chunk := await field.read_chunk(size=2**18): + async for d in field.decode_iter(chunk): + await writer.write(d) class MultipartReader: From 06c5ec98efc90cb4cb69ca64b6f34ae947465034 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 15:55:13 +0000 Subject: [PATCH 113/199] Update multipart.py --- aiohttp/multipart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 5b92a5fe0a1..b4742749ea6 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -6,7 +6,7 @@ import uuid import warnings from collections import deque -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from types import TracebackType from typing import TYPE_CHECKING, Any, Union, cast from urllib.parse import parse_qsl, unquote, urlencode From 69c16021fa64958bf952f595a64c2aec43fe6c3a Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 16:01:04 +0000 Subject: [PATCH 114/199] Update multipart.py --- aiohttp/multipart.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index b4742749ea6..a2af674227b 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -528,8 +528,8 @@ async def decode_iter(self, data: bytes) -> AsyncIterator[bytes]: """ data = self._apply_content_transfer_decoding(data) if self._needs_content_decoding(): - return await self._decode_content_async(data) - return data + yield from self._decode_content_async(data) + yield data def _decode_content(self, data: bytes) -> bytes: encoding = self.headers.get(CONTENT_ENCODING, "").lower() @@ -552,9 +552,9 @@ async def _decode_content_async(self, data: bytes) -> AsyncIterator[bytes]: encoding=encoding, suppress_deflate_header=True, ) - yield d.decompress(data, max_length=self._max_decompress_size) + yield await d.decompress(data, max_length=self._max_decompress_size) while d.data_available: - yield d.decompress(b"", max_length=self._max_decompress_size) + yield await d.decompress(b"", max_length=self._max_decompress_size) raise RuntimeError(f"unknown content encoding: {encoding}") From 36c2bc35e0006d889e29dda702aeba0ececb1c73 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 16:03:23 +0000 Subject: [PATCH 115/199] Update web_request.py --- aiohttp/web_request.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index 09126b944cf..6f34ef3f7ce 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -714,17 +714,15 @@ async def post(self) -> "MultiDictProxy[str | bytes | FileField]": tmp = await self._loop.run_in_executor( None, tempfile.TemporaryFile ) - chunk = await field.read_chunk(size=2**16) - while chunk: - chunk = await field.decode_async(chunk) - await self._loop.run_in_executor(None, tmp.write, chunk) - size += len(chunk) - if 0 < max_size < size: - await self._loop.run_in_executor(None, tmp.close) - raise HTTPRequestEntityTooLarge( - max_size=max_size, actual_size=size - ) - chunk = await field.read_chunk(size=2**16) + while chunk := await field.read_chunk(size=2**16): + async for decoded_chunk in field.decode_async(chunk): + await self._loop.run_in_executor(None, tmp.write, decoded_chunk) + size += len(decoded_chunk) + if 0 < max_size < size: + await self._loop.run_in_executor(None, tmp.close) + raise HTTPRequestEntityTooLarge( + max_size=max_size, actual_size=size + ) await self._loop.run_in_executor(None, tmp.seek, 0) if field_ct is None: From 78e709976564399e119850d12ba54edb92f7d181 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Feb 2026 16:03:57 +0000 Subject: [PATCH 116/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/web_request.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index 6f34ef3f7ce..6adb346a57e 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -716,7 +716,9 @@ async def post(self) -> "MultiDictProxy[str | bytes | FileField]": ) while chunk := await field.read_chunk(size=2**16): async for decoded_chunk in field.decode_async(chunk): - await self._loop.run_in_executor(None, tmp.write, decoded_chunk) + await self._loop.run_in_executor( + None, tmp.write, decoded_chunk + ) size += len(decoded_chunk) if 0 < max_size < size: await self._loop.run_in_executor(None, tmp.close) From 2c47750d050ca959b6f9238a71d3a39ab61acc89 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 16:04:47 +0000 Subject: [PATCH 117/199] Update test_multipart.py --- tests/test_multipart.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_multipart.py b/tests/test_multipart.py index ea184821733..14a11fc73cb 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -422,7 +422,8 @@ async def test_decode_async_with_content_transfer_encoding_base64(self) -> None: result = b"" while not obj.at_eof(): chunk = await obj.read_chunk(size=6) - result += await obj.decode_async(chunk) + async for decoded_chunk in obj.decode_iter(chunk): + result += decoded_chunk assert b"Time to Relax!" == result async def test_decode_with_content_encoding_deflate(self) -> None: From e47504c94e37aaee60a8e5e98778fd0df4bce15d Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 16:18:24 +0000 Subject: [PATCH 118/199] Update multipart.py --- aiohttp/multipart.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index a2af674227b..6b9e8d1dbb9 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -528,7 +528,8 @@ async def decode_iter(self, data: bytes) -> AsyncIterator[bytes]: """ data = self._apply_content_transfer_decoding(data) if self._needs_content_decoding(): - yield from self._decode_content_async(data) + async for d in self._decode_content_async(data): + yield d yield data def _decode_content(self, data: bytes) -> bytes: From d92ed4cc5e5736aae8b1e1826413e139e006ca2a Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 16:21:24 +0000 Subject: [PATCH 119/199] Apply suggestions from code review --- aiohttp/web_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index 6adb346a57e..0045d0370ca 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -715,7 +715,7 @@ async def post(self) -> "MultiDictProxy[str | bytes | FileField]": None, tempfile.TemporaryFile ) while chunk := await field.read_chunk(size=2**16): - async for decoded_chunk in field.decode_async(chunk): + async for decoded_chunk in field.decode_iter(chunk): await self._loop.run_in_executor( None, tmp.write, decoded_chunk ) From ad2c9ab19e67209c2c32e938e35e1b85aa330e12 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 16:37:41 +0000 Subject: [PATCH 120/199] Update multipart.py --- aiohttp/multipart.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 6b9e8d1dbb9..6c70679fd4a 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -314,7 +314,8 @@ async def read(self, *, decode: bool = False) -> bytes: data.extend(await self.read_chunk(self.chunk_size)) # https://github.com/python/mypy/issues/17537 if decode: # type: ignore[unreachable] - return b"".join(d async for d in self.decode_iter(data)) + async for d in self.decode_iter(data): + data.extend(d) return data async def read_chunk(self, size: int = chunk_size, decode: bool = False) -> bytes: From bd7c69c1e367dd177dbb5ed2434d36a7e9cf90ae Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 17:48:25 +0000 Subject: [PATCH 121/199] Update multipart.py --- aiohttp/multipart.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 6c70679fd4a..25d2b66729c 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -314,8 +314,10 @@ async def read(self, *, decode: bool = False) -> bytes: data.extend(await self.read_chunk(self.chunk_size)) # https://github.com/python/mypy/issues/17537 if decode: # type: ignore[unreachable] + decoded_data = bytearray() async for d in self.decode_iter(data): - data.extend(d) + decoded_data.extend(d) + return decoded_data return data async def read_chunk(self, size: int = chunk_size, decode: bool = False) -> bytes: From 2106fbee36afeffb743ece800d8beddf2628f19c Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 17:57:58 +0000 Subject: [PATCH 122/199] Update multipart.py --- aiohttp/multipart.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 25d2b66729c..af6cd162a0c 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -551,7 +551,7 @@ async def _decode_content_async(self, data: bytes) -> AsyncIterator[bytes]: encoding = self.headers.get(CONTENT_ENCODING, "").lower() if encoding == "identity": yield data - if encoding in {"deflate", "gzip"}: + elif encoding in {"deflate", "gzip"}: d = ZLibDecompressor( encoding=encoding, suppress_deflate_header=True, @@ -559,8 +559,8 @@ async def _decode_content_async(self, data: bytes) -> AsyncIterator[bytes]: yield await d.decompress(data, max_length=self._max_decompress_size) while d.data_available: yield await d.decompress(b"", max_length=self._max_decompress_size) - - raise RuntimeError(f"unknown content encoding: {encoding}") + else: + raise RuntimeError(f"unknown content encoding: {encoding}") def _decode_content_transfer(self, data: bytes) -> bytes: encoding = self.headers.get(CONTENT_TRANSFER_ENCODING, "").lower() From 4f7f92845a5194795149d9aca301a85c46d62683 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 3 Feb 2026 18:16:36 +0000 Subject: [PATCH 123/199] Update multipart.py --- aiohttp/multipart.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index af6cd162a0c..29e93264e85 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -533,7 +533,8 @@ async def decode_iter(self, data: bytes) -> AsyncIterator[bytes]: if self._needs_content_decoding(): async for d in self._decode_content_async(data): yield d - yield data + else: + yield data def _decode_content(self, data: bytes) -> bytes: encoding = self.headers.get(CONTENT_ENCODING, "").lower() From 26777204b9e0b91497c5215433ad6a78d983b9bc Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 4 Feb 2026 01:13:41 +0000 Subject: [PATCH 124/199] Update web_request.py --- aiohttp/web_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index 0045d0370ca..1a3d950c010 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -672,7 +672,7 @@ async def json( async def multipart(self) -> MultipartReader: """Return async iterator to process BODY as multipart.""" - return MultipartReader(self._headers, self._payload) + return MultipartReader(self._headers, self._payload, self._client_max_size) async def post(self) -> "MultiDictProxy[str | bytes | FileField]": """Return POST parameters.""" From 5cb6324fff03c115ffbdc8ba62ec32810f632bb7 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 4 Feb 2026 01:22:50 +0000 Subject: [PATCH 125/199] Update multipart.py --- aiohttp/multipart.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 29e93264e85..2836232b065 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -1,6 +1,7 @@ import base64 import binascii import json +import math import re import sys import uuid @@ -267,6 +268,7 @@ def __init__( subtype: str = "mixed", default_charset: str | None = None, max_decompress_size: int = DEFAULT_MAX_DECOMPRESS_SIZE, + client_max_size: int = math.inf ) -> None: self.headers = headers self._boundary = boundary @@ -284,6 +286,7 @@ def __init__( self._content_eof = 0 self._cache: dict[str, Any] = {} self._max_decompress_size = max_decompress_size + self._client_max_size = client_max_size def __aiter__(self) -> Self: return self @@ -312,11 +315,19 @@ async def read(self, *, decode: bool = False) -> bytes: data = bytearray() while not self._at_eof: data.extend(await self.read_chunk(self.chunk_size)) + if len(data) > self._client_max_size: + raise HttpRequestEntityTooLarge( + max_size=self._client_max_size, actual_size=len(data) + ) # https://github.com/python/mypy/issues/17537 if decode: # type: ignore[unreachable] decoded_data = bytearray() async for d in self.decode_iter(data): decoded_data.extend(d) + if len(decoded_data) > self._client_max_size: + raise HttpRequestEntityTooLarge( + max_size=self._client_max_size, actual_size=len(decoded_data) + ) return decoded_data return data From bd6b520e347dcd9031b9d4280b5c29aac0c5f28c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 4 Feb 2026 01:23:26 +0000 Subject: [PATCH 126/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/multipart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 2836232b065..d5fba174c47 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -268,7 +268,7 @@ def __init__( subtype: str = "mixed", default_charset: str | None = None, max_decompress_size: int = DEFAULT_MAX_DECOMPRESS_SIZE, - client_max_size: int = math.inf + client_max_size: int = math.inf, ) -> None: self.headers = headers self._boundary = boundary From bd625ba96f85a33b61f2f1a617cc758c7dba7898 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 4 Feb 2026 01:24:04 +0000 Subject: [PATCH 127/199] Apply suggestion from @Dreamsorcerer --- aiohttp/web_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index 1a3d950c010..a1dc1c8aa3e 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -672,7 +672,7 @@ async def json( async def multipart(self) -> MultipartReader: """Return async iterator to process BODY as multipart.""" - return MultipartReader(self._headers, self._payload, self._client_max_size) + return MultipartReader(self._headers, self._payload, client_max_size=self._client_max_size) async def post(self) -> "MultiDictProxy[str | bytes | FileField]": """Return POST parameters.""" From ef3b5d95eebf84fdb81b2ad566d01ba88551030d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 4 Feb 2026 01:24:40 +0000 Subject: [PATCH 128/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/web_request.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index a1dc1c8aa3e..5e8c5298314 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -672,7 +672,9 @@ async def json( async def multipart(self) -> MultipartReader: """Return async iterator to process BODY as multipart.""" - return MultipartReader(self._headers, self._payload, client_max_size=self._client_max_size) + return MultipartReader( + self._headers, self._payload, client_max_size=self._client_max_size + ) async def post(self) -> "MultiDictProxy[str | bytes | FileField]": """Return POST parameters.""" From c2ba043a426adfff71a20d8a01f1955d3d1ddf01 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 4 Feb 2026 01:28:41 +0000 Subject: [PATCH 129/199] Update multipart.py --- aiohttp/multipart.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index d5fba174c47..3c298bdb39e 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -40,6 +40,7 @@ payload_type, ) from .streams import StreamReader +from .web_exceptions import HttpRequestEntityTooLarge if sys.version_info >= (3, 11): from typing import Self From dcdc3863ca26689604969085e4e6921ad5019ef0 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 4 Feb 2026 01:40:30 +0000 Subject: [PATCH 130/199] Update multipart.py --- aiohttp/multipart.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 3c298bdb39e..f7a7c6df698 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -660,7 +660,13 @@ class MultipartReader: #: Body part reader class for non multipart/* content types. part_reader_cls = BodyPartReader - def __init__(self, headers: Mapping[str, str], content: StreamReader) -> None: + def __init__( + self, + headers: Mapping[str, str], + content: StreamReader, + *, + client_max_size: int = math.inf, + ) -> None: self._mimetype = parse_mimetype(headers[CONTENT_TYPE]) assert self._mimetype.type == "multipart", "multipart/* content type expected" if "boundary" not in self._mimetype.parameters: @@ -670,6 +676,7 @@ def __init__(self, headers: Mapping[str, str], content: StreamReader) -> None: self.headers = headers self._boundary = ("--" + self._get_boundary()).encode() + self._client_max_size = client_max_size self._content = content self._default_charset: str | None = None self._last_part: MultipartReader | BodyPartReader | None = None @@ -772,8 +779,10 @@ def _get_part_reader( if mimetype.type == "multipart": if self.multipart_reader_cls is None: - return type(self)(headers, self._content) - return self.multipart_reader_cls(headers, self._content) + return type(self)(headers, self._content, client_max_size=client_max_size) + return self.multipart_reader_cls( + headers, self._content, client_max_size=self._client_max_size + ) else: return self.part_reader_cls( self._boundary, @@ -781,6 +790,7 @@ def _get_part_reader( self._content, subtype=self._mimetype.subtype, default_charset=self._default_charset, + client_max_size=self._client_max_size, ) def _get_boundary(self) -> str: From 3525d3d9a2cfe7bf8ef5c03597568b4db7f34bdb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 4 Feb 2026 01:41:24 +0000 Subject: [PATCH 131/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/multipart.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index f7a7c6df698..dfa09cce7a9 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -779,7 +779,9 @@ def _get_part_reader( if mimetype.type == "multipart": if self.multipart_reader_cls is None: - return type(self)(headers, self._content, client_max_size=client_max_size) + return type(self)( + headers, self._content, client_max_size=client_max_size + ) return self.multipart_reader_cls( headers, self._content, client_max_size=self._client_max_size ) From 271d10ba58b6976020247dd7fa034db05822c42e Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 4 Feb 2026 01:42:50 +0000 Subject: [PATCH 132/199] Update multipart.py --- aiohttp/multipart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index dfa09cce7a9..1a29b22969b 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -780,7 +780,7 @@ def _get_part_reader( if mimetype.type == "multipart": if self.multipart_reader_cls is None: return type(self)( - headers, self._content, client_max_size=client_max_size + headers, self._content, client_max_size=self._client_max_size ) return self.multipart_reader_cls( headers, self._content, client_max_size=self._client_max_size From 6749571af5b73e76f82e98eb079e8d7ded251a9c Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Wed, 4 Feb 2026 01:52:41 +0000 Subject: [PATCH 133/199] Update multipart.py --- aiohttp/multipart.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 1a29b22969b..1e9107b3aad 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -40,7 +40,7 @@ payload_type, ) from .streams import StreamReader -from .web_exceptions import HttpRequestEntityTooLarge +from .web_exceptions import HTTPRequestEntityTooLarge if sys.version_info >= (3, 11): from typing import Self @@ -317,7 +317,7 @@ async def read(self, *, decode: bool = False) -> bytes: while not self._at_eof: data.extend(await self.read_chunk(self.chunk_size)) if len(data) > self._client_max_size: - raise HttpRequestEntityTooLarge( + raise HTTPRequestEntityTooLarge( max_size=self._client_max_size, actual_size=len(data) ) # https://github.com/python/mypy/issues/17537 @@ -326,7 +326,7 @@ async def read(self, *, decode: bool = False) -> bytes: async for d in self.decode_iter(data): decoded_data.extend(d) if len(decoded_data) > self._client_max_size: - raise HttpRequestEntityTooLarge( + raise HTTPRequestEntityTooLarge( max_size=self._client_max_size, actual_size=len(decoded_data) ) return decoded_data From e874ebe01fad48db3e5ad0cfc0efac2caa5f6941 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 5 Feb 2026 16:44:34 +0000 Subject: [PATCH 134/199] Apply suggestion from @Dreamsorcerer --- aiohttp/multipart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 6329c4d3190..dac71484261 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -332,7 +332,7 @@ async def read(self, *, decode: bool = False) -> bytes: return decoded_data return data - async def read_chunk(self, size: int = chunk_size, decode: bool = False) -> bytes: + async def read_chunk(self, size: int = chunk_size) -> bytes: """Reads body part content chunk of the specified size. size: chunk size From 389b0e6441dc0ed4fdba0f7072cdc8d1cbfe2630 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 5 Feb 2026 16:45:38 +0000 Subject: [PATCH 135/199] Update test_multipart.py --- tests/test_multipart.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 63d71957eff..0c266b48fc3 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -366,19 +366,6 @@ async def test_read_with_content_encoding_deflate(self) -> None: assert len(result) == len(content) # Simplifies diff on failure assert result == content - @pytest.mark.skipif(sys.version_info < (3, 11), reason="wbits not available") - async def test_read_chunk_with_content_encoding_deflate(self) -> None: - content = b"A" * 1_000_000 # Large enough to exceed max_length. - compressed = ZLibBackend.compress(content, wbits=-ZLibBackend.MAX_WBITS) - - h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) - with Stream(compressed + b"\r\n--:--") as stream: - obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) - result = b"" - while chunk := await obj.read_chunk(decode=True): - result += chunk - assert result == content - async def test_read_with_content_encoding_identity(self) -> None: thing = ( b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\x0b\xc9\xccMU" From b4207040594fc373817af9493f50d128756a6a13 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:53:43 +0000 Subject: [PATCH 136/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/http_parser.py | 4 +++- aiohttp/web_protocol.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 7b927956a5e..6e1df250310 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -462,7 +462,9 @@ def get_content_length() -> int | None: assert not self._lines assert self._payload_parser is not None try: - payload_state, data = self._payload_parser.feed_data(data[start_pos:], SEP) + payload_state, data = self._payload_parser.feed_data( + data[start_pos:], SEP + ) except Exception as underlying_exc: reraised_exc: BaseException = underlying_exc if self.payload_exception is not None: diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index e2d1759964f..abb5f86d81c 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -409,7 +409,9 @@ def connection_lost(self, exc: BaseException | None) -> None: self._payload_parser = None def set_parser( - self, parser: WebSocketReader, data_received_cb: Callable[[], None] | None = None + self, + parser: WebSocketReader, + data_received_cb: Callable[[], None] | None = None, ) -> None: assert self._payload_parser is None From 045ffb0b5111b86ca8bc4efe217824622438e11b Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 30 Mar 2026 20:00:19 +0100 Subject: [PATCH 137/199] Apply suggestion from @Dreamsorcerer --- aiohttp/compression_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index 0e3aa99fd71..a83df9ef689 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -374,4 +374,4 @@ def flush(self) -> bytes: @property def data_available(self) -> bool: - return not self._obj.needs_input and not self._obj.eof + return (not self._obj.needs_input and not self._obj.eof) or self._pending_unused_data From 1c1ea45cc61c1e3ca9dca16841fc787be8c605b8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 19:01:39 +0000 Subject: [PATCH 138/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/compression_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index 188b33f8d02..4dbe70ebd6f 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -401,4 +401,6 @@ def flush(self) -> bytes: @property def data_available(self) -> bool: - return (not self._obj.needs_input and not self._obj.eof) or self._pending_unused_data + return ( + not self._obj.needs_input and not self._obj.eof + ) or self._pending_unused_data From 7f4e807ce6044ff1b7c86358480116cc17c7f1ad Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 30 Mar 2026 22:21:39 +0100 Subject: [PATCH 139/199] Fix when compressed data just trips the high water mark --- aiohttp/_http_parser.pyx | 8 -------- tests/test_http_parser.py | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index bb5f69a96f7..b3efa71f7e9 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -312,7 +312,6 @@ cdef class HttpParser: str _path str _reason list _headers - bint _last_had_more_data list _raw_headers bint _upgraded list _messages @@ -363,7 +362,6 @@ cdef class HttpParser: self._timer = timer self._buf = bytearray() - self._last_had_more_data = False self._more_data_available = False self._paused = False self._payload = None @@ -578,15 +576,9 @@ cdef class HttpParser: if self._more_data_available: result = cb_on_body(self._cparser, EMPTY_BYTES, 0) if result is cparser.HPE_PAUSED: - self._last_had_more_data = had_more_data self._tail = data return (), False, EMPTY_BYTES # TODO: Do we need to handle error case (-1)? - # If the last pause had more data, then we probably paused at the - # end of the body. Therefore we need to continue with empty bytes. - if not data and not self._last_had_more_data: - return (), False, EMPTY_BYTES - self._last_had_more_data = False PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE) # Cache buffer pointer before PyBuffer_Release to avoid use-after-release. diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 5e2f0d865f8..8428d87fa37 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -985,6 +985,24 @@ async def test_chunk_splits_after_pause(parser: HttpRequestParser) -> None: assert result == b"b" * 50000 +async def test_compressed_256kb(response: HttpResponseParser) -> None: + original = b"x" * 256 * 1024 + compressed = zlib.compress(original) + headers = ( + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: " + str(len(compressed)).encode() + b"\r\n" + b"Content-Encoding: deflate\r\n" + b"\r\n" + ) + + messages, upgrade, tail = response.feed_data(headers + compressed) + assert len(messages) == 1 + payload = messages[0][-1] + result = await payload.read() + assert len(result) == len(original) + assert result == original + + @pytest.mark.parametrize("size", [40965, 8191]) def test_max_header_value_size_continuation( response: HttpResponseParser, size: int From 8256f07b557f617eb5131861825b37ca921be03f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 31 Mar 2026 16:19:00 -1000 Subject: [PATCH 140/199] attempt to fix connection closed cleanly while decompression still in flight --- aiohttp/client_proto.py | 76 +++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 21 deletions(-) diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index c7ea3363176..7bd4cbb2f03 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -52,6 +52,7 @@ def __init__(self, loop: asyncio.AbstractEventLoop) -> None: self._closed: None | asyncio.Future[None] = None self._connection_lost_called = False + self._connection_lost_deferred = False @property def closed(self) -> None | asyncio.Future[None]: @@ -142,25 +143,23 @@ def connection_lost(self, exc: BaseException | None) -> None: with suppress(Exception): # FIXME: log this somehow? self._payload_parser.feed_eof() - uncompleted = None - if self._parser is not None: - try: - uncompleted = self._parser.feed_eof() - except Exception as underlying_exc: - if self._payload is not None: - client_payload_exc_msg = ( - f"Response payload is not completed: {underlying_exc !r}" - ) - if not connection_closed_cleanly: - client_payload_exc_msg = ( - f"{client_payload_exc_msg !s}. " - f"{original_connection_error !r}" - ) - set_exception( - self._payload, - ClientPayloadError(client_payload_exc_msg), - underlying_exc, - ) + # If the connection closed cleanly but the parser is still + # decompressing data (reading is paused), keep the parser alive. + # The resume_reading/data_received drain cycle will continue + # feeding decompressed chunks to the reader incrementally. + # Once the parser runs out of buffered data, _close_parser + # will be called from data_received to complete the cleanup. + if ( + connection_closed_cleanly + and self._reading_paused + and self._parser is not None + ): + self._connection_lost_deferred = True + self._should_close = True + super().connection_lost(exc) + return + + uncompleted = self._close_parser(original_connection_error) if not self.is_eof(): if isinstance(original_connection_error, OSError): @@ -179,12 +178,39 @@ def connection_lost(self, exc: BaseException | None) -> None: self.set_exception(reraised_exc, underlying_non_eof_exc) self._should_close = True + + super().connection_lost(reraised_exc) + + def _close_parser( + self, original_exc: BaseException | None = None + ) -> RawResponseMessage | None: + """Feed EOF to the parser and clean up. + + Returns any uncompleted message from the parser. + """ + uncompleted = None + if self._parser is not None: + try: + uncompleted = self._parser.feed_eof() + except Exception as underlying_exc: + if self._payload is not None: + client_payload_exc_msg = ( + f"Response payload is not completed: {underlying_exc !r}" + ) + if original_exc is not None: + client_payload_exc_msg = ( + f"{client_payload_exc_msg !s}. {original_exc !r}" + ) + set_exception( + self._payload, + ClientPayloadError(client_payload_exc_msg), + underlying_exc, + ) self._parser = None self._payload = None self._payload_parser = None self._reading_paused = False - - super().connection_lost(reraised_exc) + return uncompleted def eof_received(self) -> None: # should call parser.feed_eof() most likely @@ -363,3 +389,11 @@ def data_received(self, data: bytes) -> None: if upgraded and tail: self.data_received(tail) + + # If connection_lost was deferred because the parser had pending + # decompression data, check if the parser is done draining. + # If reading is no longer paused, the parser has processed all + # buffered data -- finish the connection_lost cleanup now. + if self._connection_lost_deferred and not self._reading_paused: + self._connection_lost_deferred = False + self._close_parser() From 6b08dee0c2114ed2ebf4c11050af4d77149e743e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 31 Mar 2026 16:39:41 -1000 Subject: [PATCH 141/199] rely on timeout instead --- aiohttp/client_proto.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index 7bd4cbb2f03..9cd86258794 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -156,6 +156,7 @@ def connection_lost(self, exc: BaseException | None) -> None: ): self._connection_lost_deferred = True self._should_close = True + self._reschedule_timeout() super().connection_lost(exc) return @@ -391,9 +392,14 @@ def data_received(self, data: bytes) -> None: self.data_received(tail) # If connection_lost was deferred because the parser had pending - # decompression data, check if the parser is done draining. - # If reading is no longer paused, the parser has processed all - # buffered data -- finish the connection_lost cleanup now. - if self._connection_lost_deferred and not self._reading_paused: + # decompression data, check if the payload has reached EOF. + # EOF means _on_message_complete fired and all decompressed data + # was delivered to the reader. Clean up the parser now. + # If the data is incomplete, the read timeout will fire instead. + if ( + self._connection_lost_deferred + and self._payload is not None + and self._payload.is_eof() + ): self._connection_lost_deferred = False self._close_parser() From 97c9b1c1979b208949e6cfd6f0482bf9ee38c0c6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 31 Mar 2026 16:54:19 -1000 Subject: [PATCH 142/199] check parser as its the source of truth --- aiohttp/_http_parser.pyx | 4 ++++ aiohttp/client_proto.py | 2 +- aiohttp/http_parser.py | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index b3efa71f7e9..d4ab3a739d9 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -538,6 +538,10 @@ cdef class HttpParser: ### Public API ### + @property + def has_pending_data(self): + return bool(self._tail or self._more_data_available) + def pause_reading(self): assert self._payload is not None self._paused = True diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index 9cd86258794..078f78566fb 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -151,8 +151,8 @@ def connection_lost(self, exc: BaseException | None) -> None: # will be called from data_received to complete the cleanup. if ( connection_closed_cleanly - and self._reading_paused and self._parser is not None + and self._parser.has_pending_data ): self._connection_lost_deferred = True self._should_close = True diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 6e1df250310..a96cbf4999a 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -259,6 +259,10 @@ def parse_message(self, lines: list[bytes]) -> _MsgT: ... @abc.abstractmethod def _is_chunked_te(self, te: str) -> bool: ... + @property + def has_pending_data(self) -> bool: + return bool(self._tail or self._payload_has_more_data) + def pause_reading(self) -> None: assert self._payload_parser is not None self._payload_parser.pause_reading() From d719a0dfdcbe6237f608df8755ef1749a28bc182 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 31 Mar 2026 16:58:33 -1000 Subject: [PATCH 143/199] Revert "check parser as its the source of truth" This reverts commit 97c9b1c1979b208949e6cfd6f0482bf9ee38c0c6. --- aiohttp/_http_parser.pyx | 4 ---- aiohttp/client_proto.py | 2 +- aiohttp/http_parser.py | 4 ---- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index d4ab3a739d9..b3efa71f7e9 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -538,10 +538,6 @@ cdef class HttpParser: ### Public API ### - @property - def has_pending_data(self): - return bool(self._tail or self._more_data_available) - def pause_reading(self): assert self._payload is not None self._paused = True diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index 078f78566fb..9cd86258794 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -151,8 +151,8 @@ def connection_lost(self, exc: BaseException | None) -> None: # will be called from data_received to complete the cleanup. if ( connection_closed_cleanly + and self._reading_paused and self._parser is not None - and self._parser.has_pending_data ): self._connection_lost_deferred = True self._should_close = True diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index a96cbf4999a..6e1df250310 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -259,10 +259,6 @@ def parse_message(self, lines: list[bytes]) -> _MsgT: ... @abc.abstractmethod def _is_chunked_te(self, te: str) -> bool: ... - @property - def has_pending_data(self) -> bool: - return bool(self._tail or self._payload_has_more_data) - def pause_reading(self) -> None: assert self._payload_parser is not None self._payload_parser.pause_reading() From d6df346086853d4e27137290bcef7606ff82cb4b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 31 Mar 2026 16:58:34 -1000 Subject: [PATCH 144/199] Revert "rely on timeout instead" This reverts commit 6b08dee0c2114ed2ebf4c11050af4d77149e743e. --- aiohttp/client_proto.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index 9cd86258794..7bd4cbb2f03 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -156,7 +156,6 @@ def connection_lost(self, exc: BaseException | None) -> None: ): self._connection_lost_deferred = True self._should_close = True - self._reschedule_timeout() super().connection_lost(exc) return @@ -392,14 +391,9 @@ def data_received(self, data: bytes) -> None: self.data_received(tail) # If connection_lost was deferred because the parser had pending - # decompression data, check if the payload has reached EOF. - # EOF means _on_message_complete fired and all decompressed data - # was delivered to the reader. Clean up the parser now. - # If the data is incomplete, the read timeout will fire instead. - if ( - self._connection_lost_deferred - and self._payload is not None - and self._payload.is_eof() - ): + # decompression data, check if the parser is done draining. + # If reading is no longer paused, the parser has processed all + # buffered data -- finish the connection_lost cleanup now. + if self._connection_lost_deferred and not self._reading_paused: self._connection_lost_deferred = False self._close_parser() From e093db56da9cbba8d4342bfef7a5e11a9507b0f3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 31 Mar 2026 16:58:35 -1000 Subject: [PATCH 145/199] Revert "attempt to fix connection closed cleanly while decompression still in flight" This reverts commit 8256f07b557f617eb5131861825b37ca921be03f. --- aiohttp/client_proto.py | 76 ++++++++++++----------------------------- 1 file changed, 21 insertions(+), 55 deletions(-) diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index 7bd4cbb2f03..c7ea3363176 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -52,7 +52,6 @@ def __init__(self, loop: asyncio.AbstractEventLoop) -> None: self._closed: None | asyncio.Future[None] = None self._connection_lost_called = False - self._connection_lost_deferred = False @property def closed(self) -> None | asyncio.Future[None]: @@ -143,23 +142,25 @@ def connection_lost(self, exc: BaseException | None) -> None: with suppress(Exception): # FIXME: log this somehow? self._payload_parser.feed_eof() - # If the connection closed cleanly but the parser is still - # decompressing data (reading is paused), keep the parser alive. - # The resume_reading/data_received drain cycle will continue - # feeding decompressed chunks to the reader incrementally. - # Once the parser runs out of buffered data, _close_parser - # will be called from data_received to complete the cleanup. - if ( - connection_closed_cleanly - and self._reading_paused - and self._parser is not None - ): - self._connection_lost_deferred = True - self._should_close = True - super().connection_lost(exc) - return - - uncompleted = self._close_parser(original_connection_error) + uncompleted = None + if self._parser is not None: + try: + uncompleted = self._parser.feed_eof() + except Exception as underlying_exc: + if self._payload is not None: + client_payload_exc_msg = ( + f"Response payload is not completed: {underlying_exc !r}" + ) + if not connection_closed_cleanly: + client_payload_exc_msg = ( + f"{client_payload_exc_msg !s}. " + f"{original_connection_error !r}" + ) + set_exception( + self._payload, + ClientPayloadError(client_payload_exc_msg), + underlying_exc, + ) if not self.is_eof(): if isinstance(original_connection_error, OSError): @@ -178,39 +179,12 @@ def connection_lost(self, exc: BaseException | None) -> None: self.set_exception(reraised_exc, underlying_non_eof_exc) self._should_close = True - - super().connection_lost(reraised_exc) - - def _close_parser( - self, original_exc: BaseException | None = None - ) -> RawResponseMessage | None: - """Feed EOF to the parser and clean up. - - Returns any uncompleted message from the parser. - """ - uncompleted = None - if self._parser is not None: - try: - uncompleted = self._parser.feed_eof() - except Exception as underlying_exc: - if self._payload is not None: - client_payload_exc_msg = ( - f"Response payload is not completed: {underlying_exc !r}" - ) - if original_exc is not None: - client_payload_exc_msg = ( - f"{client_payload_exc_msg !s}. {original_exc !r}" - ) - set_exception( - self._payload, - ClientPayloadError(client_payload_exc_msg), - underlying_exc, - ) self._parser = None self._payload = None self._payload_parser = None self._reading_paused = False - return uncompleted + + super().connection_lost(reraised_exc) def eof_received(self) -> None: # should call parser.feed_eof() most likely @@ -389,11 +363,3 @@ def data_received(self, data: bytes) -> None: if upgraded and tail: self.data_received(tail) - - # If connection_lost was deferred because the parser had pending - # decompression data, check if the parser is done draining. - # If reading is no longer paused, the parser has processed all - # buffered data -- finish the connection_lost cleanup now. - if self._connection_lost_deferred and not self._reading_paused: - self._connection_lost_deferred = False - self._close_parser() From f5aa7035ed42ea0f922583915c4d600e52fb93ab Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 31 Mar 2026 17:05:27 -1000 Subject: [PATCH 146/199] one more attempt --- aiohttp/_http_parser.pyx | 7 ++++ aiohttp/client_proto.py | 78 +++++++++++++++++++++++++++++----------- aiohttp/http_parser.py | 7 ++++ 3 files changed, 71 insertions(+), 21 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index b3efa71f7e9..65b0a7f3fc4 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -538,6 +538,13 @@ cdef class HttpParser: ### Public API ### + @property + def has_pending_data(self): + """True if the parser has pending body data to decompress.""" + return self._payload is not None and bool( + self._tail or self._more_data_available + ) + def pause_reading(self): assert self._payload is not None self._paused = True diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index c7ea3363176..1b3718993a7 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -142,25 +142,24 @@ def connection_lost(self, exc: BaseException | None) -> None: with suppress(Exception): # FIXME: log this somehow? self._payload_parser.feed_eof() - uncompleted = None - if self._parser is not None: - try: - uncompleted = self._parser.feed_eof() - except Exception as underlying_exc: - if self._payload is not None: - client_payload_exc_msg = ( - f"Response payload is not completed: {underlying_exc !r}" - ) - if not connection_closed_cleanly: - client_payload_exc_msg = ( - f"{client_payload_exc_msg !s}. " - f"{original_connection_error !r}" - ) - set_exception( - self._payload, - ClientPayloadError(client_payload_exc_msg), - underlying_exc, - ) + # If the connection closed cleanly but the parser has pending + # decompression data, keep it alive and schedule a drain. + # The reader consuming data will drive resume_reading/data_received + # cycles to incrementally decompress the remaining data. + if ( + connection_closed_cleanly + and self._parser is not None + and self._parser.has_pending_data + ): + self._should_close = True + self._reschedule_timeout() + super().connection_lost(exc) + # Schedule a drain kick on the next event loop iteration. + # The reader will then drive further draining via resume_reading. + self._loop.call_soon(self.data_received, b"") + return + + uncompleted = self._close_parser(original_connection_error) if not self.is_eof(): if isinstance(original_connection_error, OSError): @@ -179,12 +178,39 @@ def connection_lost(self, exc: BaseException | None) -> None: self.set_exception(reraised_exc, underlying_non_eof_exc) self._should_close = True + + super().connection_lost(reraised_exc) + + def _close_parser( + self, original_exc: BaseException | None = None + ) -> RawResponseMessage | None: + """Feed EOF to the parser and clean up. + + Returns any uncompleted message from the parser. + """ + uncompleted = None + if self._parser is not None: + try: + uncompleted = self._parser.feed_eof() + except Exception as underlying_exc: + if self._payload is not None: + client_payload_exc_msg = ( + f"Response payload is not completed: {underlying_exc !r}" + ) + if original_exc is not None: + client_payload_exc_msg = ( + f"{client_payload_exc_msg !s}. {original_exc !r}" + ) + set_exception( + self._payload, + ClientPayloadError(client_payload_exc_msg), + underlying_exc, + ) self._parser = None self._payload = None self._payload_parser = None self._reading_paused = False - - super().connection_lost(reraised_exc) + return uncompleted def eof_received(self) -> None: # should call parser.feed_eof() most likely @@ -363,3 +389,13 @@ def data_received(self, data: bytes) -> None: if upgraded and tail: self.data_received(tail) + + # If connection_lost was deferred and the parser no longer has + # pending data, clean up. Either the message completed + # successfully or the parser drained all buffered data. + if ( + self._connection_lost_called + and self._parser is not None + and not self._parser.has_pending_data + ): + self._close_parser() diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 6e1df250310..f6201e71fb8 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -259,6 +259,13 @@ def parse_message(self, lines: list[bytes]) -> _MsgT: ... @abc.abstractmethod def _is_chunked_te(self, te: str) -> bool: ... + @property + def has_pending_data(self) -> bool: + """True if the parser has pending body data to decompress.""" + return self._payload_parser is not None and bool( + self._tail or self._payload_has_more_data + ) + def pause_reading(self) -> None: assert self._payload_parser is not None self._payload_parser.pause_reading() From cb380efa45bcc613ffe540d36049efe2db650f35 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 31 Mar 2026 17:13:14 -1000 Subject: [PATCH 147/199] Revert "one more attempt" This reverts commit f5aa7035ed42ea0f922583915c4d600e52fb93ab. --- aiohttp/_http_parser.pyx | 7 ---- aiohttp/client_proto.py | 78 +++++++++++----------------------------- aiohttp/http_parser.py | 7 ---- 3 files changed, 21 insertions(+), 71 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 65b0a7f3fc4..b3efa71f7e9 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -538,13 +538,6 @@ cdef class HttpParser: ### Public API ### - @property - def has_pending_data(self): - """True if the parser has pending body data to decompress.""" - return self._payload is not None and bool( - self._tail or self._more_data_available - ) - def pause_reading(self): assert self._payload is not None self._paused = True diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index 1b3718993a7..c7ea3363176 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -142,24 +142,25 @@ def connection_lost(self, exc: BaseException | None) -> None: with suppress(Exception): # FIXME: log this somehow? self._payload_parser.feed_eof() - # If the connection closed cleanly but the parser has pending - # decompression data, keep it alive and schedule a drain. - # The reader consuming data will drive resume_reading/data_received - # cycles to incrementally decompress the remaining data. - if ( - connection_closed_cleanly - and self._parser is not None - and self._parser.has_pending_data - ): - self._should_close = True - self._reschedule_timeout() - super().connection_lost(exc) - # Schedule a drain kick on the next event loop iteration. - # The reader will then drive further draining via resume_reading. - self._loop.call_soon(self.data_received, b"") - return - - uncompleted = self._close_parser(original_connection_error) + uncompleted = None + if self._parser is not None: + try: + uncompleted = self._parser.feed_eof() + except Exception as underlying_exc: + if self._payload is not None: + client_payload_exc_msg = ( + f"Response payload is not completed: {underlying_exc !r}" + ) + if not connection_closed_cleanly: + client_payload_exc_msg = ( + f"{client_payload_exc_msg !s}. " + f"{original_connection_error !r}" + ) + set_exception( + self._payload, + ClientPayloadError(client_payload_exc_msg), + underlying_exc, + ) if not self.is_eof(): if isinstance(original_connection_error, OSError): @@ -178,39 +179,12 @@ def connection_lost(self, exc: BaseException | None) -> None: self.set_exception(reraised_exc, underlying_non_eof_exc) self._should_close = True - - super().connection_lost(reraised_exc) - - def _close_parser( - self, original_exc: BaseException | None = None - ) -> RawResponseMessage | None: - """Feed EOF to the parser and clean up. - - Returns any uncompleted message from the parser. - """ - uncompleted = None - if self._parser is not None: - try: - uncompleted = self._parser.feed_eof() - except Exception as underlying_exc: - if self._payload is not None: - client_payload_exc_msg = ( - f"Response payload is not completed: {underlying_exc !r}" - ) - if original_exc is not None: - client_payload_exc_msg = ( - f"{client_payload_exc_msg !s}. {original_exc !r}" - ) - set_exception( - self._payload, - ClientPayloadError(client_payload_exc_msg), - underlying_exc, - ) self._parser = None self._payload = None self._payload_parser = None self._reading_paused = False - return uncompleted + + super().connection_lost(reraised_exc) def eof_received(self) -> None: # should call parser.feed_eof() most likely @@ -389,13 +363,3 @@ def data_received(self, data: bytes) -> None: if upgraded and tail: self.data_received(tail) - - # If connection_lost was deferred and the parser no longer has - # pending data, clean up. Either the message completed - # successfully or the parser drained all buffered data. - if ( - self._connection_lost_called - and self._parser is not None - and not self._parser.has_pending_data - ): - self._close_parser() diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index f6201e71fb8..6e1df250310 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -259,13 +259,6 @@ def parse_message(self, lines: list[bytes]) -> _MsgT: ... @abc.abstractmethod def _is_chunked_te(self, te: str) -> bool: ... - @property - def has_pending_data(self) -> bool: - """True if the parser has pending body data to decompress.""" - return self._payload_parser is not None and bool( - self._tail or self._payload_has_more_data - ) - def pause_reading(self) -> None: assert self._payload_parser is not None self._payload_parser.pause_reading() From bda59d231a2edabfb59fa96408f1ee5b2bec55bb Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 03:19:50 +0100 Subject: [PATCH 148/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index b3efa71f7e9..59f4630e216 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -569,6 +569,10 @@ cdef class HttpParser: char* base cdef cparser.llhttp_errno_t errno + # Proactor loop sends bytearray. + if type(data) is not bytes: + data = bytes(data) + if self._tail: data, self._tail = self._tail + data, EMPTY_BYTES From e2c7e07dbd1cc377104be1849f0434c836649beb Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 17:04:29 +0100 Subject: [PATCH 149/199] Update test_multipart.py --- tests/test_multipart.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_multipart.py b/tests/test_multipart.py index d602513fc0f..87774f34a0e 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -1736,9 +1736,15 @@ async def write(inp: bytes) -> None: output += inp h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) - writer = mock.create_autospec( - AbstractStreamWriter, write=write, spec_set=True, instance=True - ) + if sys.version_info >= (3, 12): + writer = mock.create_autospec( + AbstractStreamWriter, write=write, spec_set=True, instance=True + ) + else: + writer = mock.create_autospec( + AbstractStreamWriter, spec_set=True, instance=True + ) + writer.write.side_effect = write with Stream(compressed + b"\r\n--:--") as stream: body_part = aiohttp.BodyPartReader(BOUNDARY, h, stream) payload = BodyPartReaderPayload(body_part) From 010c507f25b6dce1ab1baf5f240ef0dba90bc22e Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 17:17:58 +0100 Subject: [PATCH 150/199] Update test_multipart.py --- tests/test_multipart.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 87774f34a0e..98eddc82368 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -1726,6 +1726,7 @@ async def test_body_part_reader_payload_as_bytes() -> None: payload.decode() +@pytest.mark.skipif(sys.version_info < (3, 11), reason="No wbits parameter") async def test_body_part_reader_payload_write() -> None: content = b"A" * 1_000_000 # Large enough to exceed max_length. compressed = ZLibBackend.compress(content, wbits=-ZLibBackend.MAX_WBITS) From 7d5a9bea2818f22ea8ab067ed318f0fb04e8198e Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 18:23:52 +0100 Subject: [PATCH 151/199] Update test_http_parser.py --- tests/test_http_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 8428d87fa37..2c15642b66d 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -2328,6 +2328,7 @@ async def test_empty_body(self, protocol: BaseProtocol) -> None: assert buf.at_eof() + @pytest.mark.xfail(platform.python_implementation() == "PyPy", reason="Broken") @pytest.mark.parametrize( "chunk_size", [1024, 2**14, 2**16], # 1KB, 16KB, 64KB From e499e20adb0d45e8ac763466a85aa44ee81d39e5 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 18:32:24 +0100 Subject: [PATCH 152/199] Update test_http_parser.py --- tests/test_http_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 2c15642b66d..c7de9a6653b 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1,6 +1,7 @@ # Tests for aiohttp/protocol.py import asyncio +import platform import re import sys import zlib From aece402ee8d87a898b14c7c695ada5b984be2311 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 20:26:12 +0100 Subject: [PATCH 153/199] Update test_http_parser.py --- tests/test_http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index c7de9a6653b..615341d4bf1 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -2329,7 +2329,7 @@ async def test_empty_body(self, protocol: BaseProtocol) -> None: assert buf.at_eof() - @pytest.mark.xfail(platform.python_implementation() == "PyPy", reason="Broken") + @pytest.mark.skipif(platform.python_implementation() == "PyPy", reason="Broken") @pytest.mark.parametrize( "chunk_size", [1024, 2**14, 2**16], # 1KB, 16KB, 64KB From 7659f5b60b401ae2da9e6b1a19ff0b3b75320d9a Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 21:40:02 +0100 Subject: [PATCH 154/199] Update test_web_protocol.py --- tests/test_web_protocol.py | 42 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/test_web_protocol.py b/tests/test_web_protocol.py index 5ae1e5dd756..f14821f5315 100644 --- a/tests/test_web_protocol.py +++ b/tests/test_web_protocol.py @@ -1,48 +1,48 @@ import asyncio -from typing import Any, cast from unittest import mock +from aiohttp.http import WebSocketReader from aiohttp.web_protocol import RequestHandler +from aiohttp.web_request import BaseRequest +from aiohttp.web_server import Server -class _DummyManager: - def __init__(self) -> None: - self.request_handler = mock.Mock() - self.request_factory = mock.Mock() +@pytest.fixture +def dummy_manager() -> Server[BaseRequest]: + return mock.create_autospec(Server[BaseRequest], spec_set=True, instance=True) -class _DummyParser: - def __init__(self) -> None: - self.received: list[bytes] = [] - - def feed_data(self, data: bytes) -> tuple[bool, bytes]: - self.received.append(data) - return False, b"" +@pytest.fixture +def dummy_reader() -> tuple[WebSocketReader, mock.Mock]: + m = mock.create_autospec(WebSocketReader, spec_set=True, instance=True) + return m, m def test_set_parser_does_not_call_data_received_cb_for_tail( loop: asyncio.AbstractEventLoop, + dummy_manager: Server[BaseRequest], + dummy_reader: tuple[WebSocketReader, mock.Mock], ) -> None: - handler: RequestHandler[Any] = RequestHandler(cast(Any, _DummyManager()), loop=loop) + handler = RequestHandler(dummy_manager, loop=loop) handler._message_tail = b"tail" cb = mock.Mock() - parser = _DummyParser() - handler.set_parser(parser, data_received_cb=cb) + handler.set_parser(dummy_reader[0], data_received_cb=cb) cb.assert_not_called() - assert parser.received == [b"tail"] + assert dummy_reader[1].feed_data.assert_called_once_with(b"tail") def test_data_received_calls_data_received_cb( loop: asyncio.AbstractEventLoop, + dummy_manager: Server[BaseRequest], + dummy_reader: tuple[WebSocketReader, mock.Mock], ) -> None: - handler: RequestHandler[Any] = RequestHandler(cast(Any, _DummyManager()), loop=loop) + handler = RequestHandler(dummy_manager, loop=loop) cb = mock.Mock() - parser = _DummyParser() - handler.set_parser(parser, data_received_cb=cb) + handler.set_parser(dummy_reader[0], data_received_cb=cb) handler.data_received(b"x") - assert cb.call_count == 1 - assert parser.received == [b"x"] + assert cb.assert_called_once() + assert dummy_reader[1].received.assert_called_once_with(b"x") From d748132966207bf6e0d71d40ac2d425e4ce36934 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 21:43:04 +0100 Subject: [PATCH 155/199] Apply suggestions from code review Co-authored-by: Sam Bull --- aiohttp/compression_utils.py | 2 +- aiohttp/multipart.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index 4dbe70ebd6f..12d4e9d3a8a 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -403,4 +403,4 @@ def flush(self) -> bytes: def data_available(self) -> bool: return ( not self._obj.needs_input and not self._obj.eof - ) or self._pending_unused_data + ) or self._pending_unused_data is not None diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index bf63ccaccd7..928fffd214a 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -270,7 +270,7 @@ def __init__( subtype: str = "mixed", default_charset: str | None = None, max_decompress_size: int = DEFAULT_MAX_DECOMPRESS_SIZE, - client_max_size: int = math.inf, + client_max_size: int = math.inf, # type: ignore[assignment] ) -> None: self.headers = headers self._boundary = boundary @@ -666,7 +666,7 @@ def __init__( headers: Mapping[str, str], content: StreamReader, *, - client_max_size: int = math.inf, + client_max_size: int = math.inf, # type: ignore[assignment] max_field_size: int = 8190, max_headers: int = 128, ) -> None: From 518bb355b1f8dd244cab050ef7fe732e2d6d4701 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 21:49:20 +0100 Subject: [PATCH 156/199] Update test_web_protocol.py --- tests/test_web_protocol.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_web_protocol.py b/tests/test_web_protocol.py index f14821f5315..cd58922c750 100644 --- a/tests/test_web_protocol.py +++ b/tests/test_web_protocol.py @@ -1,6 +1,8 @@ import asyncio from unittest import mock +import pytest + from aiohttp.http import WebSocketReader from aiohttp.web_protocol import RequestHandler from aiohttp.web_request import BaseRequest From 0e6bea55bc095b67ab24bf0ae785cb2484c95835 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 21:58:03 +0100 Subject: [PATCH 157/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index a7ee960d11e..9db8bd457f5 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -46,7 +46,7 @@ include "_headers.pxi" from aiohttp cimport _find_header -ALLOWED_UPGRADES = frozenset({"websocket"}) +cdef frozenset ALLOWED_UPGRADES = frozenset({"websocket"}) DEF DEFAULT_FREELIST_SIZE = 250 cdef extern from "Python.h": @@ -69,7 +69,6 @@ cdef object CONTENT_ENCODING = hdrs.CONTENT_ENCODING cdef object EMPTY_PAYLOAD = _EMPTY_PAYLOAD cdef object StreamReader = _StreamReader cdef object DeflateBuffer = _DeflateBuffer -cdef bytes EMPTY_BYTES = b"" # RFC 9110 singleton headers — duplicates are rejected in strict mode. # In lax mode (response parser default), the check is skipped entirely @@ -374,8 +373,8 @@ cdef class HttpParser: self._payload_exception = payload_exception self._messages = [] - self._raw_name = EMPTY_BYTES - self._raw_value = EMPTY_BYTES + self._raw_name = b"" + self._raw_value = b"" self._tail = b"" self._has_value = False self._header_name_size = 0 @@ -407,7 +406,7 @@ cdef class HttpParser: cdef _process_header(self): cdef str value - if self._raw_name is not EMPTY_BYTES: + if self._raw_name is not b"": name = find_header(self._raw_name) value = self._raw_value.decode('utf-8', 'surrogateescape') @@ -432,20 +431,20 @@ cdef class HttpParser: self._has_value = False self._header_name_size = 0 self._raw_headers.append((self._raw_name, self._raw_value)) - self._raw_name = EMPTY_BYTES - self._raw_value = EMPTY_BYTES + self._raw_name = b"" + self._raw_value = b"" cdef _on_header_field(self, char* at, size_t length): if self._has_value: self._process_header() - if self._raw_name is EMPTY_BYTES: + if self._raw_name is b"": self._raw_name = at[:length] else: self._raw_name += at[:length] cdef _on_header_value(self, char* at, size_t length): - if self._raw_value is EMPTY_BYTES: + if self._raw_value is b"": self._raw_value = at[:length] else: self._raw_value += at[:length] @@ -577,14 +576,14 @@ cdef class HttpParser: data = bytes(data) if self._tail: - data, self._tail = self._tail + data, EMPTY_BYTES + data, self._tail = self._tail + data, b"" had_more_data = self._more_data_available if self._more_data_available: - result = cb_on_body(self._cparser, EMPTY_BYTES, 0) + result = cb_on_body(self._cparser, b"", 0) if result is cparser.HPE_PAUSED: self._tail = data - return (), False, EMPTY_BYTES + return (), False, b"" # TODO: Do we need to handle error case (-1)? PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE) @@ -632,7 +631,7 @@ cdef class HttpParser: if self._upgraded: return messages, True, data[nb:] else: - return messages, False, EMPTY_BYTES + return messages, False, b"" def set_upgraded(self, val): self._upgraded = val @@ -840,7 +839,7 @@ cdef int cb_on_body(cparser.llhttp_t* parser, pyparser._payload_error = 1 pyparser._paused = False return -1 - body = EMPTY_BYTES + body = b"" if pyparser._paused: pyparser._paused = False From 73805681d6994a23f6333fd8bf420a183a51fba0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2026 20:58:45 +0000 Subject: [PATCH 158/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/_http_parser.pyx | 1 + 1 file changed, 1 insertion(+) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 9db8bd457f5..61898422b0d 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -46,6 +46,7 @@ include "_headers.pxi" from aiohttp cimport _find_header + cdef frozenset ALLOWED_UPGRADES = frozenset({"websocket"}) DEF DEFAULT_FREELIST_SIZE = 250 From 742436de3cbabff7c04c6ed9779a1afbd12eaa7a Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 22:04:45 +0100 Subject: [PATCH 159/199] Update test_web_protocol.py --- tests/test_web_protocol.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_web_protocol.py b/tests/test_web_protocol.py index cd58922c750..e3325d64090 100644 --- a/tests/test_web_protocol.py +++ b/tests/test_web_protocol.py @@ -11,7 +11,7 @@ @pytest.fixture def dummy_manager() -> Server[BaseRequest]: - return mock.create_autospec(Server[BaseRequest], spec_set=True, instance=True) + return mock.create_autospec(Server[BaseRequest], spec_set=True, instance=True) # type: ignore[no-any-return] @pytest.fixture @@ -32,7 +32,7 @@ def test_set_parser_does_not_call_data_received_cb_for_tail( handler.set_parser(dummy_reader[0], data_received_cb=cb) cb.assert_not_called() - assert dummy_reader[1].feed_data.assert_called_once_with(b"tail") + dummy_reader[1].feed_data.assert_called_once_with(b"tail") def test_data_received_calls_data_received_cb( @@ -47,4 +47,4 @@ def test_data_received_calls_data_received_cb( handler.data_received(b"x") assert cb.assert_called_once() - assert dummy_reader[1].received.assert_called_once_with(b"x") + dummy_reader[1].received.assert_called_once_with(b"x") From 4ac1cec58c5b2ded943baf3da2121a08a2f6f52d Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 22:06:13 +0100 Subject: [PATCH 160/199] Update test_web_protocol.py --- tests/test_web_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_web_protocol.py b/tests/test_web_protocol.py index e3325d64090..b0ac41f6530 100644 --- a/tests/test_web_protocol.py +++ b/tests/test_web_protocol.py @@ -11,7 +11,7 @@ @pytest.fixture def dummy_manager() -> Server[BaseRequest]: - return mock.create_autospec(Server[BaseRequest], spec_set=True, instance=True) # type: ignore[no-any-return] + return mock.create_autospec(Server[BaseRequest], request_handler=mock.Mock(), spec_set=True, instance=True) # type: ignore[no-any-return] @pytest.fixture From ae697a294c78186bc15187ce7cf59e0f53637a64 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 22:09:49 +0100 Subject: [PATCH 161/199] Update test_web_protocol.py --- tests/test_web_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_web_protocol.py b/tests/test_web_protocol.py index b0ac41f6530..89fb12c8083 100644 --- a/tests/test_web_protocol.py +++ b/tests/test_web_protocol.py @@ -46,5 +46,5 @@ def test_data_received_calls_data_received_cb( handler.set_parser(dummy_reader[0], data_received_cb=cb) handler.data_received(b"x") - assert cb.assert_called_once() + cb.assert_called_once() dummy_reader[1].received.assert_called_once_with(b"x") From a6d8f5e78d562edcb11cfd18c3563b59258c40be Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 22:28:46 +0100 Subject: [PATCH 162/199] Update test_web_protocol.py --- tests/test_web_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_web_protocol.py b/tests/test_web_protocol.py index 89fb12c8083..3f4f36160a9 100644 --- a/tests/test_web_protocol.py +++ b/tests/test_web_protocol.py @@ -11,7 +11,7 @@ @pytest.fixture def dummy_manager() -> Server[BaseRequest]: - return mock.create_autospec(Server[BaseRequest], request_handler=mock.Mock(), spec_set=True, instance=True) # type: ignore[no-any-return] + return mock.create_autospec(Server[BaseRequest], request_handler=mock.Mock(), instance=True) # type: ignore[no-any-return] @pytest.fixture From b5ea38461328fabfc6c5b81f14a4f3cec8b2c56a Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 22:35:31 +0100 Subject: [PATCH 163/199] Update test_web_protocol.py --- tests/test_web_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_web_protocol.py b/tests/test_web_protocol.py index 3f4f36160a9..959bab1f8e2 100644 --- a/tests/test_web_protocol.py +++ b/tests/test_web_protocol.py @@ -11,7 +11,7 @@ @pytest.fixture def dummy_manager() -> Server[BaseRequest]: - return mock.create_autospec(Server[BaseRequest], request_handler=mock.Mock(), instance=True) # type: ignore[no-any-return] + return mock.create_autospec(Server[BaseRequest], request_handler=mock.Mock(), request_factory=mock.Mock(), instance=True) # type: ignore[no-any-return] @pytest.fixture From c4dadbe65994a5d4cc5f73445a36d3e3aab60cba Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 22:56:44 +0100 Subject: [PATCH 164/199] Update test_web_protocol.py --- tests/test_web_protocol.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_web_protocol.py b/tests/test_web_protocol.py index 959bab1f8e2..33bcb4d5acd 100644 --- a/tests/test_web_protocol.py +++ b/tests/test_web_protocol.py @@ -17,6 +17,7 @@ def dummy_manager() -> Server[BaseRequest]: @pytest.fixture def dummy_reader() -> tuple[WebSocketReader, mock.Mock]: m = mock.create_autospec(WebSocketReader, spec_set=True, instance=True) + m.feed_data.return_value = False, b"" return m, m From ee9bc1973763557d89b4b04e00e37e5ee022a311 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 23:07:22 +0100 Subject: [PATCH 165/199] Update test_web_protocol.py --- tests/test_web_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_web_protocol.py b/tests/test_web_protocol.py index 33bcb4d5acd..9acad2f2101 100644 --- a/tests/test_web_protocol.py +++ b/tests/test_web_protocol.py @@ -48,4 +48,4 @@ def test_data_received_calls_data_received_cb( handler.data_received(b"x") cb.assert_called_once() - dummy_reader[1].received.assert_called_once_with(b"x") + dummy_reader[1].feed_data.assert_called_once_with(b"x") From 5f00f8d6ca481a7111a1a9a15b463b3729bdcc94 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 3 Apr 2026 23:25:40 +0100 Subject: [PATCH 166/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 61898422b0d..a4916347768 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -70,6 +70,7 @@ cdef object CONTENT_ENCODING = hdrs.CONTENT_ENCODING cdef object EMPTY_PAYLOAD = _EMPTY_PAYLOAD cdef object StreamReader = _StreamReader cdef object DeflateBuffer = _DeflateBuffer +cdef tuple EMPTY_FEED_DATA_RESULT = ((), False, b"") # RFC 9110 singleton headers — duplicates are rejected in strict mode. # In lax mode (response parser default), the check is skipped entirely @@ -584,7 +585,7 @@ cdef class HttpParser: result = cb_on_body(self._cparser, b"", 0) if result is cparser.HPE_PAUSED: self._tail = data - return (), False, b"" + return EMPTY_FEED_DATA_RESULT # TODO: Do we need to handle error case (-1)? PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE) @@ -631,8 +632,9 @@ cdef class HttpParser: if self._upgraded: return messages, True, data[nb:] - else: - return messages, False, b"" + if not messages: # Shortcut to reduce Python overhead + return EMPTY_FEED_DATA_RESULT + return messages, False, b"" def set_upgraded(self, val): self._upgraded = val From 54d16f976515daa39dcd642190dba7d2f51f4a01 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 4 Apr 2026 00:09:08 +0100 Subject: [PATCH 167/199] Update client_proto.py --- aiohttp/client_proto.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index c7ea3363176..c6e78e57b42 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -296,7 +296,10 @@ def _on_read_timeout(self) -> None: set_exception(self._payload, exc) def data_received(self, data: bytes) -> None: - self._reschedule_timeout() + # If no data, then we are resuming decompression. We haven't received + # data from the socket, so we can avoid the reschedule overhead. + if data: + self._reschedule_timeout() # custom payload parser - currently always WebSocketReader if self._payload_parser is not None: From a7a7c73737c49f990f4c272b3348ba4be6485592 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 4 Apr 2026 00:35:30 +0100 Subject: [PATCH 168/199] Update multipart.py --- aiohttp/multipart.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 928fffd214a..a1caee1101f 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -41,7 +41,6 @@ payload_type, ) from .streams import StreamReader -from .web_exceptions import HTTPRequestEntityTooLarge if sys.version_info >= (3, 11): from typing import Self @@ -271,6 +270,7 @@ def __init__( default_charset: str | None = None, max_decompress_size: int = DEFAULT_MAX_DECOMPRESS_SIZE, client_max_size: int = math.inf, # type: ignore[assignment] + max_size_error_cls: type[Exception] = ValueError ) -> None: self.headers = headers self._boundary = boundary @@ -289,6 +289,7 @@ def __init__( self._cache: dict[str, Any] = {} self._max_decompress_size = max_decompress_size self._client_max_size = client_max_size + self._max_size_error_cls = max_size_error_cls def __aiter__(self) -> Self: return self @@ -318,7 +319,7 @@ async def read(self, *, decode: bool = False) -> bytes: while not self._at_eof: data.extend(await self.read_chunk(self.chunk_size)) if len(data) > self._client_max_size: - raise HTTPRequestEntityTooLarge( + raise self._max_size_error_cls( max_size=self._client_max_size, actual_size=len(data) ) # https://github.com/python/mypy/issues/17537 @@ -327,7 +328,7 @@ async def read(self, *, decode: bool = False) -> bytes: async for d in self.decode_iter(data): decoded_data.extend(d) if len(decoded_data) > self._client_max_size: - raise HTTPRequestEntityTooLarge( + raise self._max_size_error_cls( max_size=self._client_max_size, actual_size=len(decoded_data) ) return decoded_data @@ -669,6 +670,7 @@ def __init__( client_max_size: int = math.inf, # type: ignore[assignment] max_field_size: int = 8190, max_headers: int = 128, + max_size_error_cls: type[Exception] = ValueError, ) -> None: self._mimetype = parse_mimetype(headers[CONTENT_TYPE]) assert self._mimetype.type == "multipart", "multipart/* content type expected" @@ -685,6 +687,7 @@ def __init__( self._last_part: MultipartReader | BodyPartReader | None = None self._max_field_size = max_field_size self._max_headers = max_headers + self._max_size_error_cls = max_size_error_cls self._at_eof = False self._at_bof = True self._unread: list[bytes] = [] @@ -790,6 +793,7 @@ def _get_part_reader( client_max_size=self._client_max_size, max_field_size=self._max_field_size, max_headers=self._max_headers, + max_size_error_cls=self._max_size_error_cls, ) return self.multipart_reader_cls( headers, @@ -797,6 +801,7 @@ def _get_part_reader( client_max_size=self._client_max_size, max_field_size=self._max_field_size, max_headers=self._max_headers, + max_size_error_cls=self._max_size_error_cls, ) else: return self.part_reader_cls( @@ -806,6 +811,7 @@ def _get_part_reader( subtype=self._mimetype.subtype, default_charset=self._default_charset, client_max_size=self._client_max_size, + max_size_error_cls=self._max_size_error_cls, ) def _get_boundary(self) -> str: From b45b20f89bdd52a2331f006eefce1ddd2b2f1b01 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2026 23:36:09 +0000 Subject: [PATCH 169/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- aiohttp/multipart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index a1caee1101f..3d6c1057c75 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -270,7 +270,7 @@ def __init__( default_charset: str | None = None, max_decompress_size: int = DEFAULT_MAX_DECOMPRESS_SIZE, client_max_size: int = math.inf, # type: ignore[assignment] - max_size_error_cls: type[Exception] = ValueError + max_size_error_cls: type[Exception] = ValueError, ) -> None: self.headers = headers self._boundary = boundary From f4b71662270466cbbb003c284c21d70df91a6538 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 4 Apr 2026 00:36:18 +0100 Subject: [PATCH 170/199] Update web_request.py --- aiohttp/web_request.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index 08dee6c6efc..d0106e47b94 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -678,6 +678,7 @@ async def multipart(self) -> MultipartReader: client_max_size=self._client_max_size, max_field_size=self._protocol.max_field_size, max_headers=self._protocol.max_headers, + max_size_error_cls=HTTPRequestEntityTooLarge, ) async def post(self) -> "MultiDictProxy[str | bytes | FileField]": From ab02eef7e7e8167ebcbffa495c9441a4c6315b15 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 4 Apr 2026 01:56:38 +0100 Subject: [PATCH 171/199] Update test_http_parser.py --- tests/test_http_parser.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 2e259fabfdb..02cd4fd1bf7 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1069,6 +1069,35 @@ async def test_chunk_splits_after_pause(parser: HttpRequestParser) -> None: assert result == b"b" * 50000 +async def test_compressed_with_tail(response: HttpResponseParser) -> None: + """Test compressed content-length body followed by a second response. + + With 2 responses arriving in one call and the first compressed, this should + trigger decompression pausing with the second response being saved as the tail. + Verify that the second response is resumed from the tail. + """ + # Must be large enough to exceed high water mark. + original = b"x" * 1024 * 1024 + compressed = zlib.compress(original) + resp1 = ( + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: " + str(len(compressed)).encode() + b"\r\n" + b"Content-Encoding: deflate\r\n" + b"\r\n" + ) + compressed + resp2 = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok" + + msgs, upgrade, tail = response.feed_data(resp1 + resp2) + payload = msgs[0][-1] + result = await payload.read() + assert len(result) == len(original) + assert result == original + + payload = response.protocol._buffer[0][-1] + result = await payload.read() + assert result == b"ok" + + async def test_compressed_256kb(response: HttpResponseParser) -> None: original = b"x" * 256 * 1024 compressed = zlib.compress(original) From d038681f90981daf46def23160a213ad02450f08 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 4 Apr 2026 00:57:21 +0000 Subject: [PATCH 172/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 02cd4fd1bf7..3c32e09834f 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1071,7 +1071,7 @@ async def test_chunk_splits_after_pause(parser: HttpRequestParser) -> None: async def test_compressed_with_tail(response: HttpResponseParser) -> None: """Test compressed content-length body followed by a second response. - + With 2 responses arriving in one call and the first compressed, this should trigger decompression pausing with the second response being saved as the tail. Verify that the second response is resumed from the tail. From ac0dc88fe38f63e63a21797d41d44aba6b045665 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 4 Apr 2026 01:57:41 +0100 Subject: [PATCH 173/199] Update _http_parser.pyx --- aiohttp/_http_parser.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index a4916347768..8300b35c247 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -299,7 +299,7 @@ cdef class HttpParser: bint _has_value int _header_name_size - object _protocol + readonly object protocol object _loop object _timer @@ -363,7 +363,7 @@ cdef class HttpParser: self._cparser.data = self self._cparser.content_length = 0 - self._protocol = protocol + self.protocol = protocol self._loop = loop self._timer = timer @@ -502,7 +502,7 @@ cdef class HttpParser: self._read_until_eof) ): payload = StreamReader( - self._protocol, timer=self._timer, loop=self._loop, + self.protocol, timer=self._timer, loop=self._loop, limit=self._limit) else: payload = EMPTY_PAYLOAD From d2220d3dccc1b95d19749c39a07eb690e3693ee4 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 4 Apr 2026 16:43:54 +0100 Subject: [PATCH 174/199] Another test --- tests/test_http_parser.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 3c32e09834f..c16e1fe6ed2 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1098,6 +1098,33 @@ async def test_compressed_with_tail(response: HttpResponseParser) -> None: assert result == b"ok" +async def test_compressed_chunked_with_pending(response: HttpResponseParser) -> None: + """Test chunked + compressed where the decompressor needs to resume from pause. + + We need to verify that chunked messages continue parsing correctly after + a pause and resume in the decompression. + """ + # Must be large enough to exceed high water mark. + original = b"A" * 1024 * 1024 + compressed = zlib.compress(original) + chunk_data = ( + hex(len(compressed))[2:].encode() + b"\r\n" + compressed + b"\r\n" + ) + headers = ( + b"HTTP/1.1 200 OK\r\n" + b"Transfer-Encoding: chunked\r\n" + b"Content-Encoding: deflate\r\n" + b"\r\n" + ) + data = headers + chunk_data + b"0\r\n\r\n" + + msgs, upgrade, tail = response.feed_data(data) + payload = msgs[0][-1] + result = await payload.read() + assert len(result) == len(original) + assert result == original + + async def test_compressed_256kb(response: HttpResponseParser) -> None: original = b"x" * 256 * 1024 compressed = zlib.compress(original) From 9f0957842d510614330c5f3c8d95278343d4ca27 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 4 Apr 2026 15:45:14 +0000 Subject: [PATCH 175/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_http_parser.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index c16e1fe6ed2..76e686bc2b8 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1107,9 +1107,7 @@ async def test_compressed_chunked_with_pending(response: HttpResponseParser) -> # Must be large enough to exceed high water mark. original = b"A" * 1024 * 1024 compressed = zlib.compress(original) - chunk_data = ( - hex(len(compressed))[2:].encode() + b"\r\n" + compressed + b"\r\n" - ) + chunk_data = hex(len(compressed))[2:].encode() + b"\r\n" + compressed + b"\r\n" headers = ( b"HTTP/1.1 200 OK\r\n" b"Transfer-Encoding: chunked\r\n" From aaefa3c303e261e388c409944ed5aed2d58b2c4f Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sat, 4 Apr 2026 18:42:37 +0100 Subject: [PATCH 176/199] Another test --- tests/test_http_parser.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 76e686bc2b8..ce6e6ee6bd1 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1123,6 +1123,27 @@ async def test_compressed_chunked_with_pending(response: HttpResponseParser) -> assert result == original +async def test_compressed_until_eof_with_pending(response: HttpResponseParser) -> None: + """Test read-until-eof + compressed with pause.""" + + # Must be large enough to exceed high water mark. + original = b"B" * 1024 * 1024 + compressed = zlib.compress(original) + # No Content-Length or Transfer-Encoding means the parser must parse until EOF. + headers = ( + b"HTTP/1.1 200 OK\r\n" + b"Content-Encoding: deflate\r\n" + b"\r\n" + ) + + msgs, upgrade, tail = response.feed_data(headers + compressed) + response.feed_eof() + payload = msgs[0][-1] + result = await payload.read() + assert len(result) == len(original) + assert result == original + + async def test_compressed_256kb(response: HttpResponseParser) -> None: original = b"x" * 256 * 1024 compressed = zlib.compress(original) @@ -2377,7 +2398,6 @@ async def test_http_payload_zstandard_many_small_frames( assert b"".join(parts) == b"".join(out._buffer) assert out.is_eof() - class TestDeflateBuffer: async def test_feed_data(self, protocol: BaseProtocol) -> None: buf = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) From 40c2c46e21c7c66751d6ea63c95b076f21e03383 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sun, 5 Apr 2026 21:48:20 +0100 Subject: [PATCH 177/199] Defer feed EOF --- aiohttp/_http_parser.pyx | 22 ++++++++++++++++++++++ aiohttp/http_parser.py | 25 ++++++++++++++++++++++--- tests/test_http_parser.py | 30 ++++++++++++++++++++---------- 3 files changed, 64 insertions(+), 13 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 8300b35c247..e21071b6834 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -323,6 +323,7 @@ cdef class HttpParser: list _messages bint _more_data_available bint _paused + bint _eof_pending object _payload bint _payload_error object _payload_exception @@ -370,6 +371,7 @@ cdef class HttpParser: self._buf = bytearray() self._more_data_available = False self._paused = False + self._eof_pending = False self._payload = None self._payload_error = 0 self._payload_exception = payload_exception @@ -560,7 +562,16 @@ cdef class HttpParser: desc = cparser.llhttp_get_error_reason(self._cparser) raise PayloadEncodingError(desc.decode('latin-1')) else: + self._eof_pending = True + while self._more_data_available: + if self._paused: + self._paused = False + return # Will resume via feed_data(b"") later + self._more_data_available = self._payload.feed_data(b"") self._payload.feed_eof() + self._payload = None + self._more_data_available = False + self._eof_pending = False elif self._started: self._on_headers_complete() if self._messages: @@ -588,6 +599,17 @@ cdef class HttpParser: return EMPTY_FEED_DATA_RESULT # TODO: Do we need to handle error case (-1)? + if self._eof_pending and not self._more_data_available: + self._payload.feed_eof() + self._payload = None + self._more_data_available = False + self._eof_pending = False + if self._messages: + messages = self._messages + self._messages = [] + return messages, False, b"" + return EMPTY_FEED_DATA_RESULT + PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE) # Cache buffer pointer before PyBuffer_Release to avoid use-after-release. base = self.py_buf.buf diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index bf87690fef3..640f725768b 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -288,7 +288,8 @@ def pause_reading(self) -> None: def feed_eof(self) -> _MsgT | None: if self._payload_parser is not None: self._payload_parser.feed_eof() - self._payload_parser = None + if self._payload_parser.done: + self._payload_parser = None else: # try to extract partial message if self._tail: @@ -810,6 +811,7 @@ def __init__( self._more_data_available = False self._trailer_lines: list[bytes] = [] self.done = False + self._eof_pending = False # payload decompression wrapper if response_with_body and compression and self._auto_decompress: @@ -841,7 +843,15 @@ def pause_reading(self) -> None: def feed_eof(self) -> None: if self._type == ParseState.PARSE_UNTIL_EOF: + self._eof_pending = True + while self._more_data_available: + if self._paused: + self._paused = False + return # Will resume via feed_data(b"") later + self._more_data_available = self.payload.feed_data(b"") self.payload.feed_eof() + self.done = True + self._eof_pending = False elif self._type == ParseState.PARSE_LENGTH: raise ContentLengthError( "Not enough data to satisfy content length header." @@ -1019,6 +1029,12 @@ def feed_data( return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" self._more_data_available = self.payload.feed_data(b"") + if self._eof_pending: + self.payload.feed_eof() + self.done = True + self._eof_pending = False + return PayloadState.PAYLOAD_COMPLETE, b"" + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" @@ -1100,9 +1116,12 @@ def feed_data(self, chunk: bytes) -> bool: def feed_eof(self) -> None: chunk = self.decompressor.flush() + # This should never contain data as we defer the call until exhausting + # the decompression. If .flush() is returning data, this may indicate a + # zip bomb vulnerability as it will decompress all remaining data at once. + assert not chunk - if chunk or self.size > 0: - self.out.feed_data(chunk) + if self.size > 0: # decompressor is not brotli unless encoding is "br" if self.encoding == "deflate" and not self.decompressor.eof: # type: ignore[union-attr] raise ContentEncodingError("deflate") diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index ce6e6ee6bd1..501c8c3d83b 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1139,6 +1139,10 @@ async def test_compressed_until_eof_with_pending(response: HttpResponseParser) - msgs, upgrade, tail = response.feed_data(headers + compressed) response.feed_eof() payload = msgs[0][-1] + + # Check that .feed_eof() hasn't decompressed entire payload into memory. + assert sum(len(b) for b in payload._buffer) <= (512 * 1024) + result = await payload.read() assert len(result) == len(original) assert result == original @@ -2428,10 +2432,10 @@ async def test_feed_eof(self, protocol: BaseProtocol) -> None: dbuf = DeflateBuffer(buf, "deflate") dbuf.decompressor = mock.Mock() - dbuf.decompressor.flush.return_value = b"line" + dbuf.decompressor.data_available = False + dbuf.decompressor.flush.return_value = b"" dbuf.feed_eof() - assert [b"line"] == list(buf._buffer) assert buf._eof async def test_feed_eof_err_deflate(self, protocol: BaseProtocol) -> None: @@ -2439,8 +2443,10 @@ async def test_feed_eof_err_deflate(self, protocol: BaseProtocol) -> None: dbuf = DeflateBuffer(buf, "deflate") dbuf.decompressor = mock.Mock() - dbuf.decompressor.flush.return_value = b"line" + dbuf.decompressor.data_available = False + dbuf.decompressor.flush.return_value = b"" dbuf.decompressor.eof = False + dbuf.size = 1 # Simulate that data was previously fed with pytest.raises(http_exceptions.ContentEncodingError): dbuf.feed_eof() @@ -2450,22 +2456,24 @@ async def test_feed_eof_no_err_gzip(self, protocol: BaseProtocol) -> None: dbuf = DeflateBuffer(buf, "gzip") dbuf.decompressor = mock.Mock() - dbuf.decompressor.flush.return_value = b"line" + dbuf.decompressor.data_available = False + dbuf.decompressor.flush.return_value = b"" dbuf.decompressor.eof = False dbuf.feed_eof() - assert [b"line"] == list(buf._buffer) + assert buf._eof async def test_feed_eof_no_err_brotli(self, protocol: BaseProtocol) -> None: buf = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) dbuf = DeflateBuffer(buf, "br") dbuf.decompressor = mock.Mock() - dbuf.decompressor.flush.return_value = b"line" + dbuf.decompressor.data_available = False + dbuf.decompressor.flush.return_value = b"" dbuf.decompressor.eof = False dbuf.feed_eof() - assert [b"line"] == list(buf._buffer) + assert buf._eof @pytest.mark.skipif(zstandard is None, reason="zstandard is not installed") async def test_feed_eof_no_err_zstandard(self, protocol: BaseProtocol) -> None: @@ -2473,11 +2481,12 @@ async def test_feed_eof_no_err_zstandard(self, protocol: BaseProtocol) -> None: dbuf = DeflateBuffer(buf, "zstd") dbuf.decompressor = mock.Mock() - dbuf.decompressor.flush.return_value = b"line" + dbuf.decompressor.data_available = False + dbuf.decompressor.flush.return_value = b"" dbuf.decompressor.eof = False dbuf.feed_eof() - assert [b"line"] == list(buf._buffer) + assert buf._eof async def test_empty_body(self, protocol: BaseProtocol) -> None: buf = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) @@ -2511,7 +2520,8 @@ async def test_streaming_decompress_large_payload( # Feed compressed data in chunks (simulating network streaming) for i in range(0, len(compressed), chunk_size): # pragma: no branch chunk = compressed[i : i + chunk_size] - dbuf.feed_data(chunk) + while dbuf.feed_data(chunk): + chunk = b"" dbuf.feed_eof() From ebc6f10cb87f4828ab1367aed275e98602f3ad86 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 5 Apr 2026 20:51:22 +0000 Subject: [PATCH 178/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_http_parser.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 501c8c3d83b..a8a512c9648 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1130,11 +1130,7 @@ async def test_compressed_until_eof_with_pending(response: HttpResponseParser) - original = b"B" * 1024 * 1024 compressed = zlib.compress(original) # No Content-Length or Transfer-Encoding means the parser must parse until EOF. - headers = ( - b"HTTP/1.1 200 OK\r\n" - b"Content-Encoding: deflate\r\n" - b"\r\n" - ) + headers = b"HTTP/1.1 200 OK\r\n" b"Content-Encoding: deflate\r\n" b"\r\n" msgs, upgrade, tail = response.feed_data(headers + compressed) response.feed_eof() @@ -2402,6 +2398,7 @@ async def test_http_payload_zstandard_many_small_frames( assert b"".join(parts) == b"".join(out._buffer) assert out.is_eof() + class TestDeflateBuffer: async def test_feed_data(self, protocol: BaseProtocol) -> None: buf = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) From 5ffa313c6c7f0afca92c0172ef846588cff1bc5c Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sun, 5 Apr 2026 22:02:42 +0100 Subject: [PATCH 179/199] Lint --- aiohttp/http_parser.py | 3 +++ tests/test_http_parser.py | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 640f725768b..8e34efc3185 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -12,6 +12,7 @@ from . import hdrs from .base_protocol import BaseProtocol +from .client_proto import ResponseHandler from .compression_utils import ( DEFAULT_MAX_DECOMPRESS_SIZE, HAS_BROTLI, @@ -701,6 +702,8 @@ class HttpResponseParser(HttpParser[RawResponseMessage]): Returns RawResponseMessage. """ + protocol: ResponseHandler + # Lax mode should only be enabled on response parser. lax = not DEBUG diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index a8a512c9648..434b4488ef3 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1125,12 +1125,11 @@ async def test_compressed_chunked_with_pending(response: HttpResponseParser) -> async def test_compressed_until_eof_with_pending(response: HttpResponseParser) -> None: """Test read-until-eof + compressed with pause.""" - # Must be large enough to exceed high water mark. original = b"B" * 1024 * 1024 compressed = zlib.compress(original) # No Content-Length or Transfer-Encoding means the parser must parse until EOF. - headers = b"HTTP/1.1 200 OK\r\n" b"Content-Encoding: deflate\r\n" b"\r\n" + headers = b"HTTP/1.1 200 OK\r\nContent-Encoding: deflate\r\n\r\n" msgs, upgrade, tail = response.feed_data(headers + compressed) response.feed_eof() From be8c7087824e13645e6ede652990f43b816ff63e Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sun, 5 Apr 2026 22:09:21 +0100 Subject: [PATCH 180/199] Fix --- aiohttp/http_parser.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 8e34efc3185..1a610d72c8e 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -5,14 +5,13 @@ from contextlib import suppress from enum import IntEnum from re import Pattern -from typing import Any, ClassVar, Final, Generic, Literal, NamedTuple, TypeVar +from typing import TYPE_CHECKING, Any, ClassVar, Final, Generic, Literal, NamedTuple, TypeVar from multidict import CIMultiDict, CIMultiDictProxy, istr from yarl import URL from . import hdrs from .base_protocol import BaseProtocol -from .client_proto import ResponseHandler from .compression_utils import ( DEFAULT_MAX_DECOMPRESS_SIZE, HAS_BROTLI, @@ -45,6 +44,9 @@ from .streams import EMPTY_PAYLOAD, StreamReader from .typedefs import RawHeaders +if TYPE_CHECKING: + from .client_proto import ResponseHandler + __all__ = ( "HeadersParser", "HttpParser", From ae82b30580ebf27d0490fd0b18c3bce03a602028 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sun, 5 Apr 2026 22:23:34 +0100 Subject: [PATCH 181/199] Fix --- aiohttp/http_parser.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 1a610d72c8e..c9b54f66e26 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -5,7 +5,16 @@ from contextlib import suppress from enum import IntEnum from re import Pattern -from typing import TYPE_CHECKING, Any, ClassVar, Final, Generic, Literal, NamedTuple, TypeVar +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, + Final, + Generic, + Literal, + NamedTuple, + TypeVar, +) from multidict import CIMultiDict, CIMultiDictProxy, istr from yarl import URL @@ -704,7 +713,7 @@ class HttpResponseParser(HttpParser[RawResponseMessage]): Returns RawResponseMessage. """ - protocol: ResponseHandler + protocol: "ResponseHandler" # Lax mode should only be enabled on response parser. lax = not DEBUG From fac2c7304a5aa62b2a098f6c2cb0d0297405d270 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sun, 5 Apr 2026 23:20:56 +0100 Subject: [PATCH 182/199] Another test --- tests/test_http_parser.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 434b4488ef3..9410bef2e8f 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1143,6 +1143,42 @@ async def test_compressed_until_eof_with_pending(response: HttpResponseParser) - assert result == original +async def test_compressed_until_eof_high_water( + response_cls: type[HttpResponseParser] +) -> None: + """Test read-until-eof + compressed with higher limit.""" + loop = asyncio.get_running_loop() + protocol = ResponseHandler(loop) + response = response_cls( + protocol, + loop, + # 512 KB limit: two 256 KB chunks fit, third triggers pause. + 2**19, + max_line_size=8190, + max_headers=128, + max_field_size=8190, + read_until_eof=True, + ) + protocol._parser = response + + # Must be large enough to exceed high water mark. + original = b"B" * 1024 * 1024 * 5 + compressed = zlib.compress(original) + # No Content-Length or Transfer-Encoding means the parser must parse until EOF. + headers = b"HTTP/1.1 200 OK\r\nContent-Encoding: deflate\r\n\r\n" + + msgs, upgrade, tail = response.feed_data(headers + compressed) + response.feed_eof() + payload = msgs[0][-1] + + # Check that .feed_eof() hasn't decompressed entire payload into memory. + assert sum(len(b) for b in payload._buffer) < (2 * 1024 * 1024) + + result = await payload.read() + assert len(result) == len(original) + assert result == original + + async def test_compressed_256kb(response: HttpResponseParser) -> None: original = b"x" * 256 * 1024 compressed = zlib.compress(original) From 94915f5b35ed7ae919ed4059c54a7be933a25bd1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 5 Apr 2026 22:21:44 +0000 Subject: [PATCH 183/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 9410bef2e8f..b7fd70b47cf 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1144,7 +1144,7 @@ async def test_compressed_until_eof_with_pending(response: HttpResponseParser) - async def test_compressed_until_eof_high_water( - response_cls: type[HttpResponseParser] + response_cls: type[HttpResponseParser], ) -> None: """Test read-until-eof + compressed with higher limit.""" loop = asyncio.get_running_loop() From b0011994d407e3fbb0cf6ca6d49410e8383b0467 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Sun, 5 Apr 2026 23:44:49 +0100 Subject: [PATCH 184/199] Decompress at limit amount to reduce pauses --- aiohttp/_http_parser.pyx | 2 +- aiohttp/http_parser.py | 6 +++++- tests/test_http_parser.py | 9 +++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index e21071b6834..7c66193f7bb 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -511,7 +511,7 @@ cdef class HttpParser: self._payload = payload if encoding is not None and self._auto_decompress: - self._payload = DeflateBuffer(payload, encoding) + self._payload = DeflateBuffer(payload, encoding, max_decompress_size=self._limit) if not self._response_with_body: payload = EMPTY_PAYLOAD diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index c9b54f66e26..20d3a854317 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -428,6 +428,7 @@ def get_content_length() -> int | None: max_line_size=self.max_line_size, max_field_size=self.max_field_size, max_trailers=max_trailers, + limit=self._limit, ) if not payload_parser.done: self._payload_parser = payload_parser @@ -450,6 +451,7 @@ def get_content_length() -> int | None: max_line_size=self.max_line_size, max_field_size=self.max_field_size, max_trailers=max_trailers, + limit=self._limit, ) elif not empty_body and length is None and self.read_until_eof: payload = StreamReader( @@ -472,6 +474,7 @@ def get_content_length() -> int | None: max_line_size=self.max_line_size, max_field_size=self.max_field_size, max_trailers=max_trailers, + limit=self._limit, ) if not payload_parser.done: self._payload_parser = payload_parser @@ -809,6 +812,7 @@ def __init__( max_line_size: int = 8190, max_field_size: int = 8190, max_trailers: int = 128, + limit: int = DEFAULT_MAX_DECOMPRESS_SIZE, ) -> None: self._length = 0 self._paused = False @@ -830,7 +834,7 @@ def __init__( # payload decompression wrapper if response_with_body and compression and self._auto_decompress: real_payload: StreamReader | DeflateBuffer = DeflateBuffer( - payload, compression + payload, compression, max_decompress_size=limit ) else: real_payload = payload diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 9410bef2e8f..23c2b5ba998 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1152,8 +1152,7 @@ async def test_compressed_until_eof_high_water( response = response_cls( protocol, loop, - # 512 KB limit: two 256 KB chunks fit, third triggers pause. - 2**19, + 2**19, # 512 KiB limit max_line_size=8190, max_headers=128, max_field_size=8190, @@ -1162,7 +1161,7 @@ async def test_compressed_until_eof_high_water( protocol._parser = response # Must be large enough to exceed high water mark. - original = b"B" * 1024 * 1024 * 5 + original = b"B" * 5 * 1024 * 1024 compressed = zlib.compress(original) # No Content-Length or Transfer-Encoding means the parser must parse until EOF. headers = b"HTTP/1.1 200 OK\r\nContent-Encoding: deflate\r\n\r\n" @@ -1172,7 +1171,9 @@ async def test_compressed_until_eof_high_water( payload = msgs[0][-1] # Check that .feed_eof() hasn't decompressed entire payload into memory. - assert sum(len(b) for b in payload._buffer) < (2 * 1024 * 1024) + assert sum(len(b) for b in payload._buffer) <= (2 * 1024 * 1024) + # Individual chunks should have been decompressed at limit amount. + assert all(len(b) == 512 * 1024 for b in payload._buffer) result = await payload.read() assert len(result) == len(original) From a767d33cb7d827cc1b8a841d1ef8e0e4ce10bb07 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 5 Apr 2026 22:48:58 +0000 Subject: [PATCH 185/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_http_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 6bf4bfd9dda..2a8e3c25eb7 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1152,7 +1152,7 @@ async def test_compressed_until_eof_high_water( response = response_cls( protocol, loop, - 2**19, # 512 KiB limit + 2**19, # 512 KiB limit max_line_size=8190, max_headers=128, max_field_size=8190, From ad2d948beb13dd41a18fc4c1f03f66c38e2c2cb5 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 6 Apr 2026 00:03:00 +0100 Subject: [PATCH 186/199] Another test --- tests/test_http_parser.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 6bf4bfd9dda..a2693973eb9 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1098,6 +1098,23 @@ async def test_compressed_with_tail(response: HttpResponseParser) -> None: assert result == b"ok" +async def test_two_content_length_responses_in_one_call( + response: HttpResponseParser, +) -> None: + """Two complete responses in a single feed_data call. + + The first payload completes with tail data for the second, hitting the + PAYLOAD_COMPLETE branch that resets the parser for the next message. + """ + resp1 = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello" + resp2 = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nworld" + + msgs, upgrade, tail = response.feed_data(resp1 + resp2) + assert len(msgs) == 2 + assert await msgs[0][-1].read() == b"hello" + assert await msgs[1][-1].read() == b"world" + + async def test_compressed_chunked_with_pending(response: HttpResponseParser) -> None: """Test chunked + compressed where the decompressor needs to resume from pause. From bf6e94cc2c3377515ef3e19755ad1550c2fb6c2f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 5 Apr 2026 16:47:59 -1000 Subject: [PATCH 187/199] ensure data is bytes --- aiohttp/_http_parser.pyx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 7c66193f7bb..5847f096161 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -577,16 +577,20 @@ cdef class HttpParser: if self._messages: return self._messages[-1][0] - def feed_data(self, data): + def feed_data(self, incoming_data): cdef: size_t data_len size_t nb char* base cdef cparser.llhttp_errno_t errno + cdef bytes data # Proactor loop sends bytearray. - if type(data) is not bytes: - data = bytes(data) + # Ensure cython sees `data` as bytes + if type(incoming_data) is not bytes: + data = bytes(incoming_data) + else: + data = incoming_data if self._tail: data, self._tail = self._tail + data, b"" From 9f6ba2fbe2bb43be9a7aa5ba2f44821282f15855 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 6 Apr 2026 19:36:09 +0100 Subject: [PATCH 188/199] Fix isal decompression and update default limits from 64KiB to 256KiB --- aiohttp/_websocket/writer.py | 2 +- aiohttp/client.py | 4 +- aiohttp/client_proto.py | 2 +- aiohttp/compression_utils.py | 8 ++- aiohttp/payload.py | 2 +- aiohttp/streams.py | 2 +- aiohttp/web_protocol.py | 2 +- aiohttp/web_ws.py | 2 +- tests/test_benchmarks_http_websocket.py | 4 +- tests/test_http_parser.py | 81 ++++++++++++++++++------- tests/test_http_writer.py | 8 +-- tests/test_multipart.py | 4 +- tests/test_payload.py | 4 +- tests/test_streams.py | 16 ++--- tests/test_web_request.py | 10 +-- tests/test_websocket_writer.py | 4 +- 16 files changed, 97 insertions(+), 58 deletions(-) diff --git a/aiohttp/_websocket/writer.py b/aiohttp/_websocket/writer.py index 1b27dff9371..df89aabbd5b 100644 --- a/aiohttp/_websocket/writer.py +++ b/aiohttp/_websocket/writer.py @@ -21,7 +21,7 @@ ) from .models import WS_DEFLATE_TRAILING, WSMsgType -DEFAULT_LIMIT: Final[int] = 2**16 +DEFAULT_LIMIT: Final[int] = 2**18 # WebSocket opcode boundary: opcodes 0-7 are data frames, 8-15 are control frames # Control frames (ping, pong, close) are never compressed diff --git a/aiohttp/client.py b/aiohttp/client.py index c3e874e650d..9bd9af10bf2 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -331,7 +331,7 @@ def __init__( trust_env: bool = False, requote_redirect_url: bool = True, trace_configs: list[TraceConfig[object]] | None = None, - read_bufsize: int = 2**16, + read_bufsize: int = 2**18, max_line_size: int = 8190, max_field_size: int = 8190, max_headers: int = 128, @@ -1226,7 +1226,7 @@ async def _ws_connect( transport = conn.transport assert transport is not None - reader = WebSocketDataQueue(conn_proto, 2**16, loop=self._loop) + reader = WebSocketDataQueue(conn_proto, 2**18, loop=self._loop) writer = WebSocketWriter( conn_proto, transport, diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index c6e78e57b42..19bd8564ca6 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -231,7 +231,7 @@ def set_response_params( read_until_eof: bool = False, auto_decompress: bool = True, read_timeout: float | None = None, - read_bufsize: int = 2**16, + read_bufsize: int = 2**18, timeout_ceil_threshold: float = 5, max_line_size: int = 8190, max_field_size: int = 8190, diff --git a/aiohttp/compression_utils.py b/aiohttp/compression_utils.py index 12d4e9d3a8a..5e12337113c 100644 --- a/aiohttp/compression_utils.py +++ b/aiohttp/compression_utils.py @@ -277,13 +277,17 @@ def __init__( self._mode = encoding_to_mode(encoding, suppress_deflate_header) self._zlib_backend: Final = ZLibBackendWrapper(ZLibBackend._zlib_backend) self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode) + self._last_empty = False def decompress_sync( self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED ) -> bytes: - return self._decompressor.decompress( + result = self._decompressor.decompress( self._decompressor.unconsumed_tail + data, max_length ) + # Only way to know that isal has no further data is checking we get no output + self._last_empty = result == b"" + return result def flush(self, length: int = 0) -> bytes: return ( @@ -294,7 +298,7 @@ def flush(self, length: int = 0) -> bytes: @property def data_available(self) -> bool: - return bool(self._decompressor.unconsumed_tail) + return bool(self._decompressor.unconsumed_tail) or not self._last_empty @property def eof(self) -> bool: diff --git a/aiohttp/payload.py b/aiohttp/payload.py index 9a8dc2f3262..71c015499a6 100644 --- a/aiohttp/payload.py +++ b/aiohttp/payload.py @@ -43,7 +43,7 @@ ) TOO_LARGE_BYTES_BODY: Final[int] = 2**20 # 1 MB -READ_SIZE: Final[int] = 2**16 # 64 KB +READ_SIZE: Final[int] = 2**18 # 256 KiB _CLOSE_FUTURES: set[asyncio.Future[None]] = set() diff --git a/aiohttp/streams.py b/aiohttp/streams.py index afefc9f0216..619f8dee4d3 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -165,7 +165,7 @@ def __repr__(self) -> str: info.append("%d bytes" % self._size) if self._eof: info.append("eof") - if self._low_water != 2**16: # default limit + if self._low_water != 2**18: # default limit info.append("low=%d high=%d" % (self._low_water, self._high_water)) if self._waiter: info.append("w=%r" % self._waiter) diff --git a/aiohttp/web_protocol.py b/aiohttp/web_protocol.py index abb5f86d81c..9785c13fa4f 100644 --- a/aiohttp/web_protocol.py +++ b/aiohttp/web_protocol.py @@ -202,7 +202,7 @@ def __init__( max_headers: int = 128, max_field_size: int = 8190, lingering_time: float = 10.0, - read_bufsize: int = 2**16, + read_bufsize: int = 2**18, auto_decompress: bool = True, timeout_ceil_threshold: float = 5, ): diff --git a/aiohttp/web_ws.py b/aiohttp/web_ws.py index 2aeeb6dec1f..1a7622b8421 100644 --- a/aiohttp/web_ws.py +++ b/aiohttp/web_ws.py @@ -383,7 +383,7 @@ def _post_start( loop = self._loop assert loop is not None - self._reader = WebSocketDataQueue(request._protocol, 2**16, loop=loop) + self._reader = WebSocketDataQueue(request._protocol, 2**18, loop=loop) parser = WebSocketReader( self._reader, self._max_msg_size, diff --git a/tests/test_benchmarks_http_websocket.py b/tests/test_benchmarks_http_websocket.py index 2aa9b8294bd..10115c1a2bd 100644 --- a/tests/test_benchmarks_http_websocket.py +++ b/tests/test_benchmarks_http_websocket.py @@ -36,8 +36,8 @@ def test_read_one_hundred_websocket_text_messages( loop: asyncio.AbstractEventLoop, benchmark: BenchmarkFixture ) -> None: """Benchmark reading 100 WebSocket text messages.""" - queue = WebSocketDataQueue(BaseProtocol(loop), 2**16, loop=loop) - reader = WebSocketReader(queue, max_msg_size=2**16) + queue = WebSocketDataQueue(BaseProtocol(loop), 2**18, loop=loop) + reader = WebSocketReader(queue, max_msg_size=2**18) raw_message = ( b'\x81~\x01!{"id":1,"src":"shellyplugus-c049ef8c30e4","dst":"aios-1453812500' b'8","result":{"name":null,"id":"shellyplugus-c049ef8c30e4","mac":"C049EF8C30E' diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 240e890161d..6a6238ac377 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -100,7 +100,7 @@ def parser( parser = request.param( protocol, loop, - 2**16, + 2**18, max_line_size=8190, max_headers=128, max_field_size=8190, @@ -128,7 +128,7 @@ def response( parser = request.param( protocol, loop, - 2**16, + 2**18, max_line_size=8190, max_headers=128, max_field_size=8190, @@ -1115,6 +1115,41 @@ async def test_two_content_length_responses_in_one_call( assert await msgs[1][-1].read() == b"world" +@pytest.mark.usefixtures("parametrize_zlib_backend") +async def test_compressed_zlib_64kb(response_cls: type[HttpResponseParser]) -> None: + loop = asyncio.get_running_loop() + protocol = ResponseHandler(loop) + response = response_cls( + protocol, + loop, + # 64KiB limit triggered a bug with isal implementation not returning all data. + 2**16, + max_line_size=8190, + max_headers=128, + max_field_size=8190, + ) + protocol._parser = response + + original = b"".join( + bytes((*range(0, i), *range(i, 0, -1))) + for _ in range(255) + for i in range(255) + ) + compressed = zlib.compress(original) + headers = ( + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: " + str(len(compressed)).encode() + b"\r\n" + b"Content-Encoding: deflate\r\n" + b"\r\n" + ) + + msgs, upgrade, tail = response.feed_data(headers + compressed) + payload = msgs[0][-1] + result = await payload.read() + assert len(result) == len(original) + assert result == original + + async def test_compressed_chunked_with_pending(response: HttpResponseParser) -> None: """Test chunked + compressed where the decompressor needs to resume from pause. @@ -1656,7 +1691,7 @@ async def test_http_response_parser_bad_chunked_strict_py( response = HttpResponseParserPy( protocol, loop, - 2**16, + 2**18, max_line_size=8190, max_field_size=8190, ) @@ -1840,7 +1875,7 @@ def test_parse_no_length_or_te_on_post( request_cls: type[HttpRequestParser], ) -> None: protocol = RequestHandler(server, loop=loop) - parser = request_cls(protocol, loop, limit=2**16) + parser = request_cls(protocol, loop, limit=2**18) protocol._parser = parser text = b"POST /test HTTP/1.1\r\n\r\n" msg, payload = parser.feed_data(text)[0][0] @@ -2123,7 +2158,7 @@ def test_parse_bad_method_for_c_parser_raises( parser = HttpRequestParserC( protocol, loop, - 2**16, + 2**18, max_line_size=8190, max_headers=128, max_field_size=8190, @@ -2145,7 +2180,7 @@ async def test_parse_eof_payload(self, protocol: BaseProtocol) -> None: assert [bytearray(b"data")] == list(out._buffer) async def test_parse_length_payload_eof(self, protocol: BaseProtocol) -> None: - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser(out, length=4, headers_parser=HeadersParser()) p.feed_data(b"da") @@ -2169,7 +2204,7 @@ async def test_parse_chunked_payload_size_data_mismatch( Regression test for #10596. """ - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser(out, chunked=True, headers_parser=HeadersParser()) # Declared chunk-size is 4 but actual data is "Hello" (5 bytes). # After consuming 4 bytes, remaining starts with "o" not "\r\n". @@ -2184,7 +2219,7 @@ async def test_parse_chunked_payload_size_data_mismatch_too_short( Regression test for #10596. """ - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser(out, chunked=True, headers_parser=HeadersParser()) # Declared chunk-size is 6 but actual data before CRLF is "Hello" (5 bytes). # Parser reads 6 bytes: "Hello\r", then expects \r\n but sees "\n0\r\n..." @@ -2206,7 +2241,7 @@ async def test_parse_chunked_payload_split_end( async def test_parse_chunked_payload_split_end2( self, protocol: BaseProtocol ) -> None: - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser(out, chunked=True, headers_parser=HeadersParser()) p.feed_data(b"4\r\nasdf\r\n0\r\n\r") p.feed_data(b"\n") @@ -2217,7 +2252,7 @@ async def test_parse_chunked_payload_split_end2( async def test_parse_chunked_payload_split_end_trailers( self, protocol: BaseProtocol ) -> None: - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser(out, chunked=True, headers_parser=HeadersParser()) p.feed_data(b"4\r\nasdf\r\n0\r\n") p.feed_data(b"Content-MD5: 912ec803b2ce49e4a541068d495ab570\r\n") @@ -2229,7 +2264,7 @@ async def test_parse_chunked_payload_split_end_trailers( async def test_parse_chunked_payload_split_end_trailers2( self, protocol: BaseProtocol ) -> None: - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser(out, chunked=True, headers_parser=HeadersParser()) p.feed_data(b"4\r\nasdf\r\n0\r\n") p.feed_data(b"Content-MD5: 912ec803b2ce49e4a541068d495ab570\r\n\r") @@ -2261,7 +2296,7 @@ async def test_parse_chunked_payload_split_end_trailers4( assert b"asdf" == b"".join(out._buffer) async def test_http_payload_parser_length(self, protocol: BaseProtocol) -> None: - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser(out, length=2, headers_parser=HeadersParser()) state, tail = p.feed_data(b"1245") assert state is PayloadState.PAYLOAD_COMPLETE @@ -2274,7 +2309,7 @@ async def test_http_payload_parser_deflate(self, protocol: BaseProtocol) -> None COMPRESSED = b"x\x9cKI,I\x04\x00\x04\x00\x01\x9b" length = len(COMPRESSED) - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser( out, length=length, compression="deflate", headers_parser=HeadersParser() ) @@ -2345,7 +2380,7 @@ async def test_http_payload_parser_deflate_split_err( async def test_http_payload_parser_length_zero( self, protocol: BaseProtocol ) -> None: - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser(out, length=0, headers_parser=HeadersParser()) assert p.done assert out.is_eof() @@ -2353,7 +2388,7 @@ async def test_http_payload_parser_length_zero( @pytest.mark.skipif(brotli is None, reason="brotli is not installed") async def test_http_payload_brotli(self, protocol: BaseProtocol) -> None: compressed = brotli.compress(b"brotli data") - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser( out, length=len(compressed), @@ -2367,7 +2402,7 @@ async def test_http_payload_brotli(self, protocol: BaseProtocol) -> None: @pytest.mark.skipif(zstandard is None, reason="zstandard is not installed") async def test_http_payload_zstandard(self, protocol: BaseProtocol) -> None: compressed = zstandard.compress(b"zstd data") - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser( out, length=len(compressed), @@ -2385,7 +2420,7 @@ async def test_http_payload_zstandard_multi_frame( frame1 = zstandard.compress(b"first") frame2 = zstandard.compress(b"second") payload = frame1 + frame2 - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser( out, length=len(payload), @@ -2402,7 +2437,7 @@ async def test_http_payload_zstandard_multi_frame_chunked( ) -> None: frame1 = zstandard.compress(b"chunk1") frame2 = zstandard.compress(b"chunk2") - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser( out, length=len(frame1) + len(frame2), @@ -2422,7 +2457,7 @@ async def test_http_payload_zstandard_frame_split_mid_chunk( frame2 = zstandard.compress(b"BBBB") combined = frame1 + frame2 split_point = len(frame1) + 3 # 3 bytes into frame2 - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser( out, length=len(combined), @@ -2440,7 +2475,7 @@ async def test_http_payload_zstandard_many_small_frames( ) -> None: parts = [f"part{i}".encode() for i in range(10)] payload = b"".join(zstandard.compress(p) for p in parts) - out = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + out = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) p = HttpPayloadParser( out, length=len(payload), @@ -2454,7 +2489,7 @@ async def test_http_payload_zstandard_many_small_frames( class TestDeflateBuffer: async def test_feed_data(self, protocol: BaseProtocol) -> None: - buf = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + buf = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) dbuf = DeflateBuffer(buf, "deflate") dbuf.decompressor = mock.Mock() @@ -2539,7 +2574,7 @@ async def test_feed_eof_no_err_zstandard(self, protocol: BaseProtocol) -> None: assert buf._eof async def test_empty_body(self, protocol: BaseProtocol) -> None: - buf = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + buf = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) dbuf = DeflateBuffer(buf, "deflate") dbuf.feed_eof() @@ -2564,7 +2599,7 @@ async def test_streaming_decompress_large_payload( original = b"A" * (3 * 2**20) compressed = zlib.compress(original) - buf = aiohttp.StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + buf = aiohttp.StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) dbuf = DeflateBuffer(buf, "deflate") # Feed compressed data in chunks (simulating network streaming) diff --git a/tests/test_http_writer.py b/tests/test_http_writer.py index e2596ea2e96..546ea60cd8b 100644 --- a/tests/test_http_writer.py +++ b/tests/test_http_writer.py @@ -1459,7 +1459,7 @@ async def test_write_drain_condition_with_small_buffer( protocol._drain_helper.reset_mock() # type: ignore[attr-defined] # Write small amount of data with drain=True but buffer under limit - small_data = b"x" * 100 # Much less than LIMIT (2**16) + small_data = b"x" * 100 # Much less than LIMIT (2**18) await msg.write(small_data, drain=True) # Drain should NOT be called because buffer_size <= LIMIT @@ -1488,7 +1488,7 @@ async def test_write_drain_condition_with_large_buffer( protocol._drain_helper.reset_mock() # type: ignore[attr-defined] # Write large amount of data with drain=True - large_data = b"x" * (2**16 + 1) # Just over LIMIT + large_data = b"x" * (2**18 + 1) # Just over LIMIT await msg.write(large_data, drain=True) # Drain should be called because drain=True AND buffer_size > LIMIT @@ -1517,12 +1517,12 @@ async def test_write_no_drain_with_large_buffer( protocol._drain_helper.reset_mock() # type: ignore[attr-defined] # Write large amount of data with drain=False - large_data = b"x" * (2**16 + 1) # Just over LIMIT + large_data = b"x" * (2**18 + 1) # Just over LIMIT await msg.write(large_data, drain=False) # Drain should NOT be called because drain=False assert not protocol._drain_helper.called # type: ignore[attr-defined] - assert msg.buffer_size == (2**16 + 1) # Buffer not reset + assert msg.buffer_size == (2**18 + 1) # Buffer not reset assert large_data in buf diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 98eddc82368..3bbc709691f 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -656,9 +656,9 @@ async def test_filename(self) -> None: assert "foo.html" == part.filename async def test_reading_long_part(self) -> None: - size = 2 * 2**16 + size = 2 * 2**18 protocol = mock.Mock(_reading_paused=False) - stream = StreamReader(protocol, 2**16, loop=asyncio.get_event_loop()) + stream = StreamReader(protocol, 2**18, loop=asyncio.get_event_loop()) stream.feed_data(b"0" * size + b"\r\n--:--") stream.feed_eof() d = CIMultiDictProxy[str](CIMultiDict()) diff --git a/tests/test_payload.py b/tests/test_payload.py index 205a3efdf81..e38335f546f 100644 --- a/tests/test_payload.py +++ b/tests/test_payload.py @@ -328,7 +328,7 @@ def mock_read(size: int | None = None) -> bytes: async def test_bytesio_payload_large_data_multiple_chunks() -> None: """Test BytesIOPayload with large data requiring multiple read chunks.""" - chunk_size = 2**16 # 64KB (READ_SIZE) + chunk_size = 2**18 # 256KiB (READ_SIZE) data = b"x" * (chunk_size + 1000) # Slightly larger than READ_SIZE payload_bytesio = payload.BytesIOPayload(io.BytesIO(data)) writer = MockStreamWriter() @@ -352,7 +352,7 @@ async def test_bytesio_payload_remaining_bytes_exhausted() -> None: async def test_iobase_payload_exact_chunk_size_limit() -> None: """Test IOBasePayload with content length matching exactly one read chunk.""" - chunk_size = 2**16 # 65536 bytes (READ_SIZE) + chunk_size = 2**18 # 256KiB (READ_SIZE) data = b"x" * chunk_size + b"extra" # Slightly larger than one read chunk p = payload.IOBasePayload(io.BytesIO(data)) writer = MockStreamWriter() diff --git a/tests/test_streams.py b/tests/test_streams.py index 52d926f6baa..c877a964ca8 100644 --- a/tests/test_streams.py +++ b/tests/test_streams.py @@ -29,7 +29,7 @@ def chunkify(seq: Sequence[_T], n: int) -> Iterator[Sequence[_T]]: async def create_stream() -> streams.StreamReader: loop = asyncio.get_event_loop() protocol = mock.Mock(_reading_paused=False) - stream = streams.StreamReader(protocol, 2**16, loop=loop) + stream = streams.StreamReader(protocol, 2**18, loop=loop) stream.feed_data(DATA) stream.feed_eof() return stream @@ -75,7 +75,7 @@ def get_memory_usage(obj: object) -> int: class TestStreamReader: DATA: bytes = b"line1\nline2\nline3\n" - def _make_one(self, limit: int = 2**16) -> streams.StreamReader: + def _make_one(self, limit: int = 2**18) -> streams.StreamReader: loop = asyncio.get_event_loop() return streams.StreamReader(mock.Mock(_reading_paused=False), limit, loop=loop) @@ -1276,7 +1276,7 @@ async def set_err() -> None: async def test_feed_data_waiters(protocol: BaseProtocol) -> None: loop = asyncio.get_event_loop() - reader = streams.StreamReader(protocol, 2**16, loop=loop) + reader = streams.StreamReader(protocol, 2**18, loop=loop) waiter = reader._waiter = loop.create_future() eof_waiter = reader._eof_waiter = loop.create_future() @@ -1304,7 +1304,7 @@ async def test_feed_data_completed_waiters(protocol: BaseProtocol) -> None: async def test_feed_eof_waiters(protocol: BaseProtocol) -> None: loop = asyncio.get_event_loop() - reader = streams.StreamReader(protocol, 2**16, loop=loop) + reader = streams.StreamReader(protocol, 2**18, loop=loop) waiter = reader._waiter = loop.create_future() eof_waiter = reader._eof_waiter = loop.create_future() @@ -1336,7 +1336,7 @@ async def test_feed_eof_cancelled(protocol: BaseProtocol) -> None: async def test_on_eof(protocol: BaseProtocol) -> None: loop = asyncio.get_event_loop() - reader = streams.StreamReader(protocol, 2**16, loop=loop) + reader = streams.StreamReader(protocol, 2**18, loop=loop) on_eof = mock.Mock() reader.on_eof(on_eof) @@ -1357,7 +1357,7 @@ async def test_on_eof_empty_reader() -> None: async def test_on_eof_exc_in_callback(protocol: BaseProtocol) -> None: loop = asyncio.get_event_loop() - reader = streams.StreamReader(protocol, 2**16, loop=loop) + reader = streams.StreamReader(protocol, 2**18, loop=loop) on_eof = mock.Mock() on_eof.side_effect = ValueError @@ -1392,7 +1392,7 @@ async def test_on_eof_eof_is_set(protocol: BaseProtocol) -> None: async def test_on_eof_eof_is_set_exception(protocol: BaseProtocol) -> None: loop = asyncio.get_event_loop() - reader = streams.StreamReader(protocol, 2**16, loop=loop) + reader = streams.StreamReader(protocol, 2**18, loop=loop) reader.feed_eof() on_eof = mock.Mock() @@ -1438,7 +1438,7 @@ async def test_set_exception_cancelled(protocol: BaseProtocol) -> None: async def test_set_exception_eof_callbacks(protocol: BaseProtocol) -> None: loop = asyncio.get_event_loop() - reader = streams.StreamReader(protocol, 2**16, loop=loop) + reader = streams.StreamReader(protocol, 2**18, loop=loop) on_eof = mock.Mock() reader.on_eof(on_eof) diff --git a/tests/test_web_request.py b/tests/test_web_request.py index 038e6c141d9..efeb6b766b0 100644 --- a/tests/test_web_request.py +++ b/tests/test_web_request.py @@ -837,7 +837,7 @@ def test_clone_headers_dict() -> None: async def test_cannot_clone_after_read(protocol: BaseProtocol) -> None: - payload = StreamReader(protocol, 2**16, loop=asyncio.get_event_loop()) + payload = StreamReader(protocol, 2**18, loop=asyncio.get_event_loop()) payload.feed_data(b"data") payload.feed_eof() req = make_mocked_request("GET", "/path", payload=payload) @@ -860,7 +860,7 @@ async def test_make_too_big_request(protocol: BaseProtocol) -> None: async def test_request_with_wrong_content_type_encoding(protocol: BaseProtocol) -> None: - payload = StreamReader(protocol, 2**16, loop=asyncio.get_event_loop()) + payload = StreamReader(protocol, 2**18, loop=asyncio.get_event_loop()) payload.feed_data(b"{}") payload.feed_eof() headers = {"Content-Type": "text/html; charset=test"} @@ -920,7 +920,7 @@ async def test_multipart_formdata(protocol: BaseProtocol) -> None: async def test_multipart_formdata_field_missing_name(protocol: BaseProtocol) -> None: # Ensure ValueError is raised when Content-Disposition has no name - payload = StreamReader(protocol, 2**16, loop=asyncio.get_event_loop()) + payload = StreamReader(protocol, 2**18, loop=asyncio.get_event_loop()) payload.feed_data( b"-----------------------------326931944431359\r\n" b"Content-Disposition: form-data\r\n" # Missing name! @@ -972,7 +972,7 @@ async def test_multipart_formdata_headers_too_many(protocol: BaseProtocol) -> No b"--b--\r\n" ) content_type = "multipart/form-data; boundary=b" - payload = StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + payload = StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) payload.feed_data(body) payload.feed_eof() req = make_mocked_request( @@ -999,7 +999,7 @@ async def test_multipart_formdata_header_too_long(protocol: BaseProtocol) -> Non b"--b--\r\n" ) content_type = "multipart/form-data; boundary=b" - payload = StreamReader(protocol, 2**16, loop=asyncio.get_running_loop()) + payload = StreamReader(protocol, 2**18, loop=asyncio.get_running_loop()) payload.feed_data(body) payload.feed_eof() req = make_mocked_request( diff --git a/tests/test_websocket_writer.py b/tests/test_websocket_writer.py index 8dffc7c015e..3b6bc98b54f 100644 --- a/tests/test_websocket_writer.py +++ b/tests/test_websocket_writer.py @@ -158,7 +158,7 @@ async def test_send_compress_cancelled( monkeypatch.setattr("aiohttp._websocket.writer.WEBSOCKET_MAX_SYNC_CHUNK_SIZE", 1024) writer = WebSocketWriter(protocol, transport, compress=15) loop = asyncio.get_running_loop() - queue = WebSocketDataQueue(mock.Mock(_reading_paused=False), 2**16, loop=loop) + queue = WebSocketDataQueue(mock.Mock(_reading_paused=False), 2**18, loop=loop) reader = WebSocketReader(queue, 50000) # Replace executor with slow one to make race condition reproducible @@ -305,7 +305,7 @@ async def test_concurrent_messages( ): writer = WebSocketWriter(protocol, transport, compress=15) loop = asyncio.get_running_loop() - queue = WebSocketDataQueue(mock.Mock(_reading_paused=False), 2**16, loop=loop) + queue = WebSocketDataQueue(mock.Mock(_reading_paused=False), 2**18, loop=loop) reader = WebSocketReader(queue, 50000) writers = [] payloads = [] From bc4dbffc04f0c22f5b9198dab0495a30b94af4aa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 18:38:13 +0000 Subject: [PATCH 189/199] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_http_parser.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 6a6238ac377..ba7f9143189 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1131,9 +1131,7 @@ async def test_compressed_zlib_64kb(response_cls: type[HttpResponseParser]) -> N protocol._parser = response original = b"".join( - bytes((*range(0, i), *range(i, 0, -1))) - for _ in range(255) - for i in range(255) + bytes((*range(0, i), *range(i, 0, -1))) for _ in range(255) for i in range(255) ) compressed = zlib.compress(original) headers = ( From 55cf9fdcf6e93c4f9d22488e1ec2c78029d6d7e1 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 6 Apr 2026 20:43:38 +0100 Subject: [PATCH 190/199] Fix --- aiohttp/streams.py | 8 ++++---- tests/test_http_parser.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aiohttp/streams.py b/aiohttp/streams.py index 619f8dee4d3..b9367066291 100644 --- a/aiohttp/streams.py +++ b/aiohttp/streams.py @@ -138,11 +138,11 @@ def __init__( self._protocol = protocol self._low_water = limit self._high_water = limit * 2 - # Ensure high_water_chunks >= 3 so it's always > low_water_chunks. - self._high_water_chunks = max(3, limit // 4) - # Use max(2, ...) because there's always at least 1 chunk split remaining + # Use max(4, ...) because there's always at least 1 chunk split remaining # (the current position), so we need low_water >= 2 to allow resume. - self._low_water_chunks = max(2, self._high_water_chunks // 2) + # limit // 16 gets us a reasonable value of 16k with default 256KiB limit. + self._high_water_chunks = max(4, limit // 16) + self._low_water_chunks = self._high_water_chunks // 2 self._loop = loop self._size = 0 self._cursor = 0 diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 6a6238ac377..0d20d579c6e 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -1178,7 +1178,7 @@ async def test_compressed_chunked_with_pending(response: HttpResponseParser) -> async def test_compressed_until_eof_with_pending(response: HttpResponseParser) -> None: """Test read-until-eof + compressed with pause.""" # Must be large enough to exceed high water mark. - original = b"B" * 1024 * 1024 + original = b"B" * 5 * 1024 * 1024 compressed = zlib.compress(original) # No Content-Length or Transfer-Encoding means the parser must parse until EOF. headers = b"HTTP/1.1 200 OK\r\nContent-Encoding: deflate\r\n\r\n" @@ -1188,7 +1188,7 @@ async def test_compressed_until_eof_with_pending(response: HttpResponseParser) - payload = msgs[0][-1] # Check that .feed_eof() hasn't decompressed entire payload into memory. - assert sum(len(b) for b in payload._buffer) <= (512 * 1024) + assert sum(len(b) for b in payload._buffer) <= (1024 * 1024) result = await payload.read() assert len(result) == len(original) From d997300b7d998c427c50801d0e359a263644f3d4 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 6 Apr 2026 21:10:41 +0100 Subject: [PATCH 191/199] Fix --- tests/test_streams.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/test_streams.py b/tests/test_streams.py index c877a964ca8..6560b4698fb 100644 --- a/tests/test_streams.py +++ b/tests/test_streams.py @@ -1545,8 +1545,8 @@ async def test_stream_reader_pause_on_high_water_chunks( ) -> None: """Test that reading is paused when chunk count exceeds high water mark.""" loop = asyncio.get_event_loop() - # Use small limit so high_water_chunks is small: limit // 4 = 10 - stream = streams.StreamReader(protocol, limit=40, loop=loop) + # Use small limit so high_water_chunks is small: limit // 16 = 10 + stream = streams.StreamReader(protocol, limit=160, loop=loop) assert stream._high_water_chunks == 10 assert stream._low_water_chunks == 5 @@ -1566,8 +1566,8 @@ async def test_stream_reader_resume_on_low_water_chunks( ) -> None: """Test that reading resumes when chunk count drops below low water mark.""" loop = asyncio.get_event_loop() - # Use small limit so high_water_chunks is small: limit // 4 = 10 - stream = streams.StreamReader(protocol, limit=40, loop=loop) + # Use small limit so high_water_chunks is small: limit // 16 = 10 + stream = streams.StreamReader(protocol, limit=160, loop=loop) assert stream._high_water_chunks == 10 assert stream._low_water_chunks == 5 @@ -1661,14 +1661,14 @@ async def test_stream_reader_resume_non_chunked_when_paused( protocol.resume_reading.assert_called() -@pytest.mark.parametrize("limit", [1, 2, 4]) +@pytest.mark.parametrize("limit", (1, 4, 7, 16)) async def test_stream_reader_small_limit_resumes_reading( protocol: mock.Mock, limit: int, ) -> None: """Test that small limits still allow resume_reading to be called. - Even with very small limits, high_water_chunks should be at least 3 + Even with very small limits, high_water_chunks should be at least 4 and low_water_chunks should be at least 2, with high > low to ensure proper flow control. """ @@ -1676,8 +1676,8 @@ async def test_stream_reader_small_limit_resumes_reading( stream = streams.StreamReader(protocol, limit=limit, loop=loop) # Verify minimum thresholds are enforced and high > low - assert stream._high_water_chunks >= 3 - assert stream._low_water_chunks >= 2 + assert stream._high_water_chunks == 4 + assert stream._low_water_chunks == 2 assert stream._high_water_chunks > stream._low_water_chunks # Set up pause/resume side effects @@ -1691,8 +1691,8 @@ def resume_reading() -> None: protocol.resume_reading.side_effect = resume_reading - # Feed 4 chunks (triggers pause at > high_water_chunks which is >= 3) - for char in b"abcd": + # Feed 5 chunks (triggers pause at > high_water_chunks which is 4) + for char in b"abcde": stream.begin_http_chunk_receiving() stream.feed_data(bytes([char])) stream.end_http_chunk_receiving() @@ -1703,7 +1703,7 @@ def resume_reading() -> None: # Read all data - should resume (chunk count drops below low_water_chunks) data = stream.read_nowait() - assert data == b"abcd" + assert data == b"abcde" assert stream._size == 0 protocol.resume_reading.assert_called() From 6079d0585c40b7c32d0c36793a52a562a38eb326 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 6 Apr 2026 21:39:59 +0100 Subject: [PATCH 192/199] Fix coverage --- aiohttp/http_parser.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 20d3a854317..4601f201122 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -496,7 +496,7 @@ def get_content_length() -> int | None: break # feed payload - elif self._payload_has_more_data or (data and start_pos < data_len): + else: assert not self._lines assert self._payload_parser is not None try: @@ -533,8 +533,6 @@ def get_content_length() -> int | None: start_pos = 0 data_len = len(data) self._payload_parser = None - else: - break if data and start_pos < data_len: data = data[start_pos:] From ecc976b8f53839b87d2f6b75a6c5cb5a8da5d448 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 6 Apr 2026 22:03:13 +0100 Subject: [PATCH 193/199] Another test --- tests/test_multipart.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 3bbc709691f..30659405599 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -1,4 +1,5 @@ import asyncio +import gzip import io import json import pathlib @@ -27,6 +28,7 @@ MultipartResponseWrapper, ) from aiohttp.streams import StreamReader +from aiohttp.web_exceptions import HTTPRequestEntityTooLarge if sys.version_info >= (3, 11): from typing import Self @@ -385,6 +387,22 @@ async def test_read_with_content_encoding_unknown(self) -> None: with pytest.raises(RuntimeError): await obj.read(decode=True) + async def test_read_decode_compressed_exceeds_max_size(self) -> None: + # Compressed data is small, but decompresses beyond client_max_size. + original = b"A" * 1024 + compressed = gzip.compress(original) + h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "gzip"})) + with Stream(compressed + b"\r\n--:--") as stream: + obj = aiohttp.BodyPartReader( + BOUNDARY, + h, + stream, + client_max_size=256, + max_size_error_cls=HTTPRequestEntityTooLarge, + ) + with pytest.raises(HTTPRequestEntityTooLarge): + await obj.read(decode=True) + async def test_read_with_content_transfer_encoding_base64(self) -> None: h = CIMultiDictProxy(CIMultiDict({CONTENT_TRANSFER_ENCODING: "base64"})) with Stream(b"VGltZSB0byBSZWxheCE=\r\n--:--") as stream: From c8bc2b37cd5cda598f9797ed7e7a954e48314b49 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 6 Apr 2026 22:22:35 +0100 Subject: [PATCH 194/199] Update setup.cfg --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index 2d7e24b0374..203c01c3754 100644 --- a/setup.cfg +++ b/setup.cfg @@ -88,6 +88,8 @@ filterwarnings = # https://github.com/spulec/freezegun/issues/508 # https://github.com/spulec/freezegun/pull/511 ignore:datetime.*utcnow\(\) is deprecated and scheduled for removal:DeprecationWarning:freezegun.api + # Weird issue in Python 3.13+ triggered in test_multipart.py + ignore:coroutine method 'aclose' of 'BodyPartReader._decode_content_async' was never awaited:RuntimeWarning junit_suite_name = aiohttp_test_suite norecursedirs = dist docs build .tox .eggs minversion = 3.8.2 From 2e006bcc32c43172bffe584dc49f3d90a146e550 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 6 Apr 2026 23:54:20 +0100 Subject: [PATCH 195/199] Fix type error --- aiohttp/multipart.py | 8 ++------ aiohttp/web_exceptions.py | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 3d6c1057c75..894143ca769 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -319,18 +319,14 @@ async def read(self, *, decode: bool = False) -> bytes: while not self._at_eof: data.extend(await self.read_chunk(self.chunk_size)) if len(data) > self._client_max_size: - raise self._max_size_error_cls( - max_size=self._client_max_size, actual_size=len(data) - ) + raise self._max_size_error_cls(self._client_max_size) # https://github.com/python/mypy/issues/17537 if decode: # type: ignore[unreachable] decoded_data = bytearray() async for d in self.decode_iter(data): decoded_data.extend(d) if len(decoded_data) > self._client_max_size: - raise self._max_size_error_cls( - max_size=self._client_max_size, actual_size=len(decoded_data) - ) + raise self._max_size_error_cls(self._client_max_size) return decoded_data return data diff --git a/aiohttp/web_exceptions.py b/aiohttp/web_exceptions.py index 782a4d39507..bd507a8813a 100644 --- a/aiohttp/web_exceptions.py +++ b/aiohttp/web_exceptions.py @@ -366,12 +366,8 @@ class HTTPPreconditionFailed(HTTPClientError): class HTTPRequestEntityTooLarge(HTTPClientError): status_code = 413 - def __init__(self, max_size: int, actual_size: int, **kwargs: Any) -> None: - kwargs.setdefault( - "text", - f"Maximum request body size {max_size} exceeded, " - f"actual body size {actual_size}", - ) + def __init__(self, max_size: int, **kwargs: Any) -> None: + kwargs.setdefault("text", f"Maximum request body size {max_size} exceeded.") super().__init__(**kwargs) From e11eda68b789a66fe2f2331317486dd2d33a4a76 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 6 Apr 2026 23:57:15 +0100 Subject: [PATCH 196/199] Fix --- aiohttp/web_request.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index d0106e47b94..b8feae19cec 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -634,9 +634,7 @@ async def read(self) -> bytes: if self._client_max_size: body_size = len(body) if body_size > self._client_max_size: - raise HTTPRequestEntityTooLarge( - max_size=self._client_max_size, actual_size=body_size - ) + raise HTTPRequestEntityTooLarge(self._client_max_size) if not chunk: break self._read_bytes = bytes(body) @@ -729,9 +727,7 @@ async def post(self) -> "MultiDictProxy[str | bytes | FileField]": size += len(decoded_chunk) if 0 < max_size < size: await self._loop.run_in_executor(None, tmp.close) - raise HTTPRequestEntityTooLarge( - max_size=max_size, actual_size=size - ) + raise HTTPRequestEntityTooLarge(max_size) await self._loop.run_in_executor(None, tmp.seek, 0) if field_ct is None: @@ -751,9 +747,7 @@ async def post(self) -> "MultiDictProxy[str | bytes | FileField]": while chunk := await field.read_chunk(): size += len(chunk) if 0 < max_size < size: - raise HTTPRequestEntityTooLarge( - max_size=max_size, actual_size=size - ) + raise HTTPRequestEntityTooLarge(max_size) raw_data.extend(chunk) value = bytearray() From ed45bde164e3b1353f61dc78f3f5c3b22d940ffd Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 7 Apr 2026 00:01:46 +0100 Subject: [PATCH 197/199] Apply suggestion from @Dreamsorcerer --- aiohttp/_http_parser.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 5847f096161..d5872b019a9 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -601,7 +601,6 @@ cdef class HttpParser: if result is cparser.HPE_PAUSED: self._tail = data return EMPTY_FEED_DATA_RESULT - # TODO: Do we need to handle error case (-1)? if self._eof_pending and not self._more_data_available: self._payload.feed_eof() From 9dabd53a80eefed912cb90cddb90c8ecc9c9fa4d Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 7 Apr 2026 00:36:07 +0100 Subject: [PATCH 198/199] Create 11966.feature.rst --- CHANGES/11966.feature.rst | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 CHANGES/11966.feature.rst diff --git a/CHANGES/11966.feature.rst b/CHANGES/11966.feature.rst new file mode 100644 index 00000000000..9f298f98e12 --- /dev/null +++ b/CHANGES/11966.feature.rst @@ -0,0 +1,8 @@ +Large overhaul of parser/decompression code. + +The zip bomb security fix in 3.13 stopped highly compressed payloads +from being decompressed, regardless of validity. Now aiohttp will +decompress such payloads in chunks of 256+ KiB, allowing safe decompression +of such payloads. + +-- by :user:`Dreamsorcerer`. From 556f95bd7fa5247da554ea7256f9dbdaeb32b637 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Tue, 7 Apr 2026 00:50:08 +0100 Subject: [PATCH 199/199] Fix --- tests/test_web_exceptions.py | 11 +++-------- tests/test_web_functional.py | 17 ++++------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/tests/test_web_exceptions.py b/tests/test_web_exceptions.py index b29de08b170..0f5d7e22ba1 100644 --- a/tests/test_web_exceptions.py +++ b/tests/test_web_exceptions.py @@ -329,14 +329,9 @@ def test_pickle(self) -> None: class TestHTTPRequestEntityTooLarge: def test_ctor(self) -> None: resp = web.HTTPRequestEntityTooLarge( - max_size=100, - actual_size=123, - headers={"X-Custom": "value"}, - reason="Too large", - ) - assert resp.text == ( - "Maximum request body size 100 exceeded, actual body size 123" + max_size=100, headers={"X-Custom": "value"}, reason="Too large" ) + assert resp.text == "Maximum request body size 100 exceeded." compare: Mapping[str, str] = {"X-Custom": "value", "Content-Type": "text/plain"} assert resp.headers == compare assert resp.reason == "Too large" @@ -344,7 +339,7 @@ def test_ctor(self) -> None: def test_pickle(self) -> None: resp = web.HTTPRequestEntityTooLarge( - 100, actual_size=123, headers={"X-Custom": "value"}, reason="Too large" + 100, headers={"X-Custom": "value"}, reason="Too large" ) resp.foo = "bar" # type: ignore[attr-defined] for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 7784a41ae7c..211594ad61c 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -1693,10 +1693,7 @@ async def handler(request: web.Request) -> NoReturn: resp = await client.post("/", data=data) assert 413 == resp.status resp_text = await resp.text() - assert "Maximum request body size 1048576 exceeded, actual body size" in resp_text - # Maximum request body size X exceeded, actual body size X - body_size = int(resp_text.split()[-1]) - assert body_size >= max_size + assert "Maximum request body size 1048576 exceeded" in resp_text resp.release() @@ -1718,7 +1715,7 @@ async def handler(request: web.Request) -> NoReturn: async with client.post("/", data=form) as resp: assert resp.status == 413 resp_text = await resp.text() - assert "Maximum request body size 1048576 exceeded, actual body size" in resp_text + assert "Maximum request body size 1048576 exceeded" in resp_text async def test_app_max_client_size_adjusted(aiohttp_client: AiohttpClient) -> None: @@ -1745,10 +1742,7 @@ async def handler(request: web.Request) -> web.Response: resp = await client.post("/", data=too_large_data) assert 413 == resp.status resp_text = await resp.text() - assert "Maximum request body size 2097152 exceeded, actual body size" in resp_text - # Maximum request body size X exceeded, actual body size X - body_size = int(resp_text.split()[-1]) - assert body_size >= custom_max_size + assert "Maximum request body size 2097152 exceeded" in resp_text resp.release() @@ -1795,10 +1789,7 @@ async def handler(request: web.Request) -> NoReturn: assert 413 == resp.status resp_text = await resp.text() - assert ( - "Maximum request body size 10 exceeded, " - "actual body size 1024" in resp_text - ) + assert "Maximum request body size 10 exceeded" in resp_text data_file = data["file"] assert isinstance(data_file, io.BytesIO) data_file.close()