From a4b5b4ac2416ae882c5a44f422aef5ca39401a13 Mon Sep 17 00:00:00 2001 From: "Aryan Singh K." <70511529+aryansk@users.noreply.github.com> Date: Sat, 15 Aug 2026 19:07:03 +0530 Subject: [PATCH] feat: support timeouts in the `send_request` context helper (#2138) Expose the HTTP clients' existing per-request `timeout` support through `SendRequestFunction` so request handlers can bound how long an extra HTTP call may take. Adds a regression test against the slow test-server endpoint. --- src/crawlee/_types.py | 3 +++ src/crawlee/crawlers/_basic/_basic_crawler.py | 2 ++ .../crawlers/_basic/test_basic_crawler.py | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index 7a1e0d7eef..19d31d3256 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -14,6 +14,7 @@ import logging import re from collections.abc import Awaitable, Coroutine, MutableMapping, Sequence + from datetime import timedelta from typing_extensions import NotRequired, Required, Self, Unpack @@ -558,6 +559,7 @@ def __call__( method: HttpMethod = 'GET', payload: HttpPayload | None = None, headers: HttpHeaders | dict[str, str] | None = None, + timeout: timedelta | None = None, ) -> Coroutine[None, None, HttpResponse]: """Call send request function. @@ -566,6 +568,7 @@ def __call__( method: The HTTP method to use. headers: The headers to include in the request. payload: The payload to include in the request. + timeout: Maximum time allowed to process the request. Returns: The HTTP response received from the server. diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 96ff205350..e082a3e37c 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -1321,6 +1321,7 @@ async def send_request( method: HttpMethod = 'GET', payload: HttpPayload | None = None, headers: HttpHeaders | dict[str, str] | None = None, + timeout: timedelta | None = None, ) -> HttpResponse: return await self._http_client.send_request( url=url, @@ -1329,6 +1330,7 @@ async def send_request( headers=headers, session=session, proxy_info=proxy_info, + timeout=timeout, ) return send_request diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 56ba257e86..95bfebe172 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -523,6 +523,25 @@ async def handler(context: BasicCrawlingContext) -> None: assert content_type == 'application/json' +async def test_send_request_respects_timeout(server_url: URL) -> None: + request_timed_out = asyncio.Event() + + crawler = BasicCrawler(max_request_retries=3) + + @crawler.router.default_handler + async def handler(context: BasicCrawlingContext) -> None: + with pytest.raises(asyncio.TimeoutError): + await context.send_request( + str(server_url / 'slow') + '?delay=2', + timeout=timedelta(milliseconds=100), + ) + request_timed_out.set() + + await crawler.run(['https://a.placeholder.com']) + + assert request_timed_out.is_set() + + @dataclass class AddRequestsTestInput: start_url: str