feat: support timeouts in the send_request context helper (#2138)

src/crawlee/_types.py

@@ -14,6 +14,7 @@
     import logging
     import re
     from collections.abc import Awaitable, Coroutine, MutableMapping, Sequence
+    from datetime import timedelta
 
     from typing_extensions import NotRequired, Required, Self, Unpack
 
@@ -558,6 +559,7 @@ def __call__(
         method: HttpMethod = 'GET',
         payload: HttpPayload | None = None,
         headers: HttpHeaders | dict[str, str] | None = None,
+        timeout: timedelta | None = None,
     ) -> Coroutine[None, None, HttpResponse]:
         """Call send request function.
 
@@ -566,6 +568,7 @@ def __call__(
             method: The HTTP method to use.
             headers: The headers to include in the request.
             payload: The payload to include in the request.
+            timeout: Maximum time allowed to process the request.
 
         Returns:
             The HTTP response received from the server.

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -1321,6 +1321,7 @@ async def send_request(
             method: HttpMethod = 'GET',
             payload: HttpPayload | None = None,
             headers: HttpHeaders | dict[str, str] | None = None,
+            timeout: timedelta | None = None,
         ) -> HttpResponse:
             return await self._http_client.send_request(
                 url=url,
@@ -1329,6 +1330,7 @@ async def send_request(
                 headers=headers,
                 session=session,
                 proxy_info=proxy_info,
+                timeout=timeout,
             )
 
         return send_request

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -523,6 +523,25 @@ async def handler(context: BasicCrawlingContext) -> None:
     assert content_type == 'application/json'
 
 
+async def test_send_request_respects_timeout(server_url: URL) -> None:
+    request_timed_out = asyncio.Event()
+
+    crawler = BasicCrawler(max_request_retries=3)
+
+    @crawler.router.default_handler
+    async def handler(context: BasicCrawlingContext) -> None:
+        with pytest.raises(asyncio.TimeoutError):
+            await context.send_request(
+                str(server_url / 'slow') + '?delay=2',
+                timeout=timedelta(milliseconds=100),
+            )
+        request_timed_out.set()
+
+    await crawler.run(['https://a.placeholder.com'])
+
+    assert request_timed_out.is_set()
+
+
 @dataclass
 class AddRequestsTestInput:
     start_url: str