test: remove external network dependency from problematic-links tests

tests/unit/conftest.py

@@ -2,6 +2,7 @@
 
 import logging
 import os
+import socket
 import warnings
 from typing import TYPE_CHECKING, Any, cast
 
@@ -18,7 +19,7 @@
 from crawlee.proxy_configuration import ProxyInfo
 from crawlee.statistics import Statistics
 from crawlee.storages import KeyValueStore
-from tests.unit.server import TestServer, app, serve_in_thread
+from tests.unit.server import TestServer, app, no_robots_app, serve_in_thread
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Callable, Iterator
@@ -174,6 +175,32 @@ def server_url(http_server: TestServer) -> URL:
     return http_server.url
 
 
+@pytest.fixture(scope='session')
+def no_robots_http_server(unused_tcp_port_factory: Callable[[], int]) -> Iterator[TestServer]:
+    """Create and start an HTTP test server that responds with 404 to robots.txt requests."""
+    config = Config(app=no_robots_app, lifespan='off', loop='asyncio', port=unused_tcp_port_factory())
+    server = TestServer(config=config)
+    yield from serve_in_thread(server)
+
+
+@pytest.fixture(scope='session')
+def no_robots_server_url(no_robots_http_server: TestServer) -> URL:
+    """Provide the base URL of the test server that has no robots.txt file."""
+    return no_robots_http_server.url
+
+
+@pytest.fixture(scope='session')
+def unreachable_url() -> Iterator[str]:
+    """Provide a URL that can never be connected to.
+
+    The port is bound for the whole session but never listened on, so nothing else can take it and every
+    connection attempt is refused immediately, without any DNS lookup or external traffic.
+    """
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(('127.0.0.1', 0))
+        yield f'http://127.0.0.1:{sock.getsockname()[1]}/'
+
+
 # It is needed only in some tests, so we use the standard `scope=function`
 @pytest.fixture
 def redirect_http_server(unused_tcp_port_factory: Callable[[], int]) -> Iterator[TestServer]:

tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py

@@ -201,8 +201,17 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
     visit.assert_has_calls(expected_visit_calls, any_order=True)
 
 
-async def test_respect_robots_txt_with_problematic_links(server_url: URL, http_client: HttpClient) -> None:
+async def test_respect_robots_txt_with_problematic_links(
+    server_url: URL,
+    no_robots_server_url: URL,
+    unreachable_url: str,
+    http_client: HttpClient,
+) -> None:
     """Test checks the crawler behavior with links that may cause problems when attempting to retrieve robots.txt."""
+    no_robots_url = str(no_robots_server_url / 'page')
+    start_url = str(
+        (server_url / 'problematic_links').with_query(unreachable_url=unreachable_url, no_robots_url=no_robots_url)
+    )
     visit = mock.Mock()
     fail = mock.Mock()
     crawler = BeautifulSoupCrawler(
@@ -220,19 +229,19 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
     async def error_handler(context: BasicCrawlingContext, _error: Exception) -> None:
         fail(context.request.url)
 
-    await crawler.run([str(server_url / 'problematic_links')])
+    await crawler.run([start_url])
 
-    # Email must be skipped
-    # https://avatars.githubusercontent.com/apify does not get robots.txt, but is correct for the crawler.
+    # Email must be skipped.
+    # An origin without robots.txt is still crawled, an unavailable file means unrestricted crawling.
     expected_visit_calls = [
-        mock.call(str(server_url / 'problematic_links')),
-        mock.call('https://avatars.githubusercontent.com/apify'),
+        mock.call(start_url),
+        mock.call(no_robots_url),
     ]
     visit.assert_has_calls(expected_visit_calls, any_order=True)
 
-    # The budplaceholder.com does not exist.
+    # The unreachable URL cannot be connected to.
     expected_fail_calls = [
-        mock.call('https://budplaceholder.com/'),
+        mock.call(unreachable_url),
     ]
     fail.assert_has_calls(expected_fail_calls, any_order=True)
 

tests/unit/crawlers/_parsel/test_parsel_crawler.py

@@ -285,8 +285,17 @@ async def request_handler(context: ParselCrawlingContext) -> None:
     visit.assert_has_calls(expected_visit_calls, any_order=True)
 
 
-async def test_respect_robots_txt_with_problematic_links(server_url: URL, http_client: HttpClient) -> None:
+async def test_respect_robots_txt_with_problematic_links(
+    server_url: URL,
+    no_robots_server_url: URL,
+    unreachable_url: str,
+    http_client: HttpClient,
+) -> None:
     """Test checks the crawler behavior with links that may cause problems when attempting to retrieve robots.txt."""
+    no_robots_url = str(no_robots_server_url / 'page')
+    start_url = str(
+        (server_url / 'problematic_links').with_query(unreachable_url=unreachable_url, no_robots_url=no_robots_url)
+    )
     visit = mock.Mock()
     fail = mock.Mock()
     crawler = ParselCrawler(
@@ -304,19 +313,19 @@ async def request_handler(context: ParselCrawlingContext) -> None:
     async def error_handler(context: BasicCrawlingContext, _error: Exception) -> None:
         fail(context.request.url)
 
-    await crawler.run([str(server_url / 'problematic_links')])
+    await crawler.run([start_url])
 
-    # Email must be skipped
-    # https://avatars.githubusercontent.com/apify does not get robots.txt, but is correct for the crawler.
+    # Email must be skipped.
+    # An origin without robots.txt is still crawled, an unavailable file means unrestricted crawling.
     expected_visit_calls = [
-        mock.call(str(server_url / 'problematic_links')),
-        mock.call('https://avatars.githubusercontent.com/apify'),
+        mock.call(start_url),
+        mock.call(no_robots_url),
     ]
     visit.assert_has_calls(expected_visit_calls, any_order=True)
 
-    # The budplaceholder.com does not exist.
+    # The unreachable URL cannot be connected to.
     expected_fail_calls = [
-        mock.call('https://budplaceholder.com/'),
+        mock.call(unreachable_url),
     ]
     fail.assert_has_calls(expected_fail_calls, any_order=True)
 

tests/unit/crawlers/_playwright/test_playwright_crawler.py

@@ -721,8 +721,16 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
     visit.assert_has_calls(expected_visit_calls, any_order=True)
 
 
-async def test_respect_robots_txt_with_problematic_links(server_url: URL) -> None:
+async def test_respect_robots_txt_with_problematic_links(
+    server_url: URL,
+    no_robots_server_url: URL,
+    unreachable_url: str,
+) -> None:
     """Test checks the crawler behavior with links that may cause problems when attempting to retrieve robots.txt."""
+    no_robots_url = str(no_robots_server_url / 'page')
+    start_url = str(
+        (server_url / 'problematic_links').with_query(unreachable_url=unreachable_url, no_robots_url=no_robots_url)
+    )
     visit = mock.Mock()
     fail = mock.Mock()
     crawler = PlaywrightCrawler(respect_robots_txt_file=True)
@@ -736,19 +744,19 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
     async def error_handler(context: BasicCrawlingContext, _error: Exception) -> None:
         fail(context.request.url)
 
-    await crawler.run([str(server_url / 'problematic_links')])
+    await crawler.run([start_url])
 
-    # Email must be skipped
-    # https://avatars.githubusercontent.com/apify does not get robots.txt, but is correct for the crawler.
+    # Email must be skipped.
+    # An origin without robots.txt is still crawled, an unavailable file means unrestricted crawling.
     expected_visit_calls = [
-        mock.call(str(server_url / 'problematic_links')),
-        mock.call('https://avatars.githubusercontent.com/apify'),
+        mock.call(start_url),
+        mock.call(no_robots_url),
     ]
     visit.assert_has_calls(expected_visit_calls, any_order=True)
 
-    # The budplaceholder.com does not exist.
+    # The unreachable URL cannot be connected to.
     expected_fail_calls = [
-        mock.call('https://budplaceholder.com/'),
+        mock.call(unreachable_url),
     ]
     fail.assert_has_calls(expected_fail_calls, any_order=True)
 

tests/unit/server.py

@@ -144,6 +144,15 @@ async def app(scope: dict[str, Any], receive: Receive, send: Send) -> None:
         await hello_world(scope, receive, send)
 
 
+async def no_robots_app(scope: dict[str, Any], receive: Receive, send: Send) -> None:
+    """ASGI application handler that behaves like `app`, except that it serves no robots.txt file."""
+    assert scope['type'] == 'http'
+    if scope['path'] == '/robots.txt':
+        await send_html_response(send, b'Not Found', status=404)
+        return
+    await app(scope, receive, send)
+
+
 async def get_cookies(scope: dict[str, Any], _receive: Receive, send: Send) -> None:
     """Handle requests to retrieve cookies sent in the request."""
     headers = get_headers_dict(scope)
@@ -302,11 +311,20 @@ async def generic_response_endpoint(_scope: dict[str, Any], _receive: Receive, s
     )
 
 
-async def problematic_links_endpoint(_scope: dict[str, Any], _receive: Receive, send: Send) -> None:
-    """Handle requests with a page containing problematic links."""
+async def problematic_links_endpoint(scope: dict[str, Any], _receive: Receive, send: Send) -> None:
+    """Handle requests with a page containing problematic links.
+
+    The links themselves are supplied by the caller through the `unreachable_url` and `no_robots_url` query
+    parameters, because they point at other test servers whose ports are only known at runtime.
+    """
+    query_params = get_query_params(scope.get('query_string', b''))
+    content = PROBLEMATIC_LINKS.format(
+        unreachable_url=query_params['unreachable_url'],
+        no_robots_url=query_params['no_robots_url'],
+    ).encode()
     await send_html_response(
         send,
-        PROBLEMATIC_LINKS,
+        content,
     )
 
 

tests/unit/server_endpoints.py

@@ -59,14 +59,14 @@
     </iframe>
 </body></html>"""
 
-PROBLEMATIC_LINKS = b"""\
+PROBLEMATIC_LINKS = """\
 <html><head>
     <title>Hello</title>
 </head>
 <body>
-    <a href="https://budplaceholder.com/">Placeholder</a>
+    <a href="{unreachable_url}">Unreachable</a>
     <a href="mailto:test@test.com">test@test.com</a>
-    <a href=https://avatars.githubusercontent.com/apify>Apify avatar/a>
+    <a href={no_robots_url}>No robots.txt/a>
 </body></html>"""
 
 NON_HREF_LINKS = b"""\