From 8cbc851954075d1c1955791f392dd45d9aed4669 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 10 Aug 2026 13:03:48 +0200 Subject: [PATCH] test: remove external network dependency from problematic-links tests --- tests/unit/conftest.py | 29 ++++++++++++++++++- .../test_beautifulsoup_crawler.py | 25 +++++++++++----- .../crawlers/_parsel/test_parsel_crawler.py | 25 +++++++++++----- .../_playwright/test_playwright_crawler.py | 24 ++++++++++----- tests/unit/server.py | 24 +++++++++++++-- tests/unit/server_endpoints.py | 6 ++-- 6 files changed, 102 insertions(+), 31 deletions(-) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 58040a1c57..0362b6253e 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -2,6 +2,7 @@ import logging import os +import socket import warnings from typing import TYPE_CHECKING, Any, cast @@ -18,7 +19,7 @@ from crawlee.proxy_configuration import ProxyInfo from crawlee.statistics import Statistics from crawlee.storages import KeyValueStore -from tests.unit.server import TestServer, app, serve_in_thread +from tests.unit.server import TestServer, app, no_robots_app, serve_in_thread if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable, Iterator @@ -174,6 +175,32 @@ def server_url(http_server: TestServer) -> URL: return http_server.url +@pytest.fixture(scope='session') +def no_robots_http_server(unused_tcp_port_factory: Callable[[], int]) -> Iterator[TestServer]: + """Create and start an HTTP test server that responds with 404 to robots.txt requests.""" + config = Config(app=no_robots_app, lifespan='off', loop='asyncio', port=unused_tcp_port_factory()) + server = TestServer(config=config) + yield from serve_in_thread(server) + + +@pytest.fixture(scope='session') +def no_robots_server_url(no_robots_http_server: TestServer) -> URL: + """Provide the base URL of the test server that has no robots.txt file.""" + return no_robots_http_server.url + + +@pytest.fixture(scope='session') +def unreachable_url() -> Iterator[str]: + """Provide a URL that can never be connected to. + + The port is bound for the whole session but never listened on, so nothing else can take it and every + connection attempt is refused immediately, without any DNS lookup or external traffic. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(('127.0.0.1', 0)) + yield f'http://127.0.0.1:{sock.getsockname()[1]}/' + + # It is needed only in some tests, so we use the standard `scope=function` @pytest.fixture def redirect_http_server(unused_tcp_port_factory: Callable[[], int]) -> Iterator[TestServer]: diff --git a/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py b/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py index 9a828b0078..261c45c1e4 100644 --- a/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py +++ b/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py @@ -201,8 +201,17 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: visit.assert_has_calls(expected_visit_calls, any_order=True) -async def test_respect_robots_txt_with_problematic_links(server_url: URL, http_client: HttpClient) -> None: +async def test_respect_robots_txt_with_problematic_links( + server_url: URL, + no_robots_server_url: URL, + unreachable_url: str, + http_client: HttpClient, +) -> None: """Test checks the crawler behavior with links that may cause problems when attempting to retrieve robots.txt.""" + no_robots_url = str(no_robots_server_url / 'page') + start_url = str( + (server_url / 'problematic_links').with_query(unreachable_url=unreachable_url, no_robots_url=no_robots_url) + ) visit = mock.Mock() fail = mock.Mock() crawler = BeautifulSoupCrawler( @@ -220,19 +229,19 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: async def error_handler(context: BasicCrawlingContext, _error: Exception) -> None: fail(context.request.url) - await crawler.run([str(server_url / 'problematic_links')]) + await crawler.run([start_url]) - # Email must be skipped - # https://avatars.githubusercontent.com/apify does not get robots.txt, but is correct for the crawler. + # Email must be skipped. + # An origin without robots.txt is still crawled, an unavailable file means unrestricted crawling. expected_visit_calls = [ - mock.call(str(server_url / 'problematic_links')), - mock.call('https://avatars.githubusercontent.com/apify'), + mock.call(start_url), + mock.call(no_robots_url), ] visit.assert_has_calls(expected_visit_calls, any_order=True) - # The budplaceholder.com does not exist. + # The unreachable URL cannot be connected to. expected_fail_calls = [ - mock.call('https://budplaceholder.com/'), + mock.call(unreachable_url), ] fail.assert_has_calls(expected_fail_calls, any_order=True) diff --git a/tests/unit/crawlers/_parsel/test_parsel_crawler.py b/tests/unit/crawlers/_parsel/test_parsel_crawler.py index 02f5b61a86..174b618465 100644 --- a/tests/unit/crawlers/_parsel/test_parsel_crawler.py +++ b/tests/unit/crawlers/_parsel/test_parsel_crawler.py @@ -285,8 +285,17 @@ async def request_handler(context: ParselCrawlingContext) -> None: visit.assert_has_calls(expected_visit_calls, any_order=True) -async def test_respect_robots_txt_with_problematic_links(server_url: URL, http_client: HttpClient) -> None: +async def test_respect_robots_txt_with_problematic_links( + server_url: URL, + no_robots_server_url: URL, + unreachable_url: str, + http_client: HttpClient, +) -> None: """Test checks the crawler behavior with links that may cause problems when attempting to retrieve robots.txt.""" + no_robots_url = str(no_robots_server_url / 'page') + start_url = str( + (server_url / 'problematic_links').with_query(unreachable_url=unreachable_url, no_robots_url=no_robots_url) + ) visit = mock.Mock() fail = mock.Mock() crawler = ParselCrawler( @@ -304,19 +313,19 @@ async def request_handler(context: ParselCrawlingContext) -> None: async def error_handler(context: BasicCrawlingContext, _error: Exception) -> None: fail(context.request.url) - await crawler.run([str(server_url / 'problematic_links')]) + await crawler.run([start_url]) - # Email must be skipped - # https://avatars.githubusercontent.com/apify does not get robots.txt, but is correct for the crawler. + # Email must be skipped. + # An origin without robots.txt is still crawled, an unavailable file means unrestricted crawling. expected_visit_calls = [ - mock.call(str(server_url / 'problematic_links')), - mock.call('https://avatars.githubusercontent.com/apify'), + mock.call(start_url), + mock.call(no_robots_url), ] visit.assert_has_calls(expected_visit_calls, any_order=True) - # The budplaceholder.com does not exist. + # The unreachable URL cannot be connected to. expected_fail_calls = [ - mock.call('https://budplaceholder.com/'), + mock.call(unreachable_url), ] fail.assert_has_calls(expected_fail_calls, any_order=True) diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index 140fda802d..cc2e474e22 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -721,8 +721,16 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: visit.assert_has_calls(expected_visit_calls, any_order=True) -async def test_respect_robots_txt_with_problematic_links(server_url: URL) -> None: +async def test_respect_robots_txt_with_problematic_links( + server_url: URL, + no_robots_server_url: URL, + unreachable_url: str, +) -> None: """Test checks the crawler behavior with links that may cause problems when attempting to retrieve robots.txt.""" + no_robots_url = str(no_robots_server_url / 'page') + start_url = str( + (server_url / 'problematic_links').with_query(unreachable_url=unreachable_url, no_robots_url=no_robots_url) + ) visit = mock.Mock() fail = mock.Mock() crawler = PlaywrightCrawler(respect_robots_txt_file=True) @@ -736,19 +744,19 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def error_handler(context: BasicCrawlingContext, _error: Exception) -> None: fail(context.request.url) - await crawler.run([str(server_url / 'problematic_links')]) + await crawler.run([start_url]) - # Email must be skipped - # https://avatars.githubusercontent.com/apify does not get robots.txt, but is correct for the crawler. + # Email must be skipped. + # An origin without robots.txt is still crawled, an unavailable file means unrestricted crawling. expected_visit_calls = [ - mock.call(str(server_url / 'problematic_links')), - mock.call('https://avatars.githubusercontent.com/apify'), + mock.call(start_url), + mock.call(no_robots_url), ] visit.assert_has_calls(expected_visit_calls, any_order=True) - # The budplaceholder.com does not exist. + # The unreachable URL cannot be connected to. expected_fail_calls = [ - mock.call('https://budplaceholder.com/'), + mock.call(unreachable_url), ] fail.assert_has_calls(expected_fail_calls, any_order=True) diff --git a/tests/unit/server.py b/tests/unit/server.py index 1bccab6308..c99daccd63 100644 --- a/tests/unit/server.py +++ b/tests/unit/server.py @@ -144,6 +144,15 @@ async def app(scope: dict[str, Any], receive: Receive, send: Send) -> None: await hello_world(scope, receive, send) +async def no_robots_app(scope: dict[str, Any], receive: Receive, send: Send) -> None: + """ASGI application handler that behaves like `app`, except that it serves no robots.txt file.""" + assert scope['type'] == 'http' + if scope['path'] == '/robots.txt': + await send_html_response(send, b'Not Found', status=404) + return + await app(scope, receive, send) + + async def get_cookies(scope: dict[str, Any], _receive: Receive, send: Send) -> None: """Handle requests to retrieve cookies sent in the request.""" headers = get_headers_dict(scope) @@ -302,11 +311,20 @@ async def generic_response_endpoint(_scope: dict[str, Any], _receive: Receive, s ) -async def problematic_links_endpoint(_scope: dict[str, Any], _receive: Receive, send: Send) -> None: - """Handle requests with a page containing problematic links.""" +async def problematic_links_endpoint(scope: dict[str, Any], _receive: Receive, send: Send) -> None: + """Handle requests with a page containing problematic links. + + The links themselves are supplied by the caller through the `unreachable_url` and `no_robots_url` query + parameters, because they point at other test servers whose ports are only known at runtime. + """ + query_params = get_query_params(scope.get('query_string', b'')) + content = PROBLEMATIC_LINKS.format( + unreachable_url=query_params['unreachable_url'], + no_robots_url=query_params['no_robots_url'], + ).encode() await send_html_response( send, - PROBLEMATIC_LINKS, + content, ) diff --git a/tests/unit/server_endpoints.py b/tests/unit/server_endpoints.py index b32d136524..e5d68ea67c 100644 --- a/tests/unit/server_endpoints.py +++ b/tests/unit/server_endpoints.py @@ -59,14 +59,14 @@ """ -PROBLEMATIC_LINKS = b"""\ +PROBLEMATIC_LINKS = """\ Hello - Placeholder + Unreachable test@test.com - Apify avatar/a> + No robots.txt/a> """ NON_HREF_LINKS = b"""\