fix(fingerprints): enforce the user agent allow list so screen constraints are respected (#2110)

`ScreenOptions` were not respected - with `strict=True` fingerprint
generation failed, otherwise the screen
constraints were silently dropped and the fingerprint got a screen
outside of the requested range.

`browserforge` turns the screen constraints into an allow list of user
agents and passes it to the header
generator, but the generator only derives browser, operating system and
device from it and then samples the
user agent freely - so the result is often outside the list, and the
fingerprint network finds no consistent
sample. `apify-fingerprint-datapoints` 0.14.0 made that deterministic
for some inputs, hence the cap in #2107.

`PatchedHeaderGenerator` now enforces the allow list: a user agent
outside it pins the next attempt to an
allowed one with the same browser, operating system and device. If
nothing is interchangeable or the attempts
run out, the wider header is returned, so inputs that worked before keep
working.

Measured on 0.14.0, before -> after:

| Case | Before | After |
| --- | --- | --- |
| `strict` Firefox + Windows, screen `600-1800 x 400-1200` | 299/300
fail, 480 ms each | 0/500 fail, 90 ms each |
| Firefox + Android, screen `300-600 x 500-1200` | ~40% of screens out
of range | 0/2000 out of range |

Also drops the `apify-fingerprint-datapoints<0.14.0` caps from #2107.

`test_fingerprint_generator_respects_screen_options_without_strict`
covers the non-strict path; it and
`test_fingerprint_generator_all_options` fail on `master` with the
0.14.0 datapoints.

Closes: #2108

*✍️ Drafted by Claude Code*

Author: vdusek

pyproject.toml

@@ -56,8 +56,7 @@ adaptive-crawler = [
     "jaro-winkler>=2.0.3",
     "playwright>=1.27.0",
     "scikit-learn>=1.6.0",
-    # TODO: Remove the upper bound, see #2108.
-    "apify_fingerprint_datapoints>=0.0.3,<0.14.0",
+    "apify_fingerprint_datapoints>=0.0.3",
     "browserforge>=1.2.4"
 ]
 pydantic-ai = ["pydantic-ai-slim[openai]>=2.1.0", "parsel>=1.10.0", "lxml[html_clean]>=5.2.0"]
@@ -70,10 +69,9 @@ cli = [
 ]
 # TODO: Remove the upper bound, see #2108.
 curl-impersonate = ["curl-cffi>=0.9.0,<0.16.0"]
-# TODO: Remove the upper bounds, see #2108.
-httpx = ["httpx[brotli,http2,zstd]>=0.27.0", "apify_fingerprint_datapoints>=0.0.2,<0.14.0", "browserforge>=1.2.3"]
+httpx = ["httpx[brotli,http2,zstd]>=0.27.0", "apify_fingerprint_datapoints>=0.0.2", "browserforge>=1.2.3"]
 parsel = ["parsel>=1.10.0"]
-playwright = ["playwright>=1.27.0", "apify_fingerprint_datapoints>=0.0.2,<0.14.0", "browserforge>=1.2.3"]
+playwright = ["playwright>=1.27.0", "apify_fingerprint_datapoints>=0.0.2", "browserforge>=1.2.3"]
 otel = [
     "opentelemetry-api>=1.34.1",
     "opentelemetry-distro[otlp]>=0.54",
@@ -90,8 +88,7 @@ sql_postgres = [
 stagehand = [
     "stagehand>=3.19.5",
     "playwright>=1.27.0",
-    # TODO: Remove the upper bound, see #2108.
-    "apify_fingerprint_datapoints>=0.0.2,<0.14.0",
+    "apify_fingerprint_datapoints>=0.0.2",
     "browserforge>=1.2.3",
 ]
 sql_sqlite = [

src/crawlee/fingerprint_suite/_browserforge_adapter.py

@@ -3,12 +3,12 @@
 import random
 from collections.abc import Iterable
 from copy import deepcopy
-from functools import reduce
+from functools import cache, reduce
 from operator import or_
 from typing import TYPE_CHECKING, Any, Literal
 
 import apify_fingerprint_datapoints
-from browserforge.bayesian_network import extract_json
+from browserforge.bayesian_network import extract_json, get_possible_values
 from browserforge.fingerprints import Fingerprint as bf_Fingerprint
 from browserforge.fingerprints import FingerprintGenerator as bf_FingerprintGenerator
 from browserforge.fingerprints import Screen
@@ -87,6 +87,17 @@ def generate(
             # headers without `sec-...` headers are valid.
             max_attempts += 50
 
+        # `browserforge` takes `user_agent` only as a hint - it derives browser, operating system and device from it,
+        # but samples the generated user agent freely. Keep the allow list to enforce it below.
+        allowed_user_agents = frozenset([user_agent] if isinstance(user_agent, str) else user_agent or [])
+
+        if allowed_user_agents:
+            # Pinning narrows the sampling to one browser version, but not to one operating system version.
+            max_attempts += 50
+
+        # Header satisfying everything but the allow list, used as a last resort below.
+        fallback_header: dict[str, str] | None = None
+
         # Use browserforge to generate headers until it satisfies our additional requirements.
         for _attempt in range(max_attempts):
             generated_header: dict[str, str] = super().generate(
@@ -114,7 +125,24 @@ def generate(
                     # Accept chromium header only with all sec headers.
                     continue
 
+                if allowed_user_agents and generated_header['User-Agent'] not in allowed_user_agents:
+                    pinned_user_agent = _pick_interchangeable_user_agent(
+                        generated_header['User-Agent'], allowed_user_agents
+                    )
+                    if pinned_user_agent is None:
+                        # Nothing interchangeable in the allow list, so no better header can be generated.
+                        return generated_header
+
+                    fallback_header = generated_header
+                    user_agent = [pinned_user_agent]
+                    continue
+
                 return generated_header
+
+        if fallback_header is not None:
+            # The allow list is only a preference, so a header satisfying everything else beats failing.
+            return fallback_header
+
         raise RuntimeError('Failed to generate header.')
 
     def _contains_all_sec_headers(self, headers: dict[str, str]) -> bool:
@@ -250,6 +278,44 @@ def generate(self, browser_type: SupportedBrowserType = 'chrome') -> dict[str, s
         return self._generator.generate(browser=[browser_type])
 
 
+def _pick_interchangeable_user_agent(generated_user_agent: str, allowed_user_agents: frozenset[str]) -> str | None:
+    """Pick a user agent from the allow list interchangeable with the generated one, `None` if there is none."""
+    generated_traits = _get_user_agent_traits(generated_user_agent)
+    if not all(generated_traits):
+        return None
+
+    candidates = [
+        allowed_user_agent
+        for allowed_user_agent in allowed_user_agents
+        if _get_user_agent_traits(allowed_user_agent) == generated_traits
+    ]
+    return random.choice(candidates) if candidates else None
+
+
+# Unbounded - only a few hundred user agents exist, and the default 128 entries would thrash on the allow list scan.
+@cache
+def _get_user_agent_traits(user_agent: str) -> tuple[frozenset[str], frozenset[str], frozenset[str]]:
+    """Get browser names, operating systems and devices the header network links to the `user_agent`.
+
+    `browserforge` derives the browser name and version, the operating system and the device from its `user_agent`
+    argument. Only the name is compared here, because the caller constrains the browser by name, so an allowed user
+    agent differing just in the browser version is a valid substitute.
+    """
+    possible_values: dict[str, Any] = {}
+    # The header network holds the user agent under a different node name for each HTTP version.
+    for node_name in ('user-agent', 'User-Agent'):
+        possible_values.update(
+            get_possible_values(bf_HeaderGenerator.header_generator_network, {node_name: (user_agent,)})
+        )
+
+    return (
+        # `*BROWSER` values are `{name}/{version}`, keep just the name.
+        frozenset(browser.split('/', maxsplit=1)[0] for browser in possible_values.get('*BROWSER', ())),
+        frozenset(possible_values.get('*OPERATING_SYSTEM', ())),
+        frozenset(possible_values.get('*DEVICE', ())),
+    )
+
+
 def get_available_header_network() -> dict:
     """Get header network that contains possible header values."""
     return extract_json(apify_fingerprint_datapoints.get_header_network())

tests/unit/fingerprint_suite/test_adapters.py

@@ -35,6 +35,30 @@ def test_fingerprint_generator_some_options_stress_test() -> None:
         assert fingerprint.screen.availWidth > 500
 
 
+def test_fingerprint_generator_respects_screen_options_without_strict() -> None:
+    """Test that screen constraints are respected without `strict`, where `browserforge` silently drops them."""
+    min_width = 300
+    max_width = 600
+    min_height = 500
+    max_height = 1200
+
+    fingerprint_generator = DefaultFingerprintGenerator(
+        header_options=HeaderGeneratorOptions(browsers=['firefox'], operating_systems=['android']),
+        screen_options=ScreenOptions(
+            min_width=min_width,
+            max_width=max_width,
+            min_height=min_height,
+            max_height=max_height,
+        ),
+    )
+
+    for _ in range(20):
+        fingerprint = fingerprint_generator.generate()
+
+        assert min_width <= fingerprint.screen.width <= max_width
+        assert min_height <= fingerprint.screen.height <= max_height
+
+
 def test_fingerprint_generator_all_options() -> None:
     """Test that header generator can work with all the options. Some most basic checks of fingerprint.