From 836f8d34497d97ef85f711ee95d8f85b84bc3722 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 3 Feb 2026 10:08:02 +0100 Subject: [PATCH] Use latest `browserforge` release and remove the workaround --- pyproject.toml | 4 +- src/crawlee/_browserforge_workaround.py | 46 ------------------- src/crawlee/browsers/__init__.py | 6 --- src/crawlee/fingerprint_suite/__init__.py | 7 --- .../_browserforge_adapter.py | 10 ++-- uv.lock | 11 +++-- 6 files changed, 11 insertions(+), 73 deletions(-) delete mode 100644 src/crawlee/_browserforge_workaround.py diff --git a/pyproject.toml b/pyproject.toml index 0950213e1c..e223c9edf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,8 +55,8 @@ adaptive-crawler = [ "jaro-winkler>=2.0.3", "playwright>=1.27.0", "scikit-learn>=1.6.0", - "apify_fingerprint_datapoints>=0.0.2", - "browserforge>=1.2.3" + "apify_fingerprint_datapoints>=0.0.3", + "browserforge>=1.2.4" ] beautifulsoup = ["beautifulsoup4[lxml]>=4.12.0", "html5lib>=1.0"] cli = ["cookiecutter>=2.6.0", "inquirer>=3.3.0", "rich>=13.9.0", "typer>=0.12.0"] diff --git a/src/crawlee/_browserforge_workaround.py b/src/crawlee/_browserforge_workaround.py deleted file mode 100644 index ed84e06baa..0000000000 --- a/src/crawlee/_browserforge_workaround.py +++ /dev/null @@ -1,46 +0,0 @@ -# ruff: noqa: N802, PLC0415 -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from collections.abc import Callable - - -def patch_browserforge() -> None: - """Patches `browserforge` to use data from `apify_fingerprint_datapoints`. - - This avoids import time or runtime file downloads. - """ - # Temporary fix until https://github.com/daijro/browserforge/pull/29 is merged - from pathlib import Path - - import apify_fingerprint_datapoints - from browserforge import download - - download.DATA_DIRS = { - 'headers': apify_fingerprint_datapoints.get_header_network().parent, - 'fingerprints': apify_fingerprint_datapoints.get_fingerprint_network().parent, - } - - def DownloadIfNotExists(**flags: bool) -> None: - pass - - download.DownloadIfNotExists: Callable[..., None] = DownloadIfNotExists - - import browserforge.bayesian_network - - class BayesianNetwork(browserforge.bayesian_network.BayesianNetwork): - def __init__(self, path: Path) -> None: - """Inverted mapping as browserforge expects somewhat renamed file names.""" - if path.name in download.DATA_FILES['headers']: - path = download.DATA_DIRS['headers'] / download.DATA_FILES['headers'][path.name] - else: - path = download.DATA_DIRS['fingerprints'] / download.DATA_FILES['fingerprints'][path.name] - super().__init__(path) - - browserforge.bayesian_network.BayesianNetwork: BayesianNetwork = BayesianNetwork - import browserforge.headers.generator - - browserforge.headers.generator.DATA_DIR = download.DATA_DIRS['headers'] - import browserforge.fingerprints.generator - - browserforge.headers.generator.DATA_DIR = download.DATA_DIRS['fingerprints'] diff --git a/src/crawlee/browsers/__init__.py b/src/crawlee/browsers/__init__.py index 50b29b9e3f..1d961d38e2 100644 --- a/src/crawlee/browsers/__init__.py +++ b/src/crawlee/browsers/__init__.py @@ -1,14 +1,8 @@ -# ruff: noqa: E402, TID252 - from crawlee._utils.try_import import install_import_hook as _install_import_hook from crawlee._utils.try_import import try_import as _try_import _install_import_hook(__name__) -# Due to patch_browserforge -from .._browserforge_workaround import patch_browserforge - -patch_browserforge() # The following imports are wrapped in try_import to handle optional dependencies, # ensuring the module can still function even if these dependencies are missing. diff --git a/src/crawlee/fingerprint_suite/__init__.py b/src/crawlee/fingerprint_suite/__init__.py index 6abdd0ebb6..4b604b1ff5 100644 --- a/src/crawlee/fingerprint_suite/__init__.py +++ b/src/crawlee/fingerprint_suite/__init__.py @@ -1,10 +1,3 @@ -# ruff: noqa: E402, TID252 - -# Due to patch_browserforge -from .._browserforge_workaround import patch_browserforge - -patch_browserforge() - from ._browserforge_adapter import BrowserforgeFingerprintGenerator as DefaultFingerprintGenerator from ._fingerprint_generator import FingerprintGenerator from ._header_generator import HeaderGenerator diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index 98460abb00..0fb3d917ba 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -5,15 +5,15 @@ from copy import deepcopy from functools import reduce from operator import or_ -from pathlib import Path from typing import TYPE_CHECKING, Any, Literal +import apify_fingerprint_datapoints from browserforge.bayesian_network import extract_json from browserforge.fingerprints import Fingerprint as bf_Fingerprint from browserforge.fingerprints import FingerprintGenerator as bf_FingerprintGenerator from browserforge.fingerprints import Screen -from browserforge.headers.generator import DATA_DIR, ListOrString from browserforge.headers.generator import HeaderGenerator as bf_HeaderGenerator +from browserforge.headers.generator import ListOrString from typing_extensions import override from crawlee._utils.docs import docs_group @@ -252,11 +252,7 @@ def generate(self, browser_type: SupportedBrowserType = 'chrome') -> dict[str, s def get_available_header_network() -> dict: """Get header network that contains possible header values.""" - if Path(DATA_DIR / 'header-network.zip').is_file(): - return extract_json(DATA_DIR / 'header-network.zip') - if Path(DATA_DIR / 'header-network-definition.zip').is_file(): - return extract_json(DATA_DIR / 'header-network-definition.zip') - raise FileNotFoundError('Missing header-network file.') + return extract_json(apify_fingerprint_datapoints.get_header_network()) def get_available_header_values(header_network: dict, node_name: str | set[str]) -> set[str]: diff --git a/uv.lock b/uv.lock index 3444fcd899..b417687e87 100644 --- a/uv.lock +++ b/uv.lock @@ -324,14 +324,15 @@ wheels = [ [[package]] name = "browserforge" -version = "1.2.3" +version = "1.2.4" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "apify-fingerprint-datapoints" }, { name = "click" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/df/5c/fe4d8cc5d5e61a5b1585190bba19d25bb76c45fdfe9c7bf264f5301fcf33/browserforge-1.2.3.tar.gz", hash = "sha256:d5bec6dffd4748b30fbac9f9c1ef33b26c01a23185240bf90011843e174b7ecc", size = 38072, upload-time = "2025-01-29T09:45:48.711Z" } +sdist = { url = "https://files.pythonhosted.org/packages/78/6f/8975af88d203efd70cc69477ebac702babef38201d04621c9583f2508f25/browserforge-1.2.4.tar.gz", hash = "sha256:05686473793769856ebd3528c69071f5be0e511260993e8b2ba839863711a0c4", size = 36700, upload-time = "2026-02-03T02:52:09.721Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/53/c60eb5bd26cf8689e361031bebc431437bc988555e80ba52d48c12c1d866/browserforge-1.2.3-py3-none-any.whl", hash = "sha256:a6c71ed4688b2f1b0bee757ca82ddad0007cbba68a71eca66ca607dde382f132", size = 39626, upload-time = "2025-01-29T09:45:47.531Z" }, + { url = "https://files.pythonhosted.org/packages/dd/35/ce962f738ae28ffce6293e7607b129075633e6bb185a5ab87e49246eedc2/browserforge-1.2.4-py3-none-any.whl", hash = "sha256:fb1c14e62ac09de221dcfc73074200269f697596c642cb200ceaab1127a17542", size = 37890, upload-time = "2026-02-03T02:52:08.745Z" }, ] [[package]] @@ -835,13 +836,13 @@ dev = [ [package.metadata] requires-dist = [ { name = "aiosqlite", marker = "extra == 'sql-sqlite'", specifier = ">=0.21.0" }, - { name = "apify-fingerprint-datapoints", marker = "extra == 'adaptive-crawler'", specifier = ">=0.0.2" }, + { name = "apify-fingerprint-datapoints", marker = "extra == 'adaptive-crawler'", specifier = ">=0.0.3" }, { name = "apify-fingerprint-datapoints", marker = "extra == 'httpx'", specifier = ">=0.0.2" }, { name = "apify-fingerprint-datapoints", marker = "extra == 'playwright'", specifier = ">=0.0.2" }, { name = "async-timeout", specifier = ">=5.0.1" }, { name = "asyncpg", marker = "extra == 'sql-postgres'", specifier = ">=0.24.0" }, { name = "beautifulsoup4", extras = ["lxml"], marker = "extra == 'beautifulsoup'", specifier = ">=4.12.0" }, - { name = "browserforge", marker = "extra == 'adaptive-crawler'", specifier = ">=1.2.3" }, + { name = "browserforge", marker = "extra == 'adaptive-crawler'", specifier = ">=1.2.4" }, { name = "browserforge", marker = "extra == 'httpx'", specifier = ">=1.2.3" }, { name = "browserforge", marker = "extra == 'playwright'", specifier = ">=1.2.3" }, { name = "cachetools", specifier = ">=5.5.0" },