-
-
+
+
-
+
A web scraping and browser automation library
-
-
+
+
-
-
-
-
-
+
+
+
+
+
Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
diff --git a/biome.json b/biome.json
deleted file mode 100644
index 8c23acdb00cb..000000000000
--- a/biome.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "formatter": {
- "includes": [
- "**",
- "!**/website/**",
- "!**/packages/**/*/dist/**",
- "!**/package.json",
- "!**/lerna.json",
- "!**/scripts/actions/docker-images/state.json"
- ],
- "formatWithErrors": true
- },
- "javascript": {
- "formatter": {
- "quoteStyle": "single",
- "semicolons": "always",
- "trailingCommas": "all",
- "lineWidth": 120,
- "indentStyle": "space",
- "indentWidth": 4,
- "quoteProperties": "preserve",
- "lineEnding": "lf"
- }
- },
- "linter": {
- "enabled": false
- }
-}
diff --git a/docs/examples/file_download.ts b/docs/examples/file_download.ts
index a6b42555e9ba..4ec682ea7002 100644
--- a/docs/examples/file_download.ts
+++ b/docs/examples/file_download.ts
@@ -2,11 +2,11 @@ import { FileDownload } from 'crawlee';
// Create a FileDownload - a custom crawler instance that will download files from URLs.
const crawler = new FileDownload({
- async requestHandler({ body, request, contentType, getKeyValueStore }) {
+ async requestHandler({ request, response, contentType, getKeyValueStore }) {
const url = new URL(request.url);
const kvs = await getKeyValueStore();
- await kvs.setValue(url.pathname.replace(/\//g, '_'), body, { contentType: contentType.type });
+ await kvs.setValue(url.pathname.replace(/\//g, '_'), response.body, { contentType: contentType.type });
},
});
diff --git a/docs/examples/file_download_stream.ts b/docs/examples/file_download_stream.ts
index a7f39a70f59a..9517531b5bd2 100644
--- a/docs/examples/file_download_stream.ts
+++ b/docs/examples/file_download_stream.ts
@@ -1,9 +1,9 @@
-import { pipeline, Transform } from 'stream';
+import { pipeline, Transform } from 'node:stream';
-import { FileDownload, type Log } from 'crawlee';
+import { FileDownload, type CrawleeLogger } from 'crawlee';
// A sample Transform stream logging the download progress.
-function createProgressTracker({ url, log, totalBytes }: { url: URL; log: Log; totalBytes: number }) {
+function createProgressTracker({ url, log, totalBytes }: { url: URL; log: CrawleeLogger; totalBytes: number }) {
let downloadedBytes = 0;
return new Transform({
@@ -23,32 +23,27 @@ function createProgressTracker({ url, log, totalBytes }: { url: URL; log: Log; t
// Create a FileDownload - a custom crawler instance that will download files from URLs.
const crawler = new FileDownload({
- async streamHandler({ stream, request, log, getKeyValueStore }) {
+ async requestHandler({ response, request, log, getKeyValueStore }) {
const url = new URL(request.url);
log.info(`Downloading ${url} to ${url.pathname.replace(/\//g, '_')}...`);
- await new Promise((resolve, reject) => {
- // With the 'response' event, we have received the headers of the response.
- stream.on('response', async (response) => {
- const kvs = await getKeyValueStore();
- await kvs.setValue(
- url.pathname.replace(/\//g, '_'),
- pipeline(
- stream,
- createProgressTracker({ url, log, totalBytes: Number(response.headers['content-length']) }),
- (error) => {
- if (error) reject(error);
- },
- ),
- { contentType: response.headers['content-type'] },
- );
-
- log.info(`Downloaded ${url} to ${url.pathname.replace(/\//g, '_')}.`);
-
- resolve();
- });
- });
+ if (!response.body) return;
+
+ const kvs = await getKeyValueStore();
+ await kvs.setValue(
+ url.pathname.replace(/\//g, '_'),
+ pipeline(
+ response.body,
+ createProgressTracker({ url, log, totalBytes: Number(response.headers.get('content-length')) }),
+ (error) => {
+ if (error) log.error(`Failed to download ${url}: ${error.message}`);
+ },
+ ),
+ response.headers.get('content-type') ? { contentType: response.headers.get('content-type')! } : {},
+ );
+
+ log.info(`Downloaded ${url} to ${url.pathname.replace(/\//g, '_')}.`);
},
});
diff --git a/docs/examples/skip-navigation.ts b/docs/examples/skip-navigation.ts
index 0bbde53c1375..867fb473271e 100644
--- a/docs/examples/skip-navigation.ts
+++ b/docs/examples/skip-navigation.ts
@@ -1,17 +1,22 @@
import { PlaywrightCrawler, KeyValueStore } from 'crawlee';
// Create a key value store for all images we find
-const imageStore = await KeyValueStore.open('images');
+const imageStore = await KeyValueStore.open({ name: 'images' });
const crawler = new PlaywrightCrawler({
async requestHandler({ request, page, sendRequest }) {
// The request should have the navigation skipped
if (request.skipNavigation) {
// Request the image and get its buffer back
- const imageResponse = await sendRequest({ responseType: 'buffer' });
-
- // Save the image in the key-value store
- await imageStore.setValue(`${request.userData.key}.png`, imageResponse.body);
+ const imageResponse = await sendRequest();
+
+ // Saves the image in the key-value store.
+ //
+ // Note: For large-scale file downloads, consider using FileDownload crawler:
+ // https://crawlee.dev/js/api/http-crawler/class/FileDownload
+ await imageStore.setValue(`${request.userData.key}.svg`, await imageResponse.bytes(), {
+ contentType: 'image/svg+xml',
+ });
// Prevent executing the rest of the code as we do not need it
return;
diff --git a/docs/experiments/systemInfoV2.mdx b/docs/experiments/systemInfoV2.mdx
deleted file mode 100644
index 93f8f27e1afe..000000000000
--- a/docs/experiments/systemInfoV2.mdx
+++ /dev/null
@@ -1,95 +0,0 @@
----
-id: experiments-system-infomation-v2
-title: System Infomation V2
-description: Improved autoscaling through cgroup aware metric collection.
----
-
-import ApiLink from '@site/src/components/ApiLink';
-
-:::caution
-
-This is an experimental feature. While we welcome testers, keep in mind that it is currently not recommended to use this in production.
-
-The API is subject to change, and we might introduce breaking changes in the future.
-
-Should you be using this, feel free to open issues on our [GitHub repository](https://github.com/apify/crawlee), and we'll take a look.
-
-:::
-
-Starting with the newest `crawlee` beta, we have introduced a new crawler option that enables an improved metric collection system.
-This new system should collect cpu and memory metrics more accurately in containerised environments by checking for cgroup enforce limits.
-
-## How to enable the experiment
-
-:::note
-
-This example shows how to enable the experiment in the `CheerioCrawler`,
-but you can apply this to any crawler type.
-
-:::
-
-```ts
-import { CheerioCrawler, Configuration } from 'crawlee';
-
-Configuration.set('systemInfoV2', true);
-
-const crawler = new CheerioCrawler({
- async requestHandler({ $, request }) {
- const title = $('title').text();
- console.log(`The title of "${request.url}" is: ${title}.`);
- },
-});
-
-await crawler.run(['https://crawlee.dev']);
-```
-
-## Other changes
-
-:::info
-
-This section is only useful if you're a tinkerer and want to see what's going on under the hood.
-
-:::
-
-The existing solution checked the bare metal metrics for how much cpu and memory was being used and how much headroom was available.
-This is an intuitive solution but unfortunately doesnt account for when there is an external limit on the amount of resources a process can consume.
-This is often the case in containerized environments where each container will have a quota for its cpu and memory usage.
-
-This experiment attempts to address this issue by introducing a new `isContainerized()` utility function and changing the way resources are collected
-when a container is detected.
-
-:::note
-
-This `isContainerized()` function is very similar to the existing `isDocker()` function however for now they both work side by side.
-If this experiment is successful, eventualy `isDocker()` may eventually be depreciated in favour of `isContainerized()`.
-
-:::
-
-### Cgroup detection
-
-On linux, to detect if cgroup is available, we check if there is a directory at `/sys/fs/cgroup`.
-If the directory exists, a version of cgroup is installed.
-Next we check the version of cgroup installed by checking for a directory at `/sys/fs/cgroup/memory/`.
-If it exists, cgroup V1 is installed. If it is missing, it is assumed cgroup V2 is installed.
-
-### CPU metric collection
-
-The existing solution worked by checking the fraction of cpu idle ticks to the total number of cpu ticks since the last profile.
-If 100000 ticks elapse and 5000 were idle, the cpu is at 95% utilisation.
-
-In this experiment, the method of cpu load calculation depends on the result of `isContainerized()` or if set, the `CRAWLEE_CONTAINERIZED` environment variable.
-If `isContainerized()` returns true, the new cgroup aware metric collection will be used over the "bare metal" numbers.
-This works by inspecting the `/sys/fs/cgroup/cpuacct/cpuacct.usage`, `/sys/fs/cgroup/cpu/cpu.cfs_quota_us` and `/sys/fs/cgroup/cpu/cpu.cfs_period_us`
-files for cgroup V1 and the `/sys/fs/cgroup/cpu.stat` and `/sys/fs/cgroup/cpu.max` files for cgroup V2.
-The actual cpu usage figure is calculated in the same manner as the "bare metal" figure by comparing the total number of ticks elapsed to the number
-of idle ticks between profiles but by using the figures from the cgroup files.
-If no cgroup quota is enforced, the "bare metal" numbers will be used.
-
-### Memory metric collection
-
-The existing solution was already cgroup aware however an improvement has been made to memory metric collection when running on windows.
-The existing solution used an external package `apify/ps-tree` to find the amount of memory crawlee and any child processes were using.
-On Windows, this package used the depreciated "WMIC" command line utility to determine memory usage.
-
-In this experiment, `apify/ps-tree` has been removed and replaced by the `packages/utils/src/internals/ps-tree.ts` file. This works in much the
-same manner however, instead of using "WMIC", it uses "powershell" to collect the same data.
\ No newline at end of file
diff --git a/docs/guides/avoid_blocking.mdx b/docs/guides/avoid_blocking.mdx
index ed10846f51e6..65cd2fc955ae 100644
--- a/docs/guides/avoid_blocking.mdx
+++ b/docs/guides/avoid_blocking.mdx
@@ -4,6 +4,8 @@ title: Avoid getting blocked
description: How to avoid getting blocked when scraping
---
+import ApiLink from '@site/src/components/ApiLink';
+
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import CodeBlock from '@theme/CodeBlock';
@@ -18,6 +20,8 @@ A scraper might get blocked for numerous reasons. Let's narrow it down to the tw
Browser fingerprint is a collection of browser attributes and significant features that can show if our browser is a bot or a real user. Moreover, most browsers have these unique features that allow the website to track the browser even within different IP addresses. This is the main reason why scrapers should change browser fingerprints while doing browser-based scraping. In return, it should significantly reduce the blocking.
+The two are not handled separately. In Crawlee a `Session` ties an IP, a cookie jar, and a fingerprint together into one consistent identity, and the `SessionPool` rotates those identities as a unit — so a fresh fingerprint always arrives with a fresh IP. This guide covers the fingerprint half; see the [session management guide](./session-management) for how to control the rotation, and the [proxy management guide](./proxy-management) for the IP half.
+
## Using browser fingerprints
Changing browser fingerprints can be a tedious job. Luckily, Crawlee provides this feature with zero configuration necessary - the usage of fingerprints is enabled by default and available in `PlaywrightCrawler` and `PuppeteerCrawler`. So whenever we build a scraper that is using one of these crawlers - the fingerprints are going to be generated for the default browser and the operating system out of the box.
@@ -56,9 +60,38 @@ On the contrary, sometimes we want to entirely disable the usage of browser fing
+## Fingerprints for HTTP crawlers
+
+Every session carries a lightweight fingerprint hint — a `browser`, `platform`, and `device` triple — that the request's HTTP client receives and applies on a best-effort basis.
+By default each session is given a realistic, randomized fingerprint (the host operating system as `platform`, with a plausible `browser`/`device` for it), and it rotates with the session just like the IP and cookies do.
+
+How much of the hint is used depends on the client. The [`impit`](impit-http-client) HTTP client maps the session's `browser` hint to a matching TLS and HTTP impersonation profile,
+so the connection's low-level signature lines up with the headers being sent.
+
+The *same* hint also drives browser crawlers, where it seeds the generated browser fingerprint. The hint only fixes the broad strokes — the browser family, operating system, and device — so a session presents a coherent profile, but it does not make the two backends produce byte-identical fingerprints: `impit` and a real browser will still differ in the finer details (a slightly different user-agent string, for example).
+
+You can pin the fingerprint explicitly through `sessionOptions` when you need a specific profile:
+
+```js
+import { CheerioCrawler, SessionPool } from 'crawlee';
+import { ImpitHttpClient } from '@crawlee/impit-client';
+
+const crawler = new CheerioCrawler({
+ httpClient: new ImpitHttpClient(),
+ sessionPool: new SessionPool({
+ sessionOptions: {
+ fingerprint: { browser: 'firefox', platform: 'windows', device: 'desktop' },
+ },
+ }),
+ requestHandler: async ({ $ }) => {
+ // requests impersonate desktop Firefox on Windows
+ },
+});
+```
+
## Camoufox
-For some protections, using our integrated solutions is not enough, one example could be the Cloudflare challenge. For such pages, you can try [Camoufox](https://camoufox.com/), a custom stealthy build of Firefox for web scraping. It might not get you through the challenge automatically, but with our `handleCloudflareChallenge` helper, it should be able to successfully mimic the required user action and get you through it.
+For some protections, using our integrated solutions is not enough, one example could be the Cloudflare challenge. For such pages, you can try [Camoufox](https://camoufox.com/), a custom stealthy build of Firefox for web scraping. It might not get you through the challenge automatically, but with our `handleCloudflareChallengeHook` post-navigation hook, it should be able to successfully mimic the required user action and get you through it. The hook also reloads the page after the challenge clears and propagates the fresh response back into the crawling context.
{PlaywrightCamoufox}
diff --git a/docs/guides/avoid_blocking_camoufox.ts b/docs/guides/avoid_blocking_camoufox.ts
index 131234578e62..01b8f8ca4dff 100644
--- a/docs/guides/avoid_blocking_camoufox.ts
+++ b/docs/guides/avoid_blocking_camoufox.ts
@@ -1,13 +1,9 @@
-import { PlaywrightCrawler } from 'crawlee';
+import { PlaywrightCrawler, handleCloudflareChallengeHook } from 'crawlee';
import { launchOptions } from 'camoufox-js';
import { firefox } from 'playwright';
const crawler = new PlaywrightCrawler({
- postNavigationHooks: [
- async ({ handleCloudflareChallenge }) => {
- await handleCloudflareChallenge();
- },
- ],
+ postNavigationHooks: [handleCloudflareChallengeHook()],
browserPoolOptions: {
// Disable the default fingerprint spoofing to avoid conflicts with Camoufox.
useFingerprints: false,
diff --git a/docs/guides/configuration.mdx b/docs/guides/configuration.mdx
index 597c3dcc2fa4..b93727c44c9e 100644
--- a/docs/guides/configuration.mdx
+++ b/docs/guides/configuration.mdx
@@ -15,13 +15,13 @@ There are three ways of changing the configuration parameters:
- using the `Configuration` class
You could also combine all the above, but you should keep in mind, that the precedence for these 3 options is the following:
-***`crawlee.json`*** < ***constructor options*** < ***environment variables***.
+***constructor options*** > ***environment variables*** > ***`crawlee.json`***.
-`crawlee.json` is a baseline. The options provided in the `Configuration` constructor will override the options provided in the JSON. Environment variables will override both.
+Constructor options have the highest priority. Environment variables override `crawlee.json`. The JSON file serves as a baseline.
## `crawlee.json`
-The first option you could use for configuring Crawlee is `crawlee.json` file. The only thing you need to do is specify the `ConfigurationOptions` in the file, place the file in the root of your project, and Crawlee will use provided options as global configuration.
+The first option you could use for configuring Crawlee is `crawlee.json` file. The only thing you need to do is specify the configuration options in the file, place the file in the root of your project, and Crawlee will use provided options as global configuration. See the `Configuration` class for the full list of supported options.
```json title="crawlee.json"
{
@@ -57,7 +57,7 @@ crawler.router.addDefaultHandler(async ({ request }) => {
await crawler.run(['https://www.example.com/1']);
```
-If you run this example (assuming you placed the `crawlee.json` file with `persistStateIntervalMillis` and `logLevel` specified there in the root of your project), you will find the `SDK_CRAWLER_STATISTICS` file in default Key-Value store,
+If you run this example (assuming you placed the `crawlee.json` file with `persistStateIntervalMillis` and `logLevel` specified there in the root of your project), you will find the `CRAWLEE_CRAWLER_STATISTICS` file in default Key-Value store,
which would show, that there's 1 finished request and crawler runtime was ~10 seconds.
This confirms that the state was persisted after 10 seconds, as it was set in `crawlee.json`.
Besides, you should see `DEBUG` logs in addition to `INFO` ones in your terminal, as `logLevel` was set to `DEBUG` in the `crawlee.json`, meaning Crawlee picked both provided options correctly.
@@ -94,7 +94,6 @@ Storage directories are purged by default. If set to `false` - local storage dir
#### `CRAWLEE_CONTAINERIZED`
-This variable is only effective when the systemInfoV2 experiment is enabled.
Changes how crawlee measures its CPU and Memory usage and limits. If unset, crawlee will determine if it is containerised using common features of containerized environments using the `isContainerized` utility function.
- A file at `/.dockerenv`.
- A file at `/proc/self/cgroup` containing `docker`.
@@ -134,24 +133,28 @@ the autoscaling feature will only use up to 2048 MB of memory.
## Configuration class
-The last option to adjust Crawlee configuration is to use the `Configuration` class in the code.
+The last option to adjust Crawlee configuration is to use the `Configuration` class in the code. Configuration is immutable — values are set via the constructor and cannot be changed afterwards.
### Global Configuration
-By default, there is a global singleton instance of `Configuration` class, it is used by the crawlers and some other classes that depend on a configurable behavior. In most cases you don't need to adjust any options there, but if needed - you can get access to it via `Configuration.getGlobalConfig()` function. Now you can easily `get` and `set` the `ConfigurationOptions`.
+By default, there is a global singleton instance of `Configuration` class, it is used by the crawlers and some other classes that depend on a configurable behavior. In most cases you don't need to adjust any options there, but if needed - you can access it via `Configuration.getGlobalConfig()`, which delegates to the global `serviceLocator` — the single source of truth for Crawlee's shared services (for example the configuration, event manager, storage backend, and logger). You can also reach the same instance directly via `serviceLocator.getConfiguration()` or swap services globally with `serviceLocator.setConfiguration(...)` before any crawler is created. Configuration values are accessible directly as properties on the instance.
```js
import { CheerioCrawler, Configuration, sleep } from 'crawlee';
// Get the global configuration
const config = Configuration.getGlobalConfig();
-// Set the 'persistStateIntervalMillis' option
-// of global configuration to 10 seconds
-config.set('persistStateIntervalMillis', 10_000);
+// Access configuration values directly as properties
+console.log(config.persistStateIntervalMillis);
-// Note, that we are not passing the configuration to the crawler
-// as it's using the global configuration
-const crawler = new CheerioCrawler();
+// To use custom configuration values, create a new Configuration instance
+const configuration = new Configuration({
+ // Set the 'persistStateIntervalMillis' option to 10 seconds
+ persistStateIntervalMillis: 10_000,
+});
+
+// Pass the configuration to the crawler
+const crawler = new CheerioCrawler({ configuration });
crawler.router.addDefaultHandler(async ({ request }) => {
// For the first request we wait for 5 seconds,
@@ -171,16 +174,14 @@ crawler.router.addDefaultHandler(async ({ request }) => {
await crawler.run(['https://www.example.com/1']);
```
-This is pretty much the same example we used for showing `crawlee.json` usage,
-but now we're using the global configuration, which is the only difference.
-If you run this example - you will find the `SDK_CRAWLER_STATISTICS` file in default Key-Value store as before,
-which would show the same number of finishes requests (one) and the same crawler runtime (~10 seconds).
-This confirms that provided parameters worked: the state was persisted after 10 seconds, as it was set in the global configuration.
+If you run this example - you will find the `CRAWLEE_CRAWLER_STATISTICS` file in default Key-Value store,
+which would show the same number of finished requests (one) and the same crawler runtime (~10 seconds).
+This confirms that provided parameters worked: the state was persisted after 10 seconds, as it was set in the configuration.
:::note
-After running the same example with commented two lines of code related to `Configuration` there will be
-no `SDK_CRAWLER_STATISTICS` file stored in the default Key-Value store:
+After running the same example without the custom configuration, there will be
+no `CRAWLEE_CRAWLER_STATISTICS` file stored in the default Key-Value store:
as we did not change the `persistStateIntervalMillis`, Crawlee used the default value of 60 seconds,
and the crawler was forcefully aborted after ~15 seconds of run time before it persisted the state for the first time.
@@ -188,19 +189,19 @@ and the crawler was forcefully aborted after ~15 seconds of run time before it p
### Custom configuration
-Alternatively, you can create a custom configuration. In this case you need to pass it to the class that is going to use it, e.g. to the crawler. Let's adjust the previous example:
+You can create a custom configuration and pass it to the crawler via the `configuration` option:
```js
import { CheerioCrawler, Configuration, sleep } from 'crawlee';
// Create new configuration
-const config = new Configuration({
+const configuration = new Configuration({
// Set the 'persistStateIntervalMillis' option to 10 seconds
persistStateIntervalMillis: 10_000,
});
-// Now we need to pass the configuration to the crawler
-const crawler = new CheerioCrawler({}, config);
+// Pass the configuration to the crawler
+const crawler = new CheerioCrawler({ configuration });
crawler.router.addDefaultHandler(async ({ request }) => {
// for the first request we wait for 5 seconds,
@@ -221,13 +222,13 @@ await crawler.run(['https://www.example.com/1']);
```
If you run this example - it would work exactly the same as before,
-with the same `SDK_CRAWLER_STATISTICS` file in default Key-Value store after the run,
+with the same `CRAWLEE_CRAWLER_STATISTICS` file in default Key-Value store after the run,
showing the same number of finished requests and the same crawler run time.
:::note
If you would not pass the configuration to the crawler, there again will be
-no `SDK_CRAWLER_STATISTICS` file stored in the default Key-Value store, this time for a different reason though.
+no `CRAWLEE_CRAWLER_STATISTICS` file stored in the default Key-Value store, this time for a different reason though.
Since we did not pass the configuration to the crawler,
the crawler will use the global configuration, which is using the default `persistStateIntervalMillis`.
So again, the run was aborted before the state was persisted for the first time.
diff --git a/docs/guides/custom-http-client/custom-http-client.mdx b/docs/guides/custom-http-client/custom-http-client.mdx
index c593ec3ba239..4e1b9f04c010 100644
--- a/docs/guides/custom-http-client/custom-http-client.mdx
+++ b/docs/guides/custom-http-client/custom-http-client.mdx
@@ -10,14 +10,34 @@ import CodeBlock from '@theme/CodeBlock';
import ImplementationSource from '!!raw-loader!./implementation.ts';
import UsageSource from '!!raw-loader!./usage.ts';
-The `BasicCrawler` class allows you to configure the HTTP client implementation using the `httpClient` constructor option. This might be useful for testing or if you need to swap out the default implementation based on `got-scraping` for something else, such as `curl-impersonate` or `axios`.
+The `BasicCrawler` class allows you to configure the HTTP client implementation using the `httpClient` constructor option. This might be useful for testing or if you need to swap out the default implementation based on `got-scraping` for something else, such as `curl-impersonate`.
-The HTTP client implementation needs to conform to the `BaseHttpClient` interface. For a rough idea on how it might look, see a skeleton implementation that uses the standard `fetch` interface:
+## Built-in HTTP clients
+
+Crawlee provides several HTTP client implementations out of the box:
+
+- **`ImpitHttpClient`** (default) - Uses the `impit` library for making requests that closely mimic browser behavior.
+- **`GotScrapingHttpClient`** - Uses the `got-scraping` library for browser-like requests with support for custom headers, browser fingerprints, and proxies. This was the default HTTP client in Crawlee v3.
+- **`FetchHttpClient`** - Simple implementation using the native `fetch` API (does not support proxies).
+
+## Implementing a custom HTTP client
+
+To create a custom HTTP client, extend the `BaseHttpClient` abstract class from `@crawlee/http-client`. The base class handles common functionality like cookie management, redirect following, session integration, proxy support, and timeout handling.
+
+Your custom implementation only needs to override the `fetch` method to perform the actual network request:
{ImplementationSource}
+By extending `BaseHttpClient`, your implementation automatically gets:
+- Cookie jar management (applying cookies before requests, saving cookies from responses)
+- Automatic redirect following (up to 10 redirects)
+- Session integration (proxy URL and cookies from session)
+- Timeout handling via AbortSignal
+- Proxy URL support
+
You may then instantiate it and pass to a crawler constructor:
{UsageSource}
-Please note that the interface is experimental and it will likely change with Crawlee version 4.
+Alternatively, you can implement the `BaseHttpClient` interface directly if you need full control over all aspects of the HTTP request handling, including cookies, redirects, and sessions. However, this approach requires implementing significantly more logic yourself.
+
diff --git a/docs/guides/custom-http-client/implementation.ts b/docs/guides/custom-http-client/implementation.ts
index 504f0b532f98..aac71784ff7e 100644
--- a/docs/guides/custom-http-client/implementation.ts
+++ b/docs/guides/custom-http-client/implementation.ts
@@ -1,122 +1,14 @@
-import type {
- BaseHttpClient,
- HttpRequest,
- HttpResponse,
- RedirectHandler,
- ResponseTypes,
- StreamingHttpResponse,
-} from '@crawlee/core';
-import { Readable } from 'node:stream';
-
-export class CustomHttpClient implements BaseHttpClient {
- async sendRequest(
- request: HttpRequest,
- ): Promise> {
- const requestHeaders = new Headers();
- for (let [headerName, headerValues] of Object.entries(request.headers ?? {})) {
- if (headerValues === undefined) {
- continue;
- }
-
- if (!Array.isArray(headerValues)) {
- headerValues = [headerValues];
- }
-
- for (const value of headerValues) {
- requestHeaders.append(headerName, value);
- }
- }
-
- const response = await fetch(request.url, {
- method: request.method,
- headers: requestHeaders,
- body: request.body as string, // TODO implement stream/generator handling
- signal: request.signal,
- // TODO implement the rest of request parameters (e.g., timeout, proxyUrl, cookieJar, ...)
- });
-
- const headers: Record = {};
-
- response.headers.forEach((value, headerName) => {
- headers[headerName] = value;
- });
-
- return {
- complete: true,
- request,
- url: response.url,
- statusCode: response.status,
- redirectUrls: [], // TODO you need to handle redirects manually to track them
- headers,
- trailers: {}, // TODO not supported by fetch
- ip: undefined,
- body:
- request.responseType === 'text'
- ? await response.text()
- : request.responseType === 'json'
- ? await response.json()
- : Buffer.from(await response.text()),
- };
- }
-
- async stream(request: HttpRequest, _onRedirect?: RedirectHandler): Promise {
- const fetchResponse = await fetch(request.url, {
- method: request.method,
- headers: new Headers(),
- body: request.body as string, // TODO implement stream/generator handling
- signal: request.signal,
- // TODO implement the rest of request parameters (e.g., timeout, proxyUrl, cookieJar, ...)
- });
-
- const headers: Record = {}; // TODO same as in sendRequest()
-
- async function* read() {
- const reader = fetchResponse.body?.getReader();
-
- const stream = new ReadableStream({
- start(controller) {
- if (!reader) {
- return null;
- }
- return pump();
- function pump(): Promise {
- return reader!.read().then(({ done, value }) => {
- // When no more data needs to be consumed, close the stream
- if (done) {
- controller.close();
- return;
- }
- // Enqueue the next data chunk into our target stream
- controller.enqueue(value);
- return pump();
- });
- }
- },
- });
-
- for await (const chunk of stream) {
- yield chunk;
- }
- }
-
- const response = {
- complete: false,
- request,
- url: fetchResponse.url,
- statusCode: fetchResponse.status,
- redirectUrls: [], // TODO you need to handle redirects manually to track them
- headers,
- trailers: {}, // TODO not supported by fetch
- ip: undefined,
- stream: Readable.from(read()),
- get downloadProgress() {
- return { percent: 0, transferred: 0 }; // TODO track this
- },
- get uploadProgress() {
- return { percent: 0, transferred: 0 }; // TODO track this
- },
- };
-
- return response;
+import { BaseHttpClient, type CustomFetchOptions } from '@crawlee/http-client';
+
+/**
+ * A simple HTTP client implementation using the native `fetch` API.
+ *
+ * Custom implementations only need to override the `fetch` method.
+ */
+export class CustomFetchClient extends BaseHttpClient {
+ protected override async fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise {
+ // The base class handles cookies, redirects, sessions, and timeouts.
+ // We only need to perform the actual network request here.
+ return fetch(request, options);
}
}
diff --git a/docs/guides/custom-http-client/usage.ts b/docs/guides/custom-http-client/usage.ts
index ebe52c236d3b..28fa63c5802a 100644
--- a/docs/guides/custom-http-client/usage.ts
+++ b/docs/guides/custom-http-client/usage.ts
@@ -1,8 +1,8 @@
import { HttpCrawler } from 'crawlee';
-import { CustomHttpClient } from './implementation.js';
+import { CustomFetchClient } from './implementation.js';
const crawler = new HttpCrawler({
- httpClient: new CustomHttpClient(),
+ httpClient: new CustomFetchClient(),
async requestHandler() {
/* ... */
},
diff --git a/docs/guides/custom-logger/custom-logger.mdx b/docs/guides/custom-logger/custom-logger.mdx
new file mode 100644
index 000000000000..8d024b6cbf64
--- /dev/null
+++ b/docs/guides/custom-logger/custom-logger.mdx
@@ -0,0 +1,88 @@
+---
+id: custom-logger
+title: Custom logger
+description: Use your own logging library (Winston, Pino, etc.) with Crawlee
+---
+
+import ApiLink from '@site/src/components/ApiLink';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import CodeBlock from '@theme/CodeBlock';
+
+import WinstonSource from '!!raw-loader!./winston.ts';
+import PinoSource from '!!raw-loader!./pino.ts';
+
+Crawlee uses `@apify/log` as its default logging library, but you can replace it with any logger you prefer, such as Winston or Pino. This is done by implementing a small adapter and passing it to the crawler.
+
+## Creating an adapter
+
+All Crawlee logging goes through the `CrawleeLogger` interface. To plug in your own logger, extend the `BaseCrawleeLogger` abstract class and implement two methods:
+
+- **`logWithLevel(level, message, data)`** — dispatches a log message to your logging library. The `level` parameter uses `LogLevel` constants (`ERROR = 1`, `SOFT_FAIL = 2`, `WARNING = 3`, `INFO = 4`, `DEBUG = 5`, `PERF = 6`). Map these to your logger's native levels. The `message` is a human-readable `string`, and `data` is an optional `Record` with structured context (e.g. `{ url, statusCode }`) — pass it to your logger as metadata or structured fields.
+- **`createChild(options)`** — returns a new child logger instance scoped to a specific component. Crawlee calls this internally to give each subsystem (e.g. `CheerioCrawler`, `AutoscaledPool`, `SessionPool`) its own identifiable logger. The `options` parameter is a `CrawleeLoggerOptions` object with a single field: `prefix` — a string label prepended to each log line from that component.
+
+All other methods (`error`, `warning`, `info`, `debug`, `exception`, `perf`, etc.) are derived automatically from `logWithLevel` — you don't need to implement them.
+
+:::info Level filtering
+
+`logWithLevel()` is called for **every** log message, regardless of the configured level. Level filtering is the responsibility of the underlying logging library (e.g. Winston's `level` option or Pino's `level` setting). This means your adapter doesn't need to check log levels — just forward everything and let the library decide what to output.
+
+:::
+
+## Injecting the logger
+
+There are two ways to inject a custom logger: per-crawler and globally.
+
+### Per-crawler logger
+
+Pass your adapter via the `logger` option in the crawler constructor. When a `logger` is provided, the crawler creates its own isolated `ServiceLocator` instance, so the custom logger is used by all internal components of that crawler (autoscaling, session pool, statistics, etc.):
+
+```ts
+import { CheerioCrawler } from 'crawlee';
+
+const crawler = new CheerioCrawler({
+ logger: new WinstonAdapter(winstonLogger),
+ async requestHandler({ log }) {
+ // `log` is a child of your custom logger, with prefix set to the crawler class name
+ log.info('Hello from my custom logger!');
+ },
+});
+```
+
+The same logger is available as `crawler.log` outside of the request handler, for example when setting up routes.
+
+### Global logger via service locator
+
+Instead of passing the logger to each crawler individually, you can set it globally via the `serviceLocator`. This is useful when you run multiple crawlers and want them all to use the same logging backend:
+
+```ts
+import { serviceLocator, CheerioCrawler, PlaywrightCrawler } from 'crawlee';
+
+// Set the logger globally — must be done before creating any crawlers
+serviceLocator.setLogger(new WinstonAdapter(winstonLogger));
+
+// Both crawlers will use the Winston logger
+const cheerioCrawler = new CheerioCrawler({ /* ... */ });
+const playwrightCrawler = new PlaywrightCrawler({ /* ... */ });
+```
+
+:::warning
+
+`serviceLocator.setLogger()` must be called **before** any crawler is created. Once a logger has been retrieved from the service locator (which happens during crawler construction), it cannot be replaced — an error will be thrown.
+
+:::
+
+## Full examples
+
+
+
+
+{WinstonSource}
+
+
+
+
+{PinoSource}
+
+
+
diff --git a/docs/guides/custom-logger/pino.ts b/docs/guides/custom-logger/pino.ts
new file mode 100644
index 000000000000..2cf60813aa74
--- /dev/null
+++ b/docs/guides/custom-logger/pino.ts
@@ -0,0 +1,48 @@
+import { CheerioCrawler, BaseCrawleeLogger, LogLevel } from 'crawlee';
+import type { CrawleeLogger, CrawleeLoggerOptions } from 'crawlee';
+import pino from 'pino';
+
+// Map Crawlee log levels to Pino levels
+const CRAWLEE_TO_PINO: Record = {
+ [LogLevel.ERROR]: 'error',
+ [LogLevel.SOFT_FAIL]: 'warn',
+ [LogLevel.WARNING]: 'warn',
+ [LogLevel.INFO]: 'info',
+ [LogLevel.DEBUG]: 'debug',
+ [LogLevel.PERF]: 'trace',
+};
+
+class PinoAdapter extends BaseCrawleeLogger {
+ constructor(
+ private logger: pino.Logger,
+ options?: Partial,
+ ) {
+ super(options);
+ }
+
+ logWithLevel(level: number, message: string, data?: Record): void {
+ const pinoLevel = CRAWLEE_TO_PINO[level] ?? 'info';
+ this.logger[pinoLevel as pino.Level](data ?? {}, message);
+ }
+
+ protected createChild(options: Partial): CrawleeLogger {
+ return new PinoAdapter(this.logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });
+ }
+}
+
+// Create a Pino logger with your preferred configuration
+const pinoLogger = pino({
+ level: 'debug',
+});
+
+// Pass the adapter to the crawler via the `logger` option
+const crawler = new CheerioCrawler({
+ logger: new PinoAdapter(pinoLogger),
+ async requestHandler({ request, $, log }) {
+ log.info(`Processing ${request.url}`);
+ const title = $('title').text();
+ log.debug('Page title extracted', { title });
+ },
+});
+
+await crawler.run(['https://crawlee.dev']);
diff --git a/docs/guides/custom-logger/winston.ts b/docs/guides/custom-logger/winston.ts
new file mode 100644
index 000000000000..9a967988b193
--- /dev/null
+++ b/docs/guides/custom-logger/winston.ts
@@ -0,0 +1,57 @@
+import { CheerioCrawler, BaseCrawleeLogger, LogLevel } from 'crawlee';
+import type { CrawleeLogger, CrawleeLoggerOptions } from 'crawlee';
+import winston from 'winston';
+
+// Map Crawlee log levels to Winston levels
+const CRAWLEE_TO_WINSTON: Record = {
+ [LogLevel.ERROR]: 'error',
+ [LogLevel.SOFT_FAIL]: 'warn',
+ [LogLevel.WARNING]: 'warn',
+ [LogLevel.INFO]: 'info',
+ [LogLevel.DEBUG]: 'debug',
+ [LogLevel.PERF]: 'debug',
+};
+
+class WinstonAdapter extends BaseCrawleeLogger {
+ constructor(
+ private logger: winston.Logger,
+ options?: Partial,
+ ) {
+ super(options);
+ }
+
+ logWithLevel(level: number, message: string, data?: Record): void {
+ const winstonLevel = CRAWLEE_TO_WINSTON[level] ?? 'info';
+ this.logger.log(winstonLevel, message, data);
+ }
+
+ protected createChild(options: Partial): CrawleeLogger {
+ return new WinstonAdapter(this.logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });
+ }
+}
+
+// Create a Winston logger with your preferred configuration
+const winstonLogger = winston.createLogger({
+ level: 'debug',
+ format: winston.format.combine(
+ winston.format.colorize(),
+ winston.format.timestamp(),
+ winston.format.printf(({ level, message, timestamp, prefix }) => {
+ const tag = prefix ? `[${prefix}] ` : '';
+ return `${timestamp} ${level}: ${tag}${message}`;
+ }),
+ ),
+ transports: [new winston.transports.Console()],
+});
+
+// Pass the adapter to the crawler via the `logger` option
+const crawler = new CheerioCrawler({
+ logger: new WinstonAdapter(winstonLogger),
+ async requestHandler({ request, $, log }) {
+ log.info(`Processing ${request.url}`);
+ const title = $('title').text();
+ log.debug('Page title extracted', { title });
+ },
+});
+
+await crawler.run(['https://crawlee.dev']);
diff --git a/docs/guides/http-clients.mdx b/docs/guides/http-clients.mdx
index 9956cb3077a3..ea751e7e9a95 100644
--- a/docs/guides/http-clients.mdx
+++ b/docs/guides/http-clients.mdx
@@ -49,7 +49,7 @@ BaseHttpClient --|> GotScrapingHttpClient
## Switching between HTTP clients
-Crawlee currently provides two main HTTP clients: `GotScrapingHttpClient`, which uses the `got-scraping` library, and `ImpitHttpClient`, which uses the `impit` library. You can switch between them by setting the `BasehttpClient` parameter when initializing a crawler class. The default HTTP client is `GotScrapingHttpClient`. For more details on anti-blocking features, see our [avoid getting blocked guide](./avoid-blocking).
+Crawlee currently provides two main HTTP clients: `GotScrapingHttpClient`, which uses the `got-scraping` library, and `ImpitHttpClient`, which uses the `impit` library. You can switch between them by setting the `BasehttpClient` parameter when initializing a crawler class. The default HTTP client is `GotScrapingHttpClient`. For more details on anti-blocking features, see our [avoid getting blocked guide](./avoid-blocking).
Below are examples of how to configure the HTTP client for the `CheerioCrawler`:
@@ -68,7 +68,7 @@ Below are examples of how to configure the HTTP client for the `GotScrapingHttpClient` is the default HTTP client, it's included with the base Crawlee installation and requires no additional packages.
+Since `GotScrapingHttpClient` is the default HTTP client, it's included with the base Crawlee installation and requires no additional packages.
For `ImpitHttpClient`, you need to install a separate `@crawlee/impit-client` package:
@@ -78,7 +78,7 @@ npm i @crawlee/impit-client
## Creating custom HTTP clients
-Crawlee provides an interface, `BaseHttpClient`, which defines the interface that all HTTP clients must implement. This allows you to create custom HTTP clients tailored to your specific requirements.
+Crawlee provides an interface, `BaseHttpClient`, which defines the interface that all HTTP clients must implement. This allows you to create custom HTTP clients tailored to your specific requirements.
HTTP clients are responsible for several key operations:
@@ -88,10 +88,10 @@ HTTP clients are responsible for several key operations:
- managing proxy configurations,
- connection pooling with timeout management.
-To create a custom HTTP client, you need to implement the `BaseHttpClient` interface. Your implementation must be async-compatible and include proper cleanup and resource management to work seamlessly with Crawlee's concurrent processing model.
+To create a custom HTTP client, you need to implement the `BaseHttpClient` interface. Your implementation must be async-compatible and include proper cleanup and resource management to work seamlessly with Crawlee's concurrent processing model.
## Conclusion
-This guide introduced you to the HTTP clients available in Crawlee and demonstrated how to switch between them, including their installation requirements and usage examples. You also learned about the responsibilities of HTTP clients and how to implement your own custom HTTP client by inheriting from the `BaseHttpClient` base class.
+This guide introduced you to the HTTP clients available in Crawlee and demonstrated how to switch between them, including their installation requirements and usage examples. You also learned about the responsibilities of HTTP clients and how to implement your own custom HTTP client by inheriting from the `BaseHttpClient` base class.
If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping!
diff --git a/docs/guides/http-clients/cheerio-got-scraping-example.ts b/docs/guides/http-clients/cheerio-got-scraping-example.ts
index a2cab0af3807..7a6b8e6ad24e 100644
--- a/docs/guides/http-clients/cheerio-got-scraping-example.ts
+++ b/docs/guides/http-clients/cheerio-got-scraping-example.ts
@@ -1,4 +1,5 @@
-import { CheerioCrawler, GotScrapingHttpClient } from 'crawlee';
+import { CheerioCrawler } from 'crawlee';
+import { GotScrapingHttpClient } from '@crawlee/got-scraping-client';
const crawler = new CheerioCrawler({
httpClient: new GotScrapingHttpClient(),
diff --git a/docs/guides/impit-http-client/basic-usage.ts b/docs/guides/impit-http-client/basic-usage.ts
index 1a8754c9fa11..51b414913bdc 100644
--- a/docs/guides/impit-http-client/basic-usage.ts
+++ b/docs/guides/impit-http-client/basic-usage.ts
@@ -7,7 +7,7 @@ const crawler = new BasicCrawler({
}),
async requestHandler({ sendRequest, log }) {
const response = await sendRequest();
- log.info('Received response', { statusCode: response.statusCode });
+ log.info('Received response', { status: response.status });
},
});
diff --git a/docs/guides/impit-http-client/impit-http-client.mdx b/docs/guides/impit-http-client/impit-http-client.mdx
index 89c71e82fa5d..5bfca4bf2d09 100644
--- a/docs/guides/impit-http-client/impit-http-client.mdx
+++ b/docs/guides/impit-http-client/impit-http-client.mdx
@@ -11,8 +11,6 @@ import CheerioCrawlerSource from '!!raw-loader!./cheerio-crawler.ts';
import HttpCrawlerSource from '!!raw-loader!./http-crawler.ts';
import AdvancedConfigSource from '!!raw-loader!./advanced-config.ts';
-## Introduction
-
The `ImpitHttpClient` is an HTTP client implementation based on the [Impit](https://github.com/apify/impit) library. It enables browser impersonation for HTTP requests, helping you bypass bot detection systems without running an actual browser.
:::info Successor to got-scraping
diff --git a/docs/guides/parallel-scraping/parallel-scraper.mjs b/docs/guides/parallel-scraping/parallel-scraper.mjs
index 6bee4f4ff13a..3b2fe80a1b13 100644
--- a/docs/guides/parallel-scraping/parallel-scraper.mjs
+++ b/docs/guides/parallel-scraping/parallel-scraper.mjs
@@ -1,5 +1,6 @@
import { fork } from 'node:child_process';
+import { FileSystemStorageBackend } from '@crawlee/fs-storage';
import { Configuration, Dataset, PlaywrightCrawler, log } from 'crawlee';
import { router } from './routes.mjs';
@@ -73,18 +74,21 @@ if (!process.env.IN_WORKER_THREAD) {
// or a configuration option. This is just for show 😈
workerLogger.setLevel(log.LEVELS.DEBUG);
- // Disable the automatic purge on start
- // This is needed when running locally, as otherwise multiple processes will try to clear the default storage (and that will cause clashes)
- Configuration.set('purgeOnStart', false);
-
// Get the request queue
const requestQueue = await getOrInitQueue(false);
- // Configure crawlee to store the worker-specific data in a separate directory (needs to be done AFTER the queue is initialized when running locally)
+ // Disable the automatic purge on start, so we don't lose the queue we prepared
const config = new Configuration({
- storageClientOptions: {
- localDataDirectory: `./storage/worker-${process.env.WORKER_INDEX}`,
- },
+ purgeOnStart: false,
+ });
+
+ // Store the worker's own internal state (its default dataset, key-value store, etc.) in a separate
+ // directory so the workers don't collide with each other. This directory is private to a single
+ // worker, so we set `requestQueueAccess: 'single'` — the concurrency-safe locking only matters for
+ // the shared `shop-urls` queue, which gets its own storage backend in `requestQueue.mjs`.
+ const storageBackend = new FileSystemStorageBackend({
+ localDataDirectory: `./storage/worker-${process.env.WORKER_INDEX}`,
+ requestQueueAccess: 'single',
});
workerLogger.debug('Setting up crawler.');
@@ -94,16 +98,16 @@ if (!process.env.IN_WORKER_THREAD) {
// Instead of the long requestHandler with
// if clauses we provide a router instance.
requestHandler: router,
- // Enable the request locking experiment so that we can actually use the queue.
- // highlight-start
- experiments: {
- requestLocking: true,
- },
// Provide the request queue we've pre-filled in previous steps
+ // highlight-start
requestQueue,
// highlight-end
// Let's also limit the crawler's concurrency, we don't want to overload a single process 🐌
maxConcurrency: 5,
+ // Use the worker-specific, concurrency-safe storage backend we created above
+ // highlight-start
+ storageBackend,
+ // highlight-end
},
config,
);
diff --git a/docs/guides/parallel-scraping/parallel-scraping.mdx b/docs/guides/parallel-scraping/parallel-scraping.mdx
index 5e05532c859b..82c0b3be9231 100644
--- a/docs/guides/parallel-scraping/parallel-scraping.mdx
+++ b/docs/guides/parallel-scraping/parallel-scraping.mdx
@@ -12,12 +12,6 @@ import AdaptedRoutesSource from '!!raw-loader!./adapted-routes.mjs';
import ParallelScraperSource from '!!raw-loader!./parallel-scraper.mjs';
import ModifiedDetailRouteSource from '!!raw-loader!./modified-detail-route.mjs';
-:::warning Experimental features ahead
-
-At the time of writing this guide (December 2023), request locking is still an experimental feature. You can read more about the experiment by visiting the [request locking experiment](../experiments/experiments-request-locking) page.
-
-:::
-
In this guide, we will walk you through how you can turn your single scraper into a scraper that can be parallelized and run in multiple instances. This guide assumes you've read and walked through our [introduction guide](../introduction/setting-up) (or have a fully-fledged scraper already built), but if you haven't done so yet, take a break, go read through all that, and come back. We'll be waiting...
*Oh, you're back already! Let's proceed in making that scraper parallel!*
@@ -66,6 +60,16 @@ The first step in our conversion process will be creating a common file (let's c
The exported function, `getOrInitQueue`, might seem like it does a lot. In essence, it just ensures the request queue is initialized, and if requested, ensures it starts off with an empty state.
+:::caution Make the shared queue concurrency-safe with `requestQueueAccess: 'shared'`
+
+Because every worker process opens this same `shop-urls` queue at the same time, it **must** use the concurrency-safe locking behavior of `FileSystemStorageBackend`. That's why `getOrInitQueue` opens the queue with a storage backend constructed with `requestQueueAccess: 'shared'`.
+
+By default, `FileSystemStorageBackend` assumes it is the *sole* consumer of a queue (`requestQueueAccess: 'single'`). On open it immediately reclaims any requests left *in progress* — great for a single-process crawl recovering after a crash, but disastrous when workers run side by side: each worker would happily grab requests another worker is still processing, so the same URL gets scraped multiple times.
+
+Setting `requestQueueAccess: 'shared'` tells the client to treat an in-progress request as a potential live peer's lock and only reclaim it once the lock expires on the wall clock, so two workers never process the same request at once.
+
+:::
+
### Adapting our previous scraper to enqueue the product URLs to the new queue
In the `src/routes.mjs` file of the scraper we previously built, we have a handler for the `CATEGORY` label. Let's adapt that handler to enqueue the product URLs to the new queue we created.
@@ -128,37 +132,44 @@ This will check how the script is executed as. If this value has _any_ value, it
We use this to ensure the parent process stays alive until all the worker processes exit. Otherwise, the worker processes would just get spawned, and lose the ability to communicate with the parent. You might not need this depending on your use case (maybe you just need to spawn workers and let them process).
-#### What's with all those `Configuration` calls?
+#### What's with all the `Configuration` and storage backend setup?
-There are three steps we want to do for the worker processes:
+There are two things we want to do for the worker processes:
-- ensure the default storages do **not** get purged on start, as otherwise we'd lose the queue we prepared
-- get the queue that supports locking from the same location as the parent process
-- initialize a special storage for worker processes so they do not collide with each other
+- get the shared queue from the same location as the parent process (it already comes with the concurrency-safe storage backend we set up in `requestQueue.mjs`)
+- ensure the default storages do **not** get purged on start, as otherwise we'd lose the queue we prepared, and give each worker its own private storage directory for its internal state so the workers don't collide with each other
In order, that's what these lines do:
```javascript title="src/parallel-scraper.mjs"
-// Disable the automatic purge on start (step 1)
-// This is needed when running locally, as otherwise multiple processes will try to clear the default storage (and that will cause clashes)
-Configuration.set('purgeOnStart', false);
+import { FileSystemStorageBackend } from '@crawlee/fs-storage';
-// Get the request queue from the parent process (step 2)
+// Get the shared request queue from the parent process (step 1)
const requestQueue = await getOrInitQueue(false);
-// Configure crawlee to store the worker-specific data in a separate directory (needs to be done AFTER the queue is initialized when running locally) (step 3)
-const config = new Configuration({
- storageClientOptions: {
- localDataDirectory: `./storage/worker-${process.env.WORKER_INDEX}`,
- },
+// Disable the automatic purge on start, so we don't lose the queue we prepared (step 2)
+const config = new Configuration({ purgeOnStart: false });
+
+// Store the worker's own internal state in a separate directory so workers don't collide (step 2,
+// cont.). This directory is private to a single worker, so we explicitly set
+// `requestQueueAccess: 'single'`.
+const storageBackend = new FileSystemStorageBackend({
+ localDataDirectory: `./storage/worker-${process.env.WORKER_INDEX}`,
+ requestQueueAccess: 'single',
});
```
-#### Enabling the request locking experiment, and telling the crawler to use the worker configuration
+:::note Why no `requestQueueAccess: 'shared'` here?
+
+Each worker's `./storage/worker-N` directory is private to that single worker — nothing else opens it — so the default `requestQueueAccess: 'single'` is exactly right. The concurrency-safe locking only matters for storage that is genuinely shared across processes, which is the `shop-urls` queue in `requestQueue.mjs`, not this per-worker internal state.
+
+:::
+
+#### Telling the crawler to use the worker configuration
-You might have noticed several lines highlighted in the code above. Those show how you can enable the request locking experiment, as well as how you provide the request queue to the crawler. You can read more about the experiment by visiting the [request locking experiment](../experiments/experiments-request-locking) page.
+You might have noticed several lines highlighted in the code above. Those show how you provide the shared request queue to the crawler.
-You might have also noticed we passed in a second parameter to the constructor of the crawler, the `config` variable we created earlier. This is needed to ensure the crawler uses the worker-specific storages for internal states, and that they do not collide with each other.
+You might have also noticed we passed in the `config` and `storageBackend` we created earlier to the crawler. These ensure the crawler uses the worker-specific storages for its own internal state (so the workers do not collide with each other), while still consuming the shared, concurrency-safe `shop-urls` queue we provided explicitly.
#### Why do we use `process.send` instead of `context.pushData`?
diff --git a/docs/guides/parallel-scraping/shared.mjs b/docs/guides/parallel-scraping/shared.mjs
index ff627fdee401..bef086bfb6e1 100644
--- a/docs/guides/parallel-scraping/shared.mjs
+++ b/docs/guides/parallel-scraping/shared.mjs
@@ -1,8 +1,19 @@
-import { RequestQueueV2 } from 'crawlee';
+import { FileSystemStorageBackend } from '@crawlee/fs-storage';
+import { RequestQueue } from 'crawlee';
-// Create the request queue that also supports parallelization
+// The request queue shared by all the parallel workers
let queue;
+// The `shop-urls` queue is opened concurrently by every worker process, so it must use the
+// concurrency-safe locking behavior. With `requestQueueAccess: 'shared'`, a request another worker
+// is still processing is treated as a live peer's lock and is not handed out again until that lock
+// expires — so two workers never scrape the same URL at once. (We point at the `./storage`
+// location, which is where this shared queue lives.)
+const sharedStorageBackend = new FileSystemStorageBackend({
+ localDataDirectory: './storage',
+ requestQueueAccess: 'shared',
+});
+
/**
* @param {boolean} makeFresh Whether the queue should be cleared before returning it
* @returns The queue
@@ -12,11 +23,11 @@ export async function getOrInitQueue(makeFresh = false) {
return queue;
}
- queue = await RequestQueueV2.open('shop-urls');
+ queue = await RequestQueue.open('shop-urls', { storageBackend: sharedStorageBackend });
if (makeFresh) {
await queue.drop();
- queue = await RequestQueueV2.open('shop-urls');
+ queue = await RequestQueue.open('shop-urls', { storageBackend: sharedStorageBackend });
}
return queue;
diff --git a/docs/guides/proxy_management.mdx b/docs/guides/proxy_management.mdx
index 8bf385f1c5b5..bc7253aa6ec9 100644
--- a/docs/guides/proxy_management.mdx
+++ b/docs/guides/proxy_management.mdx
@@ -31,7 +31,7 @@ import InspectionPuppeteerSource from '!!raw-loader!./proxy_management_inspectio
and most effective ways of preventing access to a website. It is therefore paramount for
a good web scraping library to provide easy to use but powerful tools which can work around
IP blocking. The most powerful weapon in our anti IP blocking arsenal is a
-[proxy server](https://en.wikipedia.org/wiki/Proxy_server).
+[proxy server](https://en.wikipedia.org/wiki/Proxy_server).
With Crawlee we can use our own proxy servers or proxy servers acquired from
third-party providers.
@@ -83,7 +83,7 @@ The `ProxyConfiguration` class allows you to provide a custom function to pick a
```javascript
const proxyConfiguration = new ProxyConfiguration({
- newUrlFunction: (sessionId, { request }) => {
+ newUrlFunction: ({ request } = {}) => {
if (request?.url.includes('crawlee.dev')) {
return null; // for crawlee.dev, we don't use a proxy
}
@@ -93,39 +93,10 @@ const proxyConfiguration = new ProxyConfiguration({
});
```
-The `newUrlFunction` receives two parameters - `sessionId` and `options` - and returns a string containing the proxy URL.
-
-The `sessionId` parameter is always provided and allows us to differentiate between different sessions - e.g. when Crawlee recognizes your crawlers are being blocked, it will automatically create a new session with a different id.
+The `newUrlFunction` receives a single optional `options` parameter and returns a string with the proxy URL (or `null` to skip the proxy for the current request).
The `options` parameter is an object containing a `Request`, which is the request that will be made. Note that this object is not always available, for example when we are using the `newUrl` function directly. Your custom function should therefore not rely on the `request` object being present and provide a default behavior when it is not.
-### Tiered proxies
-
-You can also provide a list of proxy tiers to the `ProxyConfiguration` class. This is useful when you want to switch between different proxies automatically based on the blocking behavior of the website.
-
-:::warning
-
-Note that the `tieredProxyUrls` option requires `ProxyConfiguration` to be used from a crawler instance ([see below](#crawler-integration)).
-
-Using this configuration through the `newUrl` calls will not yield the expected results.
-
-:::
-
-```javascript
-const proxyConfiguration = new ProxyConfiguration({
- tieredProxyUrls: [
- [null], // At first, we try to connect without a proxy
- ['http://okay-proxy.com'],
- ['http://slightly-better-proxy.com', 'http://slightly-better-proxy-2.com'],
- ['http://very-good-and-expensive-proxy.com'],
- ]
-});
-```
-
-This configuration will start with no proxy, then switch to `http://okay-proxy.com` if Crawlee recognizes we're getting blocked by the target website. If that proxy is also blocked, we will switch to one of the `slightly-better-proxy` URLs. If those are blocked, we will switch to the `very-good-and-expensive-proxy.com` URL.
-
-Crawlee also periodically probes lower tier proxies to see if they are unblocked, and if they are, it will switch back to them.
-
## Crawler integration
`ProxyConfiguration` integrates seamlessly into `HttpCrawler`, `CheerioCrawler`, `JSDOMCrawler`, `PlaywrightCrawler` and `PuppeteerCrawler`.
@@ -162,9 +133,7 @@ Our crawlers will now use the selected proxies for all connections.
## IP Rotation and session management
-`proxyConfiguration.newUrl()` allows us to pass a `sessionId` parameter. It will then be used to create a `sessionId`-`proxyUrl` pair, and subsequent `newUrl()` calls with the same `sessionId` will always return the same `proxyUrl`. This is extremely useful in scraping, because we want to create the impression of a real user. See the [session management guide](../guides/session-management) and `SessionPool` class for more information on how keeping a real session helps us avoid blocking.
-
-When no `sessionId` is provided, our proxy URLs are rotated round-robin.
+Each call to `proxyConfiguration.newUrl()` generates a new proxy URL. Crawler instances pair these URLs with `Session` instances and rotate those together with browser fingerprints, impersonated headers, and more. This is extremely useful in scraping, because we want to create the impression of a real user. See the [session management guide](../guides/session-management) and `SessionPool` class for more information on how keeping a real session helps us avoid blocking.
@@ -202,7 +171,7 @@ When no `sessionId` is provided, our proxy URLs are rotated round-robin.
## Inspecting current proxy in Crawlers
`HttpCrawler`, `CheerioCrawler`, `JSDOMCrawler`, `PlaywrightCrawler` and `PuppeteerCrawler` grant access to information about the currently used proxy
-in their `requestHandler` using a `proxyInfo` object.
+in their `requestHandler` using a `proxyInfo` object.
With the `proxyInfo` object, we can easily access the proxy URL.
diff --git a/docs/guides/proxy_management_session_cheerio.ts b/docs/guides/proxy_management_session_cheerio.ts
index bb19a5b88d35..1e23ec5d5b86 100644
--- a/docs/guides/proxy_management_session_cheerio.ts
+++ b/docs/guides/proxy_management_session_cheerio.ts
@@ -5,8 +5,7 @@ const proxyConfiguration = new ProxyConfiguration({
});
const crawler = new CheerioCrawler({
- useSessionPool: true,
- persistCookiesPerSession: true,
+ saveResponseCookies: true,
proxyConfiguration,
// ...
});
diff --git a/docs/guides/proxy_management_session_http.ts b/docs/guides/proxy_management_session_http.ts
index c8c289de4877..4677cb946273 100644
--- a/docs/guides/proxy_management_session_http.ts
+++ b/docs/guides/proxy_management_session_http.ts
@@ -5,8 +5,7 @@ const proxyConfiguration = new ProxyConfiguration({
});
const crawler = new HttpCrawler({
- useSessionPool: true,
- persistCookiesPerSession: true,
+ saveResponseCookies: true,
proxyConfiguration,
// ...
});
diff --git a/docs/guides/proxy_management_session_jsdom.ts b/docs/guides/proxy_management_session_jsdom.ts
index 98e71d904070..8162643bd1b3 100644
--- a/docs/guides/proxy_management_session_jsdom.ts
+++ b/docs/guides/proxy_management_session_jsdom.ts
@@ -5,8 +5,7 @@ const proxyConfiguration = new ProxyConfiguration({
});
const crawler = new JSDOMCrawler({
- useSessionPool: true,
- persistCookiesPerSession: true,
+ saveResponseCookies: true,
proxyConfiguration,
// ...
});
diff --git a/docs/guides/proxy_management_session_playwright.ts b/docs/guides/proxy_management_session_playwright.ts
index 70edcb79a033..c137f0191877 100644
--- a/docs/guides/proxy_management_session_playwright.ts
+++ b/docs/guides/proxy_management_session_playwright.ts
@@ -5,8 +5,7 @@ const proxyConfiguration = new ProxyConfiguration({
});
const crawler = new PlaywrightCrawler({
- useSessionPool: true,
- persistCookiesPerSession: true,
+ saveResponseCookies: true,
proxyConfiguration,
// ...
});
diff --git a/docs/guides/proxy_management_session_puppeteer.ts b/docs/guides/proxy_management_session_puppeteer.ts
index fcd1e14427f2..4e21121051a3 100644
--- a/docs/guides/proxy_management_session_puppeteer.ts
+++ b/docs/guides/proxy_management_session_puppeteer.ts
@@ -5,8 +5,7 @@ const proxyConfiguration = new ProxyConfiguration({
});
const crawler = new PuppeteerCrawler({
- useSessionPool: true,
- persistCookiesPerSession: true,
+ saveResponseCookies: true,
proxyConfiguration,
// ...
});
diff --git a/docs/guides/proxy_management_session_standalone.ts b/docs/guides/proxy_management_session_standalone.ts
index bc2010f79b18..dec095d03408 100644
--- a/docs/guides/proxy_management_session_standalone.ts
+++ b/docs/guides/proxy_management_session_standalone.ts
@@ -4,10 +4,4 @@ const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
-const sessionPool = await SessionPool.open({
- /* opts */
-});
-
-const session = await sessionPool.getSession();
-
-const proxyUrl = await proxyConfiguration.newUrl(session.id);
+const proxyUrl = await proxyConfiguration.newUrl();
diff --git a/docs/guides/remote_browser.mdx b/docs/guides/remote_browser.mdx
new file mode 100644
index 000000000000..f02d41be4b64
--- /dev/null
+++ b/docs/guides/remote_browser.mdx
@@ -0,0 +1,70 @@
+---
+id: remote-browser
+title: "Remote browser services"
+sidebar_label: "Remote browsers"
+description: Connect Crawlee crawlers to remote browser services like Browserbase, Browserless, or Steel.
+---
+
+import ApiLink from '@site/src/components/ApiLink';
+import CodeBlock from '@theme/CodeBlock';
+
+import RemoteBrowserConfigSource from '!!raw-loader!./remote_browser_config.ts';
+import RemoteBrowserProviderSource from '!!raw-loader!./remote_browser_provider.ts';
+import RemoteBrowserPuppeteerSource from '!!raw-loader!./remote_browser_puppeteer.ts';
+
+Instead of launching a local browser, Crawlee can connect to a remote browser service like [Browserbase](https://browserbase.com/), [Browserless](https://browserless.io/), [Steel](https://steel.dev/), or any service that exposes a WebSocket/CDP endpoint. The crawler manages session rotation and the request lifecycle the same way it does locally — only the browser itself runs elsewhere.
+
+Use this when you need IPs in specific regions, want to offload CPU/memory from your runner, or need stealth features the service provides.
+
+## How it works
+
+Set the crawler's `remoteBrowser` option with the connection details. The crawler builds a `RemoteBrowserPool` around its own browser plugin, so the connection is always for the matching browser — there's no plugin to construct and no way to mismatch the pool with the crawler. The pool (an `IBrowserPool` wrapping the regular `BrowserPool`) owns everything remote: resolving the endpoint, releasing sessions when browsers close, and capping how many remote browsers run at once.
+
+## Basic usage
+
+The simplest form is a static connection URL. Use this when the service exposes a single endpoint and doesn't need per-session setup.
+
+{RemoteBrowserConfigSource}
+
+`endpoint` can also be a function returning `{ url, context }`, called once per browser launch. Pair it with a `release` callback (it receives the `context`) to clean up sessions on the service side when the browser closes, crashes, or the pool is destroyed.
+
+`maxOpenBrowsers` caps the number of concurrent remote browsers — set it to the service's concurrent-session limit to avoid 429 errors. The pool enforces it inside `newPage()`, which waits for a free slot rather than overshooting.
+
+### Self-hosted
+
+Some services ship a Docker image you can run locally or on your own infrastructure. For example, [Browserless](https://www.browserless.io/) has an open-source Chromium image:
+
+```bash
+docker run -p 3000:3000 -e CONCURRENT=4 ghcr.io/browserless/chromium
+```
+
+Point the pool at the local endpoint with `endpoint: 'ws://localhost:3000'`.
+
+## Custom provider
+
+For services with a session-create / session-release lifecycle, extend `RemoteBrowserProvider` and pass the instance as the pool's `endpoint`. `connect()` runs once per browser launch and returns the connection URL plus an optional `context` object passed back to `release()`. `maxOpenBrowsers` set on the provider is adopted by the pool.
+
+{RemoteBrowserProviderSource}
+
+## Puppeteer
+
+`PuppeteerCrawler` works the same way — build the pool with a `PuppeteerPlugin`. Puppeteer connects over CDP:
+
+{RemoteBrowserPuppeteerSource}
+
+For Playwright you can choose the protocol via the `remoteBrowser.connection.protocol` option: `'cdp'` (default, `connectOverCDP()`) or `'playwright'` (`connect()`, Playwright's own WebSocket protocol).
+
+## Sharing a pool across crawlers
+
+`remoteBrowser` builds a pool the crawler owns and tears down. To share one remote pool across multiple crawlers, construct a `RemoteBrowserPool` yourself and pass it as the `browserPool` option instead — a pool supplied that way is never destroyed by the crawler, so you control its lifecycle. Use `remoteBrowser` *or* `browserPool`, not both.
+
+## Limitations
+
+- **`headless` and `launchOptions` don't apply.** The remote service controls headless mode and browser flags; configure them on the service side.
+- **`useIncognitoPages` is forced to `true`** for Playwright remote connections — `connect()` / `connectOverCDP()` don't accept persistent contexts. For state shared across requests, use the `SessionPool`.
+- **`userDataDir` has no effect** — there's no local profile when the browser runs remotely. Use the service's persistence API (e.g. Browserbase Contexts, Steel Profiles).
+
+## Further reading
+
+- `RemoteBrowserPool` API reference
+- `RemoteBrowserProvider` API reference
diff --git a/docs/guides/remote_browser_config.ts b/docs/guides/remote_browser_config.ts
new file mode 100644
index 000000000000..41f4e0542fe8
--- /dev/null
+++ b/docs/guides/remote_browser_config.ts
@@ -0,0 +1,19 @@
+import { PlaywrightCrawler } from 'crawlee';
+
+const token = process.env.BROWSERLESS_TOKEN!;
+
+const crawler = new PlaywrightCrawler({
+ // Connect to a remote browser instead of launching locally. The crawler builds the right
+ // pool for its browser — you only supply the connection details.
+ remoteBrowser: {
+ endpoint: `wss://production-sfo.browserless.io?token=${token}`,
+ // Optional — respect the service's concurrent session limit.
+ maxOpenBrowsers: 5,
+ },
+ async requestHandler({ page, request, log }) {
+ const title = await page.title();
+ log.info(`${request.loadedUrl} — "${title}"`);
+ },
+});
+
+await crawler.run(['https://crawlee.dev']);
diff --git a/docs/guides/remote_browser_provider.ts b/docs/guides/remote_browser_provider.ts
new file mode 100644
index 000000000000..45594d0fe4f4
--- /dev/null
+++ b/docs/guides/remote_browser_provider.ts
@@ -0,0 +1,46 @@
+import { RemoteBrowserProvider } from '@crawlee/browser-pool';
+import { PlaywrightCrawler } from 'crawlee';
+
+const apiKey = process.env.BROWSERBASE_API_KEY!;
+const projectId = process.env.BROWSERBASE_PROJECT_ID!;
+
+class BrowserbaseProvider extends RemoteBrowserProvider<{ id: string }> {
+ // Respect the service's concurrent session limit to avoid 429s.
+ override maxOpenBrowsers = 5;
+
+ async connect() {
+ const response = await fetch('https://api.browserbase.com/v1/sessions', {
+ method: 'POST',
+ headers: { 'x-bb-api-key': apiKey, 'Content-Type': 'application/json' },
+ body: JSON.stringify({ projectId }),
+ });
+
+ if (!response.ok) {
+ throw new Error(`Failed to create session: ${response.status} ${response.statusText}`);
+ }
+
+ const session = (await response.json()) as { id: string; connectUrl: string };
+ return { url: session.connectUrl, context: { id: session.id } };
+ }
+
+ override async release({ id }: { id: string }) {
+ await fetch(`https://api.browserbase.com/v1/sessions/${id}`, {
+ method: 'POST',
+ headers: { 'x-bb-api-key': apiKey, 'Content-Type': 'application/json' },
+ body: JSON.stringify({ status: 'REQUEST_RELEASE' }),
+ });
+ }
+}
+
+const crawler = new PlaywrightCrawler({
+ // Pass the provider as the `endpoint`; the crawler's pool calls connect()/release() per browser.
+ remoteBrowser: {
+ endpoint: new BrowserbaseProvider(),
+ },
+ async requestHandler({ page, request, log }) {
+ const title = await page.title();
+ log.info(`${request.loadedUrl} — "${title}"`);
+ },
+});
+
+await crawler.run(['https://crawlee.dev']);
diff --git a/docs/guides/remote_browser_puppeteer.ts b/docs/guides/remote_browser_puppeteer.ts
new file mode 100644
index 000000000000..2bfc14be3d65
--- /dev/null
+++ b/docs/guides/remote_browser_puppeteer.ts
@@ -0,0 +1,16 @@
+import { PuppeteerCrawler } from 'crawlee';
+
+const token = process.env.BROWSERLESS_TOKEN!;
+
+const crawler = new PuppeteerCrawler({
+ // PuppeteerCrawler connects over CDP. Same `remoteBrowser` option, matching browser guaranteed.
+ remoteBrowser: {
+ endpoint: `wss://production-sfo.browserless.io?token=${token}`,
+ },
+ async requestHandler({ page, request, log }) {
+ const title = await page.title();
+ log.info(`${request.loadedUrl} — "${title}"`);
+ },
+});
+
+await crawler.run(['https://crawlee.dev']);
diff --git a/docs/guides/request_loaders.mdx b/docs/guides/request_loaders.mdx
new file mode 100644
index 000000000000..d2be9bd6c3e6
--- /dev/null
+++ b/docs/guides/request_loaders.mdx
@@ -0,0 +1,179 @@
+---
+id: request-loaders
+title: Request loaders
+description: How to manage the requests your crawler will go through.
+---
+
+import ApiLink from '@site/src/components/ApiLink';
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import CodeBlock from '@theme/CodeBlock';
+
+import RlBasicSource from '!!raw-loader!./request_loaders_rl_basic.ts';
+import SitemapBasicSource from '!!raw-loader!./request_loaders_sitemap_basic.ts';
+import RlTandemExplicitSource from '!!raw-loader!./request_loaders_rl_tandem_explicit.ts';
+import RlTandemHelperSource from '!!raw-loader!./request_loaders_rl_tandem_helper.ts';
+import SitemapTandemExplicitSource from '!!raw-loader!./request_loaders_sitemap_tandem_explicit.ts';
+import SitemapTandemHelperSource from '!!raw-loader!./request_loaders_sitemap_tandem_helper.ts';
+
+Request loaders extend the functionality of the `RequestQueue`, providing additional tools for managing URLs and requests. If you are new to Crawlee and unfamiliar with the `RequestQueue`, consider starting with the [Request storage](./request-storage) guide first. Request loaders define how requests are fetched and stored, enabling various use cases such as reading URLs from a static list, a sitemap, an external API, or combining multiple sources together.
+
+## Overview
+
+The request loader abstractions are built around two interfaces and a couple of helpers:
+
+- `IRequestLoader`: The base interface for reading requests in a crawl.
+- `IRequestManager`: Extends `IRequestLoader` with write capabilities (adding and reclaiming requests).
+- `RequestManagerTandem`: Combines a read-only `IRequestLoader` with a writable `IRequestManager`.
+
+And the concrete request loader implementations:
+
+- `RequestList`: A lightweight implementation for managing a static list of URLs.
+- `SitemapRequestLoader`: A specialized loader that reads URLs from XML and plain-text sitemaps following the [Sitemaps protocol](https://www.sitemaps.org/protocol.html), with filtering capabilities.
+
+Below is a class diagram that illustrates the relationships between these components and the `RequestQueue`:
+
+```mermaid
+---
+config:
+ class:
+ hideEmptyMembersBox: true
+---
+
+classDiagram
+
+%% ========================
+%% Abstract interfaces
+%% ========================
+
+class IRequestLoader {
+ <>
+ + getTotalCount()
+ + getPendingCount()
+ + getHandledCount()
+ + fetchNextRequest()
+ + markRequestAsHandled()
+ + isEmpty()
+ + isFinished()
+ + toTandem()
+}
+
+class IRequestManager {
+ <>
+ + addRequest()
+ + addRequestsBatched()
+ + reclaimRequest()
+ + purge()
+}
+
+%% ========================
+%% Concrete classes
+%% ========================
+
+class RequestQueue
+
+class RequestList
+
+class SitemapRequestLoader
+
+class RequestManagerTandem
+
+%% ========================
+%% Inheritance arrows
+%% ========================
+
+IRequestLoader <|-- IRequestManager
+IRequestLoader <|.. RequestList
+IRequestLoader <|.. SitemapRequestLoader
+IRequestManager <|.. RequestQueue
+IRequestManager <|.. RequestManagerTandem
+```
+
+:::info Crawler usage
+
+A crawler reads its requests from a single `IRequestManager`, passed via the `requestManager` option. A `RequestQueue` is itself a request manager, so it can be passed directly. A read-only loader (such as `RequestList`) cannot — combine it with a queue into a tandem first, see the [Request manager tandem](#request-manager-tandem) section below.
+
+:::
+
+## Request loaders
+
+The `IRequestLoader` interface defines the foundation for fetching requests during a crawl. It provides methods for basic operations like retrieving the next request, marking requests as handled, and checking whether the loader is empty or finished. It is intentionally **read-only** — it does not allow adding new requests. Concrete implementations such as `RequestList` build on this interface to handle specific scenarios. You can create your own custom loader that reads from an external file, web endpoint, database, or any other data source.
+
+### Request list
+
+The `RequestList` manages a static list of URLs to crawl. The list is created for a single crawler run and, unlike a queue, cannot have requests added to or removed from it after initialization. It can hold a large number of URLs (even millions) with significantly lower overhead than enqueueing them one by one.
+
+Here is a basic example of working with the `RequestList`:
+
+
+ {RlBasicSource}
+
+
+### Sitemap request loader
+
+The `SitemapRequestLoader` is a specialized request loader that reads URLs from sitemaps following the [Sitemaps protocol](https://www.sitemaps.org/protocol.html). It supports both XML and plain-text sitemap formats and is particularly useful when you want to crawl a website systematically by following its sitemap structure. Loading happens in the background, so crawling can start before the sitemap is fully parsed.
+
+:::note
+
+The `SitemapRequestLoader` is designed specifically for sitemaps that follow the standard Sitemaps protocol. HTML pages containing links are not supported by this loader — those should be handled by regular crawlers using the `enqueueLinks` functionality.
+
+:::
+
+The loader supports filtering URLs using glob patterns and regular expressions, allowing you to include or exclude specific types of URLs.
+
+
+ {SitemapBasicSource}
+
+
+## Request managers
+
+The `IRequestManager` interface extends `IRequestLoader` with **write** capabilities. In addition to reading requests, a request manager can add new requests and reclaim failed ones. This is essential for dynamic crawling, where new URLs emerge during the crawl, or when requests fail and need to be retried. The `RequestQueue` is the primary built-in request manager — see the [Request storage](./request-storage) guide for details.
+
+## Request manager tandem
+
+The `RequestManagerTandem` class combines the read-only capabilities of an `IRequestLoader` (like `RequestList`) with the read-write capabilities of an `IRequestManager` (like `RequestQueue`). This is useful when you need to load initial requests from a static source (such as a file, sitemap, or database) and also dynamically add or retry requests during the crawl.
+
+Under the hood, the tandem checks whether the read-only loader still has pending requests. If so, each request from the loader is transferred to the manager (the queue) before being processed. Any newly added or reclaimed requests go directly to the manager side. Because every request passes through the queue, deduplication and retries are handled consistently and a single URL is not crawled multiple times.
+
+The easiest way to build a tandem is the `toTandem()` helper available on the loaders. Called without arguments, it pairs the loader with the default `RequestQueue`; you can also pass a specific request manager to use instead.
+
+### Request list with request queue
+
+This setup is useful when you have a static list of URLs to crawl, but also need to handle dynamic requests discovered during the crawl. Requests from the `RequestList` are processed first by being enqueued into the `RequestQueue`, which handles persistence and retries.
+
+
+
+
+ {RlTandemHelperSource}
+
+
+
+
+ {RlTandemExplicitSource}
+
+
+
+
+### Sitemap request loader with request queue
+
+Similarly, you can combine a `SitemapRequestLoader` with a `RequestQueue`. This is particularly useful when you want to crawl URLs from a sitemap while also handling dynamic requests discovered during the crawl. URLs from the sitemap are processed first by being enqueued into the queue, which handles persistence and retries.
+
+
+
+
+ {SitemapTandemHelperSource}
+
+
+
+
+ {SitemapTandemExplicitSource}
+
+
+
+
+## Conclusion
+
+This guide introduced the request loader abstractions: the read-only `IRequestLoader`, the writable `IRequestManager`, and the `RequestManagerTandem` that combines them, along with the `RequestList` and `SitemapRequestLoader` implementations. You also saw how to pair a loader with a queue using the `toTandem()` helper to handle both static and dynamically discovered requests.
+
+If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping!
diff --git a/docs/guides/request_loaders_rl_basic.ts b/docs/guides/request_loaders_rl_basic.ts
new file mode 100644
index 000000000000..bea941f734db
--- /dev/null
+++ b/docs/guides/request_loaders_rl_basic.ts
@@ -0,0 +1,15 @@
+import { RequestList } from 'crawlee';
+
+// Open a request list with a static set of URLs.
+// The name is used to persist the list's state in the default key-value store.
+const requestList = await RequestList.open('my-list', [
+ 'https://crawlee.dev/',
+ 'https://crawlee.dev/docs',
+ 'https://crawlee.dev/api',
+]);
+
+// Iterate over the requests manually (a crawler does this for you under the hood).
+for await (const request of requestList) {
+ console.log(request.url);
+ await requestList.markRequestAsHandled(request);
+}
diff --git a/docs/guides/request_loaders_rl_tandem_explicit.ts b/docs/guides/request_loaders_rl_tandem_explicit.ts
new file mode 100644
index 000000000000..8014ddc79337
--- /dev/null
+++ b/docs/guides/request_loaders_rl_tandem_explicit.ts
@@ -0,0 +1,21 @@
+import { CheerioCrawler, RequestList, RequestManagerTandem, RequestQueue } from 'crawlee';
+
+// A static list of URLs to start from (can hold millions of URLs).
+const requestList = await RequestList.open('my-list', ['https://crawlee.dev/', 'https://crawlee.dev/docs']);
+
+// A writable queue that holds requests discovered during the crawl.
+const requestQueue = await RequestQueue.open();
+
+// Combine them: the tandem reads from the list first, transferring each request
+// into the queue, and lets you enqueue new requests during the crawl.
+const requestManager = new RequestManagerTandem(requestList, requestQueue);
+
+const crawler = new CheerioCrawler({
+ requestManager,
+ async requestHandler({ enqueueLinks }) {
+ // Newly discovered links go to the queue side of the tandem.
+ await enqueueLinks();
+ },
+});
+
+await crawler.run();
diff --git a/docs/guides/request_loaders_rl_tandem_helper.ts b/docs/guides/request_loaders_rl_tandem_helper.ts
new file mode 100644
index 000000000000..8637cb510e9a
--- /dev/null
+++ b/docs/guides/request_loaders_rl_tandem_helper.ts
@@ -0,0 +1,17 @@
+import { CheerioCrawler, RequestList } from 'crawlee';
+
+// A static list of URLs to start from.
+const requestList = await RequestList.open('my-list', ['https://crawlee.dev/', 'https://crawlee.dev/docs']);
+
+// `toTandem()` is a shortcut that pairs the loader with a request queue.
+// Without arguments it opens the default `RequestQueue`.
+const requestManager = await requestList.toTandem();
+
+const crawler = new CheerioCrawler({
+ requestManager,
+ async requestHandler({ enqueueLinks }) {
+ await enqueueLinks();
+ },
+});
+
+await crawler.run();
diff --git a/docs/guides/request_loaders_sitemap_basic.ts b/docs/guides/request_loaders_sitemap_basic.ts
new file mode 100644
index 000000000000..2e76f19a1d31
--- /dev/null
+++ b/docs/guides/request_loaders_sitemap_basic.ts
@@ -0,0 +1,14 @@
+import { SitemapRequestLoader } from 'crawlee';
+
+// Open a sitemap request list. The sitemap is fetched and parsed in the background,
+// so crawling can start before the whole sitemap is loaded.
+const sitemapRequestLoader = await SitemapRequestLoader.open({
+ sitemapUrls: ['https://crawlee.dev/sitemap.xml'],
+ // Optionally filter the URLs read from the sitemap:
+ // globs: ['https://crawlee.dev/docs/**'],
+});
+
+for await (const request of sitemapRequestLoader) {
+ console.log(request.url);
+ await sitemapRequestLoader.markRequestAsHandled(request);
+}
diff --git a/docs/guides/request_loaders_sitemap_tandem_explicit.ts b/docs/guides/request_loaders_sitemap_tandem_explicit.ts
new file mode 100644
index 000000000000..48d2f936e9cd
--- /dev/null
+++ b/docs/guides/request_loaders_sitemap_tandem_explicit.ts
@@ -0,0 +1,20 @@
+import { CheerioCrawler, RequestManagerTandem, RequestQueue, SitemapRequestLoader } from 'crawlee';
+
+// Read the initial URLs from a sitemap.
+const sitemapRequestLoader = await SitemapRequestLoader.open({
+ sitemapUrls: ['https://crawlee.dev/sitemap.xml'],
+});
+
+// A writable queue for requests discovered during the crawl.
+const requestQueue = await RequestQueue.open();
+
+const requestManager = new RequestManagerTandem(sitemapRequestLoader, requestQueue);
+
+const crawler = new CheerioCrawler({
+ requestManager,
+ async requestHandler({ enqueueLinks }) {
+ await enqueueLinks();
+ },
+});
+
+await crawler.run();
diff --git a/docs/guides/request_loaders_sitemap_tandem_helper.ts b/docs/guides/request_loaders_sitemap_tandem_helper.ts
new file mode 100644
index 000000000000..bcf1c2ea0715
--- /dev/null
+++ b/docs/guides/request_loaders_sitemap_tandem_helper.ts
@@ -0,0 +1,18 @@
+import { CheerioCrawler, SitemapRequestLoader } from 'crawlee';
+
+// Read the initial URLs from a sitemap.
+const sitemapRequestLoader = await SitemapRequestLoader.open({
+ sitemapUrls: ['https://crawlee.dev/sitemap.xml'],
+});
+
+// Pair the loader with the default `RequestQueue` via the `toTandem()` shortcut.
+const requestManager = await sitemapRequestLoader.toTandem();
+
+const crawler = new CheerioCrawler({
+ requestManager,
+ async requestHandler({ enqueueLinks }) {
+ await enqueueLinks();
+ },
+});
+
+await crawler.run();
diff --git a/docs/guides/request_storage.mdx b/docs/guides/request_storage.mdx
index 8da5489b5faf..42c49ef37356 100644
--- a/docs/guides/request_storage.mdx
+++ b/docs/guides/request_storage.mdx
@@ -14,7 +14,6 @@ import BasicOperationsSource from '!!raw-loader!./request_storage_queue_basic.ts
import CrawlerExplicitSource from '!!raw-loader!./request_storage_queue_crawler_explicit.ts';
import CrawlerSource from '!!raw-loader!./request_storage_queue_crawler.ts';
-import RequestQueueListSource from '!!raw-loader!./request_storage_queue_list.ts';
import RequestQueueAddRequestsSource from '!!raw-loader!./request_storage_queue_only.ts';
Crawlee has several request storage types that are useful for specific tasks. The requests are stored on local disk to a directory defined by the `CRAWLEE_STORAGE_DIR` environment variable. If this variable is not defined, by default Crawlee sets `CRAWLEE_STORAGE_DIR` to `./storage` in the current working directory.
@@ -27,7 +26,7 @@ Each Crawlee project run is associated with a **default request queue**. Typical
In Crawlee, the request queue is represented by the `RequestQueue` class.
-The request queue is managed by `MemoryStorage` class and its data is stored in memory, while also being off-loaded to the local directory specified by the `CRAWLEE_STORAGE_DIR` environment variable as follows:
+By default, the request queue is managed by the `FileSystemStorageBackend` class and its data is stored in the local directory specified by the `CRAWLEE_STORAGE_DIR` environment variable as follows:
```text
{CRAWLEE_STORAGE_DIR}/request_queues/{QUEUE_ID}/entries.json
@@ -67,71 +66,17 @@ The following code demonstrates the usage of the request queue:
To see more detailed example of how to use the request queue with a crawler, see the [Puppeteer Crawler](/js/docs/examples/puppeteer-crawler) example.
-## Request list
+The request queue is not optimized for adding numerous URLs in a single batch — historically, requests were added one by one. To enqueue a large set of initial URLs efficiently, use the `addRequests()` method (or simply pass the URLs to `crawler.run()`), which adds requests in batches:
-The request list is not a storage per se - it represents the list of URLs to crawl that is stored in a crawler run memory (or optionally in default [Key-Value Store](../guides/result-storage#key-value-store) associated with the run, if specified). The list is used for the crawling of a large number of URLs, when we know all the URLs which should be visited by the crawler and no URLs would be added during the run. The URLs can be provided either in code or parsed from a text file hosted on the web.
+
+ {RequestQueueAddRequestsSource}
+
-Request list is created exclusively for the crawler run and only if its usage is explicitly specified in the code. Its usage is optional.
+## Reading requests from other sources
-In Crawlee, the request list is represented by the `RequestList` class.
+Sometimes you don't want to start from a dynamic queue, but from a static list of URLs (for example, parsed from a file) or from a website's sitemap. Crawlee provides **request loaders** for these read-only sources — `RequestList` and `SitemapRequestLoader` — which can be combined with a request queue when you also need to enqueue requests discovered during the crawl.
-The following code demonstrates basic operations of the request list:
-
-```javascript
-import { RequestList, PuppeteerCrawler } from 'crawlee';
-
-// Prepare the sources array with URLs to visit
-const sources = [
- { url: 'http://www.example.com/page-1' },
- { url: 'http://www.example.com/page-2' },
- { url: 'http://www.example.com/page-3' },
-];
-
-// Open the request list.
-// List name is used to persist the sources and the list state in the key-value store
-const requestList = await RequestList.open('my-list', sources);
-
-// The crawler will automatically process requests from the list
-// It's used the same way for Cheerio /Playwright crawlers.
-const crawler = new PuppeteerCrawler({
- requestList,
- async requestHandler({ page, request }) {
- // Process the page (extract data, take page screenshot, etc).
- // No more requests could be added to the request list here
- },
-});
-```
-
-## Which one to choose?
-
-When using Request queue - we would normally have several start URLs (e.g. category pages on e-commerce website) and then recursively add more (e.g. individual item pages) programmatically to the queue, it supports dynamic adding and removing of requests. No more URLs can be added to Request list after its initialization as it is immutable, URLs cannot be removed from the list either.
-
-On the other hand, the Request queue is not optimized for adding or removing numerous URLs in a batch. This is technically possible, but requests are added one by one to the queue, and thus it would take significant time with a larger number of requests. Request list however can contain even millions of URLs, and it would take significantly less time to add them to the list, compared to the queue.
-
-Note that Request queue and Request list can be used together by the same crawler. In such cases, each request from the Request list is enqueued into the Request queue first (to the foremost position in the queue, even if Request queue is not empty) and then consumed from the latter. This is necessary to avoid the same URL being processed more than once (from the list first and then possibly from the queue). In practical terms, such a combination can be useful when there are numerous initial URLs, but more URLs would be added dynamically by the crawler.
-
-:::tip
-
-In Crawlee, there is not much need to combine the request queue together with the request list (although it's technically possible).
-
-Previously there was no way to add the initial requests to the queue in batches (to add an array of requests), i.e. we could have only added the requests one by one to the queue with the help of `addRequest()` function.
-
-However, now we could use the `addRequests()` function, which adds requests in batches. Thus, instead of combining the request queue and the request list, we can use only the request queue for such use-cases now. See the examples below.
-
-:::
-
-
-
-
- {RequestQueueAddRequestsSource}
-
-
-
-
- {RequestQueueListSource}
-
-
-
+See the dedicated [Request loaders](./request-loaders) guide for details on loaders, request managers, and how to combine them with a queue into a `RequestManagerTandem`.
## Cleaning up the storages
@@ -143,4 +88,4 @@ import { purgeDefaultStorages } from 'crawlee';
await purgeDefaultStorages();
```
-Calling this function will clean up the default request storage directory (and also the request list stored in default key-value store). This is a shortcut for running (optional) `purge` method on the `StorageClient` interface, in other words it will call the `purge` method of the underlying storage implementation we are currently using. You can make sure the storage is purged only once for a given execution context if you set `onlyPurgeOnce` to `true` in the `options` object.
+Calling this function will clean up the default request storage directory (and also the request list stored in default key-value store). This is a shortcut for running (optional) `purge` method on the `StorageBackend` interface, in other words it will call the `purge` method of the underlying storage implementation we are currently using. You can make sure the storage is purged only once for a given execution context if you set `onlyPurgeOnce` to `true` in the `options` object.
diff --git a/docs/guides/request_storage_queue_basic.ts b/docs/guides/request_storage_queue_basic.ts
index 66d3d337212d..1555e5bbd595 100644
--- a/docs/guides/request_storage_queue_basic.ts
+++ b/docs/guides/request_storage_queue_basic.ts
@@ -11,7 +11,7 @@ await requestQueue.addRequests([
]);
// Open the named request queue
-const namedRequestQueue = await RequestQueue.open('named-queue');
+const namedRequestQueue = await RequestQueue.open({ name: 'named-queue' });
// Remove the named request queue
await namedRequestQueue.drop();
diff --git a/docs/guides/request_storage_queue_crawler.ts b/docs/guides/request_storage_queue_crawler.ts
index 07af11ffa712..d9c37f57f7de 100644
--- a/docs/guides/request_storage_queue_crawler.ts
+++ b/docs/guides/request_storage_queue_crawler.ts
@@ -4,7 +4,7 @@ import { CheerioCrawler } from 'crawlee';
// It's used the same way for Puppeteer/Playwright crawlers.
const crawler = new CheerioCrawler({
// Note that we're not specifying the requestQueue here
- async requestHandler({ crawler, enqueueLinks }) {
+ async requestHandler({ enqueueLinks }) {
// Add new request to the queue
await crawler.addRequests([{ url: 'https://example.com/new-page' }]);
// Add links found on page to the queue
diff --git a/docs/guides/request_storage_queue_only.ts b/docs/guides/request_storage_queue_only.ts
index 5d9a31379597..3054135504f3 100644
--- a/docs/guides/request_storage_queue_only.ts
+++ b/docs/guides/request_storage_queue_only.ts
@@ -15,7 +15,7 @@ const sources = [
// The crawler will automatically process requests from the queue.
// It's used the same way for Cheerio/Playwright crawlers
const crawler = new PuppeteerCrawler({
- async requestHandler({ crawler, enqueueLinks }) {
+ async requestHandler({ enqueueLinks }) {
// Add new request to the queue
await crawler.addRequests(['http://www.example.com/new-page']);
diff --git a/docs/guides/result_storage.mdx b/docs/guides/result_storage.mdx
index eff313aef81f..eb2b4ac507a0 100644
--- a/docs/guides/result_storage.mdx
+++ b/docs/guides/result_storage.mdx
@@ -8,7 +8,7 @@ import ApiLink from '@site/src/components/ApiLink';
Crawlee has several result storage types that are useful for specific tasks. The data is stored on a local disk to the directory defined by the `CRAWLEE_STORAGE_DIR` environment variable. If this variable is not defined, by default Crawlee sets `CRAWLEE_STORAGE_DIR` to `./storage` in the current working directory.
-Crawlee storage is managed by `MemoryStorage` class. During the crawler run all information is stored in memory, while also being off-loaded to the local files in respective storage type folders.
+By default, Crawlee storage is managed by the `FileSystemStorageBackend` class, which stores all information as local files in the respective storage type folders.
## Key-value store
@@ -110,4 +110,4 @@ import { purgeDefaultStorages } from 'crawlee';
await purgeDefaultStorages();
```
-Calling this function will clean up the default results storage directories except the `INPUT` key in default key-value store directory. This is a shortcut for running (optional) `purge` method on the `StorageClient` interface, in other words it will call the `purge` method of the underlying storage implementation we are currently using. In addition, this method will make sure the storage is purged only once for a given execution context, so it is safe to call it multiple times.
+Calling this function will clean up the default results storage directories except the `INPUT` key in default key-value store directory. This is a shortcut for running (optional) `purge` method on the `StorageBackend` interface, in other words it will call the `purge` method of the underlying storage implementation we are currently using. In addition, this method will make sure the storage is purged only once for a given execution context, so it is safe to call it multiple times.
diff --git a/docs/guides/running-in-web-server/web-server.mjs b/docs/guides/running-in-web-server/web-server.mjs
index 7c677db912bd..29e6b27367bd 100644
--- a/docs/guides/running-in-web-server/web-server.mjs
+++ b/docs/guides/running-in-web-server/web-server.mjs
@@ -1,6 +1,6 @@
import { randomUUID } from 'node:crypto';
import { CheerioCrawler, log } from 'crawlee';
-import { createServer } from 'http';
+import { createServer } from 'node:http';
// We will bind an HTTP response that we want to send to the Request.uniqueKey
const requestsToResponses = new Map();
diff --git a/docs/guides/session_management.mdx b/docs/guides/session_management.mdx
index ad0eca80e96e..b5a1ba558f55 100644
--- a/docs/guides/session_management.mdx
+++ b/docs/guides/session_management.mdx
@@ -18,18 +18,15 @@ import PlaywrightSource from '!!raw-loader!./session_management_playwright.ts';
import PuppeteerSource from '!!raw-loader!./session_management_puppeteer.ts';
import StandaloneSource from '!!raw-loader!./session_management_standalone.ts';
-`SessionPool` is a class that allows us to handle the rotation of proxy IP addresses along with cookies and other custom settings in Crawlee.
+`SessionPool` manages the rotation of proxy IP addresses, cookies, and browser fingerprints in Crawlee. A single `Session` bundles all the identifying state of one "virtual user" — its cookie jar, its proxy (and therefore its IP), and a fingerprint hint — so that everything that makes a series of requests look like it comes from one person rotates together. When a session gets blocked, the whole bundle is thrown away at once and a fresh identity takes over, rather than reusing a burnt IP with new cookies (or vice versa).
-The main benefit of using Session pool is that we can filter out blocked or non-working proxies,
-so our actor does not retry requests over known blocked/non-working proxies.
-Another benefit of using SessionPool is that we can store information tied tightly to an IP address,
-such as cookies, auth tokens, and particular headers. Having our cookies and other identifiers used only with a specific IP will reduce the chance of being blocked.
-The last but not least benefit is the even rotation of IP addresses - SessionPool picks the session randomly,
-which should prevent burning out a small pool of available IPs.
+The main benefits of the session pool are that it filters out blocked or non-working proxies so the crawler does not keep retrying over them, it keeps identity-bound state (cookies, auth tokens, headers) tied to the IP that obtained it, and it spreads requests across IPs to avoid burning a small pool. The selection strategy is configurable — see [Choosing a rotation strategy](#choosing-a-rotation-strategy) below.
-Check out the [avoid blocking guide](./avoid-blocking) for more information about blocking.
+All crawler instances now require a `SessionPool`. In most cases you do not create one yourself: you just read the `session` from the request handler and let the crawler mark it good or bad for you. You only construct a `SessionPool` explicitly when you want to override its defaults or share one instance across several crawlers.
-Now let's take a look at the examples of how to use Session pool:
+Check out the [avoid blocking guide](./avoid-blocking) for the bigger picture on why blocking happens and how fingerprints fit in.
+
+Now let's take a look at the examples of how to use the session pool:
- with `BasicCrawler`;
- with `HttpCrawler`;
- with `CheerioCrawler`;
@@ -76,6 +73,229 @@ Now let's take a look at the examples of how to use Session pool:
-These are the basics of configuring SessionPool.
-Please, bear in mind that a Session pool needs time to find working IPs and build up the pool,
-so we will probably see a lot of errors until it becomes stabilized.
+These are the basics of configuring the session pool. The rest of this guide covers how to control which session is used, what state it carries, and when it is thrown away.
+
+## How a session is retired
+
+A session stays in the pool and keeps being handed out as long as `isUsable()` returns `true`. It stops being usable — and is dropped from rotation — as soon as any of the following happens:
+
+- its **error score** reaches `maxErrorScore` (default `3`),
+- its **usage count** reaches `maxUsageCount` (default `50`),
+- it is older than `maxAgeSecs` (default `3000` seconds), or
+- it has been explicitly **retired**.
+
+You influence this with three methods on the session. `markGood()` records a successful use — it increments the usage count and heals the error score a little (by `errorScoreDecrement`, default `0.5`). `markBad()` records a failure that *might* be the session's fault and *might* just be bad luck — it raises the error score by one, so a session needs to fail repeatedly before it is dropped. `retire()` drops the session immediately and permanently; this is what you call when you are certain the identity itself is burnt (for example, a `403` response).
+
+The distinction between `markBad()` and `retire()` matters. Use `markBad()` for transient, external problems such as a timeout or a `5XX` response — the IP is probably fine and a couple of retries should not throw it away. Use `retire()` for problems that prove the session is blocked, where reusing it is pointless. Retirement is terminal: once a session is retired, a later `markGood()` will not bring it back.
+
+When using a crawler you rarely call `markGood()` yourself — the crawler calls it automatically after a successful request handler run. You only need to reach for `markBad()` / `retire()` (or let blocked status codes do it for you, see [below](#letting-blocked-responses-retire-sessions)) when you detect a problem the crawler cannot see, such as a "you are blocked" message inside an otherwise `200` response.
+
+## Managing cookies
+
+Every session owns a [`tough-cookie`](https://github.com/salesforce/tough-cookie) cookie jar, reachable as `session.cookieJar`. Cookies arriving in `Set-Cookie` response headers are stored in it automatically — this is controlled by the `saveResponseCookies` crawler option (default `true`) — so they are replayed on every later request that reuses the same session. Set `saveResponseCookies: false` to keep response cookies out of the session jar.
+
+You can also seed or read cookies yourself. `session.setCookie('name=value', url)` adds a single cookie, `session.getCookieString(url)` returns the `Cookie` header value the session would send for that URL, and `session.cookieJar` gives you the full jar for anything more involved.
+
+```js
+const crawler = new CheerioCrawler({
+ requestHandler: async ({ session, request }) => {
+ session.setCookie('consent=yes', request.url);
+ },
+});
+```
+
+### Cookie precedence and overrides
+
+When an HTTP-based crawler (or a direct `sendRequest` call) builds the outgoing `Cookie` header, it starts from a **base jar** and then overlays any cookies set on the request:
+
+- The base jar is the explicit `cookieJar` passed to `sendRequest` if you provide one, otherwise the session's own cookie jar.
+- A `Cookie` header on the request (`request.headers.Cookie`) is merged on top of that base. A cookie set this way wins over a base-jar cookie of the same name, but it is *not* persisted back into the session.
+
+So a `Cookie` request header always beats the stored cookie of the same name regardless of which jar is the base, while passing an explicit `cookieJar` swaps out the whole base for that single call. To override a single cookie for one request, set it on the request header:
+
+```js
+import { HttpCrawler } from 'crawlee';
+import { CookieJar } from 'tough-cookie';
+
+const crawler = new HttpCrawler({
+ preNavigationHooks: [
+ async ({ request }) => {
+ // wins over any same-named cookie in the session jar, for this request only
+ request.headers = { ...request.headers, Cookie: 'token=override' };
+ },
+ ],
+ requestHandler: async ({ sendRequest }) => {
+ // ...or to fully replace the jar for a single call:
+ const jar = new CookieJar();
+ await jar.setCookie('token=override', 'https://example.com');
+ await sendRequest({ url: 'https://example.com' }, { cookieJar: jar });
+ },
+});
+```
+
+A `Cookie` header you set on a request is always honored — it is never silently overwritten by the session jar.
+
+## Choosing a rotation strategy
+
+The `sessionReuseStrategy` option decides *which* session `getSession()` hands out, and it is the main lever for matching the pool's behavior to a target site. Three strategies are available, each suited to a different use case.
+
+**Maximise IP and fingerprint diversity** — use `'random'` (the default). The pool creates a brand-new session for every request until it reaches `maxPoolSize`, then picks a usable session at random. This spreads traffic as widely as possible across IPs and fingerprints and is the right default for most large crawls.
+
+**Distribute load evenly across sessions** — use `'round-robin'`. Like `random`, the pool fills up to `maxPoolSize` first, but then cycles through sessions in order instead of picking randomly. This is useful when you want every session to do roughly the same amount of work — for example, combined with `maxUsageCount` so all sessions reach their limit and rotate out at about the same time.
+
+**Use a single IP until it breaks** — use `'use-until-failure'`. The pool returns the *same* session on every call and only moves to the next one once the current session is retired. This is the strategy for sites that reward consistency: where switching IP mid-flow looks suspicious, where you have logged in and want to stay logged in, or where you simply want to squeeze a working proxy for as long as it lasts before paying for another.
+
+
+
+
+```js
+import { SessionPool } from 'crawlee';
+
+const sessionPool = new SessionPool({
+ sessionReuseStrategy: 'random',
+});
+```
+
+
+
+
+```js
+import { SessionPool } from 'crawlee';
+
+const sessionPool = new SessionPool({
+ sessionReuseStrategy: 'round-robin',
+ // make every session retire after the same amount of work
+ sessionOptions: { maxUsageCount: 100 },
+});
+```
+
+
+
+
+```js
+import { SessionPool } from 'crawlee';
+
+const sessionPool = new SessionPool({
+ sessionReuseStrategy: 'use-until-failure',
+});
+```
+
+
+
+
+Whichever strategy you pick, you can cap how hard each session works through `sessionOptions`. Set `maxUsageCount` when you know a site starts blocking after roughly _N_ requests from one IP, `maxAgeSecs` when sessions should be cycled on a time basis, and `maxErrorScore` to control how forgiving the pool is about intermittent failures before dropping a session.
+
+```js
+const sessionPool = new SessionPool({
+ maxPoolSize: 25,
+ sessionOptions: {
+ maxAgeSecs: 600,
+ maxUsageCount: 150, // e.g. when you know the site blocks after ~150 requests
+ },
+});
+```
+
+## Letting blocked responses retire sessions
+
+You do not have to inspect every response by hand. Crawlers treat a configurable set of HTTP status codes as proof that a session is blocked and retire it automatically, retrying the request with a fresh session. This is controlled by the `blockedStatusCodes` crawler option (default `[401, 403, 429]`).
+
+```js
+import { CheerioCrawler } from 'crawlee';
+
+const crawler = new CheerioCrawler({
+ // a 403 or 429 will retire the current session and retry on a new one
+ blockedStatusCodes: [403, 429],
+ requestHandler: async ({ session, request }) => {
+ // session is already a working, non-blocked one
+ },
+});
+```
+
+For sites that respond with a `200` page that is actually a bot wall (Cloudflare challenges, Google's rate-limit page), set `retryOnBlocked: true` to have the crawler detect those by content and retry as well. For deeper anti-blocking measures see the [avoid blocking guide](./avoid-blocking).
+
+## Sharing a session pool between crawlers
+
+A `SessionPool` instance can be shared across multiple crawlers by passing the same object to each crawler's `sessionPool` option. This is useful in multi-stage scrapers — for example a fast `CheerioCrawler` that discovers links and a `PlaywrightCrawler` that renders detail pages — where you want both stages to reuse the same proven, non-blocked identities and their cookies instead of each warming up its own pool from scratch.
+
+```js
+import { CheerioCrawler, PlaywrightCrawler, SessionPool } from 'crawlee';
+
+const sessionPool = new SessionPool({ maxPoolSize: 100 });
+
+const listingCrawler = new CheerioCrawler({ sessionPool, requestHandler: async () => { /* ... */ } });
+const detailCrawler = new PlaywrightCrawler({ sessionPool, requestHandler: async () => { /* ... */ } });
+```
+
+A pool you construct yourself is owned by you, not the crawler — the crawler will never tear it down or reset it between runs. Call `teardown()` when you are done with it to persist its final state and stop listening for persistence events.
+
+## Custom session pools
+
+A crawler accepts any object implementing the `ISessionPool` interface as its `sessionPool` option, not just the built-in `SessionPool`. The contract is intentionally tiny — a single `getSession()` / `getSession(id)` method that hands out an `ISession` for a request. This lets you plug in a remote, shared, or database-backed session strategy without subclassing `SessionPool` or copying its internals.
+
+```ts
+import { BasicCrawler, Session, type ISessionPool } from 'crawlee';
+
+class MySessionPool implements ISessionPool {
+ private readonly sessions = new Map();
+
+ async getSession(sessionId?: string): Promise {
+ if (sessionId) {
+ const existing = this.sessions.get(sessionId);
+ return existing?.isUsable() ? existing : undefined;
+ }
+
+ const usable = [...this.sessions.values()].find((s) => s.isUsable());
+ if (usable) return usable;
+
+ const fresh = new Session();
+ this.sessions.set(fresh.id, fresh);
+ return fresh;
+ }
+}
+
+const crawler = new BasicCrawler({
+ sessionPool: new MySessionPool(),
+ requestHandler: async ({ session }) => {
+ // session is a Session instance, use it as usual
+ },
+});
+```
+
+The returned objects just need to implement `ISession` — the crawler only calls `markGood()`, `markBad()`, `retire()`, and reads `cookieJar`, `proxyInfo`, and `fingerprint`, all of which are part of that interface.
+
+## Pinning a request to a specific session
+
+By default the pool decides which session a request gets. Setting `request.sessionId` overrides that and forces the request — and all of its retries — onto the session with that id. You can create a custom named session with `addSession()`, giving each its own proxy, cookies, or fingerprint. Because a session bundles a proxy, this is how you bind specific requests to specific proxies.
+
+One important consequence: if a named session is retired — whether through accumulated `markBad()` calls, hitting `maxUsageCount`, or an explicit `retire()` — any subsequent `getSession(id)` call for that id returns `undefined`.
+The crawler treats that as a `MissingSessionError`, counts it as a regular request error, and retries the request with the same `sessionId`. If the session stays retired, retries keep failing and the request eventually exhausts `maxRequestRetries`.
+When a named session can be retired, handle this in your `errorHandler`: either recreate the session via `addSession()` with the same id, or clear `request.sessionId` to let the pool assign a fresh one.
+
+A common usage pattern is escalating between proxy "tiers": add a cheap session and a premium one, start requests on the cheap session, and reassign `request.sessionId` to the premium one in an `errorHandler` so the retry goes out over the better proxy.
+
+```ts
+import { BasicCrawler, SessionPool } from 'crawlee';
+
+const proxyInfoFromUrl = (proxyUrl: string) => {
+ const { username, password, hostname, port } = new URL(proxyUrl);
+ return { url: proxyUrl, username, password, hostname, port };
+};
+
+const sessionPool = new SessionPool();
+await sessionPool.addSession({ id: 'cheap', proxyInfo: proxyInfoFromUrl('http://cheap-proxy.com') });
+await sessionPool.addSession({ id: 'premium', proxyInfo: proxyInfoFromUrl('http://expensive-proxy.com') });
+
+const crawler = new BasicCrawler({
+ sessionPool,
+ retryOnBlocked: true,
+ requestHandler: async ({ sendRequest, request }) => {
+ await sendRequest({ url: request.url });
+ },
+ errorHandler: async ({ request }) => {
+ request.sessionId = 'premium'; // escalate the retry to the premium proxy
+ },
+});
+
+await crawler.run([{ url: 'https://example.com', sessionId: 'cheap' }]);
+```
+
diff --git a/docs/guides/session_management_basic.ts b/docs/guides/session_management_basic.ts
index c7b7ec37c361..948f75ea0388 100644
--- a/docs/guides/session_management_basic.ts
+++ b/docs/guides/session_management_basic.ts
@@ -1,33 +1,29 @@
-import { BasicCrawler, ProxyConfiguration } from 'crawlee';
-import { gotScraping } from 'got-scraping';
+import { BasicCrawler, ProxyConfiguration, SessionPool } from 'crawlee';
+import { Impit } from 'impit';
+import { Cookie } from 'tough-cookie';
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new BasicCrawler({
- // Activates the Session pool (default is true).
- useSessionPool: true,
// Overrides default Session pool configuration.
- sessionPoolOptions: { maxPoolSize: 100 },
+ sessionPool: new SessionPool({ maxPoolSize: 100 }),
async requestHandler({ request, session }) {
const { url } = request;
- const requestOptions = {
- url,
- // We use session id in order to have the same proxyUrl
- // for all the requests using the same session.
- proxyUrl: await proxyConfiguration.newUrl(session?.id),
- throwHttpErrors: false,
+ const client = new Impit({
+ proxyUrl: await proxyConfiguration.newUrl(),
+ ignoreTlsErrors: true,
headers: {
// If you want to use the cookieJar.
// This way you get the Cookie headers string from session.
- Cookie: session?.getCookieString(url),
+ Cookie: session?.cookieJar.getCookieStringSync(url) ?? '',
},
- };
+ });
let response;
try {
- response = await gotScraping(requestOptions);
+ response = await client.fetch(url);
} catch (e) {
if (e === 'SomeNetworkError') {
// If a network error happens, such as timeout, socket hangup, etc.
@@ -38,10 +34,7 @@ const crawler = new BasicCrawler({
throw e;
}
- // Automatically retires the session based on response HTTP status code.
- session?.retireOnBlockedStatusCodes(response.statusCode);
-
- if (response.body.includes('You are blocked!')) {
+ if ((await response.text()).includes('You are blocked!')) {
// You are sure it is blocked.
// This will throw away the session.
session?.retire();
@@ -51,6 +44,17 @@ const crawler = new BasicCrawler({
// No need to call session.markGood -> BasicCrawler calls it for you.
// If you want to use the CookieJar in session you need.
- session?.setCookiesFromResponse(response);
+ if (response.headers.has('set-cookie')) {
+ const newCookies = response.headers
+ .get('set-cookie')
+ ?.split(';')
+ .map((x) => Cookie.parse(x));
+
+ for (const cookie of newCookies ?? []) {
+ if (cookie) {
+ await session?.cookieJar?.setCookie(cookie, url);
+ }
+ }
+ }
},
});
diff --git a/docs/guides/session_management_cheerio.ts b/docs/guides/session_management_cheerio.ts
index 7f8b2f90a09a..bda80505992d 100644
--- a/docs/guides/session_management_cheerio.ts
+++ b/docs/guides/session_management_cheerio.ts
@@ -1,4 +1,4 @@
-import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
+import { CheerioCrawler, ProxyConfiguration, SessionPool } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
/* opts */
@@ -7,13 +7,11 @@ const proxyConfiguration = new ProxyConfiguration({
const crawler = new CheerioCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
- // Activates the Session pool (default is true).
- useSessionPool: true,
// Overrides default Session pool configuration.
- sessionPoolOptions: { maxPoolSize: 100 },
+ sessionPool: new SessionPool({ maxPoolSize: 100 }),
// Set to true if you want the crawler to save cookies per session,
// and set the cookie header to request automatically (default is true).
- persistCookiesPerSession: true,
+ saveResponseCookies: true,
async requestHandler({ session, $ }) {
const title = $('title').text();
diff --git a/docs/guides/session_management_http.ts b/docs/guides/session_management_http.ts
index 9c684bcb0566..bb55dc3e69da 100644
--- a/docs/guides/session_management_http.ts
+++ b/docs/guides/session_management_http.ts
@@ -1,4 +1,4 @@
-import { HttpCrawler, ProxyConfiguration } from 'crawlee';
+import { HttpCrawler, ProxyConfiguration, SessionPool } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
/* opts */
@@ -7,15 +7,13 @@ const proxyConfiguration = new ProxyConfiguration({
const crawler = new HttpCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
- // Activates the Session pool (default is true).
- useSessionPool: true,
// Overrides default Session pool configuration.
- sessionPoolOptions: { maxPoolSize: 100 },
+ sessionPool: new SessionPool({ maxPoolSize: 100 }),
// Set to true if you want the crawler to save cookies per session,
// and set the cookie header to request automatically (default is true).
- persistCookiesPerSession: true,
+ saveResponseCookies: true,
async requestHandler({ session, body }) {
- const title = (body as string).match(/(.*?)<\/title>/)?.[1];
+ const title = /(.*?)<\/title>/.exec(body as string)?.[1];
if (title === 'Blocked') {
session?.retire();
diff --git a/docs/guides/session_management_jsdom.ts b/docs/guides/session_management_jsdom.ts
index ef55b6632640..ee8e7cfffda3 100644
--- a/docs/guides/session_management_jsdom.ts
+++ b/docs/guides/session_management_jsdom.ts
@@ -1,4 +1,4 @@
-import { JSDOMCrawler, ProxyConfiguration } from 'crawlee';
+import { JSDOMCrawler, ProxyConfiguration, SessionPool } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
/* opts */
@@ -7,13 +7,11 @@ const proxyConfiguration = new ProxyConfiguration({
const crawler = new JSDOMCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
- // Activates the Session pool (default is true).
- useSessionPool: true,
// Overrides default Session pool configuration.
- sessionPoolOptions: { maxPoolSize: 100 },
+ sessionPool: new SessionPool({ maxPoolSize: 100 }),
// Set to true if you want the crawler to save cookies per session,
// and set the cookie header to request automatically (default is true).
- persistCookiesPerSession: true,
+ saveResponseCookies: true,
async requestHandler({ session, window }) {
const title = window.document.title;
diff --git a/docs/guides/session_management_playwright.ts b/docs/guides/session_management_playwright.ts
index f4f2f7c80f6f..01749fccddbb 100644
--- a/docs/guides/session_management_playwright.ts
+++ b/docs/guides/session_management_playwright.ts
@@ -1,4 +1,4 @@
-import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
+import { PlaywrightCrawler, ProxyConfiguration, SessionPool } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
/* opts */
@@ -7,13 +7,11 @@ const proxyConfiguration = new ProxyConfiguration({
const crawler = new PlaywrightCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
- // Activates the Session pool (default is true).
- useSessionPool: true,
// Overrides default Session pool configuration
- sessionPoolOptions: { maxPoolSize: 100 },
+ sessionPool: new SessionPool({ maxPoolSize: 100 }),
// Set to true if you want the crawler to save cookies per session,
// and set the cookies to page before navigation automatically (default is true).
- persistCookiesPerSession: true,
+ saveResponseCookies: true,
async requestHandler({ page, session }) {
const title = await page.title();
diff --git a/docs/guides/session_management_puppeteer.ts b/docs/guides/session_management_puppeteer.ts
index 63b342146397..76ad3fcc7ee5 100644
--- a/docs/guides/session_management_puppeteer.ts
+++ b/docs/guides/session_management_puppeteer.ts
@@ -1,4 +1,4 @@
-import { PuppeteerCrawler, ProxyConfiguration } from 'crawlee';
+import { PuppeteerCrawler, ProxyConfiguration, SessionPool } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
/* opts */
@@ -7,13 +7,11 @@ const proxyConfiguration = new ProxyConfiguration({
const crawler = new PuppeteerCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxyConfiguration,
- // Activates the Session pool (default is true).
- useSessionPool: true,
// Overrides default Session pool configuration
- sessionPoolOptions: { maxPoolSize: 100 },
+ sessionPool: new SessionPool({ maxPoolSize: 100 }),
// Set to true if you want the crawler to save cookies per session,
// and set the cookies to page before navigation automatically (default is true).
- persistCookiesPerSession: true,
+ saveResponseCookies: true,
async requestHandler({ page, session }) {
const title = await page.title();
diff --git a/docs/guides/session_management_standalone.ts b/docs/guides/session_management_standalone.ts
index c6fa33d82170..8ac133d9501c 100644
--- a/docs/guides/session_management_standalone.ts
+++ b/docs/guides/session_management_standalone.ts
@@ -5,17 +5,16 @@ const sessionPoolOptions = {
maxPoolSize: 100,
};
-// Open Session Pool.
-const sessionPool = await SessionPool.open(sessionPoolOptions);
+const sessionPool = new SessionPool(sessionPoolOptions);
// Get session.
const session = await sessionPool.getSession();
// Increase the errorScore.
-session.markBad();
+session?.markBad();
// Throw away the session.
-session.retire();
+session?.retire();
// Lower the errorScore and mark the session good.
-session.markGood();
+session?.markGood();
diff --git a/docs/package.json b/docs/package.json
index 26a6039ff021..8f5150c62336 100644
--- a/docs/package.json
+++ b/docs/package.json
@@ -2,7 +2,6 @@
"name": "crawlee-docs",
"description": "Documentation and examples for Crawlee. This package is not published to npm, only used locally for TS build checks.",
"type": "module",
- "packageManager": "yarn@4.10.3",
"scripts": {
"typecheck": "tsc --noEmit"
},
@@ -10,8 +9,19 @@
"typescript": "^5.9.3"
},
"dependencies": {
+ "@crawlee/browser-pool": "workspace:*",
+ "@crawlee/core": "workspace:*",
+ "@crawlee/got-scraping-client": "workspace:*",
+ "@crawlee/http-client": "workspace:*",
+ "@crawlee/impit-client": "workspace:*",
+ "@crawlee/stagehand": "workspace:*",
+ "apify": "*",
+ "crawlee": "workspace:*",
+ "impit": "^0.14.2",
+ "pino": "^9.6.0",
"playwright-extra": "^4.3.6",
"puppeteer-extra": "^3.3.6",
- "puppeteer-extra-plugin-stealth": "^2.11.2"
+ "puppeteer-extra-plugin-stealth": "^2.11.2",
+ "winston": "^3.17.0"
}
}
diff --git a/docs/public-api/README.md b/docs/public-api/README.md
new file mode 100644
index 000000000000..da55871b7e32
--- /dev/null
+++ b/docs/public-api/README.md
@@ -0,0 +1,37 @@
+# Public API surface maps
+
+Each `*.api.md` file in this folder is a generated **map of the public, type-level
+interface** of one publishable `@crawlee/*` package — every exported class, method,
+property, function, and type, with full signatures. These reports define **where we
+promise backwards compatibility**.
+
+They are produced by [API Extractor](https://api-extractor.com/) from the built
+`dist/index.d.ts` of each package.
+
+## Workflow
+
+- After changing any package's public surface, regenerate the reports and commit them:
+
+ ```sh
+ pnpm build # the reports are generated from dist/
+ pnpm api:extract
+ ```
+
+- CI runs `pnpm api:check`, which fails if a committed report is out of date. A failing
+ check means you changed the public API: either that change is intentional (commit the
+ updated report — reviewers will see the surface diff) or it was accidental (fix it).
+
+## Notes
+
+- `docs/public-api/temp/` holds intermediate reports and is git-ignored.
+- `@crawlee/cli` and `@crawlee/templates` are deliberately excluded — they are tooling
+ (a CLI binary and project scaffolding), not an importable API where we promise BC. The
+ exclude list lives in `scripts/api-extractor/run.ts`.
+- The generator lives in `scripts/api-extractor/`. It temporarily strips the build's
+ injected `// @ts-ignore` comment lines from the `.d.ts` files (restoring them
+ afterwards) because API Extractor's AST walker trips over some of them; a small number
+ of packages additionally need a sanitized-mirror fallback. See the comments in
+ `scripts/api-extractor/run.ts` for details.
+- Symbols tagged `@internal` still show up here if they are exported. Shrinking these
+ reports (hiding internals, e.g. via `@internal` + a trimmed rollup, or by not exporting
+ them at all) is the goal tracked in issue #3109.
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md
new file mode 100644
index 000000000000..3adf0364d9c8
--- /dev/null
+++ b/docs/public-api/crawlee-basic.api.md
@@ -0,0 +1,335 @@
+## API Report File for "@crawlee/basic"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import type { AddRequestsBatchedOptions } from '@crawlee/core';
+import type { AddRequestsBatchedResult } from '@crawlee/core';
+import { AnyPredicate } from 'ow';
+import { ArrayPredicate } from 'ow';
+import { AutoscaledPool } from '@crawlee/core';
+import { AutoscaledPoolOptions } from '@crawlee/core';
+import type { Awaitable } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/types';
+import { BasePredicate } from 'ow';
+import type { BatchAddRequestsResult } from '@crawlee/types';
+import { BooleanPredicate } from 'ow';
+import { Cheerio } from '@crawlee/utils';
+import { CheerioAPI } from '@crawlee/utils';
+import { CheerioRoot } from '@crawlee/utils';
+import type { Configuration } from '@crawlee/core';
+import { ContextPipeline } from '@crawlee/core';
+import type { CrawleeLogger } from '@crawlee/core';
+import type { CrawlingContext } from '@crawlee/core';
+import { Dataset } from '@crawlee/core';
+import type { DatasetExportOptions } from '@crawlee/core';
+import type { Dictionary } from '@crawlee/types';
+import { Element as Element_2 } from '@crawlee/utils';
+import type { EnqueueLinksOptions } from '@crawlee/core';
+import type { EventManager } from '@crawlee/core';
+import type { FinalStatistics } from '@crawlee/core';
+import type { GetUserDataFromRequest } from '@crawlee/core';
+import { IRequestLoader } from '@crawlee/core';
+import { IRequestManager } from '@crawlee/core';
+import type { ISession } from '@crawlee/types';
+import type { ISessionPool } from '@crawlee/types';
+import { NumberPredicate } from 'ow';
+import { ObjectPredicate } from 'ow';
+import { Predicate } from 'ow';
+import { ProxyConfiguration } from '@crawlee/core';
+import type { ProxyInfo } from '@crawlee/types';
+import type { ReadonlyDeep } from 'type-fest';
+import { Request as Request_2 } from '@crawlee/core';
+import { RequestQueue } from '@crawlee/core';
+import type { RequestsLike } from '@crawlee/core';
+import { RobotsTxtFile } from '@crawlee/utils';
+import type { RouterHandler } from '@crawlee/core';
+import type { RouterRoutes } from '@crawlee/core';
+import type { SetRequired } from 'type-fest';
+import type { SetStatusMessageOptions } from '@crawlee/types';
+import type { SkippedRequestCallback } from '@crawlee/core';
+import type { Source } from '@crawlee/core';
+import { Statistics } from '@crawlee/core';
+import type { StatisticsOptions } from '@crawlee/core';
+import type { StatisticState } from '@crawlee/core';
+import type { StorageBackend } from '@crawlee/types';
+import type { StorageIdentifier } from '@crawlee/core';
+import { StringPredicate } from 'ow';
+import { TimeoutError } from '@apify/timeout';
+
+// @public
+export class BasicCrawler, ExtendedContext extends Context = Context & ContextExtension> {
+ constructor(options?: BasicCrawlerOptions & RequireContextPipeline);
+ // @internal
+ protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator;
+ // (undocumented)
+ protected additionalHttpErrorStatusCodes: Set;
+ addRequests(requests: ReadonlyDeep, options?: CrawlerAddRequestsOptions): Promise;
+ autoscaledPool?: AutoscaledPool;
+ // (undocumented)
+ protected autoscaledPoolOptions: AutoscaledPoolOptions;
+ get basicContextPipeline(): ContextPipeline<{
+ request: Request_2;
+ }, CrawlingContext>;
+ // (undocumented)
+ protected blockedStatusCodes: Set;
+ protected buildBasicContextPipeline(): ContextPipeline<{
+ request: Request_2;
+ }, CrawlingContext>;
+ protected buildContextPipeline(): ContextPipeline;
+ // (undocumented)
+ protected calculateEnqueuedRequestLimit(explicitLimit?: number): Promise;
+ // (undocumented)
+ protected _canRequestBeRetried(request: Request_2, error: Error): boolean;
+ // (undocumented)
+ get contextPipeline(): ContextPipeline;
+ // (undocumented)
+ protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
+ protected _defaultIsFinishedFunction(): Promise;
+ protected delayRequest(request: Request_2, source: IRequestManager): boolean;
+ // (undocumented)
+ protected domainAccessedTime: Map;
+ // @internal
+ protected enqueueLinksWithCrawlDepth(options: SetRequired, request: Request_2, requestManager: IRequestManager): Promise;
+ // (undocumented)
+ protected errorHandler?: ErrorHandler;
+ exportData(path: string, format?: 'json' | 'csv', options?: DatasetExportOptions): Promise;
+ // (undocumented)
+ protected failedRequestHandler?: ErrorHandler;
+ protected _fetchNextRequest(): Promise | null>;
+ // (undocumented)
+ protected _getCookieHeaderFromRequest(request: Request_2): string;
+ getData(...args: Parameters): ReturnType;
+ getDataset(identifier?: string | StorageIdentifier): Promise;
+ protected _getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
+ // (undocumented)
+ protected getPendingRequestCountApproximation(): Promise;
+ getRequestManager(): Promise;
+ // @deprecated (undocumented)
+ getRequestQueue(): Promise;
+ // (undocumented)
+ protected getRobotsTxtFileForUrl(url: string): Promise;
+ // (undocumented)
+ protected handledRequestsCount: number;
+ // (undocumented)
+ protected _handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise;
+ protected handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager, request: Request_2): Promise;
+ // (undocumented)
+ protected handleSkippedRequest(options: Parameters[0]): Promise;
+ // (undocumented)
+ hasFinishedBefore: boolean;
+ // (undocumented)
+ protected httpClient: BaseHttpClient;
+ // (undocumented)
+ protected ignoreHttpErrorStatusCodes: Set;
+ protected _init(): Promise;
+ // (undocumented)
+ protected internalTimeoutMillis: number;
+ protected isErrorStatusCode(status: number): boolean;
+ protected isProxyError(error: Error): boolean;
+ protected _isTaskReadyFunction(): Promise;
+ protected _loadHandledRequestCount(): Promise;
+ // (undocumented)
+ get log(): CrawleeLogger;
+ // (undocumented)
+ protected maxCrawlDepth?: number;
+ // (undocumented)
+ protected maxRequestRetries: number;
+ // (undocumented)
+ protected maxRequestsPerCrawl?: number;
+ // (undocumented)
+ protected onSkippedRequest?: SkippedRequestCallback;
+ // (undocumented)
+ protected static optionsShape: {
+ contextPipelineBuilder: ObjectPredicate
+ 
- 
- 
+