From 8d17e11d360432f29e58f302680b14afda78979b Mon Sep 17 00:00:00 2001 From: Iain Date: Wed, 12 Aug 2026 23:32:36 +0100 Subject: [PATCH] fix(core): accept the documented requestManager crawler option --- .../src/internals/basic-crawler.ts | 1 + packages/core/src/validators.ts | 4 ++++ test/core/crawlers/basic_crawler.test.ts | 19 ++++++++++++++++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index 76f8dbabde55..30e63887aa05 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -587,6 +587,7 @@ export class BasicCrawler `Expected argument '${label}' to be a RequestQueue, got something else.`, }), + requestManager: (value: Dictionary) => ({ + validator: ow.isValid(value, ow.object.hasKeys('fetchNextRequest', 'addRequest', 'addRequestsBatched')), + message: (label: string) => `Expected argument '${label}' to be an IRequestManager, got something else.`, + }), }; diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts index dfb480e2c407..a9865a688064 100644 --- a/test/core/crawlers/basic_crawler.test.ts +++ b/test/core/crawlers/basic_crawler.test.ts @@ -26,7 +26,7 @@ import { RequestValidationError, Router, } from '@crawlee/basic'; -import { RequestState, SessionPool, Statistics } from '@crawlee/core'; +import { RequestManagerTandem, RequestState, SessionPool, Statistics } from '@crawlee/core'; import type { Dictionary } from '@crawlee/utils'; import { RobotsTxtFile, sleep } from '@crawlee/utils'; import express from 'express'; @@ -95,6 +95,23 @@ describe('BasicCrawler', () => { expect(process.listenerCount('SIGINT')).toBe(count - 1); }); + test('should accept a request manager and crawl from it', async () => { + const requestList = await RequestList.open(null, ['https://example.com/1']); + const requestQueue = await RequestQueue.open(); + const processed: string[] = []; + + const basicCrawler = new BasicCrawler({ + requestManager: new RequestManagerTandem(requestList, requestQueue), + requestHandler: async ({ request }) => { + processed.push(request.url); + }, + }); + + await basicCrawler.run(); + + expect(processed).toEqual(['https://example.com/1']); + }); + test('should run in parallel thru all the requests', async () => { const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index}` })); const sourcesCopy = JSON.parse(JSON.stringify(sources));