diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index 76f8dbabde55..30e63887aa05 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -587,6 +587,7 @@ export class BasicCrawler `Expected argument '${label}' to be a RequestQueue, got something else.`, }), + requestManager: (value: Dictionary) => ({ + validator: ow.isValid(value, ow.object.hasKeys('fetchNextRequest', 'addRequest', 'addRequestsBatched')), + message: (label: string) => `Expected argument '${label}' to be an IRequestManager, got something else.`, + }), }; diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts index dfb480e2c407..a9865a688064 100644 --- a/test/core/crawlers/basic_crawler.test.ts +++ b/test/core/crawlers/basic_crawler.test.ts @@ -26,7 +26,7 @@ import { RequestValidationError, Router, } from '@crawlee/basic'; -import { RequestState, SessionPool, Statistics } from '@crawlee/core'; +import { RequestManagerTandem, RequestState, SessionPool, Statistics } from '@crawlee/core'; import type { Dictionary } from '@crawlee/utils'; import { RobotsTxtFile, sleep } from '@crawlee/utils'; import express from 'express'; @@ -95,6 +95,23 @@ describe('BasicCrawler', () => { expect(process.listenerCount('SIGINT')).toBe(count - 1); }); + test('should accept a request manager and crawl from it', async () => { + const requestList = await RequestList.open(null, ['https://example.com/1']); + const requestQueue = await RequestQueue.open(); + const processed: string[] = []; + + const basicCrawler = new BasicCrawler({ + requestManager: new RequestManagerTandem(requestList, requestQueue), + requestHandler: async ({ request }) => { + processed.push(request.url); + }, + }); + + await basicCrawler.run(); + + expect(processed).toEqual(['https://example.com/1']); + }); + test('should run in parallel thru all the requests', async () => { const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index}` })); const sourcesCopy = JSON.parse(JSON.stringify(sources));