From 8c0110de5a61508ac4ed75daeac51695b217d894 Mon Sep 17 00:00:00 2001 From: Akhil Trivedi Date: Thu, 16 Jul 2026 17:14:05 +0530 Subject: [PATCH 1/4] feat(core): provide requests to skipped callbacks --- .../src/internals/basic-crawler.ts | 35 ++++++++++++------- .../src/internals/browser-crawler.ts | 2 +- .../core/src/enqueue_links/enqueue_links.ts | 31 ++++++++-------- packages/core/src/enqueue_links/shared.ts | 14 ++++---- .../src/internals/http-crawler.ts | 2 +- .../internals/adaptive-playwright-crawler.ts | 2 +- test/core/crawlers/basic_crawler.test.ts | 11 ++++-- 7 files changed, 55 insertions(+), 42 deletions(-) diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index e1384776b3f6..7387f02eaa77 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -15,7 +15,6 @@ import type { IRequestManager, LoadedContext, ProxyInfo, - Request, RequestsLike, RequestTransform, RestrictedCrawlingContext, @@ -41,6 +40,7 @@ import { mergeCookies, NonRetryableError, purgeDefaultStorages, + Request, RequestListAdapter, RequestManagerTandem, RequestProvider, @@ -1205,8 +1205,14 @@ export class BasicCrawler(); - const skippedBecauseOfMaxCrawlDepth = new Set(); + const skippedBecauseOfRobots = new Map(); + const skippedBecauseOfMaxCrawlDepth = new Map(); + + const normalizeSkippedRequest = (request: string | Source): Request => { + return request instanceof Request + ? request + : new Request(typeof request === 'string' ? { url: request } : { ...request, url: request.url! }); + }; const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this); const maxCrawlDepth = this.maxCrawlDepth; @@ -1221,16 +1227,17 @@ export class BasicCrawler maxCrawlDepth) { - skippedBecauseOfMaxCrawlDepth.add(url); + skippedBecauseOfMaxCrawlDepth.set(url, skippedRequest); continue; } if (await isAllowedBasedOnRobotsTxtFile(url)) { yield request; } else { - skippedBecauseOfRobots.add(url); + skippedBecauseOfRobots.set(url, skippedRequest); } } } @@ -1255,17 +1262,19 @@ export class BasicCrawler 0 ) { await Promise.all( - [...skippedBecauseOfRobots] - .map((url) => { - return this.handleSkippedRequest({ url, reason: 'robotsTxt' }); + [...skippedBecauseOfRobots.values()] + .map((request) => { + return this.handleSkippedRequest({ request, reason: 'robotsTxt' }); }) .concat( skippedBecauseOfLimit.map((request) => { - const url = typeof request === 'string' ? request : request.url!; - return this.handleSkippedRequest({ url, reason: 'limit' }); + return this.handleSkippedRequest({ + request: normalizeSkippedRequest(request), + reason: 'limit', + }); }), - [...skippedBecauseOfMaxCrawlDepth].map((url) => { - return this.handleSkippedRequest({ url, reason: 'depth' }); + [...skippedBecauseOfMaxCrawlDepth.values()].map((request) => { + return this.handleSkippedRequest({ request, reason: 'depth' }); }), ), ); @@ -1581,7 +1590,7 @@ export class BasicCrawler 0) { await Promise.all( skippedRequests.map((request) => { return onSkippedRequest({ - url: request.url, - reason: request.skippedReason ?? reason, + request: + request instanceof Request + ? request + : new Request(typeof request === 'string' ? { url: request } : { ...request, url: request.url! }), + reason: (request as { skippedReason?: SkippedRequestReason }).skippedReason ?? reason, }) as Promise; }), ); @@ -464,7 +464,7 @@ export async function enqueueLinks( } async function createFilteredRequests() { - const skippedRequests: string[] = []; + const skippedRequests: Request[] = []; // No user provided patterns means we can skip an extra filtering step if (urlPatternObjects.length === 0) { @@ -473,7 +473,7 @@ export async function enqueueLinks( enqueueStrategyPatterns, urlExcludePatternObjects, options.strategy, - (url) => skippedRequests.push(url), + (request) => skippedRequests.push(request), ); } @@ -483,17 +483,14 @@ export async function enqueueLinks( urlPatternObjects, urlExcludePatternObjects, options.strategy, - (url) => skippedRequests.push(url), + (request) => skippedRequests.push(request), ); // ...then filter them by the enqueue links strategy (making this an AND check) - const filtered = filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns, (url) => - skippedRequests.push(url), + const filtered = filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns, (request) => + skippedRequests.push(request), ); - await reportSkippedRequests( - skippedRequests.map((url) => ({ url })), - 'filters', - ); + await reportSkippedRequests(skippedRequests, 'filters'); return filtered; } @@ -506,7 +503,7 @@ export async function enqueueLinks( if (requestsOverLimit?.length !== undefined && requestsOverLimit.length > 0) { await reportSkippedRequests( - requestsOverLimit.map((r) => ({ url: typeof r === 'string' ? r : r.url! })), + requestsOverLimit.map((request) => (typeof request === 'string' ? { url: request } : request)), 'enqueueLimit', ); } diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts index eae7603135b2..a6ca40e8eaca 100644 --- a/packages/core/src/enqueue_links/shared.ts +++ b/packages/core/src/enqueue_links/shared.ts @@ -49,7 +49,7 @@ export type RegExpInput = RegExp | RegExpObject; export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'redirect' | 'depth'; -export type SkippedRequestCallback = (args: { url: string; reason: SkippedRequestReason }) => Awaitable; +export type SkippedRequestCallback = (args: { request: Request; reason: SkippedRequestReason }) => Awaitable; /** * @ignore @@ -171,18 +171,18 @@ export function createRequests( urlPatternObjects?: UrlPatternObject[], excludePatternObjects: UrlPatternObject[] = [], strategy?: EnqueueLinksOptions['strategy'], - onSkippedUrl?: (url: string) => void, + onSkippedRequest?: (request: Request) => void, ): Request[] { const excludePatternObjectMatchers = excludePatternObjects.map(createPatternObjectMatcher); const urlPatternObjectMatchers = urlPatternObjects?.map(createPatternObjectMatcher); return requestOptions .map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts })) - .filter(({ url }) => { + .filter(({ url, opts }) => { const matchesExcludePatterns = excludePatternObjectMatchers.some(({ match }) => match(url)); if (matchesExcludePatterns) { - onSkippedUrl?.(url); + onSkippedRequest?.(new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : opts)); } return !matchesExcludePatterns; @@ -205,7 +205,7 @@ export function createRequests( } // didn't match any positive pattern - onSkippedUrl?.(url); + onSkippedRequest?.(new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : opts)); return null; }) .filter((request) => request) as Request[]; @@ -214,7 +214,7 @@ export function createRequests( export function filterRequestsByPatterns( requests: Request[], patterns?: UrlPatternObject[], - onSkippedUrl?: (url: string) => void, + onSkippedRequest?: (request: Request) => void, ): Request[] { if (!patterns?.length) { return requests; @@ -229,7 +229,7 @@ export function filterRequestsByPatterns( if (matchingPattern !== undefined) { filtered.push(request); } else { - onSkippedUrl?.(request.url); + onSkippedRequest?.(request); } } diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts index 9fc59af379f7..f3f7ced0f7eb 100644 --- a/packages/http-crawler/src/internals/http-crawler.ts +++ b/packages/http-crawler/src/internals/http-crawler.ts @@ -560,7 +560,7 @@ export class HttpCrawler< request.noRetry = true; request.state = RequestState.SKIPPED; - await this.handleSkippedRequest({ url: request.url, reason: 'redirect' }); + await this.handleSkippedRequest({ request, reason: 'redirect' }); return; } diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts index 7f9c91fe65a3..6fa8fa54856f 100644 --- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts +++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts @@ -641,7 +641,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler { request.noRetry = true; request.state = RequestState.SKIPPED; - await this.handleSkippedRequest({ url: request.url, reason: 'redirect' }); + await this.handleSkippedRequest({ request, reason: 'redirect' }); return; } diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts index 57323e591fa9..8b59f1215077 100644 --- a/test/core/crawlers/basic_crawler.test.ts +++ b/test/core/crawlers/basic_crawler.test.ts @@ -260,6 +260,7 @@ describe('BasicCrawler', () => { options = { urls: ['https://example.com/1/', 'https://example.com/2/'], onSkippedRequest: onSkippedRequestMock, + userData: { source: 'crawl-depth-test' }, }; request = new Request({ url: 'https://example.com/', crawlDepth: 2 }); requestQueue = { @@ -287,8 +288,14 @@ describe('BasicCrawler', () => { const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]); expect(skippedRequests).toHaveLength(2); - expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'depth' }); - expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'depth' }); + expect(skippedRequests[0]).toMatchObject({ reason: 'depth', request: { url: 'https://example.com/1/' } }); + expect(skippedRequests[1]).toMatchObject({ reason: 'depth', request: { url: 'https://example.com/2/' } }); + expect(skippedRequests[0].request).toBeInstanceOf(Request); + expect(skippedRequests[1].request).toBeInstanceOf(Request); + expect(skippedRequests[0].request.userData).toMatchObject({ source: 'crawl-depth-test' }); + expect(skippedRequests[1].request.userData).toMatchObject({ source: 'crawl-depth-test' }); + expect(skippedRequests[0]).not.toHaveProperty('url'); + expect(skippedRequests[1]).not.toHaveProperty('url'); }); it('should respect user provided transformRequestFunction', async () => { From 7b61581dbc91090e6999999beb391d84bee5fe15 Mon Sep 17 00:00:00 2001 From: Akhil Trivedi Date: Thu, 30 Jul 2026 19:43:35 +0530 Subject: [PATCH 2/4] fix(core): preserve skipped callback url --- .../src/internals/basic-crawler.ts | 4 ++-- .../core/src/enqueue_links/enqueue_links.ts | 19 +++++++++++++------ packages/core/src/enqueue_links/shared.ts | 11 +++++++++-- test/core/crawlers/basic_crawler.test.ts | 14 ++++++++++---- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index 7387f02eaa77..b894a09480e7 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -1160,7 +1160,7 @@ export class BasicCrawler[0]): Promise { + protected async handleSkippedRequest(options: Omit[0], 'url'>): Promise { if (options.reason === 'limit') { this.logOncePerRun( 'maxRequestsPerCrawl', @@ -1176,7 +1176,7 @@ export class BasicCrawler 0) { await Promise.all( skippedRequests.map((request) => { + const skippedRequest = + request instanceof Request + ? request + : new Request( + typeof request === 'string' ? { url: request } : { ...request, url: request.url! }, + ); + return onSkippedRequest({ - request: - request instanceof Request - ? request - : new Request(typeof request === 'string' ? { url: request } : { ...request, url: request.url! }), + url: skippedRequest.url, + request: skippedRequest, reason: (request as { skippedReason?: SkippedRequestReason }).skippedReason ?? reason, }) as Promise; }), @@ -486,8 +491,10 @@ export async function enqueueLinks( (request) => skippedRequests.push(request), ); // ...then filter them by the enqueue links strategy (making this an AND check) - const filtered = filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns, (request) => - skippedRequests.push(request), + const filtered = filterRequestsByPatterns( + generatedRequestsFromUserFilters, + enqueueStrategyPatterns, + (request) => skippedRequests.push(request), ); await reportSkippedRequests(skippedRequests, 'filters'); diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts index a6ca40e8eaca..19de7496d39a 100644 --- a/packages/core/src/enqueue_links/shared.ts +++ b/packages/core/src/enqueue_links/shared.ts @@ -49,7 +49,12 @@ export type RegExpInput = RegExp | RegExpObject; export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'redirect' | 'depth'; -export type SkippedRequestCallback = (args: { request: Request; reason: SkippedRequestReason }) => Awaitable; +export type SkippedRequestCallback = (args: { + /** @deprecated Use `request.url` instead. */ + url: string; + request: Request; + reason: SkippedRequestReason; +}) => Awaitable; /** * @ignore @@ -182,7 +187,9 @@ export function createRequests( const matchesExcludePatterns = excludePatternObjectMatchers.some(({ match }) => match(url)); if (matchesExcludePatterns) { - onSkippedRequest?.(new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : opts)); + onSkippedRequest?.( + new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : opts), + ); } return !matchesExcludePatterns; diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts index 8b59f1215077..4892ec1e7993 100644 --- a/test/core/crawlers/basic_crawler.test.ts +++ b/test/core/crawlers/basic_crawler.test.ts @@ -288,14 +288,20 @@ describe('BasicCrawler', () => { const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]); expect(skippedRequests).toHaveLength(2); - expect(skippedRequests[0]).toMatchObject({ reason: 'depth', request: { url: 'https://example.com/1/' } }); - expect(skippedRequests[1]).toMatchObject({ reason: 'depth', request: { url: 'https://example.com/2/' } }); + expect(skippedRequests[0]).toMatchObject({ + url: 'https://example.com/1/', + reason: 'depth', + request: { url: 'https://example.com/1/' }, + }); + expect(skippedRequests[1]).toMatchObject({ + url: 'https://example.com/2/', + reason: 'depth', + request: { url: 'https://example.com/2/' }, + }); expect(skippedRequests[0].request).toBeInstanceOf(Request); expect(skippedRequests[1].request).toBeInstanceOf(Request); expect(skippedRequests[0].request.userData).toMatchObject({ source: 'crawl-depth-test' }); expect(skippedRequests[1].request.userData).toMatchObject({ source: 'crawl-depth-test' }); - expect(skippedRequests[0]).not.toHaveProperty('url'); - expect(skippedRequests[1]).not.toHaveProperty('url'); }); it('should respect user provided transformRequestFunction', async () => { From 54b31051eddca5c85d3416dbc4b86b1d592c4d5b Mon Sep 17 00:00:00 2001 From: Akhil Trivedi Date: Wed, 5 Aug 2026 10:17:38 +0530 Subject: [PATCH 3/4] test(core): update skipped request assertions to verify request property --- test/core/crawlers/basic_crawler.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts index 33ccb4ac4c9a..498c747a8003 100644 --- a/test/core/crawlers/basic_crawler.test.ts +++ b/test/core/crawlers/basic_crawler.test.ts @@ -2246,9 +2246,10 @@ describe('BasicCrawler', () => { ]; for (const mock of [crawlerOnSkippedRequest, userOnSkippedRequest]) { - expect(mock.mock.calls.map((call) => call[0]).sort((a, b) => a.url.localeCompare(b.url))).toEqual( - skipped, - ); + const calls = mock.mock.calls.map((call) => call[0]).sort((a, b) => a.url.localeCompare(b.url)); + expect(calls).toMatchObject(skipped); + expect(calls[0].request).toBeInstanceOf(Request); + expect(calls[1].request).toBeInstanceOf(Request); } }); From 97f06e2e1a11d4459a5fbe4ae430e3cdd590c6e9 Mon Sep 17 00:00:00 2001 From: Akhil Trivedi Date: Wed, 5 Aug 2026 10:18:10 +0530 Subject: [PATCH 4/4] test(core): update transformRequestFunction skip assertions to match request property --- test/core/crawlers/basic_crawler.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts index 498c747a8003..164e198644ad 100644 --- a/test/core/crawlers/basic_crawler.test.ts +++ b/test/core/crawlers/basic_crawler.test.ts @@ -336,8 +336,10 @@ describe('BasicCrawler', () => { const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]); expect(skippedRequests).toHaveLength(2); - expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'filters' }); - expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'filters' }); + expect(skippedRequests[0]).toMatchObject({ url: 'https://example.com/1/', reason: 'filters' }); + expect(skippedRequests[1]).toMatchObject({ url: 'https://example.com/2/', reason: 'filters' }); + expect(skippedRequests[0].request).toBeInstanceOf(Request); + expect(skippedRequests[1].request).toBeInstanceOf(Request); }); });