From 52dc085fd9f568e4a1adc0942744d74d1c6b4fa4 Mon Sep 17 00:00:00 2001 From: Ayush7614 Date: Thu, 30 Jul 2026 20:15:19 +0530 Subject: [PATCH 1/2] fix: route robots.txt through proxy and apply MITM TLS per proxy URL respectRobotsTxtFile fetched robots.txt without the crawler proxy, leaking the real IP and missing geo-specific rules. Also MITM TLS disabling was configuration-wide, so mixed MITM and normal proxies shared one incorrect setting. Fetch robots via ProxyConfiguration and gate rejectUnauthorized / ignoreHTTPSErrors on the selected proxy URL. --- .../src/internals/basic-crawler.ts | 24 ++++++++++++- .../src/internals/browser-crawler.ts | 6 ++-- packages/core/src/proxy_configuration.ts | 36 ++++++++++++++++++- .../src/internals/http-crawler.ts | 7 ++-- test/core/proxy_configuration.test.ts | 32 +++++++++++++++++ 5 files changed, 95 insertions(+), 10 deletions(-) diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index 0b48a3080d95..56cea20987f8 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -14,6 +14,7 @@ import type { IRequestList, IRequestManager, LoadedContext, + ProxyConfiguration, ProxyInfo, Request, RequestsLike, @@ -1459,7 +1460,8 @@ export class BasicCrawler { + const proxyConfiguration = (this as { proxyConfiguration?: ProxyConfiguration }).proxyConfiguration; + if (!proxyConfiguration) { + return undefined; + } + + try { + const proxyInfo = await proxyConfiguration.newProxyInfo(undefined, { + request: { url } as Request, + }); + return proxyInfo?.url; + } catch { + return undefined; + } + } + protected async _pauseOnMigration() { if (this.autoscaledPool) { // if run wasn't called, this is going to crash diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts index a8fae22a83d4..65d76ed02613 100644 --- a/packages/browser-crawler/src/internals/browser-crawler.ts +++ b/packages/browser-crawler/src/internals/browser-crawler.ts @@ -542,7 +542,7 @@ export abstract class BrowserCrawler< newPageOptions.proxyUrl = proxyInfo?.url; newPageOptions.proxyTier = proxyInfo?.proxyTier; - if (this.proxyConfiguration.isManInTheMiddle) { + if (proxyInfo?.isManInTheMiddle || this.proxyConfiguration.isProxyManInTheMiddle(proxyInfo?.url)) { /** * @see https://playwright.dev/docs/api/class-browser/#browser-new-context * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md @@ -766,8 +766,8 @@ export abstract class BrowserCrawler< launchContext.proxyUrl = proxyInfo?.url; launchContextExtends.proxyInfo = proxyInfo; - // Disable SSL verification for MITM proxies - if (this.proxyConfiguration.isManInTheMiddle) { + // Disable SSL verification only for the selected MITM proxy URL. + if (proxyInfo?.isManInTheMiddle || this.proxyConfiguration.isProxyManInTheMiddle(proxyInfo?.url)) { /** * @see https://playwright.dev/docs/api/class-browser/#browser-new-context * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md diff --git a/packages/core/src/proxy_configuration.ts b/packages/core/src/proxy_configuration.ts index 132d923aca83..a1097c1ac751 100644 --- a/packages/core/src/proxy_configuration.ts +++ b/packages/core/src/proxy_configuration.ts @@ -40,6 +40,15 @@ export interface ProxyConfigurationOptions { * Use `null` as a proxy URL to disable the proxy for the given tier. */ tieredProxyUrls?: UrlList[]; + + /** + * Subset of proxy URLs that perform TLS interception (MITM). + * When the currently selected proxy URL is listed here, crawlers disable certificate verification + * for that request only. This allows mixing MITM and normal proxies in one configuration. + * + * When omitted, the {@apilink ProxyConfiguration.isManInTheMiddle} flag applies to every proxy URL. + */ + manInTheMiddleProxyUrls?: string[]; } export interface TieredProxy { @@ -112,6 +121,12 @@ export interface ProxyInfo { * Proxy tier for the current proxy, if applicable (only for `tieredProxyUrls`). */ proxyTier?: number; + + /** + * Whether this specific proxy URL performs TLS interception (MITM). + * Used by crawlers to decide whether certificate verification should be disabled for the request. + */ + isManInTheMiddle?: boolean; } interface TieredProxyOptions { @@ -205,6 +220,7 @@ export class ProxyConfiguration { protected nextCustomUrlIndex = 0; protected proxyUrls?: UrlList; protected tieredProxyUrls?: UrlList[]; + protected manInTheMiddleProxyUrls?: Set; protected usedProxyUrls = new Map(); protected newUrlFunction?: ProxyConfigurationFunction; protected log = log.child({ prefix: 'ProxyConfiguration' }); @@ -240,10 +256,11 @@ export class ProxyConfiguration { tieredProxyUrls: ow.optional.array.nonEmpty.ofType( ow.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null)), ), + manInTheMiddleProxyUrls: ow.optional.array.ofType(ow.string.url), }), ); - const { proxyUrls, newUrlFunction, tieredProxyUrls } = options; + const { proxyUrls, newUrlFunction, tieredProxyUrls, manInTheMiddleProxyUrls } = options; if ([proxyUrls, newUrlFunction, tieredProxyUrls].filter((x) => x).length > 1) this._throwCannotCombineCustomMethods(); @@ -252,6 +269,22 @@ export class ProxyConfiguration { this.proxyUrls = proxyUrls; this.newUrlFunction = newUrlFunction; this.tieredProxyUrls = tieredProxyUrls; + this.manInTheMiddleProxyUrls = manInTheMiddleProxyUrls?.length + ? new Set(manInTheMiddleProxyUrls) + : undefined; + } + + /** + * Returns whether the given proxy URL should be treated as a MITM (TLS-intercepting) proxy. + * Prefers {@apilink ProxyConfigurationOptions.manInTheMiddleProxyUrls} when configured; + * otherwise falls back to the configuration-wide {@apilink ProxyConfiguration.isManInTheMiddle} flag. + */ + isProxyManInTheMiddle(proxyUrl?: string | null): boolean { + if (!proxyUrl) return false; + if (this.manInTheMiddleProxyUrls) { + return this.manInTheMiddleProxyUrls.has(proxyUrl); + } + return this.isManInTheMiddle; } /** @@ -296,6 +329,7 @@ export class ProxyConfiguration { hostname, port: port!, proxyTier: tier, + isManInTheMiddle: this.isProxyManInTheMiddle(url), }; } diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts index be969884d92e..e70e363cd81f 100644 --- a/packages/http-crawler/src/internals/http-crawler.ts +++ b/packages/http-crawler/src/internals/http-crawler.ts @@ -824,11 +824,8 @@ export class HttpCrawler< // Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header Reflect.deleteProperty(requestOptions.headers!, 'cookie'); - // TODO this is incorrect, the check for man in the middle needs to be done - // on individual proxy level, not on the `proxyConfiguration` level, - // because users can use normal + MITM proxies in a single configuration. - // Disable SSL verification for MITM proxies - if (this.proxyConfiguration && this.proxyConfiguration.isManInTheMiddle) { + // Disable SSL verification only for the selected MITM proxy URL (not the whole configuration). + if (proxyUrl && this.proxyConfiguration?.isProxyManInTheMiddle(proxyUrl)) { requestOptions.https = { ...requestOptions.https, rejectUnauthorized: false, diff --git a/test/core/proxy_configuration.test.ts b/test/core/proxy_configuration.test.ts index e70344109488..841c75cd506b 100644 --- a/test/core/proxy_configuration.test.ts +++ b/test/core/proxy_configuration.test.ts @@ -20,6 +20,7 @@ describe('ProxyConfiguration', () => { username: '', password: '', port: '1111', + isManInTheMiddle: false, }; expect(await proxyConfiguration.newProxyInfo(sessionId)).toEqual(proxyInfo); }); @@ -35,10 +36,41 @@ describe('ProxyConfiguration', () => { username: 'user@name', password: 'pass@word', port: '1111', + isManInTheMiddle: false, }; expect(await proxyConfiguration.newProxyInfo(sessionId)).toEqual(proxyInfo); }); + test('isProxyManInTheMiddle respects per-URL MITM list', async () => { + const mitmUrl = 'http://mitm-proxy.com:8000'; + const normalUrl = 'http://normal-proxy.com:8000'; + const proxyConfiguration = new ProxyConfiguration({ + proxyUrls: [mitmUrl, normalUrl], + manInTheMiddleProxyUrls: [mitmUrl], + }); + + expect(proxyConfiguration.isProxyManInTheMiddle(mitmUrl)).toBe(true); + expect(proxyConfiguration.isProxyManInTheMiddle(normalUrl)).toBe(false); + expect(proxyConfiguration.isProxyManInTheMiddle(undefined)).toBe(false); + + const mitmInfo = await proxyConfiguration.newProxyInfo(sessionId); + // newProxyInfo rotates; force checks via helper + explicit construction path + expect(proxyConfiguration.isProxyManInTheMiddle(mitmInfo!.url)).toBe( + mitmInfo!.url === mitmUrl, + ); + expect(mitmInfo!.isManInTheMiddle).toBe(mitmInfo!.url === mitmUrl); + }); + + test('isProxyManInTheMiddle falls back to configuration-wide flag', () => { + const proxyConfiguration = new ProxyConfiguration({ + proxyUrls: ['http://proxy.com:1111'], + }); + proxyConfiguration.isManInTheMiddle = true; + + expect(proxyConfiguration.isProxyManInTheMiddle('http://proxy.com:1111')).toBe(true); + expect(proxyConfiguration.isProxyManInTheMiddle('http://other.com:1111')).toBe(true); + }); + test('should throw on invalid newUrlFunction', async () => { const newUrlFunction = () => { return 'http://proxy.com:1111*invalid_url'; From 827a6e1233471cfaff52731effc50fb0989b940d Mon Sep 17 00:00:00 2001 From: Ayush7614 Date: Thu, 30 Jul 2026 20:16:03 +0530 Subject: [PATCH 2/2] test(basic-crawler): cover robots.txt fetch through proxy configuration --- test/core/crawlers/basic_crawler.test.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts index d2324e6cc11a..ccb9e0e159a4 100644 --- a/test/core/crawlers/basic_crawler.test.ts +++ b/test/core/crawlers/basic_crawler.test.ts @@ -1732,6 +1732,29 @@ describe('BasicCrawler', () => { expect(addRequestsBatchedSpy).toHaveBeenCalledOnce(); }); + test('fetches robots.txt through the crawler proxy configuration', async () => { + const proxyUrl = 'http://user:pass@proxy.example:8000'; + const findSpy = vitest.spyOn(RobotsTxtFile, 'find').mockResolvedValue( + RobotsTxtFile.from('http://example.com/robots.txt', 'User-agent: *\nAllow: /\n'), + ); + + class ProxiedBasicCrawler extends BasicCrawler { + proxyConfiguration = { + newProxyInfo: async () => ({ url: proxyUrl }), + } as any; + } + + const crawler = new ProxiedBasicCrawler({ + respectRobotsTxtFile: true, + requestHandler: async () => {}, + }); + + await (crawler as any).getRobotsTxtFileForUrl('http://example.com/page'); + + expect(findSpy).toHaveBeenCalledWith('http://example.com/page', proxyUrl); + findSpy.mockRestore(); + }); + test.each([ { testName: 'custom user-agent robots.txt rules',