fix: route robots.txt through proxy and apply MITM TLS per proxy URL

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -14,6 +14,7 @@ import type {
     IRequestList,
     IRequestManager,
     LoadedContext,
+    ProxyConfiguration,
     ProxyInfo,
     Request,
     RequestsLike,
@@ -1459,7 +1460,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
                 return cachedRobotsTxtFile;
             }
 
-            const robotsTxtFile = await RobotsTxtFile.find(url);
+            const proxyUrl = await this.getProxyUrlForRobotsTxt(url);
+            const robotsTxtFile = await RobotsTxtFile.find(url, proxyUrl);
             this.robotsTxtFileCache.add(origin, robotsTxtFile);
 
             return robotsTxtFile;
@@ -1469,6 +1471,26 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
         }
     }
 
+    /**
+     * Resolves a proxy URL for fetching robots.txt using the crawler's proxy configuration when available.
+     * This keeps robots lookups on the same network path as page requests (geo rules, IP allowlists, no direct IP leak).
+     */
+    protected async getProxyUrlForRobotsTxt(url: string): Promise<string | undefined> {
+        const proxyConfiguration = (this as { proxyConfiguration?: ProxyConfiguration }).proxyConfiguration;
+        if (!proxyConfiguration) {
+            return undefined;
+        }
+
+        try {
+            const proxyInfo = await proxyConfiguration.newProxyInfo(undefined, {
+                request: { url } as Request,
+            });
+            return proxyInfo?.url;
+        } catch {
+            return undefined;
+        }
+    }
+
     protected async _pauseOnMigration() {
         if (this.autoscaledPool) {
             // if run wasn't called, this is going to crash

packages/browser-crawler/src/internals/browser-crawler.ts

@@ -542,7 +542,7 @@ export abstract class BrowserCrawler<
             newPageOptions.proxyUrl = proxyInfo?.url;
             newPageOptions.proxyTier = proxyInfo?.proxyTier;
 
-            if (this.proxyConfiguration.isManInTheMiddle) {
+            if (proxyInfo?.isManInTheMiddle || this.proxyConfiguration.isProxyManInTheMiddle(proxyInfo?.url)) {
                 /**
                  * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
                  * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
@@ -766,8 +766,8 @@ export abstract class BrowserCrawler<
             launchContext.proxyUrl = proxyInfo?.url;
             launchContextExtends.proxyInfo = proxyInfo;
 
-            // Disable SSL verification for MITM proxies
-            if (this.proxyConfiguration.isManInTheMiddle) {
+            // Disable SSL verification only for the selected MITM proxy URL.
+            if (proxyInfo?.isManInTheMiddle || this.proxyConfiguration.isProxyManInTheMiddle(proxyInfo?.url)) {
                 /**
                  * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
                  * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md

packages/core/src/proxy_configuration.ts

@@ -40,6 +40,15 @@ export interface ProxyConfigurationOptions {
      * Use `null` as a proxy URL to disable the proxy for the given tier.
      */
     tieredProxyUrls?: UrlList[];
+
+    /**
+     * Subset of proxy URLs that perform TLS interception (MITM).
+     * When the currently selected proxy URL is listed here, crawlers disable certificate verification
+     * for that request only. This allows mixing MITM and normal proxies in one configuration.
+     *
+     * When omitted, the {@apilink ProxyConfiguration.isManInTheMiddle} flag applies to every proxy URL.
+     */
+    manInTheMiddleProxyUrls?: string[];
 }
 
 export interface TieredProxy {
@@ -112,6 +121,12 @@ export interface ProxyInfo {
      * Proxy tier for the current proxy, if applicable (only for `tieredProxyUrls`).
      */
     proxyTier?: number;
+
+    /**
+     * Whether this specific proxy URL performs TLS interception (MITM).
+     * Used by crawlers to decide whether certificate verification should be disabled for the request.
+     */
+    isManInTheMiddle?: boolean;
 }
 
 interface TieredProxyOptions {
@@ -205,6 +220,7 @@ export class ProxyConfiguration {
     protected nextCustomUrlIndex = 0;
     protected proxyUrls?: UrlList;
     protected tieredProxyUrls?: UrlList[];
+    protected manInTheMiddleProxyUrls?: Set<string>;
     protected usedProxyUrls = new Map<string, string | null>();
     protected newUrlFunction?: ProxyConfigurationFunction;
     protected log = log.child({ prefix: 'ProxyConfiguration' });
@@ -240,10 +256,11 @@ export class ProxyConfiguration {
                 tieredProxyUrls: ow.optional.array.nonEmpty.ofType(
                     ow.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null)),
                 ),
+                manInTheMiddleProxyUrls: ow.optional.array.ofType(ow.string.url),
             }),
         );
 
-        const { proxyUrls, newUrlFunction, tieredProxyUrls } = options;
+        const { proxyUrls, newUrlFunction, tieredProxyUrls, manInTheMiddleProxyUrls } = options;
 
         if ([proxyUrls, newUrlFunction, tieredProxyUrls].filter((x) => x).length > 1)
             this._throwCannotCombineCustomMethods();
@@ -252,6 +269,22 @@ export class ProxyConfiguration {
         this.proxyUrls = proxyUrls;
         this.newUrlFunction = newUrlFunction;
         this.tieredProxyUrls = tieredProxyUrls;
+        this.manInTheMiddleProxyUrls = manInTheMiddleProxyUrls?.length
+            ? new Set(manInTheMiddleProxyUrls)
+            : undefined;
+    }
+
+    /**
+     * Returns whether the given proxy URL should be treated as a MITM (TLS-intercepting) proxy.
+     * Prefers {@apilink ProxyConfigurationOptions.manInTheMiddleProxyUrls} when configured;
+     * otherwise falls back to the configuration-wide {@apilink ProxyConfiguration.isManInTheMiddle} flag.
+     */
+    isProxyManInTheMiddle(proxyUrl?: string | null): boolean {
+        if (!proxyUrl) return false;
+        if (this.manInTheMiddleProxyUrls) {
+            return this.manInTheMiddleProxyUrls.has(proxyUrl);
+        }
+        return this.isManInTheMiddle;
     }
 
     /**
@@ -296,6 +329,7 @@ export class ProxyConfiguration {
             hostname,
             port: port!,
             proxyTier: tier,
+            isManInTheMiddle: this.isProxyManInTheMiddle(url),
         };
     }
 

packages/http-crawler/src/internals/http-crawler.ts

@@ -824,11 +824,8 @@ export class HttpCrawler<
         // Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header
         Reflect.deleteProperty(requestOptions.headers!, 'cookie');
 
-        // TODO this is incorrect, the check for man in the middle needs to be done
-        //   on individual proxy level, not on the `proxyConfiguration` level,
-        //   because users can use normal + MITM proxies in a single configuration.
-        // Disable SSL verification for MITM proxies
-        if (this.proxyConfiguration && this.proxyConfiguration.isManInTheMiddle) {
+        // Disable SSL verification only for the selected MITM proxy URL (not the whole configuration).
+        if (proxyUrl && this.proxyConfiguration?.isProxyManInTheMiddle(proxyUrl)) {
             requestOptions.https = {
                 ...requestOptions.https,
                 rejectUnauthorized: false,

test/core/crawlers/basic_crawler.test.ts

@@ -1732,6 +1732,29 @@ describe('BasicCrawler', () => {
             expect(addRequestsBatchedSpy).toHaveBeenCalledOnce();
         });
 
+        test('fetches robots.txt through the crawler proxy configuration', async () => {
+            const proxyUrl = 'http://user:pass@proxy.example:8000';
+            const findSpy = vitest.spyOn(RobotsTxtFile, 'find').mockResolvedValue(
+                RobotsTxtFile.from('http://example.com/robots.txt', 'User-agent: *\nAllow: /\n'),
+            );
+
+            class ProxiedBasicCrawler extends BasicCrawler {
+                proxyConfiguration = {
+                    newProxyInfo: async () => ({ url: proxyUrl }),
+                } as any;
+            }
+
+            const crawler = new ProxiedBasicCrawler({
+                respectRobotsTxtFile: true,
+                requestHandler: async () => {},
+            });
+
+            await (crawler as any).getRobotsTxtFileForUrl('http://example.com/page');
+
+            expect(findSpy).toHaveBeenCalledWith('http://example.com/page', proxyUrl);
+            findSpy.mockRestore();
+        });
+
         test.each([
             {
                 testName: 'custom user-agent robots.txt rules',

test/core/proxy_configuration.test.ts

@@ -20,6 +20,7 @@ describe('ProxyConfiguration', () => {
             username: '',
             password: '',
             port: '1111',
+            isManInTheMiddle: false,
         };
         expect(await proxyConfiguration.newProxyInfo(sessionId)).toEqual(proxyInfo);
     });
@@ -35,10 +36,41 @@ describe('ProxyConfiguration', () => {
             username: 'user@name',
             password: 'pass@word',
             port: '1111',
+            isManInTheMiddle: false,
         };
         expect(await proxyConfiguration.newProxyInfo(sessionId)).toEqual(proxyInfo);
     });
 
+    test('isProxyManInTheMiddle respects per-URL MITM list', async () => {
+        const mitmUrl = 'http://mitm-proxy.com:8000';
+        const normalUrl = 'http://normal-proxy.com:8000';
+        const proxyConfiguration = new ProxyConfiguration({
+            proxyUrls: [mitmUrl, normalUrl],
+            manInTheMiddleProxyUrls: [mitmUrl],
+        });
+
+        expect(proxyConfiguration.isProxyManInTheMiddle(mitmUrl)).toBe(true);
+        expect(proxyConfiguration.isProxyManInTheMiddle(normalUrl)).toBe(false);
+        expect(proxyConfiguration.isProxyManInTheMiddle(undefined)).toBe(false);
+
+        const mitmInfo = await proxyConfiguration.newProxyInfo(sessionId);
+        // newProxyInfo rotates; force checks via helper + explicit construction path
+        expect(proxyConfiguration.isProxyManInTheMiddle(mitmInfo!.url)).toBe(
+            mitmInfo!.url === mitmUrl,
+        );
+        expect(mitmInfo!.isManInTheMiddle).toBe(mitmInfo!.url === mitmUrl);
+    });
+
+    test('isProxyManInTheMiddle falls back to configuration-wide flag', () => {
+        const proxyConfiguration = new ProxyConfiguration({
+            proxyUrls: ['http://proxy.com:1111'],
+        });
+        proxyConfiguration.isManInTheMiddle = true;
+
+        expect(proxyConfiguration.isProxyManInTheMiddle('http://proxy.com:1111')).toBe(true);
+        expect(proxyConfiguration.isProxyManInTheMiddle('http://other.com:1111')).toBe(true);
+    });
+
     test('should throw on invalid newUrlFunction', async () => {
         const newUrlFunction = () => {
             return 'http://proxy.com:1111*invalid_url';