diff --git a/package-lock.json b/package-lock.json index 042fcc3a..2154ca04 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "commander": "^14.0.3", "js-yaml": "^4.1.0", "turndown": "^7.2.2", + "undici": "^7.24.5", "ws": "^8.18.0" }, "bin": { @@ -3514,6 +3515,15 @@ "node": ">=14.17" } }, + "node_modules/undici": { + "version": "7.24.5", + "resolved": "https://registry.npmmirror.com/undici/-/undici-7.24.5.tgz", + "integrity": "sha512-3IWdCpjgxp15CbJnsi/Y9TCDE7HWVN19j1hmzVhoAkY/+CJx449tVxT5wZc1Gwg8J+P0LWvzlBzxYRnHJ+1i7Q==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", diff --git a/package.json b/package.json index 18faef5e..a62f5568 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "commander": "^14.0.3", "js-yaml": "^4.1.0", "turndown": "^7.2.2", + "undici": "^7.24.5", "ws": "^8.18.0" }, "devDependencies": { diff --git a/src/pipeline/steps/fetch.ts b/src/pipeline/steps/fetch.ts index edecda63..c0df217c 100644 --- a/src/pipeline/steps/fetch.ts +++ b/src/pipeline/steps/fetch.ts @@ -8,8 +8,17 @@ import type { IPage } from '../../types.js'; import { render } from '../template.js'; import { isRecord, mapConcurrent } from '../../utils.js'; - - +import { ProxyAgent, fetch as undiciFetch } from 'undici'; + +/** Returns a fetch function that routes through the system proxy if configured. */ +function getProxyFetch(): typeof fetch { + const proxyUrl = process.env.HTTPS_PROXY ?? process.env.https_proxy + ?? process.env.HTTP_PROXY ?? process.env.http_proxy + ?? process.env.ALL_PROXY ?? process.env.all_proxy; + if (!proxyUrl) return fetch; + const dispatcher = new ProxyAgent(proxyUrl); + return (input, init) => undiciFetch(input as Parameters[0], { ...(init as object), dispatcher }) as unknown as Promise; +} /** Single URL fetch helper */ async function fetchSingle( @@ -29,7 +38,8 @@ async function fetchSingle( } if (page === null) { - const resp = await fetch(finalUrl, { method: method.toUpperCase(), headers: renderedHeaders }); + const proxyFetch = getProxyFetch(); + const resp = await proxyFetch(finalUrl, { method: method.toUpperCase(), headers: renderedHeaders }); if (!resp.ok) { throw new CliError('FETCH_ERROR', `HTTP ${resp.status} ${resp.statusText} from ${finalUrl}`); }