From ca2deecec7a8bc93506c6f16eacc637dbdc44615 Mon Sep 17 00:00:00 2001 From: Nonumpa Date: Thu, 4 Jun 2026 07:23:26 +0000 Subject: [PATCH 01/11] feat(browser): add backend factory for local vs cloudflare CDP A new launchOrConnect() helper picks between puppeteer.launch() (local chromium) and puppeteer-core.connect() (Cloudflare Browser Rendering WebSocket CDP) based on BROWSER_BACKEND env. Default stays local so existing deployments are unchanged. Cloudflare credentials live in CLOUDFLARE_ACCOUNT_ID + CLOUDFLARE_API_TOKEN and the required token scope is Browser Rendering: Edit. keep_alive defaults to the documented 600000 ms (10 min) ceiling and can be overridden via CLOUDFLARE_KEEP_ALIVE_MS. puppeteer-core is added as a dependency at the same major version as puppeteer to keep CDP wire protocols aligned. --- package-lock.json | 1 + package.json | 1 + src/lib/__tests__/launchOrConnect.test.js | 83 +++++++++++++++++++++++ src/lib/launchOrConnect.js | 42 ++++++++++++ 4 files changed, 127 insertions(+) create mode 100644 src/lib/__tests__/launchOrConnect.test.js create mode 100644 src/lib/launchOrConnect.js diff --git a/package-lock.json b/package-lock.json index 7c412db..d4778aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "google-protobuf": "^3.9.1", "node-fetch": "^2.2.0", "puppeteer": "^24.10.0", + "puppeteer-core": "^24.43.1", "rollbar": "^2.5.1", "unfurl.js": "^5.1.0" }, diff --git a/package.json b/package.json index 9eb826b..08c67b1 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "google-protobuf": "^3.9.1", "node-fetch": "^2.2.0", "puppeteer": "^24.10.0", + "puppeteer-core": "^24.43.1", "rollbar": "^2.5.1", "unfurl.js": "^5.1.0" }, diff --git a/src/lib/__tests__/launchOrConnect.test.js b/src/lib/__tests__/launchOrConnect.test.js new file mode 100644 index 0000000..1a17273 --- /dev/null +++ b/src/lib/__tests__/launchOrConnect.test.js @@ -0,0 +1,83 @@ +jest.mock('puppeteer', () => ({ launch: jest.fn() }), { virtual: true }); +jest.mock('puppeteer-core', () => ({ connect: jest.fn() }), { virtual: true }); + +describe('launchOrConnect', () => { + let launchOrConnect; + let puppeteer; + let puppeteerCore; + const ORIGINAL_ENV = process.env; + + beforeEach(() => { + jest.resetModules(); + process.env = { ...ORIGINAL_ENV }; + delete process.env.BROWSER_BACKEND; + delete process.env.CLOUDFLARE_ACCOUNT_ID; + delete process.env.CLOUDFLARE_API_TOKEN; + delete process.env.CLOUDFLARE_KEEP_ALIVE_MS; + delete process.env.PUPPETEER_EXECUTABLE_PATH; + puppeteer = require('puppeteer'); + puppeteerCore = require('puppeteer-core'); + launchOrConnect = require('../launchOrConnect'); + }); + + afterAll(() => { + process.env = ORIGINAL_ENV; + }); + + it('uses puppeteer.launch when BROWSER_BACKEND is unset', () => { + puppeteer.launch.mockReturnValue('local-browser'); + const result = launchOrConnect(); + expect(puppeteer.launch).toHaveBeenCalledTimes(1); + expect(puppeteerCore.connect).not.toHaveBeenCalled(); + expect(result).toBe('local-browser'); + }); + + it('uses puppeteer.launch when BROWSER_BACKEND=local', () => { + process.env.BROWSER_BACKEND = 'local'; + puppeteer.launch.mockReturnValue('local-browser'); + launchOrConnect(); + expect(puppeteer.launch).toHaveBeenCalledTimes(1); + expect(puppeteerCore.connect).not.toHaveBeenCalled(); + }); + + it('uses puppeteer-core.connect when BROWSER_BACKEND=cloudflare', () => { + process.env.BROWSER_BACKEND = 'cloudflare'; + process.env.CLOUDFLARE_ACCOUNT_ID = 'acc123'; + process.env.CLOUDFLARE_API_TOKEN = 'token456'; + puppeteerCore.connect.mockReturnValue('remote-browser'); + + const result = launchOrConnect(); + + expect(puppeteerCore.connect).toHaveBeenCalledTimes(1); + const arg = puppeteerCore.connect.mock.calls[0][0]; + expect(arg.browserWSEndpoint).toBe( + 'wss://api.cloudflare.com/client/v4/accounts/acc123/browser-rendering/devtools/browser?keep_alive=600000' + ); + expect(arg.headers).toEqual({ Authorization: 'Bearer token456' }); + expect(puppeteer.launch).not.toHaveBeenCalled(); + expect(result).toBe('remote-browser'); + }); + + it('honors CLOUDFLARE_KEEP_ALIVE_MS override', () => { + process.env.BROWSER_BACKEND = 'cloudflare'; + process.env.CLOUDFLARE_ACCOUNT_ID = 'acc'; + process.env.CLOUDFLARE_API_TOKEN = 'tok'; + process.env.CLOUDFLARE_KEEP_ALIVE_MS = '60000'; + puppeteerCore.connect.mockReturnValue('remote-browser'); + + launchOrConnect(); + + const arg = puppeteerCore.connect.mock.calls[0][0]; + expect(arg.browserWSEndpoint).toContain('keep_alive=60000'); + }); + + it('throws if BROWSER_BACKEND=cloudflare but credentials missing', () => { + process.env.BROWSER_BACKEND = 'cloudflare'; + expect(() => launchOrConnect()).toThrow(/CLOUDFLARE_ACCOUNT_ID/); + }); + + it('throws on unknown BROWSER_BACKEND value', () => { + process.env.BROWSER_BACKEND = 'banana'; + expect(() => launchOrConnect()).toThrow(/banana/); + }); +}); diff --git a/src/lib/launchOrConnect.js b/src/lib/launchOrConnect.js new file mode 100644 index 0000000..2b16db4 --- /dev/null +++ b/src/lib/launchOrConnect.js @@ -0,0 +1,42 @@ +const puppeteer = require('puppeteer'); +const puppeteerCore = require('puppeteer-core'); + +const LOCAL_LAUNCH_OPTIONS = { + acceptInsecureCerts: true, + executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined, + args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'], +}; + +function buildCloudflareConnectOptions() { + const accountId = process.env.CLOUDFLARE_ACCOUNT_ID; + const token = process.env.CLOUDFLARE_API_TOKEN; + if (!accountId || !token) { + throw new Error( + 'BROWSER_BACKEND=cloudflare requires CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN' + ); + } + // keep_alive caps at 600000 ms (10 min) per CF docs: + // https://developers.cloudflare.com/browser-run/limits/ + const keepAliveMs = process.env.CLOUDFLARE_KEEP_ALIVE_MS || '600000'; + return { + browserWSEndpoint: + `wss://api.cloudflare.com/client/v4/accounts/${accountId}` + + `/browser-rendering/devtools/browser?keep_alive=${keepAliveMs}`, + headers: { Authorization: `Bearer ${token}` }, + acceptInsecureCerts: true, + }; +} + +function launchOrConnect() { + const backend = process.env.BROWSER_BACKEND || 'local'; + switch (backend) { + case 'local': + return puppeteer.launch(LOCAL_LAUNCH_OPTIONS); + case 'cloudflare': + return puppeteerCore.connect(buildCloudflareConnectOptions()); + default: + throw new Error(`unknown BROWSER_BACKEND: ${backend}`); + } +} + +module.exports = launchOrConnect; From 0c95d0660fc1c799c10216ac1581ba6b55b06229 Mon Sep 17 00:00:00 2001 From: Nonumpa Date: Thu, 4 Jun 2026 07:23:26 +0000 Subject: [PATCH 02/11] feat(scrap): use backend factory for browser instance scrap.js now obtains its browser via launchOrConnect(), so the same binary can run with local chromium (default) or Cloudflare Browser Rendering (BROWSER_BACKEND=cloudflare). All downstream code paths - targetcreated handler, disconnected reconnect, page-level operations - work identically against either backend because puppeteer.connect() returns the same Browser interface as puppeteer.launch(). --- src/lib/scrap.js | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/lib/scrap.js b/src/lib/scrap.js index 22e75a2..c13d78a 100644 --- a/src/lib/scrap.js +++ b/src/lib/scrap.js @@ -1,8 +1,8 @@ /* eslint-env browser */ const path = require('path'); const fs = require('fs'); -const puppeteer = require('puppeteer'); const { TimeoutError } = require('puppeteer'); +const launchOrConnect = require('./launchOrConnect'); const ResolveError = require('./ResolveError'); const ScrapResult = require('./ScrapResult'); const rollbar = require('./rollbar'); @@ -20,16 +20,7 @@ let isBrowserClosing = false; * Launch Google Chrome and sets browserPromise */ function launchBrowser() { - browserPromise = puppeteer.launch({ - acceptInsecureCerts: true, - executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined, - args: [ - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-dev-shm-usage', // https://github.com/puppeteer/puppeteer/issues/1321#issuecomment-378361236 - ], - // devtools: true, - }); + browserPromise = launchOrConnect(); browserPromise.then(browser => { // eslint-disable-next-line no-console console.log(`Browser launched successfully.`); From 26f875682e40a627deaf3d5ffbeebe36a3aa70f5 Mon Sep 17 00:00:00 2001 From: Nonumpa Date: Thu, 4 Jun 2026 07:23:26 +0000 Subject: [PATCH 03/11] docs: document BROWSER_BACKEND env and Cloudflare setup Note Workers Paid plan requirement (Free tier 10 min/day is insufficient for production URL resolution) and indicative cost at 10k/100k URLs per day so operators can size their Cloudflare account before flipping the flag. --- .env.sample | 10 ++++++++++ README.md | 29 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/.env.sample b/.env.sample index e5264a7..3688af8 100644 --- a/.env.sample +++ b/.env.sample @@ -1,3 +1,13 @@ # Rollbar. When not given, disables Rollbar ROLLBAR_TOKEN= ROLLBAR_ENV=localhost + +# Browser backend: 'local' (default) or 'cloudflare' +BROWSER_BACKEND=local + +# Required when BROWSER_BACKEND=cloudflare +CLOUDFLARE_ACCOUNT_ID= +CLOUDFLARE_API_TOKEN= + +# Optional: Cloudflare keep_alive in ms. Default 600000 (10 min, current CF max) +# CLOUDFLARE_KEEP_ALIVE_MS=600000 diff --git a/README.md b/README.md index ad88356..059a634 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,35 @@ $ npm start After editing `proto` files, run `npm run compile` to generate corresponding Javascript binary. +## Running with Cloudflare Browser Rendering + +Instead of launching a local Chromium, url-resolver can talk to Cloudflare Browser Rendering over its WebSocket CDP endpoint, offloading the ~500 MB chromium process to Cloudflare's edge. + +Set in `.env`: + +``` +BROWSER_BACKEND=cloudflare +CLOUDFLARE_ACCOUNT_ID= +CLOUDFLARE_API_TOKEN= +``` + +The token needs the **Browser Rendering: Edit** scope at the account level. Create it under **My Profile → API Tokens → Custom token**. + +### Plan requirement + +Workers Free only allows 10 minutes of browser time per day, which is insufficient for production URL resolution. The **Workers Paid** plan ($5/month) is required, with $0.09 per browser-hour beyond the 10 hours included monthly. + +Indicative cost at 5 seconds per resolution: + +| Volume | Browser-hours/month | Estimated cost | +|---|---|---| +| 10,000 URLs/day | ~417 | ~$37 | +| 100,000 URLs/day | ~4,170 | ~$374 | + +See https://developers.cloudflare.com/browser-run/pricing/ for current rates. + +Local Chromium remains the default and works as a fallback — the Docker image still bundles it, so unsetting `BROWSER_BACKEND` rolls back instantly without redeploy. + ## Build Directly use docker to build image. From 133db958018d7a9529d80b8039bf9dcc76b2426c Mon Sep 17 00:00:00 2001 From: Nonumpa Date: Fri, 5 Jun 2026 07:50:51 +0000 Subject: [PATCH 04/11] feat(resolver): cap concurrent scraps with SCRAP_MAX_CONCURRENCY A five-URL request fans out into five concurrent puppeteer pages, each holding ~50 MB. Under bursty load this adds enough memory pressure to OOM the container; puppeteer auto-reconnects on disconnect without backoff, so the cycle repeats. Wrap each scrap call in a module-level p-limit semaphore so the cap holds across all in-flight gRPC streams, not per call. Default 3, override with SCRAP_MAX_CONCURRENCY. --- package-lock.json | 42 ++++++++++-- package.json | 3 +- src/resolvers/__tests__/resolveUrls.test.js | 35 ++++++++++ src/resolvers/resolveUrls.js | 74 ++++++++++++--------- 4 files changed, 114 insertions(+), 40 deletions(-) diff --git a/package-lock.json b/package-lock.json index d4778aa..4729ec1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,8 +14,9 @@ "dotenv": "^6.0.0", "google-protobuf": "^3.9.1", "node-fetch": "^2.2.0", + "p-limit": "^3.1.0", "puppeteer": "^24.10.0", - "puppeteer-core": "^24.43.1", + "puppeteer-core": "^24.43.0", "rollbar": "^2.5.1", "unfurl.js": "^5.1.0" }, @@ -8622,16 +8623,18 @@ } }, "node_modules/p-limit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", - "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", - "dev": true, + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", "license": "MIT", "dependencies": { - "p-try": "^1.0.0" + "yocto-queue": "^0.1.0" }, "engines": { - "node": ">=4" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/p-locate": { @@ -8647,6 +8650,19 @@ "node": ">=4" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/p-try": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", @@ -11909,6 +11925,18 @@ "fd-slicer": "~1.1.0" } }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/zod": { "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", diff --git a/package.json b/package.json index 08c67b1..cdd10dd 100644 --- a/package.json +++ b/package.json @@ -18,8 +18,9 @@ "dotenv": "^6.0.0", "google-protobuf": "^3.9.1", "node-fetch": "^2.2.0", + "p-limit": "^3.1.0", "puppeteer": "^24.10.0", - "puppeteer-core": "^24.43.1", + "puppeteer-core": "^24.43.0", "rollbar": "^2.5.1", "unfurl.js": "^5.1.0" }, diff --git a/src/resolvers/__tests__/resolveUrls.test.js b/src/resolvers/__tests__/resolveUrls.test.js index d7b2dc0..6bd9b76 100644 --- a/src/resolvers/__tests__/resolveUrls.test.js +++ b/src/resolvers/__tests__/resolveUrls.test.js @@ -250,4 +250,39 @@ Array [ }) .catch(err => done.fail(err)); }); + + it('caps concurrent scrap operations at SCRAP_MAX_CONCURRENCY default 3', done => { + normalize.mockImplementation(url => url); + unshorten.mockImplementation(async url => url); + parseMeta.mockImplementation(() => + Promise.resolve(new ScrapResult({ canonical: 'partial' })) + ); + + let active = 0; + let max = 0; + scrap.mockImplementation(async url => { + active++; + if (active > max) max = active; + await new Promise(r => setImmediate(r)); + await new Promise(r => setImmediate(r)); + active--; + return scrap.getResult(url); + }); + + const urls = ['u1', 'u2', 'u3', 'u4', 'u5']; + const call = { + request: { urls }, + write: jest.fn(), + end: jest.fn(), + }; + + resolveUrls(call) + .then(() => { + expect(max).toBe(3); + expect(scrap).toHaveBeenCalledTimes(urls.length); + expect(call.write).toHaveBeenCalledTimes(urls.length); + done(); + }) + .catch(err => done.fail(err)); + }); }); diff --git a/src/resolvers/resolveUrls.js b/src/resolvers/resolveUrls.js index b7c7a94..bbc0760 100644 --- a/src/resolvers/resolveUrls.js +++ b/src/resolvers/resolveUrls.js @@ -1,3 +1,4 @@ +const pLimit = require('p-limit'); const scrap = require('../lib/scrap'); const unshorten = require('../lib/unshorten'); const normalize = require('../lib/normalize'); @@ -5,47 +6,56 @@ const parseMeta = require('../lib/parseMeta'); const ResolveError = require('../lib/ResolveError'); const ScrapResult = require('../lib/ScrapResult'); +const SCRAP_MAX_CONCURRENCY = + parseInt(process.env.SCRAP_MAX_CONCURRENCY, 10) || 3; + +// Server-wide cap on concurrent scrap operations to bound puppeteer memory. +const limit = pLimit(SCRAP_MAX_CONCURRENCY); + function resolveUrls(call) { const { urls } = call.request; return Promise.all( - urls.map(async url => { - let fetchResult; - try { - // Normalize and unshorten URLs, update fetchResult - const normalized = normalize(url); - fetchResult = new ScrapResult({ canonical: normalized }); + urls.map(url => + limit(async () => { + let fetchResult; + try { + // Normalize and unshorten URLs, update fetchResult + const normalized = normalize(url); + fetchResult = new ScrapResult({ canonical: normalized }); - const unshortened = await unshorten(normalized); - fetchResult = new ScrapResult({ canonical: unshortened }); + const unshortened = await unshorten(normalized); + fetchResult = new ScrapResult({ canonical: unshortened }); - // Fetch info from page - fetchResult = await parseMeta(unshortened); + // Fetch info from page + fetchResult = await parseMeta(unshortened); - if (fetchResult.isIncomplete) { - fetchResult.merge(await scrap(unshortened)); - } + if (fetchResult.isIncomplete) { + fetchResult.merge(await scrap(unshortened)); + } - call.write({ - ...fetchResult, - top_image_url: fetchResult.topImageUrl, - url, // Provide the most original url - successfully_resolved: true, - }); - } catch (e) { - // eslint-disable-next-line no-console - console.error('[resolvedUrls]', url, e); - let errMsg; - if (e instanceof ResolveError) { - errMsg = e.returnedError; + call.write({ + ...fetchResult, + top_image_url: fetchResult.topImageUrl, + url, // Provide the most original url + successfully_resolved: true, + }); + } catch (e) { + // eslint-disable-next-line no-console + console.error('[resolvedUrls]', url, e); + let errMsg; + if (e instanceof ResolveError) { + errMsg = e.returnedError; + } + call.write({ + ...fetchResult, // Still try return available fetch result + url, + error: errMsg, + }); } - call.write({ - ...fetchResult, // Still try return available fetch result - url, - error: errMsg, - }); - } - }) + }) + ) ).then(() => call.end()); } module.exports = { resolveUrls }; +module.exports.SCRAP_MAX_CONCURRENCY = SCRAP_MAX_CONCURRENCY; From a32dccb151d346fec23d12703fade4d7b7f1adcf Mon Sep 17 00:00:00 2001 From: Nonumpa Date: Fri, 5 Jun 2026 07:51:14 +0000 Subject: [PATCH 05/11] feat(scrap): abort image, media, and font requests during scrap Loading these resource types only inflates JS heap and network sockets; none of title, summary, canonical, or og:image-driven topImageUrl reads them. With image loading skipped, the topImageUrl fallback that walks by rendered size returns the first image instead of the largest, so sites without og:image lose the "largest image" semantics. Sites with og:image (the common case) are unaffected. Default block list image,media,font; clear via SCRAP_BLOCK_RESOURCES= to disable. --- src/lib/__tests__/scrap.test.js | 127 ++++++++++++++++++++++++++++++++ src/lib/scrap.js | 20 +++++ 2 files changed, 147 insertions(+) create mode 100644 src/lib/__tests__/scrap.test.js diff --git a/src/lib/__tests__/scrap.test.js b/src/lib/__tests__/scrap.test.js new file mode 100644 index 0000000..2021802 --- /dev/null +++ b/src/lib/__tests__/scrap.test.js @@ -0,0 +1,127 @@ +jest.mock( + 'puppeteer', + () => ({ + TimeoutError: class TimeoutError extends Error {}, + launch: jest.fn(), + }), + { virtual: true } +); +jest.mock('puppeteer-core', () => ({ connect: jest.fn() }), { virtual: true }); +jest.mock('../launchOrConnect', () => jest.fn()); +jest.mock('../rollbar', () => ({ warn: jest.fn(), error: jest.fn() })); + +describe('scrap request interception', () => { + const ORIGINAL_ENV = process.env; + let launchOrConnect; + let mockPage; + let mockBrowser; + let requestHandler; + + function setupMocks() { + requestHandler = undefined; + mockPage = { + setRequestInterception: jest.fn().mockResolvedValue(), + on: jest.fn((event, handler) => { + if (event === 'request') requestHandler = handler; + }), + goto: jest.fn().mockResolvedValue({ + headers: () => ({ 'content-type': 'text/html' }), + status: () => 200, + }), + content: jest.fn().mockResolvedValue(''), + setJavaScriptEnabled: jest.fn().mockResolvedValue(), + reload: jest.fn().mockResolvedValue(), + setContent: jest.fn().mockResolvedValue(), + waitForNavigation: jest.fn().mockResolvedValue(), + evaluate: jest + .fn() + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce('https://canonical.test/') + .mockResolvedValueOnce('https://image.test/img.jpg') + .mockResolvedValueOnce({ title: 'T', textContent: 'Body' }), + close: jest.fn().mockResolvedValue(), + }; + mockBrowser = { + newPage: jest.fn().mockResolvedValue(mockPage), + on: jest.fn(), + }; + } + + beforeEach(() => { + jest.resetModules(); + process.env = { ...ORIGINAL_ENV }; + delete process.env.SCRAP_BLOCK_RESOURCES; + setupMocks(); + launchOrConnect = require('../launchOrConnect'); + launchOrConnect.mockReturnValue(Promise.resolve(mockBrowser)); + }); + + afterAll(() => { + process.env = ORIGINAL_ENV; + }); + + it('blocks image, media, font by default and continues other types', async () => { + const scrap = require('../scrap'); + await scrap('https://example.test/'); + + expect(mockPage.setRequestInterception).toHaveBeenCalledWith(true); + expect(requestHandler).toBeDefined(); + + ['image', 'media', 'font'].forEach(type => { + const req = { + resourceType: () => type, + abort: jest.fn(), + continue: jest.fn(), + }; + requestHandler(req); + expect(req.abort).toHaveBeenCalledTimes(1); + expect(req.continue).not.toHaveBeenCalled(); + }); + + ['document', 'stylesheet', 'script', 'xhr', 'fetch'].forEach(type => { + const req = { + resourceType: () => type, + abort: jest.fn(), + continue: jest.fn(), + }; + requestHandler(req); + expect(req.continue).toHaveBeenCalledTimes(1); + expect(req.abort).not.toHaveBeenCalled(); + }); + }); + + it('honors SCRAP_BLOCK_RESOURCES override', async () => { + process.env.SCRAP_BLOCK_RESOURCES = 'script, xhr'; + const scrap = require('../scrap'); + await scrap('https://example.test/'); + + expect(mockPage.setRequestInterception).toHaveBeenCalledWith(true); + + const reqImage = { + resourceType: () => 'image', + abort: jest.fn(), + continue: jest.fn(), + }; + requestHandler(reqImage); + expect(reqImage.continue).toHaveBeenCalled(); + + const reqScript = { + resourceType: () => 'script', + abort: jest.fn(), + continue: jest.fn(), + }; + requestHandler(reqScript); + expect(reqScript.abort).toHaveBeenCalled(); + }); + + it('skips interception when SCRAP_BLOCK_RESOURCES is empty', async () => { + process.env.SCRAP_BLOCK_RESOURCES = ''; + const scrap = require('../scrap'); + await scrap('https://example.test/'); + + expect(mockPage.setRequestInterception).not.toHaveBeenCalled(); + expect(requestHandler).toBeUndefined(); + }); +}); diff --git a/src/lib/scrap.js b/src/lib/scrap.js index c13d78a..495cee6 100644 --- a/src/lib/scrap.js +++ b/src/lib/scrap.js @@ -13,6 +13,14 @@ const { ResolveError: ResolveErrorEnum } = require('./resolve_error_pb'); const FETCHING_TIMEOUT = 5000; const PROCESSING_TIMEOUT = 1000; +const SCRAP_BLOCK_RESOURCES = (process.env.SCRAP_BLOCK_RESOURCES === undefined + ? 'image,media,font' + : process.env.SCRAP_BLOCK_RESOURCES +) + .split(',') + .map(s => s.trim()) + .filter(Boolean); + let browserPromise; let isBrowserClosing = false; @@ -88,6 +96,17 @@ async function scrap(url) { const browser = await browserPromise; const page = await browser.newPage(); + if (SCRAP_BLOCK_RESOURCES.length > 0) { + await page.setRequestInterception(true); + page.on('request', req => { + if (SCRAP_BLOCK_RESOURCES.includes(req.resourceType())) { + req.abort(); + } else { + req.continue(); + } + }); + } + // Automaticaly accept all alert() or confirm() page.on('dialog', async dialog => { // eslint-disable-next-line no-console @@ -306,6 +325,7 @@ async function scrap(url) { } module.exports = scrap; +module.exports.SCRAP_BLOCK_RESOURCES = SCRAP_BLOCK_RESOURCES; scrap.getBrowserPromise = () => browserPromise; From 12f75df6a9f9cd6490a4926b9a90e66928b0c6d6 Mon Sep 17 00:00:00 2001 From: Nonumpa Date: Fri, 5 Jun 2026 07:53:09 +0000 Subject: [PATCH 06/11] docs: document SCRAP_MAX_CONCURRENCY and SCRAP_BLOCK_RESOURCES Document the concurrency cap and resource block list with the topImageUrl fallback caveat so operators can tune or disable them per workload. --- .env.sample | 8 ++++++++ README.md | 13 ++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.env.sample b/.env.sample index 3688af8..09bf29a 100644 --- a/.env.sample +++ b/.env.sample @@ -11,3 +11,11 @@ CLOUDFLARE_API_TOKEN= # Optional: Cloudflare keep_alive in ms. Default 600000 (10 min, current CF max) # CLOUDFLARE_KEEP_ALIVE_MS=600000 + +# Optional: max concurrent scrap operations server-wide. Default 3. +# Lower = less puppeteer memory pressure, slower tail latency for batched URLs. +# SCRAP_MAX_CONCURRENCY=3 + +# Optional: comma-separated puppeteer resourceType list to block during scrap. +# Default: image,media,font. Set to empty string to disable. +# SCRAP_BLOCK_RESOURCES=image,media,font diff --git a/README.md b/README.md index 059a634..022085e 100644 --- a/README.md +++ b/README.md @@ -77,4 +77,15 @@ Directly use docker to build image. ```bash $ docker build -t cofacts/url-resolver:latest . -``` \ No newline at end of file +``` + +## Resource limits + +Two env vars bound puppeteer memory at scrap time. Both are optional. + +| Variable | Default | Description | +|---|---|---| +| `SCRAP_MAX_CONCURRENCY` | `3` | Maximum concurrent `scrap()` operations across all gRPC calls. Each in-flight scrap holds one puppeteer page (image, JS heap, network sockets). Lower this if the server OOMs; raise it if tail latency dominates and there is RAM headroom. | +| `SCRAP_BLOCK_RESOURCES` | `image,media,font` | Comma-separated [puppeteer resourceType](https://pptr.dev/api/puppeteer.httprequest.resourcetype) list aborted before they hit the wire. Set to empty string to load every resource. The default leaves `document`, `stylesheet`, `script`, and `xhr` intact, so Readability.js sees the same DOM. | + +`og:image` is read from the meta tag in HTML, not from a loaded image, so the default block list does not affect `topImageUrl` for sites that expose Open Graph metadata. Sites without `og:image` fall back to scanning `` tags by rendered size; with image loading blocked, all images report `0x0`, so the fallback returns the first `` instead of the largest. \ No newline at end of file From 40654cc46cff998f6ee42909be5ba09b0a216bdd Mon Sep 17 00:00:00 2001 From: Nonumpa Date: Fri, 5 Jun 2026 16:10:58 +0000 Subject: [PATCH 07/11] fix(scrap): swallow puppeteer abort/continue rejections page.on('request', ...) handlers ignore the promises returned by req.abort()/req.continue(). If the page closes or the request was already resolved mid-flight (navigation cancellation, target closed), the rejection surfaces as an unhandled promise rejection on Node 24. --- src/lib/__tests__/scrap.test.js | 37 ++++++++++++++++++++++++++------- src/lib/scrap.js | 5 +++-- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/lib/__tests__/scrap.test.js b/src/lib/__tests__/scrap.test.js index 2021802..82e8634 100644 --- a/src/lib/__tests__/scrap.test.js +++ b/src/lib/__tests__/scrap.test.js @@ -72,8 +72,8 @@ describe('scrap request interception', () => { ['image', 'media', 'font'].forEach(type => { const req = { resourceType: () => type, - abort: jest.fn(), - continue: jest.fn(), + abort: jest.fn().mockResolvedValue(), + continue: jest.fn().mockResolvedValue(), }; requestHandler(req); expect(req.abort).toHaveBeenCalledTimes(1); @@ -83,8 +83,8 @@ describe('scrap request interception', () => { ['document', 'stylesheet', 'script', 'xhr', 'fetch'].forEach(type => { const req = { resourceType: () => type, - abort: jest.fn(), - continue: jest.fn(), + abort: jest.fn().mockResolvedValue(), + continue: jest.fn().mockResolvedValue(), }; requestHandler(req); expect(req.continue).toHaveBeenCalledTimes(1); @@ -101,16 +101,16 @@ describe('scrap request interception', () => { const reqImage = { resourceType: () => 'image', - abort: jest.fn(), - continue: jest.fn(), + abort: jest.fn().mockResolvedValue(), + continue: jest.fn().mockResolvedValue(), }; requestHandler(reqImage); expect(reqImage.continue).toHaveBeenCalled(); const reqScript = { resourceType: () => 'script', - abort: jest.fn(), - continue: jest.fn(), + abort: jest.fn().mockResolvedValue(), + continue: jest.fn().mockResolvedValue(), }; requestHandler(reqScript); expect(reqScript.abort).toHaveBeenCalled(); @@ -124,4 +124,25 @@ describe('scrap request interception', () => { expect(mockPage.setRequestInterception).not.toHaveBeenCalled(); expect(requestHandler).toBeUndefined(); }); + + it('swallows abort/continue rejection when page closes mid-flight', async () => { + const scrap = require('../scrap'); + await scrap('https://example.test/'); + + const rejectedAbort = { + resourceType: () => 'image', + abort: jest.fn().mockRejectedValue(new Error('Target closed')), + continue: jest.fn().mockResolvedValue(), + }; + const rejectedContinue = { + resourceType: () => 'document', + abort: jest.fn().mockResolvedValue(), + continue: jest.fn().mockRejectedValue(new Error('Target closed')), + }; + + expect(() => requestHandler(rejectedAbort)).not.toThrow(); + expect(() => requestHandler(rejectedContinue)).not.toThrow(); + + await new Promise(r => setImmediate(r)); + }); }); diff --git a/src/lib/scrap.js b/src/lib/scrap.js index 495cee6..59c2bb2 100644 --- a/src/lib/scrap.js +++ b/src/lib/scrap.js @@ -99,10 +99,11 @@ async function scrap(url) { if (SCRAP_BLOCK_RESOURCES.length > 0) { await page.setRequestInterception(true); page.on('request', req => { + // abort()/continue() reject if the page closes mid-flight; swallow. if (SCRAP_BLOCK_RESOURCES.includes(req.resourceType())) { - req.abort(); + req.abort().catch(() => {}); } else { - req.continue(); + req.continue().catch(() => {}); } }); } From ab9adcebb41fcc367a54e18e14128d2db6c430cf Mon Sep 17 00:00:00 2001 From: Nonumpa Date: Fri, 5 Jun 2026 16:11:04 +0000 Subject: [PATCH 08/11] fix(resolver): limit only scrap() instead of the whole url pipeline The semaphore exists to bound puppeteer page memory, but wrapping the entire per-url pipeline also serialised normalize/unshorten/parseMeta. A few slow metadata-only urls could then block unrelated fast results without ever opening Chromium. --- src/resolvers/__tests__/resolveUrls.test.js | 28 +++++--- src/resolvers/resolveUrls.js | 74 ++++++++++----------- 2 files changed, 54 insertions(+), 48 deletions(-) diff --git a/src/resolvers/__tests__/resolveUrls.test.js b/src/resolvers/__tests__/resolveUrls.test.js index 6bd9b76..ef6d801 100644 --- a/src/resolvers/__tests__/resolveUrls.test.js +++ b/src/resolvers/__tests__/resolveUrls.test.js @@ -251,21 +251,28 @@ Array [ .catch(err => done.fail(err)); }); - it('caps concurrent scrap operations at SCRAP_MAX_CONCURRENCY default 3', done => { + it('caps concurrent scrap() at SCRAP_MAX_CONCURRENCY without limiting parseMeta', done => { normalize.mockImplementation(url => url); unshorten.mockImplementation(async url => url); - parseMeta.mockImplementation(() => - Promise.resolve(new ScrapResult({ canonical: 'partial' })) - ); - let active = 0; - let max = 0; + let parseMetaActive = 0; + let parseMetaMax = 0; + parseMeta.mockImplementation(async () => { + parseMetaActive++; + if (parseMetaActive > parseMetaMax) parseMetaMax = parseMetaActive; + await new Promise(r => setImmediate(r)); + parseMetaActive--; + return new ScrapResult({ canonical: 'partial' }); + }); + + let scrapActive = 0; + let scrapMax = 0; scrap.mockImplementation(async url => { - active++; - if (active > max) max = active; + scrapActive++; + if (scrapActive > scrapMax) scrapMax = scrapActive; await new Promise(r => setImmediate(r)); await new Promise(r => setImmediate(r)); - active--; + scrapActive--; return scrap.getResult(url); }); @@ -278,7 +285,8 @@ Array [ resolveUrls(call) .then(() => { - expect(max).toBe(3); + expect(scrapMax).toBe(3); + expect(parseMetaMax).toBe(urls.length); expect(scrap).toHaveBeenCalledTimes(urls.length); expect(call.write).toHaveBeenCalledTimes(urls.length); done(); diff --git a/src/resolvers/resolveUrls.js b/src/resolvers/resolveUrls.js index bbc0760..baf0101 100644 --- a/src/resolvers/resolveUrls.js +++ b/src/resolvers/resolveUrls.js @@ -15,45 +15,43 @@ const limit = pLimit(SCRAP_MAX_CONCURRENCY); function resolveUrls(call) { const { urls } = call.request; return Promise.all( - urls.map(url => - limit(async () => { - let fetchResult; - try { - // Normalize and unshorten URLs, update fetchResult - const normalized = normalize(url); - fetchResult = new ScrapResult({ canonical: normalized }); - - const unshortened = await unshorten(normalized); - fetchResult = new ScrapResult({ canonical: unshortened }); - - // Fetch info from page - fetchResult = await parseMeta(unshortened); - - if (fetchResult.isIncomplete) { - fetchResult.merge(await scrap(unshortened)); - } - - call.write({ - ...fetchResult, - top_image_url: fetchResult.topImageUrl, - url, // Provide the most original url - successfully_resolved: true, - }); - } catch (e) { - // eslint-disable-next-line no-console - console.error('[resolvedUrls]', url, e); - let errMsg; - if (e instanceof ResolveError) { - errMsg = e.returnedError; - } - call.write({ - ...fetchResult, // Still try return available fetch result - url, - error: errMsg, - }); + urls.map(async url => { + let fetchResult; + try { + // Normalize and unshorten URLs, update fetchResult + const normalized = normalize(url); + fetchResult = new ScrapResult({ canonical: normalized }); + + const unshortened = await unshorten(normalized); + fetchResult = new ScrapResult({ canonical: unshortened }); + + // Fetch info from page + fetchResult = await parseMeta(unshortened); + + if (fetchResult.isIncomplete) { + fetchResult.merge(await limit(() => scrap(unshortened))); + } + + call.write({ + ...fetchResult, + top_image_url: fetchResult.topImageUrl, + url, // Provide the most original url + successfully_resolved: true, + }); + } catch (e) { + // eslint-disable-next-line no-console + console.error('[resolvedUrls]', url, e); + let errMsg; + if (e instanceof ResolveError) { + errMsg = e.returnedError; } - }) - ) + call.write({ + ...fetchResult, // Still try return available fetch result + url, + error: errMsg, + }); + } + }) ).then(() => call.end()); } From 5c2dff836b01e751da0a1c5f3e4825e425e465ef Mon Sep 17 00:00:00 2001 From: Nonumpa Date: Tue, 9 Jun 2026 10:52:00 +0000 Subject: [PATCH 09/11] fix(scrap): lazy connect for cloudflare backend The eager launchBrowser() at module load opens a remote browser session on process start even before any request arrives. The disconnected handler then immediately reconnects, so when Cloudflare auto-closes the session after the keep_alive inactivity window (max 10 min), we acquire a fresh session and the cycle repeats forever, burning quota and browser-time at idle. Split the lifecycle by backend: local stays eager (no billing, hot crash recovery wanted), cloudflare connects lazily on first scrap() and clears browserPromise on disconnect so the next scrap reconnects on demand. Also surface launchOrConnect rejections via a catch handler that resets browserPromise, so a transient acquire failure does not pin the resolver to a rejected promise. --- src/lib/__tests__/scrap.test.js | 114 ++++++++++++++++++++++++++++++++ src/lib/scrap.js | 85 +++++++++++++++--------- 2 files changed, 168 insertions(+), 31 deletions(-) diff --git a/src/lib/__tests__/scrap.test.js b/src/lib/__tests__/scrap.test.js index 82e8634..f96ca97 100644 --- a/src/lib/__tests__/scrap.test.js +++ b/src/lib/__tests__/scrap.test.js @@ -146,3 +146,117 @@ describe('scrap request interception', () => { await new Promise(r => setImmediate(r)); }); }); + +describe('scrap backend lifecycle', () => { + const ORIGINAL_ENV = process.env; + let launchOrConnect; + let mockBrowser; + + function buildMockBrowser() { + return { + newPage: jest.fn().mockResolvedValue({ + setRequestInterception: jest.fn().mockResolvedValue(), + on: jest.fn(), + goto: jest.fn().mockResolvedValue({ + headers: () => ({ 'content-type': 'text/html' }), + status: () => 200, + }), + content: jest.fn().mockResolvedValue(''), + setJavaScriptEnabled: jest.fn().mockResolvedValue(), + reload: jest.fn().mockResolvedValue(), + setContent: jest.fn().mockResolvedValue(), + waitForNavigation: jest.fn().mockResolvedValue(), + evaluate: jest + .fn() + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce('https://canonical.test/') + .mockResolvedValueOnce('https://image.test/img.jpg') + .mockResolvedValueOnce({ title: 'T', textContent: 'Body' }), + close: jest.fn().mockResolvedValue(), + }), + on: jest.fn(), + }; + } + + function getDisconnectHandler(browser) { + const call = browser.on.mock.calls.find( + ([event]) => event === 'disconnected' + ); + return call && call[1]; + } + + beforeEach(() => { + jest.resetModules(); + process.env = { ...ORIGINAL_ENV }; + delete process.env.BROWSER_BACKEND; + delete process.env.SCRAP_BLOCK_RESOURCES; + mockBrowser = buildMockBrowser(); + launchOrConnect = require('../launchOrConnect'); + launchOrConnect.mockReturnValue(Promise.resolve(mockBrowser)); + }); + + afterAll(() => { + process.env = ORIGINAL_ENV; + }); + + it('eagerly launches at module load when BROWSER_BACKEND is local (default)', () => { + require('../scrap'); + expect(launchOrConnect).toHaveBeenCalledTimes(1); + }); + + it('does not launch at module load when BROWSER_BACKEND=cloudflare', () => { + process.env.BROWSER_BACKEND = 'cloudflare'; + require('../scrap'); + expect(launchOrConnect).not.toHaveBeenCalled(); + }); + + it('lazily connects on first scrap() when BROWSER_BACKEND=cloudflare', async () => { + process.env.BROWSER_BACKEND = 'cloudflare'; + const scrap = require('../scrap'); + expect(launchOrConnect).not.toHaveBeenCalled(); + + await scrap('https://example.test/'); + expect(launchOrConnect).toHaveBeenCalledTimes(1); + }); + + it('does not reconnect on disconnect when BROWSER_BACKEND=cloudflare', async () => { + process.env.BROWSER_BACKEND = 'cloudflare'; + const scrap = require('../scrap'); + await scrap('https://example.test/'); + expect(launchOrConnect).toHaveBeenCalledTimes(1); + + const disconnect = getDisconnectHandler(mockBrowser); + expect(disconnect).toBeDefined(); + disconnect(); + + expect(launchOrConnect).toHaveBeenCalledTimes(1); + expect(scrap.getBrowserPromise()).toBeUndefined(); + }); + + it('reconnects on demand after cloudflare disconnect', async () => { + process.env.BROWSER_BACKEND = 'cloudflare'; + const scrap = require('../scrap'); + await scrap('https://example.test/'); + + getDisconnectHandler(mockBrowser)(); + expect(scrap.getBrowserPromise()).toBeUndefined(); + + mockBrowser = buildMockBrowser(); + launchOrConnect.mockReturnValue(Promise.resolve(mockBrowser)); + + await scrap('https://example.test/'); + expect(launchOrConnect).toHaveBeenCalledTimes(2); + }); + + it('reconnects immediately on disconnect when BROWSER_BACKEND=local', async () => { + const scrap = require('../scrap'); + await scrap('https://example.test/'); + expect(launchOrConnect).toHaveBeenCalledTimes(1); + + getDisconnectHandler(mockBrowser)(); + + expect(launchOrConnect).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/lib/scrap.js b/src/lib/scrap.js index 59c2bb2..1e12673 100644 --- a/src/lib/scrap.js +++ b/src/lib/scrap.js @@ -24,45 +24,67 @@ const SCRAP_BLOCK_RESOURCES = (process.env.SCRAP_BLOCK_RESOURCES === undefined let browserPromise; let isBrowserClosing = false; +const isCloudflare = (process.env.BROWSER_BACKEND || 'local') === 'cloudflare'; + /** * Launch Google Chrome and sets browserPromise */ function launchBrowser() { browserPromise = launchOrConnect(); - browserPromise.then(browser => { - // eslint-disable-next-line no-console - console.log(`Browser launched successfully.`); - - // Some page may use window.open() to open extra pages. - // We should close them when such page is detected. - // - browser.on('targetcreated', async target => { - const opener = target.opener(); - if (opener) { - // eslint-disable-next-line no-console - console.info( - `[targetcreated] Extra page "${target.url()}" opened by "${opener.url()}". Closing.` - ); - - const page = await target.page(); - if (page) page.close(); - } - }); - - // Google Chrome sometimes crashes, needs re-launch - // https://github.com/cofacts/url-resolver/issues/9 - // - browser.on('disconnected', () => { - // Ignore the case when close() is invoked - if (isBrowserClosing) return; + browserPromise + .then(browser => { + // eslint-disable-next-line no-console + console.log(`Browser launched successfully.`); + + // Some page may use window.open() to open extra pages. + // We should close them when such page is detected. + // + browser.on('targetcreated', async target => { + const opener = target.opener(); + if (opener) { + // eslint-disable-next-line no-console + console.info( + `[targetcreated] Extra page "${target.url()}" opened by "${opener.url()}". Closing.` + ); + + const page = await target.page(); + if (page) page.close(); + } + }); - rollbar.warn('Puppeteer disconnected from Chrome'); - launchBrowser(); + browser.on('disconnected', () => { + // Ignore the case when close() is invoked + if (isBrowserClosing) return; + + if (isCloudflare) { + // Cloudflare auto-closes the remote session after keep_alive + // inactivity. Reset so the next scrap() reconnects on demand + // instead of burning a fresh session per idle timeout cycle. + browserPromise = undefined; + return; + } + + // Local Chrome sometimes crashes; reconnect eagerly. + // https://github.com/cofacts/url-resolver/issues/9 + rollbar.warn('Puppeteer disconnected from Chrome'); + launchBrowser(); + }); + }) + .catch(err => { + rollbar.error(err, '[scrap] launchOrConnect failed'); + browserPromise = undefined; }); - }); } -launchBrowser(); +function getBrowser() { + if (!browserPromise) launchBrowser(); + return browserPromise; +} + +// Local backend eagerly launches so the first request does not pay chromium +// startup latency. Cloudflare backend connects lazily on demand because each +// session is billed and auto-closes on idle. +if (!isCloudflare) launchBrowser(); const readabilityJsStr = fs.readFileSync( path.join(__dirname, '../vendor/Readability.js'), @@ -93,7 +115,7 @@ async function stop(page) { * @return {Promise} */ async function scrap(url) { - const browser = await browserPromise; + const browser = await getBrowser(); const page = await browser.newPage(); if (SCRAP_BLOCK_RESOURCES.length > 0) { @@ -336,6 +358,7 @@ scrap.getBrowserPromise = () => browserPromise; */ scrap.closeBrowser = async () => { isBrowserClosing = true; + if (!browserPromise) return; const browser = await browserPromise; browser.close(); }; From aa818129c36f7534bd5f00ad46d276f6eedf1bce Mon Sep 17 00:00:00 2001 From: Nonumpa Date: Tue, 9 Jun 2026 12:52:07 +0000 Subject: [PATCH 10/11] fix(cloudflare): default CLOUDFLARE_KEEP_ALIVE_MS to 60s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous 600000 ms (10 min) default was chosen when scrap eagerly relaunched on disconnect, so the keep_alive window only controlled churn frequency, not the bill — one session was always open either way. With lazy lifecycle (5c2dff8), the session only exists during/around traffic, and the keep_alive window is now a trade-off between scrap-to-scrap reuse and billed idle tail after the last request (CF bills until session timeout). Use CF's own 60s default; operators can widen for sparse traffic via CLOUDFLARE_KEEP_ALIVE_MS. --- .env.sample | 6 ++++-- src/lib/__tests__/launchOrConnect.test.js | 6 +++--- src/lib/launchOrConnect.js | 8 ++++++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.env.sample b/.env.sample index 09bf29a..b41ed7a 100644 --- a/.env.sample +++ b/.env.sample @@ -9,8 +9,10 @@ BROWSER_BACKEND=local CLOUDFLARE_ACCOUNT_ID= CLOUDFLARE_API_TOKEN= -# Optional: Cloudflare keep_alive in ms. Default 600000 (10 min, current CF max) -# CLOUDFLARE_KEEP_ALIVE_MS=600000 +# Optional: Cloudflare keep_alive in ms. Default 60000 (matches CF default). +# Wider = more session reuse across scraps, longer billed idle tail per cluster. +# Max 600000 (10 min) per CF docs. +# CLOUDFLARE_KEEP_ALIVE_MS=60000 # Optional: max concurrent scrap operations server-wide. Default 3. # Lower = less puppeteer memory pressure, slower tail latency for batched URLs. diff --git a/src/lib/__tests__/launchOrConnect.test.js b/src/lib/__tests__/launchOrConnect.test.js index 1a17273..c5cddc4 100644 --- a/src/lib/__tests__/launchOrConnect.test.js +++ b/src/lib/__tests__/launchOrConnect.test.js @@ -51,7 +51,7 @@ describe('launchOrConnect', () => { expect(puppeteerCore.connect).toHaveBeenCalledTimes(1); const arg = puppeteerCore.connect.mock.calls[0][0]; expect(arg.browserWSEndpoint).toBe( - 'wss://api.cloudflare.com/client/v4/accounts/acc123/browser-rendering/devtools/browser?keep_alive=600000' + 'wss://api.cloudflare.com/client/v4/accounts/acc123/browser-rendering/devtools/browser?keep_alive=60000' ); expect(arg.headers).toEqual({ Authorization: 'Bearer token456' }); expect(puppeteer.launch).not.toHaveBeenCalled(); @@ -62,13 +62,13 @@ describe('launchOrConnect', () => { process.env.BROWSER_BACKEND = 'cloudflare'; process.env.CLOUDFLARE_ACCOUNT_ID = 'acc'; process.env.CLOUDFLARE_API_TOKEN = 'tok'; - process.env.CLOUDFLARE_KEEP_ALIVE_MS = '60000'; + process.env.CLOUDFLARE_KEEP_ALIVE_MS = '600000'; puppeteerCore.connect.mockReturnValue('remote-browser'); launchOrConnect(); const arg = puppeteerCore.connect.mock.calls[0][0]; - expect(arg.browserWSEndpoint).toContain('keep_alive=60000'); + expect(arg.browserWSEndpoint).toContain('keep_alive=600000'); }); it('throws if BROWSER_BACKEND=cloudflare but credentials missing', () => { diff --git a/src/lib/launchOrConnect.js b/src/lib/launchOrConnect.js index 2b16db4..1987903 100644 --- a/src/lib/launchOrConnect.js +++ b/src/lib/launchOrConnect.js @@ -15,9 +15,13 @@ function buildCloudflareConnectOptions() { 'BROWSER_BACKEND=cloudflare requires CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN' ); } - // keep_alive caps at 600000 ms (10 min) per CF docs: + // keep_alive sets the inactivity window before Cloudflare closes the remote + // session. Default 60000 ms (matches CF default); cap is 600000 ms (10 min). + // Wider window = more session reuse across scraps but longer billed idle + // tail after the last request, since the open session keeps consuming + // browser-hours until it times out. // https://developers.cloudflare.com/browser-run/limits/ - const keepAliveMs = process.env.CLOUDFLARE_KEEP_ALIVE_MS || '600000'; + const keepAliveMs = process.env.CLOUDFLARE_KEEP_ALIVE_MS || '60000'; return { browserWSEndpoint: `wss://api.cloudflare.com/client/v4/accounts/${accountId}` + From a19cb106e742a2167b474af6220c08d9766f8c6d Mon Sep 17 00:00:00 2001 From: Nonumpa Date: Thu, 18 Jun 2026 04:17:58 +0000 Subject: [PATCH 11/11] refactor: rename scrap to scrape across the codebase The repo has used scrap as the verb/noun for 'fetch + render + extract' since the first commit, but the correct English spelling is scrape. Rename module files (scrap.js, ScrapResult.js, mocks, tests), function and class identifiers, env var prefixes (SCRAPE_MAX_CONCURRENCY, SCRAPE_BLOCK_RESOURCES), the proto enum (UNKNOWN_SCRAPE_ERROR), README prose, and snapshot fixtures. Wire-format ResolveError numeric values are unchanged, so existing gRPC clients keep working without redeploy. Downstream code that imports the symbolic name UNKNOWN_SCRAP_ERROR will need to update on next proto re-sync. --- .env.sample | 10 +-- README.md | 8 +- src/lib/{ScrapResult.js => ScrapeResult.js} | 22 +++--- src/lib/__mocks__/{scrap.js => scrape.js} | 4 +- .../__snapshots__/parseMeta.test.js.snap | 78 +++++++++---------- .../{scrap.test.js => scrape.test.js} | 60 +++++++------- src/lib/launchOrConnect.js | 2 +- src/lib/parseMeta.js | 6 +- src/lib/{scrap.js => scrape.js} | 50 ++++++------ src/resolvers/__tests__/browser.test.js | 4 +- src/resolvers/__tests__/resolveUrls.test.js | 72 ++++++++--------- src/resolvers/browser.js | 2 +- src/resolvers/resolveUrls.js | 20 ++--- src/typeDefs/resolve_error.proto | 2 +- 14 files changed, 170 insertions(+), 170 deletions(-) rename src/lib/{ScrapResult.js => ScrapeResult.js} (74%) rename src/lib/__mocks__/{scrap.js => scrape.js} (93%) rename src/lib/__tests__/{scrap.test.js => scrape.test.js} (83%) rename src/lib/{scrap.js => scrape.js} (87%) diff --git a/.env.sample b/.env.sample index b41ed7a..29ede16 100644 --- a/.env.sample +++ b/.env.sample @@ -10,14 +10,14 @@ CLOUDFLARE_ACCOUNT_ID= CLOUDFLARE_API_TOKEN= # Optional: Cloudflare keep_alive in ms. Default 60000 (matches CF default). -# Wider = more session reuse across scraps, longer billed idle tail per cluster. +# Wider = more session reuse across scrapes, longer billed idle tail per cluster. # Max 600000 (10 min) per CF docs. # CLOUDFLARE_KEEP_ALIVE_MS=60000 -# Optional: max concurrent scrap operations server-wide. Default 3. +# Optional: max concurrent scrape operations server-wide. Default 3. # Lower = less puppeteer memory pressure, slower tail latency for batched URLs. -# SCRAP_MAX_CONCURRENCY=3 +# SCRAPE_MAX_CONCURRENCY=3 -# Optional: comma-separated puppeteer resourceType list to block during scrap. +# Optional: comma-separated puppeteer resourceType list to block during scrape. # Default: image,media,font. Set to empty string to disable. -# SCRAP_BLOCK_RESOURCES=image,media,font +# SCRAPE_BLOCK_RESOURCES=image,media,font diff --git a/README.md b/README.md index 022085e..1c5b7b5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build Status](https://travis-ci.org/cofacts/url-resolver.svg?branch=master)](https://travis-ci.org/cofacts/url-resolver) [![Coverage Status](https://coveralls.io/repos/github/cofacts/url-resolver/badge.svg?branch=master)](https://coveralls.io/github/cofacts/url-resolver?branch=master) -A gRPC service that scraps the specified URL and returns scrapped result and summary extracted by +A gRPC service that scrapes the specified URL and returns scraped result and summary extracted by [Readability.js] ## Usage @@ -81,11 +81,11 @@ $ docker build -t cofacts/url-resolver:latest . ## Resource limits -Two env vars bound puppeteer memory at scrap time. Both are optional. +Two env vars bound puppeteer memory at scrape time. Both are optional. | Variable | Default | Description | |---|---|---| -| `SCRAP_MAX_CONCURRENCY` | `3` | Maximum concurrent `scrap()` operations across all gRPC calls. Each in-flight scrap holds one puppeteer page (image, JS heap, network sockets). Lower this if the server OOMs; raise it if tail latency dominates and there is RAM headroom. | -| `SCRAP_BLOCK_RESOURCES` | `image,media,font` | Comma-separated [puppeteer resourceType](https://pptr.dev/api/puppeteer.httprequest.resourcetype) list aborted before they hit the wire. Set to empty string to load every resource. The default leaves `document`, `stylesheet`, `script`, and `xhr` intact, so Readability.js sees the same DOM. | +| `SCRAPE_MAX_CONCURRENCY` | `3` | Maximum concurrent `scrape()` operations across all gRPC calls. Each in-flight scrape holds one puppeteer page (image, JS heap, network sockets). Lower this if the server OOMs; raise it if tail latency dominates and there is RAM headroom. | +| `SCRAPE_BLOCK_RESOURCES` | `image,media,font` | Comma-separated [puppeteer resourceType](https://pptr.dev/api/puppeteer.httprequest.resourcetype) list aborted before they hit the wire. Set to empty string to load every resource. The default leaves `document`, `stylesheet`, `script`, and `xhr` intact, so Readability.js sees the same DOM. | `og:image` is read from the meta tag in HTML, not from a loaded image, so the default block list does not affect `topImageUrl` for sites that expose Open Graph metadata. Sites without `og:image` fall back to scanning `` tags by rendered size; with image loading blocked, all images report `0x0`, so the fallback returns the first `` instead of the largest. \ No newline at end of file diff --git a/src/lib/ScrapResult.js b/src/lib/ScrapeResult.js similarity index 74% rename from src/lib/ScrapResult.js rename to src/lib/ScrapeResult.js index 0f22b44..3ab0691 100644 --- a/src/lib/ScrapResult.js +++ b/src/lib/ScrapeResult.js @@ -1,4 +1,4 @@ -// When one of these fields is undefined, the scrap result is considered incomplete. +// When one of these fields is undefined, the scrape result is considered incomplete. const REQUIRED_FIELDS = ['canonical', 'topImageUrl', 'title', 'summary']; // Merge strategy: use longer when merge() @@ -10,7 +10,7 @@ const PREFER_CURRENT_FIELDS = ['canonical', 'topImageUrl']; // Merge strategy: always use new field const PREFER_NEW_FIELDS = ['status']; -class ScrapResult { +class ScrapeResult { constructor(init) { Object.assign(this, init); } @@ -19,27 +19,27 @@ class ScrapResult { * Merge each field using the strategy defined above. * For other fields, they are kept intact. * - * @param {ScrapResult} scrapResult - * @return {ScrapResult} + * @param {ScrapeResult} scrapeResult + * @return {ScrapeResult} */ - merge(scrapResult) { + merge(scrapeResult) { PREFER_CURRENT_FIELDS.forEach(field => { if (typeof this[field] === 'undefined') { - this[field] = scrapResult[field]; + this[field] = scrapeResult[field]; } }); USE_LONGER_FIELDS.forEach(field => { if ( - typeof scrapResult[field] === 'string' && - (this[field] || '').length < scrapResult[field].length + typeof scrapeResult[field] === 'string' && + (this[field] || '').length < scrapeResult[field].length ) { - this[field] = scrapResult[field]; + this[field] = scrapeResult[field]; } }); PREFER_NEW_FIELDS.forEach(field => { - this[field] = scrapResult[field]; + this[field] = scrapeResult[field]; }); } @@ -83,4 +83,4 @@ class ScrapResult { status; } -module.exports = ScrapResult; +module.exports = ScrapeResult; diff --git a/src/lib/__mocks__/scrap.js b/src/lib/__mocks__/scrape.js similarity index 93% rename from src/lib/__mocks__/scrap.js rename to src/lib/__mocks__/scrape.js index 6f8a030..9656998 100644 --- a/src/lib/__mocks__/scrap.js +++ b/src/lib/__mocks__/scrape.js @@ -1,7 +1,7 @@ -const ScrapResult = require('../ScrapResult'); +const ScrapeResult = require('../ScrapeResult'); const getResult = url => - new ScrapResult({ + new ScrapeResult({ title: 't', summary: 's', canonical: url, diff --git a/src/lib/__tests__/__snapshots__/parseMeta.test.js.snap b/src/lib/__tests__/__snapshots__/parseMeta.test.js.snap index 9c98ee7..72f6de9 100644 --- a/src/lib/__tests__/__snapshots__/parseMeta.test.js.snap +++ b/src/lib/__tests__/__snapshots__/parseMeta.test.js.snap @@ -1,7 +1,7 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`parseMeta parses metadata: http://ezp9.com/p128163.asp 1`] = ` -ScrapResult { +ScrapeResult { "canonical": "http://ezp9.com/p128163.asp", "html": undefined, "status": undefined, @@ -14,7 +14,7 @@ ScrapResult { exports[`parseMeta parses metadata: http://ezp9.com/p128163.asp incomplete? 1`] = `false`; exports[`parseMeta parses metadata: http://goez1.com/p117046.asp 1`] = ` -ScrapResult { +ScrapeResult { "canonical": "http://goez1.com/p117046.asp", "html": undefined, "status": undefined, @@ -27,7 +27,7 @@ ScrapResult { exports[`parseMeta parses metadata: http://goez1.com/p117046.asp incomplete? 1`] = `false`; exports[`parseMeta parses metadata: http://mp.weixin.qq.com/s/Mg_F1Dbegi0k1i8DRvMORA 1`] = ` -ScrapResult { +ScrapeResult { "canonical": "http://mp.weixin.qq.com/s?__biz=MzIxNTA1OTk3MQ==&mid=2654202378&idx=1&sn=a81dd7ab7f0707221ed508279b3848c7&chksm=8c595166bb2ed8704eb9459c3e5deb51fddf7bf8d266f29dbdf7fc45ec7d9d14a6e1041e8099#rd", "html": undefined, "status": undefined, @@ -40,7 +40,7 @@ ScrapResult { exports[`parseMeta parses metadata: http://mp.weixin.qq.com/s/Mg_F1Dbegi0k1i8DRvMORA incomplete? 1`] = `false`; exports[`parseMeta parses metadata: http://www.cocomy.net/post/457479?from=app_line 1`] = ` -ScrapResult { +ScrapeResult { "canonical": "https://www.cocomy.net/post/457479", "html": undefined, "status": undefined, @@ -53,7 +53,7 @@ ScrapResult { exports[`parseMeta parses metadata: http://www.cocomy.net/post/457479?from=app_line incomplete? 1`] = `false`; exports[`parseMeta parses metadata: http://www.rumtoast.com/7450/%E5%90%8D%E5%AE%B6%E5%81%BD%E8%AA%9E%E9%8C%84%EF%BC%8F%E5%8F%B0%E7%81%A3%E8%80%81%E4%BA%BA-%E6%A5%8A%E6%8C%AF%E6%98%8E%E6%95%99%E6%8E%88%E8%A8%98%E8%BF%B0-%EF%BC%9A%E6%98%AF%E6%97%A5%E6%9C%AC%E4%BA%BA 1`] = ` -ScrapResult { +ScrapeResult { "canonical": "https://www.rumtoast.com/7450/名家偽語錄／台灣老人-楊振明教授記述-：是日本人", "html": undefined, "status": undefined, @@ -66,7 +66,7 @@ ScrapResult { exports[`parseMeta parses metadata: http://www.rumtoast.com/7450/%E5%90%8D%E5%AE%B6%E5%81%BD%E8%AA%9E%E9%8C%84%EF%BC%8F%E5%8F%B0%E7%81%A3%E8%80%81%E4%BA%BA-%E6%A5%8A%E6%8C%AF%E6%98%8E%E6%95%99%E6%8E%88%E8%A8%98%E8%BF%B0-%EF%BC%9A%E6%98%AF%E6%97%A5%E6%9C%AC%E4%BA%BA incomplete? 1`] = `false`; exports[`parseMeta parses metadata: http://www.setn.com/News.aspx?NewsID=122876 1`] = ` -ScrapResult { +ScrapeResult { "canonical": "https://www.setn.com/News.aspx?NewsID=122876", "html": undefined, "status": undefined, @@ -79,7 +79,7 @@ ScrapResult { exports[`parseMeta parses metadata: http://www.setn.com/News.aspx?NewsID=122876 incomplete? 1`] = `false`; exports[`parseMeta parses metadata: https://jerrynest.io/line-girl/ 1`] = ` -ScrapResult { +ScrapeResult { "canonical": "https://jerrynest.io/line-girl/", "html": "
分享就免費送貼圖主題、18 禁影片？透過文本分析了解 Line@ 機器人酒店攬客手法