diff --git a/.env.sample b/.env.sample index e5264a7..29ede16 100644 --- a/.env.sample +++ b/.env.sample @@ -1,3 +1,23 @@ # Rollbar. When not given, disables Rollbar ROLLBAR_TOKEN= ROLLBAR_ENV=localhost + +# Browser backend: 'local' (default) or 'cloudflare' +BROWSER_BACKEND=local + +# Required when BROWSER_BACKEND=cloudflare +CLOUDFLARE_ACCOUNT_ID= +CLOUDFLARE_API_TOKEN= + +# Optional: Cloudflare keep_alive in ms. Default 60000 (matches CF default). +# Wider = more session reuse across scrapes, longer billed idle tail per cluster. +# Max 600000 (10 min) per CF docs. +# CLOUDFLARE_KEEP_ALIVE_MS=60000 + +# Optional: max concurrent scrape operations server-wide. Default 3. +# Lower = less puppeteer memory pressure, slower tail latency for batched URLs. +# SCRAPE_MAX_CONCURRENCY=3 + +# Optional: comma-separated puppeteer resourceType list to block during scrape. +# Default: image,media,font. Set to empty string to disable. +# SCRAPE_BLOCK_RESOURCES=image,media,font diff --git a/README.md b/README.md index ad88356..1c5b7b5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build Status](https://travis-ci.org/cofacts/url-resolver.svg?branch=master)](https://travis-ci.org/cofacts/url-resolver) [![Coverage Status](https://coveralls.io/repos/github/cofacts/url-resolver/badge.svg?branch=master)](https://coveralls.io/github/cofacts/url-resolver?branch=master) -A gRPC service that scraps the specified URL and returns scrapped result and summary extracted by +A gRPC service that scrapes the specified URL and returns scraped result and summary extracted by [Readability.js] ## Usage @@ -42,10 +42,50 @@ $ npm start After editing `proto` files, run `npm run compile` to generate corresponding Javascript binary. +## Running with Cloudflare Browser Rendering + +Instead of launching a local Chromium, url-resolver can talk to Cloudflare Browser Rendering over its WebSocket CDP endpoint, offloading the ~500 MB chromium process to Cloudflare's edge. + +Set in `.env`: + +``` +BROWSER_BACKEND=cloudflare +CLOUDFLARE_ACCOUNT_ID= +CLOUDFLARE_API_TOKEN= +``` + +The token needs the **Browser Rendering: Edit** scope at the account level. Create it under **My Profile → API Tokens → Custom token**. + +### Plan requirement + +Workers Free only allows 10 minutes of browser time per day, which is insufficient for production URL resolution. The **Workers Paid** plan ($5/month) is required, with $0.09 per browser-hour beyond the 10 hours included monthly. + +Indicative cost at 5 seconds per resolution: + +| Volume | Browser-hours/month | Estimated cost | +|---|---|---| +| 10,000 URLs/day | ~417 | ~$37 | +| 100,000 URLs/day | ~4,170 | ~$374 | + +See https://developers.cloudflare.com/browser-run/pricing/ for current rates. + +Local Chromium remains the default and works as a fallback — the Docker image still bundles it, so unsetting `BROWSER_BACKEND` rolls back instantly without redeploy. + ## Build Directly use docker to build image. ```bash $ docker build -t cofacts/url-resolver:latest . -``` \ No newline at end of file +``` + +## Resource limits + +Two env vars bound puppeteer memory at scrape time. Both are optional. + +| Variable | Default | Description | +|---|---|---| +| `SCRAPE_MAX_CONCURRENCY` | `3` | Maximum concurrent `scrape()` operations across all gRPC calls. Each in-flight scrape holds one puppeteer page (image, JS heap, network sockets). Lower this if the server OOMs; raise it if tail latency dominates and there is RAM headroom. | +| `SCRAPE_BLOCK_RESOURCES` | `image,media,font` | Comma-separated [puppeteer resourceType](https://pptr.dev/api/puppeteer.httprequest.resourcetype) list aborted before they hit the wire. Set to empty string to load every resource. The default leaves `document`, `stylesheet`, `script`, and `xhr` intact, so Readability.js sees the same DOM. | + +`og:image` is read from the meta tag in HTML, not from a loaded image, so the default block list does not affect `topImageUrl` for sites that expose Open Graph metadata. Sites without `og:image` fall back to scanning `` tags by rendered size; with image loading blocked, all images report `0x0`, so the fallback returns the first `` instead of the largest. \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 7c412db..4729ec1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,9 @@ "dotenv": "^6.0.0", "google-protobuf": "^3.9.1", "node-fetch": "^2.2.0", + "p-limit": "^3.1.0", "puppeteer": "^24.10.0", + "puppeteer-core": "^24.43.0", "rollbar": "^2.5.1", "unfurl.js": "^5.1.0" }, @@ -8621,16 +8623,18 @@ } }, "node_modules/p-limit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", - "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", - "dev": true, + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", "license": "MIT", "dependencies": { - "p-try": "^1.0.0" + "yocto-queue": "^0.1.0" }, "engines": { - "node": ">=4" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/p-locate": { @@ -8646,6 +8650,19 @@ "node": ">=4" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/p-try": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", @@ -11908,6 +11925,18 @@ "fd-slicer": "~1.1.0" } }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/zod": { "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", diff --git a/package.json b/package.json index 9eb826b..cdd10dd 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,9 @@ "dotenv": "^6.0.0", "google-protobuf": "^3.9.1", "node-fetch": "^2.2.0", + "p-limit": "^3.1.0", "puppeteer": "^24.10.0", + "puppeteer-core": "^24.43.0", "rollbar": "^2.5.1", "unfurl.js": "^5.1.0" }, diff --git a/src/lib/ScrapResult.js b/src/lib/ScrapeResult.js similarity index 74% rename from src/lib/ScrapResult.js rename to src/lib/ScrapeResult.js index 0f22b44..3ab0691 100644 --- a/src/lib/ScrapResult.js +++ b/src/lib/ScrapeResult.js @@ -1,4 +1,4 @@ -// When one of these fields is undefined, the scrap result is considered incomplete. +// When one of these fields is undefined, the scrape result is considered incomplete. const REQUIRED_FIELDS = ['canonical', 'topImageUrl', 'title', 'summary']; // Merge strategy: use longer when merge() @@ -10,7 +10,7 @@ const PREFER_CURRENT_FIELDS = ['canonical', 'topImageUrl']; // Merge strategy: always use new field const PREFER_NEW_FIELDS = ['status']; -class ScrapResult { +class ScrapeResult { constructor(init) { Object.assign(this, init); } @@ -19,27 +19,27 @@ class ScrapResult { * Merge each field using the strategy defined above. * For other fields, they are kept intact. * - * @param {ScrapResult} scrapResult - * @return {ScrapResult} + * @param {ScrapeResult} scrapeResult + * @return {ScrapeResult} */ - merge(scrapResult) { + merge(scrapeResult) { PREFER_CURRENT_FIELDS.forEach(field => { if (typeof this[field] === 'undefined') { - this[field] = scrapResult[field]; + this[field] = scrapeResult[field]; } }); USE_LONGER_FIELDS.forEach(field => { if ( - typeof scrapResult[field] === 'string' && - (this[field] || '').length < scrapResult[field].length + typeof scrapeResult[field] === 'string' && + (this[field] || '').length < scrapeResult[field].length ) { - this[field] = scrapResult[field]; + this[field] = scrapeResult[field]; } }); PREFER_NEW_FIELDS.forEach(field => { - this[field] = scrapResult[field]; + this[field] = scrapeResult[field]; }); } @@ -83,4 +83,4 @@ class ScrapResult { status; } -module.exports = ScrapResult; +module.exports = ScrapeResult; diff --git a/src/lib/__mocks__/scrap.js b/src/lib/__mocks__/scrape.js similarity index 93% rename from src/lib/__mocks__/scrap.js rename to src/lib/__mocks__/scrape.js index 6f8a030..9656998 100644 --- a/src/lib/__mocks__/scrap.js +++ b/src/lib/__mocks__/scrape.js @@ -1,7 +1,7 @@ -const ScrapResult = require('../ScrapResult'); +const ScrapeResult = require('../ScrapeResult'); const getResult = url => - new ScrapResult({ + new ScrapeResult({ title: 't', summary: 's', canonical: url, diff --git a/src/lib/__tests__/__snapshots__/parseMeta.test.js.snap b/src/lib/__tests__/__snapshots__/parseMeta.test.js.snap index 9c98ee7..72f6de9 100644 --- a/src/lib/__tests__/__snapshots__/parseMeta.test.js.snap +++ b/src/lib/__tests__/__snapshots__/parseMeta.test.js.snap @@ -1,7 +1,7 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`parseMeta parses metadata: http://ezp9.com/p128163.asp 1`] = ` -ScrapResult { +ScrapeResult { "canonical": "http://ezp9.com/p128163.asp", "html": undefined, "status": undefined, @@ -14,7 +14,7 @@ ScrapResult { exports[`parseMeta parses metadata: http://ezp9.com/p128163.asp incomplete? 1`] = `false`; exports[`parseMeta parses metadata: http://goez1.com/p117046.asp 1`] = ` -ScrapResult { +ScrapeResult { "canonical": "http://goez1.com/p117046.asp", "html": undefined, "status": undefined, @@ -27,7 +27,7 @@ ScrapResult { exports[`parseMeta parses metadata: http://goez1.com/p117046.asp incomplete? 1`] = `false`; exports[`parseMeta parses metadata: http://mp.weixin.qq.com/s/Mg_F1Dbegi0k1i8DRvMORA 1`] = ` -ScrapResult { +ScrapeResult { "canonical": "http://mp.weixin.qq.com/s?__biz=MzIxNTA1OTk3MQ==&mid=2654202378&idx=1&sn=a81dd7ab7f0707221ed508279b3848c7&chksm=8c595166bb2ed8704eb9459c3e5deb51fddf7bf8d266f29dbdf7fc45ec7d9d14a6e1041e8099#rd", "html": undefined, "status": undefined, @@ -40,7 +40,7 @@ ScrapResult { exports[`parseMeta parses metadata: http://mp.weixin.qq.com/s/Mg_F1Dbegi0k1i8DRvMORA incomplete? 1`] = `false`; exports[`parseMeta parses metadata: http://www.cocomy.net/post/457479?from=app_line 1`] = ` -ScrapResult { +ScrapeResult { "canonical": "https://www.cocomy.net/post/457479", "html": undefined, "status": undefined, @@ -53,7 +53,7 @@ ScrapResult { exports[`parseMeta parses metadata: http://www.cocomy.net/post/457479?from=app_line incomplete? 1`] = `false`; exports[`parseMeta parses metadata: http://www.rumtoast.com/7450/%E5%90%8D%E5%AE%B6%E5%81%BD%E8%AA%9E%E9%8C%84%EF%BC%8F%E5%8F%B0%E7%81%A3%E8%80%81%E4%BA%BA-%E6%A5%8A%E6%8C%AF%E6%98%8E%E6%95%99%E6%8E%88%E8%A8%98%E8%BF%B0-%EF%BC%9A%E6%98%AF%E6%97%A5%E6%9C%AC%E4%BA%BA 1`] = ` -ScrapResult { +ScrapeResult { "canonical": "https://www.rumtoast.com/7450/名家偽語錄／台灣老人-楊振明教授記述-：是日本人", "html": undefined, "status": undefined, @@ -66,7 +66,7 @@ ScrapResult { exports[`parseMeta parses metadata: http://www.rumtoast.com/7450/%E5%90%8D%E5%AE%B6%E5%81%BD%E8%AA%9E%E9%8C%84%EF%BC%8F%E5%8F%B0%E7%81%A3%E8%80%81%E4%BA%BA-%E6%A5%8A%E6%8C%AF%E6%98%8E%E6%95%99%E6%8E%88%E8%A8%98%E8%BF%B0-%EF%BC%9A%E6%98%AF%E6%97%A5%E6%9C%AC%E4%BA%BA incomplete? 1`] = `false`; exports[`parseMeta parses metadata: http://www.setn.com/News.aspx?NewsID=122876 1`] = ` -ScrapResult { +ScrapeResult { "canonical": "https://www.setn.com/News.aspx?NewsID=122876", "html": undefined, "status": undefined, @@ -79,7 +79,7 @@ ScrapResult { exports[`parseMeta parses metadata: http://www.setn.com/News.aspx?NewsID=122876 incomplete? 1`] = `false`; exports[`parseMeta parses metadata: https://jerrynest.io/line-girl/ 1`] = ` -ScrapResult { +ScrapeResult { "canonical": "https://jerrynest.io/line-girl/", "html": "
分享就免費送貼圖主題、18 禁影片？透過文本分析了解 Line@ 機器人酒店攬客手法