feat: add Cloudflare browser backend and bound Puppeteer resource usage

.env.sample

@@ -1,3 +1,23 @@
 # Rollbar. When not given, disables Rollbar
 ROLLBAR_TOKEN=
 ROLLBAR_ENV=localhost
+
+# Browser backend: 'local' (default) or 'cloudflare'
+BROWSER_BACKEND=local
+
+# Required when BROWSER_BACKEND=cloudflare
+CLOUDFLARE_ACCOUNT_ID=
+CLOUDFLARE_API_TOKEN=
+
+# Optional: Cloudflare keep_alive in ms. Default 60000 (matches CF default).
+# Wider = more session reuse across scrapes, longer billed idle tail per cluster.
+# Max 600000 (10 min) per CF docs.
+# CLOUDFLARE_KEEP_ALIVE_MS=60000
+
+# Optional: max concurrent scrape operations server-wide. Default 3.
+# Lower = less puppeteer memory pressure, slower tail latency for batched URLs.
+# SCRAPE_MAX_CONCURRENCY=3
+
+# Optional: comma-separated puppeteer resourceType list to block during scrape.
+# Default: image,media,font. Set to empty string to disable.
+# SCRAPE_BLOCK_RESOURCES=image,media,font

README.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://travis-ci.org/cofacts/url-resolver.svg?branch=master)](https://travis-ci.org/cofacts/url-resolver) [![Coverage Status](https://coveralls.io/repos/github/cofacts/url-resolver/badge.svg?branch=master)](https://coveralls.io/github/cofacts/url-resolver?branch=master)
 
-A gRPC service that scraps the specified URL and returns scrapped result and summary extracted by
+A gRPC service that scrapes the specified URL and returns scraped result and summary extracted by
 [Readability.js]
 
 ## Usage
@@ -42,10 +42,50 @@ $ npm start
 
 After editing `proto` files, run `npm run compile` to generate corresponding Javascript binary.
 
+## Running with Cloudflare Browser Rendering
+
+Instead of launching a local Chromium, url-resolver can talk to Cloudflare Browser Rendering over its WebSocket CDP endpoint, offloading the ~500 MB chromium process to Cloudflare's edge.
+
+Set in `.env`:
+
+```
+BROWSER_BACKEND=cloudflare
+CLOUDFLARE_ACCOUNT_ID=<your account ID>
+CLOUDFLARE_API_TOKEN=<API token with Browser Rendering: Edit permission>
+```
+
+The token needs the **Browser Rendering: Edit** scope at the account level. Create it under **My Profile → API Tokens → Custom token**.
+
+### Plan requirement
+
+Workers Free only allows 10 minutes of browser time per day, which is insufficient for production URL resolution. The **Workers Paid** plan ($5/month) is required, with $0.09 per browser-hour beyond the 10 hours included monthly.
+
+Indicative cost at 5 seconds per resolution:
+
+| Volume | Browser-hours/month | Estimated cost |
+|---|---|---|
+| 10,000 URLs/day | ~417 | ~$37 |
+| 100,000 URLs/day | ~4,170 | ~$374 |
+
+See https://developers.cloudflare.com/browser-run/pricing/ for current rates.
+
+Local Chromium remains the default and works as a fallback — the Docker image still bundles it, so unsetting `BROWSER_BACKEND` rolls back instantly without redeploy.
+
 ## Build
 
 Directly use docker to build image.
 
 ```bash
 $ docker build -t cofacts/url-resolver:latest .
-```
+```
+
+## Resource limits
+
+Two env vars bound puppeteer memory at scrape time. Both are optional.
+
+| Variable | Default | Description |
+|---|---|---|
+| `SCRAPE_MAX_CONCURRENCY` | `3` | Maximum concurrent `scrape()` operations across all gRPC calls. Each in-flight scrape holds one puppeteer page (image, JS heap, network sockets). Lower this if the server OOMs; raise it if tail latency dominates and there is RAM headroom. |
+| `SCRAPE_BLOCK_RESOURCES` | `image,media,font` | Comma-separated [puppeteer resourceType](https://pptr.dev/api/puppeteer.httprequest.resourcetype) list aborted before they hit the wire. Set to empty string to load every resource. The default leaves `document`, `stylesheet`, `script`, and `xhr` intact, so Readability.js sees the same DOM. |
+
+`og:image` is read from the meta tag in HTML, not from a loaded image, so the default block list does not affect `topImageUrl` for sites that expose Open Graph metadata. Sites without `og:image` fall back to scanning `<img>` tags by rendered size; with image loading blocked, all images report `0x0`, so the fallback returns the first `<img>` instead of the largest.

package.json

@@ -18,7 +18,9 @@
     "dotenv": "^6.0.0",
     "google-protobuf": "^3.9.1",
     "node-fetch": "^2.2.0",
+    "p-limit": "^3.1.0",
     "puppeteer": "^24.10.0",
+    "puppeteer-core": "^24.43.0",
     "rollbar": "^2.5.1",
     "unfurl.js": "^5.1.0"
   },

src/lib/ScrapResult.js → src/lib/ScrapeResult.js

@@ -1,4 +1,4 @@
-// When one of these fields is undefined, the scrap result is considered incomplete.
+// When one of these fields is undefined, the scrape result is considered incomplete.
 const REQUIRED_FIELDS = ['canonical', 'topImageUrl', 'title', 'summary'];
 
 // Merge strategy: use longer when merge()
@@ -10,7 +10,7 @@ const PREFER_CURRENT_FIELDS = ['canonical', 'topImageUrl'];
 // Merge strategy: always use new field
 const PREFER_NEW_FIELDS = ['status'];
 
-class ScrapResult {
+class ScrapeResult {
   constructor(init) {
     Object.assign(this, init);
   }
@@ -19,27 +19,27 @@ class ScrapResult {
    * Merge each field using the strategy defined above.
    * For other fields, they are kept intact.
    *
-   * @param {ScrapResult} scrapResult
-   * @return {ScrapResult}
+   * @param {ScrapeResult} scrapeResult
+   * @return {ScrapeResult}
    */
-  merge(scrapResult) {
+  merge(scrapeResult) {
     PREFER_CURRENT_FIELDS.forEach(field => {
       if (typeof this[field] === 'undefined') {
-        this[field] = scrapResult[field];
+        this[field] = scrapeResult[field];
       }
     });
 
     USE_LONGER_FIELDS.forEach(field => {
       if (
-        typeof scrapResult[field] === 'string' &&
-        (this[field] || '').length < scrapResult[field].length
+        typeof scrapeResult[field] === 'string' &&
+        (this[field] || '').length < scrapeResult[field].length
       ) {
-        this[field] = scrapResult[field];
+        this[field] = scrapeResult[field];
       }
     });
 
     PREFER_NEW_FIELDS.forEach(field => {
-      this[field] = scrapResult[field];
+      this[field] = scrapeResult[field];
     });
   }
 
@@ -83,4 +83,4 @@ class ScrapResult {
   status;
 }
 
-module.exports = ScrapResult;
+module.exports = ScrapeResult;

src/lib/__mocks__/scrap.js → src/lib/__mocks__/scrape.js

@@ -1,7 +1,7 @@
-const ScrapResult = require('../ScrapResult');
+const ScrapeResult = require('../ScrapeResult');
 
 const getResult = url =>
-  new ScrapResult({
+  new ScrapeResult({
     title: 't',
     summary: 's',
     canonical: url,