diff --git a/README.md b/README.md index 3842ad3..0a0f5a0 100644 --- a/README.md +++ b/README.md @@ -352,7 +352,8 @@ node dist/index.js news ``` The crawler uses a lazy headless Chromium instance (or plain HTTP fallback when -Playwright is not installed), scrolls each source page, extracts likely +the Chromium binary has not been downloaded, e.g. `npx playwright install` was +never run, or when launching the browser fails), scrolls each source page, extracts likely article/event links, then follows source-local event and article URLs within the configured depth/page budget. Detail pages are captured from their own heading/metadata/body text, URLs are normalized, and items are deduplicated by diff --git a/package-lock.json b/package-lock.json index 2b44ae0..6b24843 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "Tradefast", - "version": "0.4.1", + "version": "0.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "Tradefast", - "version": "0.4.1", + "version": "0.5.0", "license": "MIT", "dependencies": { "@apollo/server": "^5.5.1", @@ -102,7 +102,6 @@ "resolved": "https://registry.npmjs.org/@apollo/server/-/server-5.5.1.tgz", "integrity": "sha512-Rn3g5TJQsMSUY23CWZTghWdBWyjX7dP1eaEBPkvmM2RHi82cDcpgTIkSCbGvtTUEGjwopLv1AAooU/n7iIZ20A==", "license": "MIT", - "peer": true, "dependencies": { "@apollo/cache-control-types": "^1.0.3", "@apollo/server-gateway-interface": "^2.0.0", @@ -369,8 +368,7 @@ "version": "0.2.17", "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.2.17.tgz", "integrity": "sha512-qEpKRT2oUaWDH6tjRxLHjdzMqRUGYDnGZlKrnL4dJ77JVMcP2Hpo3NYnOSPKdZdeec57B6QPprCUFg0picx5Pw==", - "license": "Apache-2.0", - "peer": true + "license": "Apache-2.0" }, "node_modules/@esbuild-kit/core-utils": { "version": "3.3.2", @@ -1399,7 +1397,6 @@ "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-11.1.24.tgz", "integrity": "sha512-9zHxaDDM+oXW9As6UsP5yYB+UqczBmpeSCIFWdPEtEukMnZhxODG1BBjaUcdBB8Sc1uzojSJSJlp3yFp853t1g==", "license": "MIT", - "peer": true, "dependencies": { "file-type": "21.3.4", "iterare": "1.2.1", @@ -1432,7 +1429,6 @@ "integrity": "sha512-K4bzT+lEdd0Hhcsw3jtk56QAW6s6skK3ViN7hIROSN0kUf4ROwWEAKopJID6yhPQxB45kDtP2wEcjzE8171J3g==", "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "@nuxt/opencollective": "0.4.1", "fast-safe-stringify": "2.1.1", @@ -1473,7 +1469,6 @@ "resolved": "https://registry.npmjs.org/@nestjs/graphql/-/graphql-13.4.2.tgz", "integrity": "sha512-MIaMIaV9o3Tj2LsoGGwhISTZVXEIfDK8rDXplE3tSYULj6cXSY1dofOSLMF/aY+BZLwlrN4BUUowgu8qNdDZFg==", "license": "MIT", - "peer": true, "dependencies": { "@graphql-tools/merge": "9.1.9", "@graphql-tools/schema": "10.0.33", @@ -1560,7 +1555,6 @@ "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-11.1.24.tgz", "integrity": "sha512-CeMKbRBm05aOBiWhIHWO2xDeHbxynBF9ySQv3gRjObz2N5+uJnYriAYkHvVqvC4JIydmMPmT5VdICFNlNz3qyA==", "license": "MIT", - "peer": true, "dependencies": { "cors": "2.8.6", "express": "5.2.1", @@ -2107,7 +2101,6 @@ "integrity": "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@types/node": "*", "pg-protocol": "*", @@ -2127,7 +2120,6 @@ "integrity": "sha512-ch0qJdr2JY0r04NXSprbK6TXOgnaJ1Tz23fm5W+z0/CBah6BSBc3n96h7K9GOtwh0HrilNWHIBzE1Ko4Dcw/Wg==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -3151,7 +3143,6 @@ "dev": true, "hasInstallScript": true, "license": "MIT", - "peer": true, "bin": { "esbuild": "bin/esbuild" }, @@ -3258,7 +3249,6 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", - "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -3618,7 +3608,6 @@ "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.14.0.tgz", "integrity": "sha512-BBvQ/406p+4CZbTpCbVPSxfzrZrbnuWSP1ELYgyS6B+hNeKzgrdB4JczCa5VZUBQrDa9hUngm0KnexY6pJRN5Q==", "license": "MIT", - "peer": true, "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } @@ -3794,7 +3783,6 @@ "resolved": "https://registry.npmjs.org/ink/-/ink-5.2.1.tgz", "integrity": "sha512-BqcUyWrG9zq5HIwW6JcfFHsIYebJkWWb4fczNah1goUO0vv5vneIlfwuS85twyJ5hYR/y18FlAYUxrO9ChIWVg==", "license": "MIT", - "peer": true, "dependencies": { "@alcalzone/ansi-tokenize": "^0.1.3", "ansi-escapes": "^7.0.0", @@ -4523,7 +4511,6 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.21.0.tgz", "integrity": "sha512-AUP1EYJuHraQGsVoCQVIcM7TEJVGtDzxWtGFZd8rds9d+CCXlU5Js1rYgfLNvxy9iJrpHjGrRjoi/3BT9fRyiA==", "license": "MIT", - "peer": true, "dependencies": { "pg-connection-string": "^2.13.0", "pg-pool": "^3.14.0", @@ -4621,7 +4608,6 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -4710,7 +4696,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", @@ -4879,7 +4864,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -4934,8 +4918,7 @@ "version": "0.2.2", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", - "license": "Apache-2.0", - "peer": true + "license": "Apache-2.0" }, "node_modules/resolve-from": { "version": "5.0.0", @@ -5081,6 +5064,7 @@ "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", "license": "Apache-2.0", + "peer": true, "dependencies": { "tslib": "^2.1.0" } @@ -6260,7 +6244,6 @@ "integrity": "sha512-mdoNxBC/cSQObGGVQ5Bpn5i+yv7j68gk3Nfm3wFjcJg3Z0Mix9jzAFfP12prmm5eVGmDKtp0yyArrs0Q+8gZHg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "~0.28.0" }, @@ -6851,7 +6834,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -6928,7 +6910,6 @@ "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", @@ -7637,7 +7618,6 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", "license": "MIT", - "peer": true, "engines": { "node": ">=10.0.0" }, diff --git a/src/services/news-crawler.ts b/src/services/news-crawler.ts index ffd702a..262f26b 100644 --- a/src/services/news-crawler.ts +++ b/src/services/news-crawler.ts @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { existsSync } from 'node:fs'; import { readFile } from 'node:fs/promises'; import configuredNewsSources from '../config/news-sources.json' with { type: 'json' }; @@ -142,16 +143,72 @@ export async function createNewsCrawler(): Promise { }); } -async function detectPlaywrightFetcher(): Promise { +export async function detectPlaywrightFetcher(): Promise { try { const { chromium } = await import('playwright'); - chromium.executablePath(); - return new PlaywrightNewsPageFetcher(); + // executablePath() only computes where the Chromium binary is *expected* to + // live — it does not verify the file is actually there. When the browser was + // never downloaded (e.g. `npx playwright install` was not run), it returns a + // non-empty path to a missing file, so we must check the binary exists on + // disk before choosing Playwright. Otherwise every source fails at + // launch time with "browserType.launch: Executable doesn't exist at ...". + const executablePath = chromium.executablePath(); + if (!executablePath || !existsSync(executablePath)) { + return new HttpNewsPageFetcher(); + } + return new ResilientNewsPageFetcher(new PlaywrightNewsPageFetcher()); } catch { return new HttpNewsPageFetcher(); } } +/** + * Wraps the Playwright fetcher and falls back to the HTTP fetcher when the + * headless browser cannot be launched (e.g. Chromium was only partially + * installed). The first time a launch fails, every subsequent fetch is served + * by the HTTP fallback so a single missing browser does not fail every source. + */ +export class ResilientNewsPageFetcher implements NewsPageFetcher { + readonly name = 'resilient-news'; + private readonly fallback = new HttpNewsPageFetcher(); + private useFallback = false; + + constructor(private readonly primary: NewsPageFetcher) {} + + async fetch(source: NewsSource, options: NewsFetchOptions): Promise { + if (this.useFallback) return this.fallback.fetch(source, options); + try { + return await this.primary.fetch(source, options); + } catch (error) { + if (isBrowserLaunchError(error)) { + this.useFallback = true; + await this.primary.close().catch(() => {}); + return this.fallback.fetch(source, options); + } + throw error; + } + } + + async close(): Promise { + await this.primary.close().catch(() => {}); + await this.fallback.close().catch(() => {}); + } +} + +/** + * Detects the Playwright error raised when the Chromium binary is missing or + * cannot be started, so the crawler can switch to the HTTP fallback instead of + * reporting every source as failed. + */ +function isBrowserLaunchError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes('browserType.launch') || + message.includes("Executable doesn't exist") || + message.includes('playwright install') + ); +} + export class NewsCrawler { constructor( readonly sources: readonly NewsSource[], diff --git a/tests/news-crawler.test.ts b/tests/news-crawler.test.ts index 4a5d867..ca6eb49 100644 --- a/tests/news-crawler.test.ts +++ b/tests/news-crawler.test.ts @@ -5,7 +5,9 @@ import { TradefastStore } from '../src/db/store.js'; import { DEFAULT_NEWS_SOURCES, NewsCrawler, + ResilientNewsPageFetcher, type NewsCandidate, + type NewsFetchOptions, type NewsItem, type NewsPageFetcher, } from '../src/services/news-crawler.js'; @@ -222,6 +224,62 @@ describe('NewsCrawler', () => { }); }); +describe('ResilientNewsPageFetcher', () => { + const source = DEFAULT_NEWS_SOURCES[0]; + const options: NewsFetchOptions = { + timeoutMs: 1_000, + scrollPasses: 0, + settleMs: 0, + maxCandidates: 4, + }; + + it('falls back to HTTP fetching when the browser cannot be launched', async () => { + const primary: NewsPageFetcher = { + name: 'broken-playwright', + fetch: vi.fn(async () => { + throw new Error( + "browserType.launch: Executable doesn't exist at C:\\\\ms-playwright\\\\chrome.exe", + ); + }), + close: vi.fn(async () => {}), + }; + const fetcher = new ResilientNewsPageFetcher(primary); + const fetchSpy = vi + .spyOn(globalThis, 'fetch') + .mockImplementation(async () => + new Response('Markets', { + headers: { 'content-type': 'text/html' }, + }), + ); + + try { + const first = await fetcher.fetch(source, options); + expect(first.pageTitle).toBe('Markets'); + // After the first launch failure, the primary fetcher is not retried. + await fetcher.fetch(source, options); + expect(primary.fetch).toHaveBeenCalledOnce(); + expect(primary.close).toHaveBeenCalledOnce(); + expect(fetchSpy).toHaveBeenCalledTimes(2); + } finally { + fetchSpy.mockRestore(); + } + }); + + it('rethrows non-launch errors instead of masking them', async () => { + const primary: NewsPageFetcher = { + name: 'flaky-playwright', + fetch: vi.fn(async () => { + throw new Error('net::ERR_CONNECTION_REFUSED'); + }), + close: vi.fn(async () => {}), + }; + const fetcher = new ResilientNewsPageFetcher(primary); + + await expect(fetcher.fetch(source, options)).rejects.toThrow('net::ERR_CONNECTION_REFUSED'); + expect(primary.close).not.toHaveBeenCalled(); + }); +}); + describe('TradefastStore news persistence', () => { let handle: DbHandle; let store: TradefastStore;