From b3803ca193fd53f7c5444ba0233277541806b1a5 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 31 May 2026 07:00:58 +0000 Subject: [PATCH 1/3] Initial commit with task details Adding .gitkeep for PR creation (default mode). This file will be removed when the task is complete. Issue: https://github.com/Payel-git-ol/Tradefast/issues/29 --- .gitkeep | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitkeep diff --git a/.gitkeep b/.gitkeep new file mode 100644 index 0000000..972c064 --- /dev/null +++ b/.gitkeep @@ -0,0 +1 @@ +# .gitkeep file auto-generated at 2026-05-31T07:00:58.765Z for PR creation at branch issue-29-8fbc824ebc18 for issue https://github.com/Payel-git-ol/Tradefast/issues/29 \ No newline at end of file From 7148b612015c492eafa9d1e5f2cf5cafeb13273c Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 31 May 2026 07:04:10 +0000 Subject: [PATCH 2/3] Fix /news failing when Chromium binary is not installed chromium.executablePath() returns the expected binary path without verifying the file exists, so detectPlaywrightFetcher chose the Playwright fetcher even when 'npx playwright install' was never run. Every source then failed at launch with "browserType.launch: Executable doesn't exist at ...". - Verify the Chromium binary exists on disk before selecting Playwright; otherwise use the HTTP fallback fetcher. - Add ResilientNewsPageFetcher that switches to the HTTP fallback at runtime if launching the browser still fails, so a single missing browser no longer fails every source. - Add tests covering the launch-failure fallback and non-launch error propagation; clarify README wording. Fixes #29 --- README.md | 3 +- package-lock.json | 30 +++-------------- src/services/news-crawler.ts | 63 ++++++++++++++++++++++++++++++++++-- tests/news-crawler.test.ts | 58 +++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 3842ad3..0a0f5a0 100644 --- a/README.md +++ b/README.md @@ -352,7 +352,8 @@ node dist/index.js news ``` The crawler uses a lazy headless Chromium instance (or plain HTTP fallback when -Playwright is not installed), scrolls each source page, extracts likely +the Chromium binary has not been downloaded, e.g. `npx playwright install` was +never run, or when launching the browser fails), scrolls each source page, extracts likely article/event links, then follows source-local event and article URLs within the configured depth/page budget. Detail pages are captured from their own heading/metadata/body text, URLs are normalized, and items are deduplicated by diff --git a/package-lock.json b/package-lock.json index 2b44ae0..6b24843 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "Tradefast", - "version": "0.4.1", + "version": "0.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "Tradefast", - "version": "0.4.1", + "version": "0.5.0", "license": "MIT", "dependencies": { "@apollo/server": "^5.5.1", @@ -102,7 +102,6 @@ "resolved": "https://registry.npmjs.org/@apollo/server/-/server-5.5.1.tgz", "integrity": "sha512-Rn3g5TJQsMSUY23CWZTghWdBWyjX7dP1eaEBPkvmM2RHi82cDcpgTIkSCbGvtTUEGjwopLv1AAooU/n7iIZ20A==", "license": "MIT", - "peer": true, "dependencies": { "@apollo/cache-control-types": "^1.0.3", "@apollo/server-gateway-interface": "^2.0.0", @@ -369,8 +368,7 @@ "version": "0.2.17", "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.2.17.tgz", "integrity": "sha512-qEpKRT2oUaWDH6tjRxLHjdzMqRUGYDnGZlKrnL4dJ77JVMcP2Hpo3NYnOSPKdZdeec57B6QPprCUFg0picx5Pw==", - "license": "Apache-2.0", - "peer": true + "license": "Apache-2.0" }, "node_modules/@esbuild-kit/core-utils": { "version": "3.3.2", @@ -1399,7 +1397,6 @@ "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-11.1.24.tgz", "integrity": "sha512-9zHxaDDM+oXW9As6UsP5yYB+UqczBmpeSCIFWdPEtEukMnZhxODG1BBjaUcdBB8Sc1uzojSJSJlp3yFp853t1g==", "license": "MIT", - "peer": true, "dependencies": { "file-type": "21.3.4", "iterare": "1.2.1", @@ -1432,7 +1429,6 @@ "integrity": "sha512-K4bzT+lEdd0Hhcsw3jtk56QAW6s6skK3ViN7hIROSN0kUf4ROwWEAKopJID6yhPQxB45kDtP2wEcjzE8171J3g==", "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "@nuxt/opencollective": "0.4.1", "fast-safe-stringify": "2.1.1", @@ -1473,7 +1469,6 @@ "resolved": "https://registry.npmjs.org/@nestjs/graphql/-/graphql-13.4.2.tgz", "integrity": "sha512-MIaMIaV9o3Tj2LsoGGwhISTZVXEIfDK8rDXplE3tSYULj6cXSY1dofOSLMF/aY+BZLwlrN4BUUowgu8qNdDZFg==", "license": "MIT", - "peer": true, "dependencies": { "@graphql-tools/merge": "9.1.9", "@graphql-tools/schema": "10.0.33", @@ -1560,7 +1555,6 @@ "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-11.1.24.tgz", "integrity": "sha512-CeMKbRBm05aOBiWhIHWO2xDeHbxynBF9ySQv3gRjObz2N5+uJnYriAYkHvVqvC4JIydmMPmT5VdICFNlNz3qyA==", "license": "MIT", - "peer": true, "dependencies": { "cors": "2.8.6", "express": "5.2.1", @@ -2107,7 +2101,6 @@ "integrity": "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@types/node": "*", "pg-protocol": "*", @@ -2127,7 +2120,6 @@ "integrity": "sha512-ch0qJdr2JY0r04NXSprbK6TXOgnaJ1Tz23fm5W+z0/CBah6BSBc3n96h7K9GOtwh0HrilNWHIBzE1Ko4Dcw/Wg==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -3151,7 +3143,6 @@ "dev": true, "hasInstallScript": true, "license": "MIT", - "peer": true, "bin": { "esbuild": "bin/esbuild" }, @@ -3258,7 +3249,6 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", - "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -3618,7 +3608,6 @@ "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.14.0.tgz", "integrity": "sha512-BBvQ/406p+4CZbTpCbVPSxfzrZrbnuWSP1ELYgyS6B+hNeKzgrdB4JczCa5VZUBQrDa9hUngm0KnexY6pJRN5Q==", "license": "MIT", - "peer": true, "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } @@ -3794,7 +3783,6 @@ "resolved": "https://registry.npmjs.org/ink/-/ink-5.2.1.tgz", "integrity": "sha512-BqcUyWrG9zq5HIwW6JcfFHsIYebJkWWb4fczNah1goUO0vv5vneIlfwuS85twyJ5hYR/y18FlAYUxrO9ChIWVg==", "license": "MIT", - "peer": true, "dependencies": { "@alcalzone/ansi-tokenize": "^0.1.3", "ansi-escapes": "^7.0.0", @@ -4523,7 +4511,6 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.21.0.tgz", "integrity": "sha512-AUP1EYJuHraQGsVoCQVIcM7TEJVGtDzxWtGFZd8rds9d+CCXlU5Js1rYgfLNvxy9iJrpHjGrRjoi/3BT9fRyiA==", "license": "MIT", - "peer": true, "dependencies": { "pg-connection-string": "^2.13.0", "pg-pool": "^3.14.0", @@ -4621,7 +4608,6 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -4710,7 +4696,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", @@ -4879,7 +4864,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -4934,8 +4918,7 @@ "version": "0.2.2", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", - "license": "Apache-2.0", - "peer": true + "license": "Apache-2.0" }, "node_modules/resolve-from": { "version": "5.0.0", @@ -5081,6 +5064,7 @@ "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", "license": "Apache-2.0", + "peer": true, "dependencies": { "tslib": "^2.1.0" } @@ -6260,7 +6244,6 @@ "integrity": "sha512-mdoNxBC/cSQObGGVQ5Bpn5i+yv7j68gk3Nfm3wFjcJg3Z0Mix9jzAFfP12prmm5eVGmDKtp0yyArrs0Q+8gZHg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "~0.28.0" }, @@ -6851,7 +6834,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -6928,7 +6910,6 @@ "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", @@ -7637,7 +7618,6 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", "license": "MIT", - "peer": true, "engines": { "node": ">=10.0.0" }, diff --git a/src/services/news-crawler.ts b/src/services/news-crawler.ts index ffd702a..262f26b 100644 --- a/src/services/news-crawler.ts +++ b/src/services/news-crawler.ts @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { existsSync } from 'node:fs'; import { readFile } from 'node:fs/promises'; import configuredNewsSources from '../config/news-sources.json' with { type: 'json' }; @@ -142,16 +143,72 @@ export async function createNewsCrawler(): Promise { }); } -async function detectPlaywrightFetcher(): Promise { +export async function detectPlaywrightFetcher(): Promise { try { const { chromium } = await import('playwright'); - chromium.executablePath(); - return new PlaywrightNewsPageFetcher(); + // executablePath() only computes where the Chromium binary is *expected* to + // live — it does not verify the file is actually there. When the browser was + // never downloaded (e.g. `npx playwright install` was not run), it returns a + // non-empty path to a missing file, so we must check the binary exists on + // disk before choosing Playwright. Otherwise every source fails at + // launch time with "browserType.launch: Executable doesn't exist at ...". + const executablePath = chromium.executablePath(); + if (!executablePath || !existsSync(executablePath)) { + return new HttpNewsPageFetcher(); + } + return new ResilientNewsPageFetcher(new PlaywrightNewsPageFetcher()); } catch { return new HttpNewsPageFetcher(); } } +/** + * Wraps the Playwright fetcher and falls back to the HTTP fetcher when the + * headless browser cannot be launched (e.g. Chromium was only partially + * installed). The first time a launch fails, every subsequent fetch is served + * by the HTTP fallback so a single missing browser does not fail every source. + */ +export class ResilientNewsPageFetcher implements NewsPageFetcher { + readonly name = 'resilient-news'; + private readonly fallback = new HttpNewsPageFetcher(); + private useFallback = false; + + constructor(private readonly primary: NewsPageFetcher) {} + + async fetch(source: NewsSource, options: NewsFetchOptions): Promise { + if (this.useFallback) return this.fallback.fetch(source, options); + try { + return await this.primary.fetch(source, options); + } catch (error) { + if (isBrowserLaunchError(error)) { + this.useFallback = true; + await this.primary.close().catch(() => {}); + return this.fallback.fetch(source, options); + } + throw error; + } + } + + async close(): Promise { + await this.primary.close().catch(() => {}); + await this.fallback.close().catch(() => {}); + } +} + +/** + * Detects the Playwright error raised when the Chromium binary is missing or + * cannot be started, so the crawler can switch to the HTTP fallback instead of + * reporting every source as failed. + */ +function isBrowserLaunchError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes('browserType.launch') || + message.includes("Executable doesn't exist") || + message.includes('playwright install') + ); +} + export class NewsCrawler { constructor( readonly sources: readonly NewsSource[], diff --git a/tests/news-crawler.test.ts b/tests/news-crawler.test.ts index 4a5d867..ca6eb49 100644 --- a/tests/news-crawler.test.ts +++ b/tests/news-crawler.test.ts @@ -5,7 +5,9 @@ import { TradefastStore } from '../src/db/store.js'; import { DEFAULT_NEWS_SOURCES, NewsCrawler, + ResilientNewsPageFetcher, type NewsCandidate, + type NewsFetchOptions, type NewsItem, type NewsPageFetcher, } from '../src/services/news-crawler.js'; @@ -222,6 +224,62 @@ describe('NewsCrawler', () => { }); }); +describe('ResilientNewsPageFetcher', () => { + const source = DEFAULT_NEWS_SOURCES[0]; + const options: NewsFetchOptions = { + timeoutMs: 1_000, + scrollPasses: 0, + settleMs: 0, + maxCandidates: 4, + }; + + it('falls back to HTTP fetching when the browser cannot be launched', async () => { + const primary: NewsPageFetcher = { + name: 'broken-playwright', + fetch: vi.fn(async () => { + throw new Error( + "browserType.launch: Executable doesn't exist at C:\\\\ms-playwright\\\\chrome.exe", + ); + }), + close: vi.fn(async () => {}), + }; + const fetcher = new ResilientNewsPageFetcher(primary); + const fetchSpy = vi + .spyOn(globalThis, 'fetch') + .mockImplementation(async () => + new Response('Markets', { + headers: { 'content-type': 'text/html' }, + }), + ); + + try { + const first = await fetcher.fetch(source, options); + expect(first.pageTitle).toBe('Markets'); + // After the first launch failure, the primary fetcher is not retried. + await fetcher.fetch(source, options); + expect(primary.fetch).toHaveBeenCalledOnce(); + expect(primary.close).toHaveBeenCalledOnce(); + expect(fetchSpy).toHaveBeenCalledTimes(2); + } finally { + fetchSpy.mockRestore(); + } + }); + + it('rethrows non-launch errors instead of masking them', async () => { + const primary: NewsPageFetcher = { + name: 'flaky-playwright', + fetch: vi.fn(async () => { + throw new Error('net::ERR_CONNECTION_REFUSED'); + }), + close: vi.fn(async () => {}), + }; + const fetcher = new ResilientNewsPageFetcher(primary); + + await expect(fetcher.fetch(source, options)).rejects.toThrow('net::ERR_CONNECTION_REFUSED'); + expect(primary.close).not.toHaveBeenCalled(); + }); +}); + describe('TradefastStore news persistence', () => { let handle: DbHandle; let store: TradefastStore; From 528e40e97081264bce3df013af192d9656fbd401 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 31 May 2026 07:07:59 +0000 Subject: [PATCH 3/3] Revert "Initial commit with task details" This reverts commit b3803ca193fd53f7c5444ba0233277541806b1a5. --- .gitkeep | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .gitkeep diff --git a/.gitkeep b/.gitkeep deleted file mode 100644 index 972c064..0000000 --- a/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -# .gitkeep file auto-generated at 2026-05-31T07:00:58.765Z for PR creation at branch issue-29-8fbc824ebc18 for issue https://github.com/Payel-git-ol/Tradefast/issues/29 \ No newline at end of file