Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,8 @@ node dist/index.js news
```

The crawler uses a lazy headless Chromium instance (or plain HTTP fallback when
Playwright is not installed), scrolls each source page, extracts likely
the Chromium binary has not been downloaded, e.g. `npx playwright install` was
never run, or when launching the browser fails), scrolls each source page, extracts likely
article/event links, then follows source-local event and article URLs within the
configured depth/page budget. Detail pages are captured from their own
heading/metadata/body text, URLs are normalized, and items are deduplicated by
Expand Down
30 changes: 5 additions & 25 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

63 changes: 60 additions & 3 deletions src/services/news-crawler.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { createHash } from 'node:crypto';
import { existsSync } from 'node:fs';
import { readFile } from 'node:fs/promises';

import configuredNewsSources from '../config/news-sources.json' with { type: 'json' };
Expand Down Expand Up @@ -142,16 +143,72 @@ export async function createNewsCrawler(): Promise<NewsCrawler> {
});
}

async function detectPlaywrightFetcher(): Promise<NewsPageFetcher> {
export async function detectPlaywrightFetcher(): Promise<NewsPageFetcher> {
try {
const { chromium } = await import('playwright');
chromium.executablePath();
return new PlaywrightNewsPageFetcher();
// executablePath() only computes where the Chromium binary is *expected* to
// live — it does not verify the file is actually there. When the browser was
// never downloaded (e.g. `npx playwright install` was not run), it returns a
// non-empty path to a missing file, so we must check the binary exists on
// disk before choosing Playwright. Otherwise every source fails at
// launch time with "browserType.launch: Executable doesn't exist at ...".
const executablePath = chromium.executablePath();
if (!executablePath || !existsSync(executablePath)) {
return new HttpNewsPageFetcher();
}
return new ResilientNewsPageFetcher(new PlaywrightNewsPageFetcher());
} catch {
return new HttpNewsPageFetcher();
}
}

/**
* Wraps the Playwright fetcher and falls back to the HTTP fetcher when the
* headless browser cannot be launched (e.g. Chromium was only partially
* installed). The first time a launch fails, every subsequent fetch is served
* by the HTTP fallback so a single missing browser does not fail every source.
*/
export class ResilientNewsPageFetcher implements NewsPageFetcher {
readonly name = 'resilient-news';
private readonly fallback = new HttpNewsPageFetcher();
private useFallback = false;

constructor(private readonly primary: NewsPageFetcher) {}

async fetch(source: NewsSource, options: NewsFetchOptions): Promise<NewsPageSnapshot> {
if (this.useFallback) return this.fallback.fetch(source, options);
try {
return await this.primary.fetch(source, options);
} catch (error) {
if (isBrowserLaunchError(error)) {
this.useFallback = true;
await this.primary.close().catch(() => {});
return this.fallback.fetch(source, options);
}
throw error;
}
}

async close(): Promise<void> {
await this.primary.close().catch(() => {});
await this.fallback.close().catch(() => {});
}
}

/**
* Detects the Playwright error raised when the Chromium binary is missing or
* cannot be started, so the crawler can switch to the HTTP fallback instead of
* reporting every source as failed.
*/
function isBrowserLaunchError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
return (
message.includes('browserType.launch') ||
message.includes("Executable doesn't exist") ||
message.includes('playwright install')
);
}

export class NewsCrawler {
constructor(
readonly sources: readonly NewsSource[],
Expand Down
58 changes: 58 additions & 0 deletions tests/news-crawler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ import { TradefastStore } from '../src/db/store.js';
import {
DEFAULT_NEWS_SOURCES,
NewsCrawler,
ResilientNewsPageFetcher,
type NewsCandidate,
type NewsFetchOptions,
type NewsItem,
type NewsPageFetcher,
} from '../src/services/news-crawler.js';
Expand Down Expand Up @@ -222,6 +224,62 @@ describe('NewsCrawler', () => {
});
});

describe('ResilientNewsPageFetcher', () => {
const source = DEFAULT_NEWS_SOURCES[0];
const options: NewsFetchOptions = {
timeoutMs: 1_000,
scrollPasses: 0,
settleMs: 0,
maxCandidates: 4,
};

it('falls back to HTTP fetching when the browser cannot be launched', async () => {
const primary: NewsPageFetcher = {
name: 'broken-playwright',
fetch: vi.fn(async () => {
throw new Error(
"browserType.launch: Executable doesn't exist at C:\\\\ms-playwright\\\\chrome.exe",
);
}),
close: vi.fn(async () => {}),
};
const fetcher = new ResilientNewsPageFetcher(primary);
const fetchSpy = vi
.spyOn(globalThis, 'fetch')
.mockImplementation(async () =>
new Response('<html><head><title>Markets</title></head><body></body></html>', {
headers: { 'content-type': 'text/html' },
}),
);

try {
const first = await fetcher.fetch(source, options);
expect(first.pageTitle).toBe('Markets');
// After the first launch failure, the primary fetcher is not retried.
await fetcher.fetch(source, options);
expect(primary.fetch).toHaveBeenCalledOnce();
expect(primary.close).toHaveBeenCalledOnce();
expect(fetchSpy).toHaveBeenCalledTimes(2);
} finally {
fetchSpy.mockRestore();
}
});

it('rethrows non-launch errors instead of masking them', async () => {
const primary: NewsPageFetcher = {
name: 'flaky-playwright',
fetch: vi.fn(async () => {
throw new Error('net::ERR_CONNECTION_REFUSED');
}),
close: vi.fn(async () => {}),
};
const fetcher = new ResilientNewsPageFetcher(primary);

await expect(fetcher.fetch(source, options)).rejects.toThrow('net::ERR_CONNECTION_REFUSED');
expect(primary.close).not.toHaveBeenCalled();
});
});

describe('TradefastStore news persistence', () => {
let handle: DbHandle;
let store: TradefastStore;
Expand Down
Loading