From b096db5ab9949e0e1326d70733f0beb39b13a734 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 31 May 2026 08:06:49 +0000 Subject: [PATCH 1/3] Initial commit with task details Adding .gitkeep for PR creation (default mode). This file will be removed when the task is complete. Issue: https://github.com/Payel-git-ol/Tradefast/issues/33 --- .gitkeep | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitkeep diff --git a/.gitkeep b/.gitkeep new file mode 100644 index 0000000..5964d94 --- /dev/null +++ b/.gitkeep @@ -0,0 +1 @@ +# .gitkeep file auto-generated at 2026-05-31T08:06:49.631Z for PR creation at branch issue-33-feb048cfde11 for issue https://github.com/Payel-git-ol/Tradefast/issues/33 \ No newline at end of file From a36fb1fc188e7ad4b4b8d9197c22cbc3812390fc Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 31 May 2026 08:27:53 +0000 Subject: [PATCH 2/3] Add whole-internet Web Search platform via server-side GraphQL Integrate a Google web search (modelled on web-agent-master/google-search, scraped with Playwright + DuckDuckGo HTML fallback) as additional support on top of the curated news sources, exposed through the backend. - src/services/web-search.ts: googleSearch() API, pluggable WebSearchEngine (Playwright Google / DuckDuckGo HTTP / resilient), WebSearchProvider adapting hits to SearchResult, offline/CI-safe (empty list on failure). - src/services/search.ts: CompositeSearchProvider merges curated + web hits, de-duplicates by URL, sorts by score. - src/cli/sources.ts: 'web-search' ResearchPlatformId (not a SourceGroup), selectablePlatformIds/getPlatformLabel/getPlatformDescription/isWebSearchEnabled, DEFAULT_ENABLED_PLATFORMS. PlatformSelector + App wire the new toggle. - pipeline/collector + app/tradefast run web search when the platform is enabled. - backend: SearchResultDto + webSearch GraphQL query (resolver/repository/facade), cli graphql repository + WEB_SEARCH_QUERY; search runs server-side. - Declare design:paramtypes for the @Args() params since esbuild can't emit it. - Tests for web-search service, composite provider, selector platforms, and the backend/graphql webSearch path. README + version bump to 0.7.0. Closes #33 --- README.md | 25 +- package-lock.json | 4 +- package.json | 2 +- src/app/tradefast.ts | 31 ++ src/backend/graphql/dto/search-result.dto.ts | 20 + src/backend/graphql/facade.ts | 3 + src/backend/graphql/index.ts | 1 + src/backend/graphql/repository.ts | 12 + src/backend/graphql/tradefast.resolver.ts | 22 +- src/cli/App.tsx | 46 ++- src/cli/components/PlatformSelector.tsx | 16 +- src/cli/graphql/index.ts | 1 + src/cli/graphql/repository.ts | 6 + src/cli/graphql/web-search.ts | 25 ++ src/cli/preferences.ts | 6 +- src/cli/sources.ts | 57 ++- src/config.ts | 2 +- src/index.tsx | 13 +- src/pipeline/collector.ts | 25 +- src/services/search.ts | 33 ++ src/services/web-search.ts | 367 +++++++++++++++++++ tests/backend.test.ts | 11 + tests/graphql-repository.test.ts | 8 + tests/sources.test.ts | 34 ++ tests/web-search.test.ts | 143 ++++++++ 25 files changed, 865 insertions(+), 48 deletions(-) create mode 100644 src/backend/graphql/dto/search-result.dto.ts create mode 100644 src/cli/graphql/web-search.ts create mode 100644 src/services/web-search.ts create mode 100644 tests/web-search.test.ts diff --git a/README.md b/README.md index 62f7f89..7210ab4 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,12 @@ interactive terminal UI. Normal (fast) to Max (full comment graph), persisted across sessions. - **Selectable research platforms**: `/serching-platforms` lets you toggle source groups on/off — skip Reddit, use only calendars, or go all-in. +- **Whole-internet Web Search**: a `Web Search` platform (in the same + `/serching-platforms` pop-up) layers a Google web search — modelled on + [web-agent-master/google-search](https://github.com/web-agent-master/google-search) + and scraped with Playwright, with a DuckDuckGo HTML fallback — *on top of* the + curated sources, so research is no longer limited to the built-in feeds. The + search runs **server-side** through the GraphQL `webSearch` query. - **Reddit-specific extraction**: specialized parser follows comment-thread links to other Reddit posts and external articles. - **13 strategies** computed from textbook technical indicators (SMA, EMA, RSI, @@ -72,7 +78,8 @@ interactive terminal UI. facade onto GraphQL DTOs, and the frontend `GraphqlTradefastRepository` (`src/cli/graphql/`) is the only place the CLI talks to the API. Each GraphQL class lives in its own file on both sides. The headless `status`, `strategies` - and `clear` commands make their requests through this path. + and `clear` commands make their requests through this path, as does the + whole-internet `webSearch` query so web search executes server-side. - **Dockerised**: `docker compose up` brings up PostgreSQL and the CLI. --- @@ -129,7 +136,7 @@ TRADEFAST_MARKET_SOURCE=synthetic TRADEFAST_DATA_DIR=:memory: node dist/index.js | `/operating-mode [name]` | Open the trading-style selector pop-up, or switch directly by name (`long-term`, `medium-term`, `scalping`). Applies that horizon's timeframe. | | `/operating-mode-time [tf]` | Open the timeframe selector, or set it directly (`1m`–`1d`) to fine-tune within the current mode. | | `/serching-level [level]` | Set crawl depth/resolution: `normal` (fast, depth 2), `high` (deep, depth 4), or `max` (full graph, depth 8 with comment traversal). Opens a pop-up without argument. | -| `/serching-platforms` | Toggle source groups on/off: economic calendars, news portals, crypto news, Reddit communities, crypto communities, exchange communities. Opens a multi-select pop-up. | +| `/serching-platforms` | Toggle research platforms on/off: economic calendars, news portals, crypto news, Reddit communities, crypto communities, exchange communities, plus **Web Search** (whole-internet Google/DuckDuckGo). Opens a multi-select pop-up. | | `/currency [symbol]` | Run a full forecast for a single symbol with news sentiment and price chart. | | `/exchange [name]` | Switch the venue/data source: `binance`, `okx`, `bybit`, `mexc` (crypto spot) or `pocketoption` (forex binary options). Pocket Option swaps in forex majors and renders an expiry **Time** column instead of TP/SL. Opens a pop-up without an argument. | | `/ratings` | Show source credibility ratings. Subcommands: `correct`, `incorrect`, `loud-claim`, or a numeric grade (`/ratings "Хабр" -1`). | @@ -294,7 +301,7 @@ All configuration is environment-driven (see `.env.example`): | `TRADEFAST_AI_MODEL` | `claude-4.7-opus` | Model for per-symbol advice and cross-symbol correction. | | `TRADEFAST_SKIP_AI_VALIDATION` | `0` | Set to `1` to skip the cross-symbol AI correction step. | | `TRADEFAST_SEARCHING_LEVEL`| _(unset → pop-up)_ | Preset depth: `normal`, `high`, or `max`. Skips the pop-up. | -| `TRADEFAST_SEARCHING_PLATFORMS` | _(unset → pop-up)_ | Comma-separated source groups to enable (e.g. `news-portals,reddit-communities`). Skips the pop-up. | +| `TRADEFAST_SEARCHING_PLATFORMS` | _(unset → pop-up)_ | Comma-separated platforms to enable (e.g. `news-portals,reddit-communities,web-search`). Add `web-search` for whole-internet search. Skips the pop-up. | The market source falls back gracefully: `resilient` uses live Binance data and transparently switches to deterministic synthetic candles if the network is @@ -344,6 +351,13 @@ limit. Sources are grouped into 6 platform groups: | `crypto-communities` | 6 crypto subreddits (Bitcoin, ethereum, CryptoMarkets, defi, Altcoin, CryptoTechnology) — kind: `reddit`. | | `exchange-communities` | Binance, Bybit, OKX, MEXC blogs + their subreddits. | +Beyond these curated groups, the **`web-search`** platform adds a whole-internet +Google web search (DuckDuckGo HTML fallback) that runs server-side via the +GraphQL `webSearch` query, so `/start`, `/update` and `/currency` can pull in +results from outside the configured feeds. It is **not** a news group — it has no +curated `sources.json` entry — so it appears in `/serching-platforms` as its own +toggle and contributes results to the general search table alongside the crawl. + Sources with `kind: "reddit"` use a specialized extractor that reads Reddit threads, extracts the self-text/comments, and follows comment-thread links to other Reddit posts and external articles. This produces rich cross-referenced @@ -425,9 +439,10 @@ exposes these tools to the AI: - **`run_serching_level`** — sets crawl depth/resolution. The AI can request `normal` (fast, shallow), `high` (deep), or `max` (exhaustive) depending on how much context it needs. -- **`run_serching_platforms`** — enables/disables source groups. The AI can +- **`run_serching_platforms`** — enables/disables research platforms. The AI can toggle specific groups (e.g. enable only `economic-calendars` for - fundamentals) to tailor research scope. + fundamentals) or the whole-internet `web-search` platform to tailor research + scope. - **`run_ratings_adjust`** — modifies a source's credibility rating. Parameters: `source` (title or ID) and `grade` (integer, e.g. `-1` = −1%). Triggered by user feedback like *"Хабр -1"* or *"понизь рейтинг Хабру"*. diff --git a/package-lock.json b/package-lock.json index 6b24843..ce0caab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "Tradefast", - "version": "0.5.0", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "Tradefast", - "version": "0.5.0", + "version": "0.7.0", "license": "MIT", "dependencies": { "@apollo/server": "^5.5.1", diff --git a/package.json b/package.json index a2ae694..6d163a3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "Tradefast", - "version": "0.6.0", + "version": "0.7.0", "description": "TRADEFΛST — a disciplined crypto market-research CLI: strategies, analytics, risk and AI-assisted research, in the style of Gemini CLI.", "type": "module", "license": "MIT", diff --git a/src/app/tradefast.ts b/src/app/tradefast.ts index d75a95d..49f3b91 100644 --- a/src/app/tradefast.ts +++ b/src/app/tradefast.ts @@ -21,6 +21,9 @@ import { import { ALL_STRATEGIES } from '../strategies/registry.js'; import { computeCrowdConsensus, type InstrumentConsensus } from '../services/news-consensus.js'; import { SourceRatingService } from '../services/source-ratings.js'; +import { isWebSearchEnabled } from '../cli/sources.js'; +import { createWebSearchProvider } from '../services/web-search.js'; +import type { SearchResult } from '../services/search.js'; export interface StatusReport { driver: string; @@ -107,6 +110,19 @@ export class Tradefast { (this.config as any).interval = interval; } + /** Updates the enabled research platforms (used by the `/serching-platforms` + * selector). The "Web Search" toggle here drives whether subsequent /start, + * /update and /currency runs search the whole Internet in addition to the + * curated knowledge base. */ + setSearchingPlatforms(platforms: readonly string[]): void { + (this.config as any).searchingPlatforms = [...platforms]; + } + + /** Whether whole-internet web search is currently enabled in the config. */ + private get webSearchEnabled(): boolean { + return isWebSearchEnabled(this.config.searchingPlatforms ?? []); + } + /** Records the active operating mode (trading style). The associated timeframe * is applied separately via {@link setInterval} so analysis actually shifts * to the chosen horizon. */ @@ -129,6 +145,7 @@ export class Tradefast { limit: this.config.candleLimit, accountBalance: this.config.accountBalance, exchange: this.config.exchange, + webSearch: this.webSearchEnabled, ...extra, }, onProgress, @@ -145,6 +162,7 @@ export class Tradefast { limit: this.config.candleLimit, accountBalance: this.config.accountBalance, exchange: this.config.exchange, + webSearch: this.webSearchEnabled, ...extra, }, onProgress, @@ -194,6 +212,7 @@ export class Tradefast { limit: this.config.candleLimit, accountBalance: this.config.accountBalance, exchange: this.config.exchange, + webSearch: this.webSearchEnabled, ...extra, }, onProgress, @@ -215,6 +234,18 @@ export class Tradefast { return { symbol, report, price, newsConsensus: currencyNews, candles }; } + /** + * Whole-internet web search exposed for the backend `webSearch` GraphQL query. + * Runs server-side through the Playwright/HTTP {@link WebSearchProvider} so the + * CLI can search the entire Internet, not just the curated sources. Returns an + * empty list on failure rather than throwing. + */ + async search(query: string, limit = 10): Promise { + const provider = createWebSearchProvider({ limit }); + const results = await provider.search(query); + return results.slice(0, limit); + } + /** Read back persisted candles for chart rendering. */ getCandles(symbol: string, interval: string): Promise { return this.store.getCandles(symbol, interval); diff --git a/src/backend/graphql/dto/search-result.dto.ts b/src/backend/graphql/dto/search-result.dto.ts new file mode 100644 index 0000000..00c2721 --- /dev/null +++ b/src/backend/graphql/dto/search-result.dto.ts @@ -0,0 +1,20 @@ +import { Field, Float, ObjectType } from '@nestjs/graphql'; + +/** A single whole-internet web search hit, returned by the `webSearch` query. */ +@ObjectType() +export class SearchResultDto { + @Field(() => String) + source!: string; + + @Field(() => String) + title!: string; + + @Field(() => String, { nullable: true }) + url!: string | null; + + @Field(() => String, { nullable: true }) + snippet!: string | null; + + @Field(() => Float) + score!: number; +} diff --git a/src/backend/graphql/facade.ts b/src/backend/graphql/facade.ts index 759f7e3..e76a65b 100644 --- a/src/backend/graphql/facade.ts +++ b/src/backend/graphql/facade.ts @@ -1,5 +1,6 @@ import type { StatusReport } from '../../app/tradefast.js'; import type { RunReport } from '../../pipeline/collector.js'; +import type { SearchResult } from '../../services/search.js'; /** DI token under which the application facade is provided to the resolver. */ export const TRADEFAST_FACADE = Symbol('TRADEFAST_FACADE'); @@ -16,4 +17,6 @@ export interface TradefastApiFacade { start(): Promise; update(): Promise; clear(): Promise; + /** Whole-internet web search executed server-side. */ + search(query: string, limit?: number): Promise; } diff --git a/src/backend/graphql/index.ts b/src/backend/graphql/index.ts index be4c588..ef16d76 100644 --- a/src/backend/graphql/index.ts +++ b/src/backend/graphql/index.ts @@ -2,6 +2,7 @@ export { TRADEFAST_FACADE, type TradefastApiFacade } from './facade.js'; export { AnalyticsDto } from './dto/analytics.dto.js'; export { RunReportDto } from './dto/run-report.dto.js'; +export { SearchResultDto } from './dto/search-result.dto.js'; export { StatusDto } from './dto/status.dto.js'; export { StrategyDto } from './dto/strategy.dto.js'; export { SymbolRunDto } from './dto/symbol-run.dto.js'; diff --git a/src/backend/graphql/repository.ts b/src/backend/graphql/repository.ts index 59861e6..1bdd24b 100644 --- a/src/backend/graphql/repository.ts +++ b/src/backend/graphql/repository.ts @@ -2,6 +2,7 @@ import { Inject, Injectable } from '@nestjs/common'; import type { RunReport } from '../../pipeline/collector.js'; import { RunReportDto } from './dto/run-report.dto.js'; +import { SearchResultDto } from './dto/search-result.dto.js'; import { StatusDto } from './dto/status.dto.js'; import { StrategyDto } from './dto/strategy.dto.js'; import { TRADEFAST_FACADE, type TradefastApiFacade } from './facade.js'; @@ -52,6 +53,17 @@ export class TradefastRepository { clear(): Promise { return this.tradefast.clear(); } + + async webSearch(query: string, limit?: number): Promise { + const results = await this.tradefast.search(query, limit); + return results.map((result) => ({ + source: result.source, + title: result.title, + url: result.url ?? null, + snippet: result.snippet ?? null, + score: result.score, + })); + } } /** Map a pipeline {@link RunReport} into its GraphQL DTO. */ diff --git a/src/backend/graphql/tradefast.resolver.ts b/src/backend/graphql/tradefast.resolver.ts index e423064..352becb 100644 --- a/src/backend/graphql/tradefast.resolver.ts +++ b/src/backend/graphql/tradefast.resolver.ts @@ -1,7 +1,9 @@ +import 'reflect-metadata'; import { Inject } from '@nestjs/common'; -import { Int, Mutation, Query, Resolver } from '@nestjs/graphql'; +import { Args, Int, Mutation, Query, Resolver } from '@nestjs/graphql'; import { RunReportDto } from './dto/run-report.dto.js'; +import { SearchResultDto } from './dto/search-result.dto.js'; import { StatusDto } from './dto/status.dto.js'; import { StrategyDto } from './dto/strategy.dto.js'; import { TradefastRepository } from './repository.js'; @@ -24,6 +26,15 @@ export class TradefastResolver { return this.repository.strategies(); } + /** Whole-internet web search ("Web Search" platform), executed server-side. */ + @Query(() => [SearchResultDto]) + webSearch( + @Args('query', { type: () => String }) query: string, + @Args('limit', { type: () => Int, nullable: true }) limit?: number, + ): Promise { + return this.repository.webSearch(query, limit ?? undefined); + } + @Mutation(() => RunReportDto) start(): Promise { return this.repository.start(); @@ -39,3 +50,12 @@ export class TradefastResolver { return this.repository.clear(); } } + +// The build pipeline (esbuild via tsup/vitest) cannot emit `design:paramtypes`, +// which `@Args()` reflects at schema-build time. Every other field passes an +// explicit `@Field(() => …)` thunk and so never needs it, but `@Args()` reads +// the reflected parameter list *before* honouring its explicit `type` option and +// would otherwise crash on `undefined[index]`. Declaring the metadata by hand — +// exactly what `emitDecoratorMetadata` would generate — keeps the schema build +// reflection-free while preserving the explicit `() => String` / `() => Int` types. +Reflect.defineMetadata('design:paramtypes', [String, Number], TradefastResolver.prototype, 'webSearch'); diff --git a/src/cli/App.tsx b/src/cli/App.tsx index 642d046..8db3fb4 100644 --- a/src/cli/App.tsx +++ b/src/cli/App.tsx @@ -22,7 +22,13 @@ import { getExchange, exchangeNames, isKnownExchange, isBinaryOptions, type Exch import { getInterval, intervalNames, type IntervalName } from './intervals.js'; import { getMode, modeNames, type ModeName } from './modes.js'; import { searchLevelNames, getSearchLevel, type SearchLevelName } from './search-level.js'; -import { sourceGroupIds, getSourceGroup, resolveSourceIds, DEFAULT_ENABLED_GROUPS, type SourceGroupId } from './sources.js'; +import { + resolveSourceIds, + selectablePlatformIds, + getPlatformLabel, + DEFAULT_ENABLED_PLATFORMS, + type ResearchPlatformId, +} from './sources.js'; export interface AppProps { app: Tradefast; @@ -65,14 +71,15 @@ export function App({ app, version, apiUrl, promptOperatingMode }: AppProps): Re const [searchingLevel, setSearchingLevel] = useState(() => getSearchLevel(app.config.searchingLevel).name as SearchLevelName); const [platformSelectorOpen, setPlatformSelectorOpen] = useState(false); const [platformCursorIndex, setPlatformCursorIndex] = useState(0); - const [enabledPlatforms, setEnabledPlatforms] = useState(() => { + const [enabledPlatforms, setEnabledPlatforms] = useState(() => { if (Array.isArray(app.config.searchingPlatforms) && app.config.searchingPlatforms.length > 0) { - const valid = app.config.searchingPlatforms.filter((g): g is SourceGroupId => - (sourceGroupIds() as string[]).includes(g), + const selectable = selectablePlatformIds() as string[]; + const valid = app.config.searchingPlatforms.filter((g): g is ResearchPlatformId => + selectable.includes(g), ); - return valid.length > 0 ? valid : [...DEFAULT_ENABLED_GROUPS]; + return valid.length > 0 ? valid : [...DEFAULT_ENABLED_PLATFORMS]; } - return [...DEFAULT_ENABLED_GROUPS]; + return [...DEFAULT_ENABLED_PLATFORMS]; }); const [busy, setBusy] = useState(false); const newsCrawlOptions = useCallback( @@ -214,7 +221,7 @@ export function App({ app, version, apiUrl, promptOperatingMode }: AppProps): Re ); const togglePlatform = useCallback( - (id: SourceGroupId) => { + (id: ResearchPlatformId) => { setEnabledPlatforms((prev) => prev.includes(id) ? prev.filter((g) => g !== id) : [...prev, id], ); @@ -226,11 +233,12 @@ export function App({ app, version, apiUrl, promptOperatingMode }: AppProps): Re () => { setPlatformSelectorOpen(false); void saveSearchingPlatforms(enabledPlatforms); + app.setSearchingPlatforms(enabledPlatforms); const count = resolveSourceIds(enabledPlatforms).length; - const groups = enabledPlatforms.map((id) => getSourceGroup(id)?.label ?? id).join(', '); + const groups = enabledPlatforms.map((id) => getPlatformLabel(id)).join(', '); push({ kind: 'text', text: `Research platforms: ${groups} (${count} sources)`, color: theme.colors.info }); }, - [enabledPlatforms, push, theme], + [app, enabledPlatforms, push, theme], ); const applyCurrency = useCallback( @@ -355,13 +363,13 @@ export function App({ app, version, apiUrl, promptOperatingMode }: AppProps): Re if (key.escape) { setPlatformSelectorOpen(false); } else if (key.upArrow) { - setPlatformCursorIndex((index) => (index - 1 + sourceGroupIds().length) % sourceGroupIds().length); + setPlatformCursorIndex((index) => (index - 1 + selectablePlatformIds().length) % selectablePlatformIds().length); } else if (key.downArrow) { - setPlatformCursorIndex((index) => (index + 1) % sourceGroupIds().length); + setPlatformCursorIndex((index) => (index + 1) % selectablePlatformIds().length); } else if (key.return) { applyPlatforms(); } else if (_input === ' ') { - togglePlatform(sourceGroupIds()[platformCursorIndex]); + togglePlatform(selectablePlatformIds()[platformCursorIndex]); } return; } @@ -505,7 +513,7 @@ export function App({ app, version, apiUrl, promptOperatingMode }: AppProps): Re return; } if (args[0] === 'check') { - const groups = enabledPlatforms.map((id) => getSourceGroup(id)?.label ?? id).join(', '); + const groups = enabledPlatforms.map((id) => getPlatformLabel(id)).join(', '); const count = resolveSourceIds(enabledPlatforms).length; push({ kind: 'text', text: `Research platforms: ${groups} (${count} sources)`, color: theme.colors.info }); return; @@ -747,12 +755,14 @@ export function App({ app, version, apiUrl, promptOperatingMode }: AppProps): Re case 'run_serching_platforms': { const groups = toolArgs.groups as string[] | undefined; if (groups && Array.isArray(groups) && groups.length > 0) { - const valid = groups.filter((g) => sourceGroupIds().includes(g as SourceGroupId)); + const selectable = selectablePlatformIds() as string[]; + const valid = groups.filter((g) => selectable.includes(g)) as ResearchPlatformId[]; if (valid.length > 0) { - setEnabledPlatforms(valid as SourceGroupId[]); - void saveSearchingPlatforms(valid as SourceGroupId[]); - const count = resolveSourceIds(valid as SourceGroupId[]).length; - return `Research platforms set to ${valid.length} group(s) (${count} sources).`; + setEnabledPlatforms(valid); + void saveSearchingPlatforms(valid); + app.setSearchingPlatforms(valid); + const count = resolveSourceIds(valid).length; + return `Research platforms set to ${valid.length} platform(s) (${count} sources).`; } } setPlatformCursorIndex(0); diff --git a/src/cli/components/PlatformSelector.tsx b/src/cli/components/PlatformSelector.tsx index 9e2d34f..c82b6a7 100644 --- a/src/cli/components/PlatformSelector.tsx +++ b/src/cli/components/PlatformSelector.tsx @@ -1,7 +1,12 @@ import { Box, Text } from 'ink'; import React from 'react'; -import { sourceGroupIds, getSourceGroup, type SourceGroupId } from '../sources.js'; +import { + selectablePlatformIds, + getPlatformLabel, + getPlatformDescription, + type ResearchPlatformId, +} from '../sources.js'; import type { CliTheme } from '../theme.js'; export function PlatformSelector({ @@ -11,7 +16,7 @@ export function PlatformSelector({ }: { theme: CliTheme; cursorIndex: number; - enabledGroups: SourceGroupId[]; + enabledGroups: ResearchPlatformId[]; }): React.ReactElement { return ( Space=toggle, Enter=done, Esc=cancel - {sourceGroupIds().map((id, index) => { - const group = getSourceGroup(id)!; + {selectablePlatformIds().map((id, index) => { const checked = enabledGroups.includes(id); const focused = index === cursorIndex; @@ -37,9 +41,9 @@ export function PlatformSelector({ {focused ? '> ' : ' '} {checked ? '[x]' : '[ ]'} {' '} - {group.label} + {getPlatformLabel(id)} {' '} - {group.description} + {getPlatformDescription(id)} ); })} diff --git a/src/cli/graphql/index.ts b/src/cli/graphql/index.ts index 1c8d1c2..1822ca1 100644 --- a/src/cli/graphql/index.ts +++ b/src/cli/graphql/index.ts @@ -7,3 +7,4 @@ export type { Strategy } from './strategy.js'; export type { SymbolRun } from './symbol-run.js'; export type { RunReport } from './run-report.js'; export type { TableCount } from './table-count.js'; +export type { WebSearchHit } from './web-search.js'; diff --git a/src/cli/graphql/repository.ts b/src/cli/graphql/repository.ts index 89d60e8..dc20d23 100644 --- a/src/cli/graphql/repository.ts +++ b/src/cli/graphql/repository.ts @@ -3,6 +3,7 @@ import { GraphqlClient } from './client.js'; import { START_MUTATION, UPDATE_MUTATION, type RunReport, type StartResult, type UpdateResult } from './run-report.js'; import { STATUS_QUERY, type Status, type StatusResult } from './status.js'; import { STRATEGIES_QUERY, type Strategy, type StrategiesResult } from './strategy.js'; +import { WEB_SEARCH_QUERY, type WebSearchHit, type WebSearchResult } from './web-search.js'; /** * The frontend repository. Every API request the CLI makes goes through this @@ -41,4 +42,9 @@ export class GraphqlTradefastRepository { const data = await this.client.request(CLEAR_MUTATION); return data.clear; } + + async webSearch(query: string, limit?: number): Promise { + const data = await this.client.request(WEB_SEARCH_QUERY, { query, limit }); + return data.webSearch; + } } diff --git a/src/cli/graphql/web-search.ts b/src/cli/graphql/web-search.ts new file mode 100644 index 0000000..756b60b --- /dev/null +++ b/src/cli/graphql/web-search.ts @@ -0,0 +1,25 @@ +/** A single whole-internet web search hit returned by the backend. */ +export interface WebSearchHit { + source: string; + title: string; + url: string | null; + snippet: string | null; + score: number; +} + +/** Run a whole-internet web search ("Web Search" platform) on the backend. */ +export const WEB_SEARCH_QUERY = ` + query WebSearch($query: String!, $limit: Int) { + webSearch(query: $query, limit: $limit) { + source + title + url + snippet + score + } + } +`; + +export interface WebSearchResult { + webSearch: WebSearchHit[]; +} diff --git a/src/cli/preferences.ts b/src/cli/preferences.ts index e308bb6..8eca3d6 100644 --- a/src/cli/preferences.ts +++ b/src/cli/preferences.ts @@ -7,7 +7,7 @@ import type { ThemeName } from './theme.js'; import type { ExchangeName } from './exchanges.js'; import type { ModeName } from './modes.js'; import type { SearchLevelName } from './search-level.js'; -import type { SourceGroupId } from './sources.js'; +import type { ResearchPlatformId } from './sources.js'; const PREF_PATH = join(homedir(), '.tradefast', 'preferences.json'); @@ -17,7 +17,7 @@ export interface UserPreferences { interval?: IntervalName; mode?: ModeName; searchingLevel?: SearchLevelName; - searchingPlatforms?: SourceGroupId[]; + searchingPlatforms?: ResearchPlatformId[]; } export async function loadPreferences(): Promise { @@ -57,6 +57,6 @@ export async function saveSearchingLevel(name: SearchLevelName): Promise { await savePreferences({ searchingLevel: name }); } -export async function saveSearchingPlatforms(groups: SourceGroupId[]): Promise { +export async function saveSearchingPlatforms(groups: ResearchPlatformId[]): Promise { await savePreferences({ searchingPlatforms: groups }); } diff --git a/src/cli/sources.ts b/src/cli/sources.ts index 328b4c6..850b94c 100644 --- a/src/cli/sources.ts +++ b/src/cli/sources.ts @@ -117,12 +117,16 @@ export function getSourceGroup(id: string): SourceGroup | undefined { return SOURCE_GROUPS.find((g) => g.id === id); } -/** All enabled source IDs from the enabled group IDs. */ +/** + * All enabled news source IDs from the enabled platforms. Non-group platforms + * (e.g. `web-search`) contribute no curated source IDs and are simply ignored + * here — they are fulfilled by their own provider, not the news crawler. + */ export function resolveSourceIds( - enabledGroups: SourceGroupId[], + enabledGroups: readonly string[], ): string[] { return SOURCE_GROUPS - .filter((g) => enabledGroups.includes(g.id as SourceGroupId)) + .filter((g) => enabledGroups.includes(g.id)) .flatMap((g) => g.sourceIds); } @@ -135,3 +139,50 @@ export const DEFAULT_ENABLED_GROUPS: SourceGroupId[] = [ 'crypto-communities', 'exchange-communities', ]; + +/** + * The identifier of the whole-internet web search. It is intentionally NOT a + * {@link SourceGroup}: it has no curated `sourceIds` and is fulfilled by the + * Playwright/HTTP {@link WebSearchProvider} rather than the news crawler. Keeping + * it separate preserves the strict news-group contract while still letting the + * selector and config treat it as one more toggleable platform. + */ +export const WEB_SEARCH_PLATFORM_ID = 'web-search' as const; + +/** A research platform shown in `/serching-platforms`: a news group or web search. */ +export type ResearchPlatformId = SourceGroupId | typeof WEB_SEARCH_PLATFORM_ID; + +const WEB_SEARCH_PLATFORM = { + id: WEB_SEARCH_PLATFORM_ID, + label: 'Web Search', + description: 'Search the entire Internet (Google via Playwright, DuckDuckGo HTML fallback)', +} as const; + +/** Every platform the selector offers: the news groups followed by web search. */ +export const selectablePlatformIds = (): ResearchPlatformId[] => [ + ...sourceGroupIds(), + WEB_SEARCH_PLATFORM_ID, +]; + +/** The display label for any platform id (news group or web search). */ +export function getPlatformLabel(id: ResearchPlatformId): string { + if (id === WEB_SEARCH_PLATFORM_ID) return WEB_SEARCH_PLATFORM.label; + return getSourceGroup(id)?.label ?? id; +} + +/** The description line for any platform id (news group or web search). */ +export function getPlatformDescription(id: ResearchPlatformId): string { + if (id === WEB_SEARCH_PLATFORM_ID) return WEB_SEARCH_PLATFORM.description; + return getSourceGroup(id)?.description ?? ''; +} + +/** Whether whole-internet web search is among the enabled platforms. */ +export function isWebSearchEnabled(platforms: readonly string[]): boolean { + return platforms.includes(WEB_SEARCH_PLATFORM_ID); +} + +/** Default-enabled platforms: every news group plus whole-internet web search. */ +export const DEFAULT_ENABLED_PLATFORMS: ResearchPlatformId[] = [ + ...DEFAULT_ENABLED_GROUPS, + WEB_SEARCH_PLATFORM_ID, +]; diff --git a/src/config.ts b/src/config.ts index 912afc4..adeba04 100644 --- a/src/config.ts +++ b/src/config.ts @@ -73,7 +73,7 @@ export function loadConfig(overrides: Partial = {}): TradefastC searchingPlatforms: overrides.searchingPlatforms ?? ( process.env.TRADEFAST_SEARCHING_PLATFORMS ? process.env.TRADEFAST_SEARCHING_PLATFORMS.split(',').map(s => s.trim()).filter(Boolean) - : ['economic-calendars', 'news-portals', 'reddit-communities', 'exchange-communities'] + : ['economic-calendars', 'news-portals', 'reddit-communities', 'exchange-communities', 'web-search'] ), apiEnabled: overrides.apiEnabled ?? envFlag('TRADEFAST_API', true), apiHost: overrides.apiHost ?? process.env.TRADEFAST_API_HOST ?? '127.0.0.1', diff --git a/src/index.tsx b/src/index.tsx index 029a739..f52db4a 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -17,7 +17,7 @@ import { getExchange, exchangeNames, type ExchangeName } from './cli/exchanges.j import { getInterval, intervalNames } from './cli/intervals.js'; import { getMode, modeNames, type ModeName } from './cli/modes.js'; import { getSearchLevel, searchLevelNames, type SearchLevelName } from './cli/search-level.js'; -import { sourceGroupIds, resolveSourceIds, type SourceGroupId } from './cli/sources.js'; +import { resolveSourceIds, selectablePlatformIds, type ResearchPlatformId } from './cli/sources.js'; import { loadPreferences, saveTheme, saveExchange, saveInterval, saveMode, saveSearchingLevel, saveSearchingPlatforms } from './cli/preferences.js'; import { renderBacktestLines } from './cli/backtest-log.js'; import { renderTradeLogLines } from './cli/trade-log.js'; @@ -123,16 +123,17 @@ async function runHeadless(command: string): Promise { if (name === 'serching-platforms') { if (args.length > 0) { const groups = args[0].split(',').map((s) => s.trim()).filter(Boolean); - const valid = groups.filter((g) => sourceGroupIds().includes(g as SourceGroupId)); + const selectable = selectablePlatformIds() as string[]; + const valid = groups.filter((g) => selectable.includes(g)) as ResearchPlatformId[]; if (valid.length === 0) { - process.stderr.write(`No valid platform groups. Available: ${sourceGroupIds().join(', ')}\n`); + process.stderr.write(`No valid platforms. Available: ${selectablePlatformIds().join(', ')}\n`); return 1; } - await saveSearchingPlatforms(valid as SourceGroupId[]); - const count = resolveSourceIds(valid as SourceGroupId[]).length; + await saveSearchingPlatforms(valid); + const count = resolveSourceIds(valid).length; process.stdout.write(`Research platforms: ${valid.join(', ')} (${count} sources)\n`); } else { - process.stdout.write(`Platform groups: ${sourceGroupIds().join(', ')}\n`); + process.stdout.write(`Platforms: ${selectablePlatformIds().join(', ')}\n`); } return 0; } diff --git a/src/pipeline/collector.ts b/src/pipeline/collector.ts index 19c838d..a8aa2f0 100644 --- a/src/pipeline/collector.ts +++ b/src/pipeline/collector.ts @@ -4,7 +4,8 @@ import { AnalyticsService, type SymbolAnalysis } from '../services/analytics.js' import { createAdvisor, type AiAdvisor, ValidationAdvisor, type ValidationResult } from '../services/ai-advisor.js'; import { createResilientMarketSourceFor, type MarketDataSource } from '../services/market-data.js'; import { createScraper, type Scraper } from '../services/scraping.js'; -import { KnowledgeBaseSearch, type SearchProvider } from '../services/search.js'; +import { CompositeSearchProvider, KnowledgeBaseSearch, type SearchProvider } from '../services/search.js'; +import { createWebSearchProvider } from '../services/web-search.js'; import { buildForecast } from '../strategies/forecast.js'; export interface CollectOptions { @@ -17,6 +18,12 @@ export interface CollectOptions { exchange?: string; /** When true, skip the extra AI validation API call (used when AI chat runs the command itself). */ skipAiValidation?: boolean; + /** + * When true, layer whole-internet web search on top of the curated knowledge + * base for each symbol (the `/serching-platforms` "Web Search" toggle). Runs + * server-side via the {@link WebSearchProvider}. + */ + webSearch?: boolean; } /** A single human-readable progress step, streamed to the UI as work proceeds. */ @@ -75,6 +82,12 @@ export class CollectionPipeline { private readonly search: SearchProvider = new KnowledgeBaseSearch(), private readonly advisor: AiAdvisor = createAdvisor(), private readonly scraper: Scraper | null = createScraper(), + /** + * Whole-internet web search provider, merged with {@link search} when a run + * enables `webSearch`. Injectable for tests; lazily created from the + * Playwright/HTTP engine when omitted. + */ + private readonly webSearchProvider: SearchProvider | null = null, ) {} async collect(kind: RunKind, options: CollectOptions, onProgress?: ProgressListener): Promise { @@ -98,6 +111,14 @@ export class CollectionPipeline { emit({ phase: 'wipe', message: 'Cleared previous run data (general search table preserved)' }); } + // When the run opts into whole-internet web search, merge it with the + // curated knowledge base as *additional* support; otherwise search the + // knowledge base alone. The web provider is created lazily so runs that + // leave "Web Search" off never touch Playwright. + const searchProvider: SearchProvider = options.webSearch + ? new CompositeSearchProvider(this.search, this.webSearchProvider ?? createWebSearchProvider()) + : this.search; + const runId = await this.store.createRun(kind, symbols); const reports: SymbolReport[] = []; let searchCount = 0; @@ -135,7 +156,7 @@ export class CollectionPipeline { emit({ phase: 'search', symbol, message: `Indexing references for ${symbol}` }); const query = `${symbol} ${analysis.analytics.strongestStrategy ?? 'trading strategy'}`; - const results = await this.search.search(query, symbol); + const results = await searchProvider.search(query, symbol); for (const r of results) { await this.store.saveSearchResult(r); } diff --git a/src/services/search.ts b/src/services/search.ts index 243b41d..6d9ecb5 100644 --- a/src/services/search.ts +++ b/src/services/search.ts @@ -128,6 +128,39 @@ export class KnowledgeBaseSearch implements SearchProvider { } } +/** + * Fans a query out to several {@link SearchProvider}s and merges their hits into + * a single ranked list. This is how whole-internet "Web Search" is layered on + * top of the curated {@link KnowledgeBaseSearch} as *additional* support: both + * providers run, their results are concatenated, de-duplicated by URL/title and + * re-sorted by score. A provider that throws is skipped so one flaky source + * never sinks the whole search. + */ +export class CompositeSearchProvider implements SearchProvider { + readonly name = 'composite'; + private readonly providers: SearchProvider[]; + + constructor(...providers: SearchProvider[]) { + this.providers = providers.filter(Boolean); + } + + async search(query: string, symbol?: string): Promise { + const settled = await Promise.allSettled(this.providers.map((p) => p.search(query, symbol))); + const merged: SearchResult[] = []; + const seen = new Set(); + for (const outcome of settled) { + if (outcome.status !== 'fulfilled') continue; + for (const result of outcome.value) { + const key = (result.url ?? result.title).toLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + merged.push(result); + } + } + return merged.sort((a, b) => b.score - a.score); + } +} + /** Normalised keyword-overlap relevance in the range [0, 1]. */ function relevance(entry: KnowledgeEntry, terms: string[]): number { if (terms.length === 0) return 0; diff --git a/src/services/web-search.ts b/src/services/web-search.ts new file mode 100644 index 0000000..8083843 --- /dev/null +++ b/src/services/web-search.ts @@ -0,0 +1,367 @@ +import { existsSync } from 'node:fs'; +import { createRequire } from 'node:module'; + +import type { SearchProvider, SearchResult } from './search.js'; + +/** ESM-safe `require`, used only to resolve Playwright's synchronous `executablePath()`. */ +const nodeRequire = createRequire(import.meta.url); + +/** + * Whole-internet web search, integrated as additional support for the curated + * {@link KnowledgeBaseSearch}. It mirrors the public API of the + * [`web-agent-master/google-search`](https://github.com/web-agent-master/google-search) + * library — `googleSearch(query, options)` returning `{ query, results }` with + * `{ title, link, snippet }` hits — so that library can be slotted in behind the + * same {@link WebSearchEngine} interface without touching callers. + * + * The library drives Google through Playwright. We do the same when a Chromium + * binary is available, and otherwise fall back to a JavaScript-free HTTP engine + * (DuckDuckGo's HTML endpoint), exactly like the news crawler degrades from + * Playwright to plain `fetch`. The result is a web search that works whether or + * not a browser is installed, and that returns an empty list — never throws — + * when the network is unavailable, so the collection pipeline stays resilient. + */ + +/** A single web hit. Mirrors `web-agent-master/google-search`'s `SearchResult`. */ +export interface WebSearchHit { + title: string; + link: string; + snippet: string; +} + +/** The response envelope. Mirrors `web-agent-master/google-search`'s `SearchResponse`. */ +export interface WebSearchResponse { + query: string; + results: WebSearchHit[]; +} + +/** Options accepted by {@link googleSearch}. Mirrors the library's `CommandOptions`. */ +export interface WebSearchOptions { + /** Maximum number of hits to return. Defaults to 10. */ + limit?: number; + /** Per-request timeout in milliseconds. Defaults to 20s. */ + timeoutMs?: number; + /** Locale hint sent as the `accept-language` header. Defaults to `en-US`. */ + locale?: string; +} + +const DEFAULT_LIMIT = 10; +const DEFAULT_TIMEOUT_MS = 20_000; +const DEFAULT_LOCALE = 'en-US'; + +/** + * A pluggable engine that turns a query into raw web hits. Implementations are + * injectable so the provider is fully testable with offline doubles. + */ +export interface WebSearchEngine { + readonly name: string; + search(query: string, options: Required): Promise; +} + +/** + * Google search backed by Playwright/Chromium — the strategy the + * `web-agent-master/google-search` library uses. Chromium is imported lazily so + * environments without the binary can still run every other command. + */ +export class PlaywrightGoogleSearchEngine implements WebSearchEngine { + readonly name = 'google-playwright'; + // Typed as unknown to avoid a hard compile-time dependency on Playwright's types. + private browser: unknown = null; + + async search(query: string, options: Required): Promise { + const browser = await this.ensureBrowser(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const page = await (browser as any).newPage(); + try { + await page.setExtraHTTPHeaders({ 'accept-language': `${options.locale},en;q=0.9` }); + const url = `https://www.google.com/search?q=${encodeURIComponent(query)}&num=${options.limit}&hl=en`; + await page.goto(url, { waitUntil: 'domcontentloaded', timeout: options.timeoutMs }); + const hits = (await page.evaluate(extractGoogleHitsFromPage, options.limit)) as WebSearchHit[]; + return hits.slice(0, options.limit); + } finally { + await page.close(); + } + } + + async close(): Promise { + if (!this.browser) return; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await (this.browser as any).close(); + this.browser = null; + } + + private async ensureBrowser(): Promise { + if (this.browser) return this.browser; + const { chromium } = await import('playwright'); + this.browser = await chromium.launch({ headless: true }); + return this.browser; + } +} + +/** + * JavaScript-free fallback engine that queries DuckDuckGo's HTML endpoint with a + * plain `fetch` and parses the static markup with regular expressions. Activated + * when Chromium is unavailable, mirroring the news crawler's HTTP fallback, so + * "Web Search" keeps working without a headless browser. + */ +export class HttpWebSearchEngine implements WebSearchEngine { + readonly name = 'duckduckgo-http'; + + async search(query: string, options: Required): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), options.timeoutMs); + try { + const response = await fetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`, { + signal: controller.signal, + headers: { + 'accept-language': `${options.locale},en;q=0.9`, + 'user-agent': 'TradefastWebSearch/0.1 (+https://github.com/Payel-git-ol/Tradefast)', + }, + }); + const html = await response.text(); + return parseDuckDuckGoHtml(html, options.limit); + } finally { + clearTimeout(timeout); + } + } +} + +/** + * Wraps the Playwright engine and falls back to the HTTP engine when the + * headless browser cannot be launched (e.g. Chromium was never installed). The + * first launch failure switches every subsequent query to the HTTP fallback so a + * single missing browser does not fail every search. Mirrors + * {@link ResilientNewsPageFetcher}. + */ +export class ResilientWebSearchEngine implements WebSearchEngine { + readonly name = 'resilient-web-search'; + private readonly fallback = new HttpWebSearchEngine(); + private useFallback = false; + + constructor(private readonly primary: WebSearchEngine = new PlaywrightGoogleSearchEngine()) {} + + async search(query: string, options: Required): Promise { + if (this.useFallback) return this.fallback.search(query, options); + try { + return await this.primary.search(query, options); + } catch (error) { + if (isBrowserLaunchError(error)) { + this.useFallback = true; + return this.fallback.search(query, options); + } + throw error; + } + } +} + +/** + * Programmatic entry point mirroring `web-agent-master/google-search`'s + * `googleSearch(query, options)`. Returns the query alongside its hits. + */ +export async function googleSearch( + query: string, + options: WebSearchOptions = {}, + engine: WebSearchEngine = detectWebSearchEngine(), +): Promise { + const resolved = resolveOptions(options); + const results = await engine.search(query, resolved); + return { query, results: results.slice(0, resolved.limit) }; +} + +/** + * Adapts the web search engine to the pipeline's {@link SearchProvider} + * interface, so whole-internet hits land in the same `search_results` table as + * the curated knowledge base. Network failures degrade to an empty list rather + * than aborting a collection run. + */ +export class WebSearchProvider implements SearchProvider { + readonly name = 'web-search'; + + constructor( + private readonly engine: WebSearchEngine = detectWebSearchEngine(), + private readonly options: WebSearchOptions = {}, + ) {} + + async search(query: string, symbol?: string): Promise { + let hits: WebSearchHit[]; + try { + const { results } = await googleSearch(query, this.options, this.engine); + hits = results; + } catch { + // Whole-internet search is best-effort: a flaky network must never abort a run. + return []; + } + + const total = hits.length || 1; + return hits.map((hit, index) => ({ + query, + symbol, + source: this.name, + title: hit.title, + url: hit.link || undefined, + snippet: hit.snippet || undefined, + // Rank decays with position so the most relevant hit scores highest. + score: Number(((total - index) / total).toFixed(3)), + })); + } +} + +/** + * Chooses the engine the same way the crawler chooses a fetcher: prefer the + * Playwright/Google strategy when a Chromium binary is actually on disk, + * otherwise use the HTTP fallback. `executablePath()` only reports where the + * binary is *expected* to live, so the file must be verified to exist. + */ +export function detectWebSearchEngine(): WebSearchEngine { + try { + const playwright = requirePlaywright(); + const executablePath = playwright?.chromium?.executablePath?.(); + if (executablePath && existsSync(executablePath)) { + return new ResilientWebSearchEngine(new PlaywrightGoogleSearchEngine()); + } + } catch { + // Fall through to the HTTP engine. + } + return new HttpWebSearchEngine(); +} + +/** Factory used by the pipeline; isolates engine selection for easy stubbing in tests. */ +export function createWebSearchProvider(options: WebSearchOptions = {}): WebSearchProvider { + return new WebSearchProvider(detectWebSearchEngine(), options); +} + +// --- internals -------------------------------------------------------------- + +function resolveOptions(options: WebSearchOptions): Required { + const limit = Number.isFinite(options.limit) && (options.limit as number) > 0 + ? Math.floor(options.limit as number) + : DEFAULT_LIMIT; + const timeoutMs = Number.isFinite(options.timeoutMs) && (options.timeoutMs as number) > 0 + ? Math.floor(options.timeoutMs as number) + : DEFAULT_TIMEOUT_MS; + return { limit, timeoutMs, locale: options.locale ?? DEFAULT_LOCALE }; +} + +/** + * Synchronously resolves the optional Playwright dependency via `require` when + * available. Returns `null` when the module (or `require`) is absent, so engine + * detection degrades cleanly to the HTTP fallback. + */ +function requirePlaywright(): { chromium?: { executablePath?: () => string } } | null { + try { + // `executablePath()` is synchronous, so resolving the module via `require` avoids an async import here. + return nodeRequire('playwright') as { chromium?: { executablePath?: () => string } }; + } catch { + return null; + } +} + +function isBrowserLaunchError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes('browserType.launch') || + message.includes("Executable doesn't exist") || + message.includes('playwright install') + ); +} + +/** + * Parses DuckDuckGo's HTML SERP. Results live in `` anchors + * whose `href` is a `/l/?uddg=` redirect; snippets follow in + * ``. Both classes are extracted independently and + * zipped by order, which is robust to the surrounding markup changing. + */ +export function parseDuckDuckGoHtml(html: string, limit: number): WebSearchHit[] { + const hits: WebSearchHit[] = []; + const seen = new Set(); + + const linkRe = /]*class="[^"]*\bresult__a\b[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi; + const snippetRe = /]*class="[^"]*\bresult__snippet\b[^"]*"[^>]*>([\s\S]*?)<\/a>/gi; + + const snippets: string[] = []; + let sm: RegExpExecArray | null; + while ((sm = snippetRe.exec(html)) !== null) snippets.push(stripHtml(sm[1])); + + let lm: RegExpExecArray | null; + let index = 0; + while ((lm = linkRe.exec(html)) !== null && hits.length < limit) { + const link = decodeDuckDuckGoHref(lm[1]); + const title = stripHtml(lm[2]); + if (!link || !title) { + index++; + continue; + } + if (seen.has(link)) { + index++; + continue; + } + seen.add(link); + hits.push({ title, link, snippet: snippets[index] ?? '' }); + index++; + } + + return hits; +} + +/** Resolves DuckDuckGo's `/l/?uddg=` redirect wrapper to the real destination URL. */ +function decodeDuckDuckGoHref(href: string): string { + try { + const url = new URL(href, 'https://duckduckgo.com'); + const target = url.searchParams.get('uddg'); + if (target) return decodeURIComponent(target); + return url.protocol === 'http:' || url.protocol === 'https:' ? url.toString() : ''; + } catch { + return ''; + } +} + +/** Removes tags and decodes the handful of HTML entities DuckDuckGo emits. */ +function stripHtml(value: string): string { + return value + .replace(/<[^>]*>/g, '') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'|'/g, "'") + .replace(/ /g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +/** + * Extracts organic results from a Google SERP. Runs inside the browser via + * `page.evaluate`, so it must be self-contained and reference only DOM globals. + */ +function extractGoogleHitsFromPage(limit: number): WebSearchHit[] { + const doc = (globalThis as { document?: any }).document; + if (!doc?.querySelectorAll) return []; + const clean = (value: unknown): string => + typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : ''; + + const out: WebSearchHit[] = []; + const seen = new Set(); + + for (const heading of Array.from(doc.querySelectorAll('a h3')) as any[]) { + if (out.length >= limit) break; + const anchor = heading.closest?.('a'); + const href: string | undefined = anchor?.getAttribute?.('href'); + if (!href || !/^https?:\/\//i.test(href)) continue; + if (/^https?:\/\/(?:www\.)?google\./i.test(href)) continue; + if (seen.has(href)) continue; + const title = clean(heading.textContent); + if (!title) continue; + seen.add(href); + + const block = anchor.closest?.('div.g') ?? anchor.parentElement?.parentElement; + let snippet = ''; + const snippetNode = + block?.querySelector?.('div[data-sncf], div[data-content-feature], .VwiC3b, .yXK7lf') ?? + block?.querySelector?.('span'); + if (snippetNode) snippet = clean(snippetNode.textContent); + + out.push({ title, link: href, snippet }); + } + + return out; +} diff --git a/tests/backend.test.ts b/tests/backend.test.ts index 9fcc0d7..f1bb0d3 100644 --- a/tests/backend.test.ts +++ b/tests/backend.test.ts @@ -43,6 +43,9 @@ const facade: TradefastApiFacade = { interval: '1h', }), clear: async () => 1, + search: async () => [ + { query: 'btc', source: 'web-search', title: 'BTC rallies', url: 'https://example.com/btc', snippet: 'up', score: 1 }, + ], }; describe('Nest GraphQL resolver', () => { @@ -69,6 +72,14 @@ describe('Nest GraphQL resolver', () => { await expect(resolver.clear()).resolves.toBe(1); }); + it('runs whole-internet web search server-side via the webSearch query', async () => { + const resolver = new TradefastResolver(new TradefastRepository(facade)); + + await expect(resolver.webSearch('btc', 5)).resolves.toEqual([ + { source: 'web-search', title: 'BTC rallies', url: 'https://example.com/btc', snippet: 'up', score: 1 }, + ]); + }); + it('serves the GraphQL schema over HTTP', async () => { const backend = await startTradefastBackend(facade); diff --git a/tests/graphql-repository.test.ts b/tests/graphql-repository.test.ts index caec888..c099dec 100644 --- a/tests/graphql-repository.test.ts +++ b/tests/graphql-repository.test.ts @@ -56,6 +56,9 @@ const facade: TradefastApiFacade = { interval: '1h', }), clear: async () => 4, + search: async (query: string) => [ + { query, source: 'web-search', title: 'BTC rallies', url: 'https://example.com/btc', snippet: 'up', score: 1 }, + ], }; describe('GraphqlTradefastRepository', () => { @@ -83,6 +86,11 @@ describe('GraphqlTradefastRepository', () => { await expect(repository.update()).resolves.toMatchObject({ runId: 3, kind: 'update' }); await expect(repository.clear()).resolves.toBe(4); + + const hits = await repository.webSearch('btc', 5); + expect(hits).toEqual([ + { source: 'web-search', title: 'BTC rallies', url: 'https://example.com/btc', snippet: 'up', score: 1 }, + ]); } finally { await backend.close(); } diff --git a/tests/sources.test.ts b/tests/sources.test.ts index b705b48..3956ae1 100644 --- a/tests/sources.test.ts +++ b/tests/sources.test.ts @@ -3,9 +3,15 @@ import { describe, expect, it } from 'vitest'; import { DEFAULT_NEWS_SOURCES } from '../src/services/news-crawler.js'; import { DEFAULT_ENABLED_GROUPS, + DEFAULT_ENABLED_PLATFORMS, + getPlatformDescription, + getPlatformLabel, getSourceGroup, + isWebSearchEnabled, resolveSourceIds, + selectablePlatformIds, sourceGroupIds, + WEB_SEARCH_PLATFORM_ID, } from '../src/cli/sources.js'; describe('research platform groups', () => { @@ -43,3 +49,31 @@ describe('research platform groups', () => { expect(new Set(resolveSourceIds(DEFAULT_ENABLED_GROUPS))).toEqual(sourceIds); }); }); + +describe('selectable research platforms', () => { + it('offers every news group followed by Web Search', () => { + expect(selectablePlatformIds()).toEqual([...sourceGroupIds(), WEB_SEARCH_PLATFORM_ID]); + }); + + it('keeps Web Search out of the strict news-group contract', () => { + // Web Search must not be a SourceGroup, so sourceGroupIds() stays unchanged. + expect(sourceGroupIds()).not.toContain(WEB_SEARCH_PLATFORM_ID); + expect(getSourceGroup(WEB_SEARCH_PLATFORM_ID)).toBeUndefined(); + }); + + it('labels and describes Web Search and the news groups', () => { + expect(getPlatformLabel(WEB_SEARCH_PLATFORM_ID)).toBe('Web Search'); + expect(getPlatformDescription(WEB_SEARCH_PLATFORM_ID)).toMatch(/entire Internet/i); + expect(getPlatformLabel('news-portals')).toBe(getSourceGroup('news-portals')!.label); + }); + + it('contributes no curated sources for the Web Search platform', () => { + expect(resolveSourceIds([WEB_SEARCH_PLATFORM_ID])).toEqual([]); + }); + + it('enables Web Search by default and detects it', () => { + expect(DEFAULT_ENABLED_PLATFORMS).toContain(WEB_SEARCH_PLATFORM_ID); + expect(isWebSearchEnabled(DEFAULT_ENABLED_PLATFORMS)).toBe(true); + expect(isWebSearchEnabled(sourceGroupIds())).toBe(false); + }); +}); diff --git a/tests/web-search.test.ts b/tests/web-search.test.ts new file mode 100644 index 0000000..68f9fce --- /dev/null +++ b/tests/web-search.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from 'vitest'; + +import { CompositeSearchProvider, KnowledgeBaseSearch, type SearchProvider } from '../src/services/search.js'; +import { + googleSearch, + parseDuckDuckGoHtml, + WebSearchProvider, + type WebSearchEngine, + type WebSearchHit, +} from '../src/services/web-search.js'; + +/** A deterministic offline engine so the suite never touches the network. */ +class StubEngine implements WebSearchEngine { + readonly name = 'stub'; + constructor(private readonly hits: WebSearchHit[]) {} + async search(_query: string, options: { limit: number }): Promise { + return this.hits.slice(0, options.limit); + } +} + +const HITS: WebSearchHit[] = [ + { title: 'Bitcoin breakout analysis', link: 'https://example.com/btc', snippet: 'BTC clears resistance.' }, + { title: 'Crypto market recap', link: 'https://example.com/recap', snippet: 'Markets rally.' }, +]; + +describe('googleSearch (web-agent-master/google-search API)', () => { + it('returns the query alongside its hits, capped by limit', async () => { + const response = await googleSearch('bitcoin', { limit: 1 }, new StubEngine(HITS)); + expect(response.query).toBe('bitcoin'); + expect(response.results).toHaveLength(1); + expect(response.results[0]).toEqual(HITS[0]); + }); + + it('defaults to ten hits when no limit is given', async () => { + const many = Array.from({ length: 25 }, (_, i) => ({ + title: `Result ${i}`, + link: `https://example.com/${i}`, + snippet: '', + })); + const response = await googleSearch('q', {}, new StubEngine(many)); + expect(response.results).toHaveLength(10); + }); +}); + +describe('WebSearchProvider', () => { + it('maps hits into SearchResult rows with decaying scores', async () => { + const provider = new WebSearchProvider(new StubEngine(HITS)); + const results = await provider.search('bitcoin', 'BTCUSDT'); + + expect(results).toHaveLength(2); + expect(results[0]).toMatchObject({ + query: 'bitcoin', + symbol: 'BTCUSDT', + source: 'web-search', + title: 'Bitcoin breakout analysis', + url: 'https://example.com/btc', + snippet: 'BTC clears resistance.', + }); + // The first hit must outrank the second. + expect(results[0].score).toBeGreaterThan(results[1].score); + }); + + it('degrades to an empty list when the engine throws', async () => { + const failing: WebSearchEngine = { + name: 'boom', + search: async () => { + throw new Error('network down'); + }, + }; + const provider = new WebSearchProvider(failing); + await expect(provider.search('bitcoin')).resolves.toEqual([]); + }); +}); + +describe('CompositeSearchProvider', () => { + it('layers web search on top of the curated knowledge base', async () => { + const composite = new CompositeSearchProvider( + new KnowledgeBaseSearch(), + new WebSearchProvider(new StubEngine(HITS)), + ); + const results = await composite.search('trend following momentum', 'BTCUSDT'); + + const sources = new Set(results.map((r) => r.source)); + expect(sources.has('knowledge-base')).toBe(true); + expect(sources.has('web-search')).toBe(true); + // Sorted by score descending. + for (let i = 1; i < results.length; i++) { + expect(results[i - 1].score).toBeGreaterThanOrEqual(results[i].score); + } + }); + + it('survives a provider that throws', async () => { + const failing: SearchProvider = { + name: 'boom', + search: async () => { + throw new Error('boom'); + }, + }; + const composite = new CompositeSearchProvider(new KnowledgeBaseSearch(), failing); + const results = await composite.search('rsi mean reversion'); + expect(results.length).toBeGreaterThan(0); + }); + + it('de-duplicates hits that share a URL', async () => { + const dup: WebSearchHit[] = [ + { title: 'A', link: 'https://example.com/x', snippet: '' }, + { title: 'A copy', link: 'https://example.com/x', snippet: '' }, + ]; + const composite = new CompositeSearchProvider(new WebSearchProvider(new StubEngine(dup))); + const results = await composite.search('q'); + expect(results).toHaveLength(1); + }); +}); + +describe('parseDuckDuckGoHtml (HTTP fallback engine)', () => { + it('extracts titles, decoded links and snippets from the HTML SERP', () => { + const html = ` +
+ Bitcoin & markets + BTC clears resistance today +
+
+ Ethereum news + ETH update +
+ `; + const hits = parseDuckDuckGoHtml(html, 10); + expect(hits).toHaveLength(2); + expect(hits[0]).toEqual({ + title: 'Bitcoin & markets', + link: 'https://example.com/btc', + snippet: 'BTC clears resistance today', + }); + expect(hits[1].link).toBe('https://example.com/eth'); + }); + + it('honours the limit', () => { + const html = Array.from({ length: 5 }, (_, i) => + `Title ${i}`, + ).join('\n'); + expect(parseDuckDuckGoHtml(html, 2)).toHaveLength(2); + }); +}); From 66b388750014740ddb2613b558e36370979e50b6 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 31 May 2026 08:31:46 +0000 Subject: [PATCH 3/3] Revert "Initial commit with task details" This reverts commit b096db5ab9949e0e1326d70733f0beb39b13a734. --- .gitkeep | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .gitkeep diff --git a/.gitkeep b/.gitkeep deleted file mode 100644 index 5964d94..0000000 --- a/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -# .gitkeep file auto-generated at 2026-05-31T08:06:49.631Z for PR creation at branch issue-33-feb048cfde11 for issue https://github.com/Payel-git-ol/Tradefast/issues/33 \ No newline at end of file