From d9f7b6671acf2b748c61d984e1c13d861485c4c5 Mon Sep 17 00:00:00 2001 From: aimerdoux Date: Wed, 3 Jun 2026 19:12:57 -0700 Subject: [PATCH] fix(inference): url-prefetch follows meta-refresh + flags parked/empty sites The onboarding Pillar-1 URL fetch only followed HTTP redirects, so sites using (parked domains + some real sites) returned an empty shell and the model guessed the company from the domain name. Now: follow one meta-refresh hop; mark pages with <120 chars of readable text as FETCH FAILED (parked/empty/JS-rendered); detect common parking fingerprints (IONOS/GoDaddy/ Sedo/"defaultsite"/"coming soon"). Verified live: apple.com -> real content; a parked IONOS domain -> COULD_NOT_READ_SITE through the Realtime onboarding worker. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../inference-server/src/lib/url-prefetch.ts | 64 +++++++++++++++---- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/packages/inference-server/src/lib/url-prefetch.ts b/packages/inference-server/src/lib/url-prefetch.ts index 1fdb3cf3..d8055ae9 100644 --- a/packages/inference-server/src/lib/url-prefetch.ts +++ b/packages/inference-server/src/lib/url-prefetch.ts @@ -32,6 +32,18 @@ const FETCH_TIMEOUT_MS = 5_000; const MAX_BYTES = 50 * 1024; const MAX_EXTRACT_CHARS = 30 * 1024; const UA = "WaveX-OS-Onboarding/1.0 (+https://github.com/aimerdoux/wavex-os)"; +// Minimum readable text (after stripping the [META] prefix) for a page to +// count as "real" content. Below this, the site is almost certainly parked, +// a redirect stub, or JS-rendered with no SSR — feeding it to the model just +// invites a confident hallucination, so we mark it FETCH FAILED instead. +const MIN_CONTENT_CHARS = 120; +// `` — common for parked +// domains and some real sites. fetch() follows HTTP redirects but not this. +const META_REFRESH_REGEX = /]+http-equiv=["']?refresh["']?[^>]*content=["'][^"']*url=([^"'>\s]+)/i; +// Parked / placeholder fingerprints (IONOS, GoDaddy, Sedo, generic "buy this +// domain" pages). If the only content we can read is a parking page, the model +// must not infer a real company from it. +const PARKED_RE = /\b(este dominio ya|buy this domain|this domain (is|may be) for sale|domain is parked|parkingcrew|sedoparking|godaddy\.com\/domains|hugedomains|is registered with ionos|defaultsite|under construction|coming soon)\b/i; const PRIVATE_HOST_PATTERNS = [ /^localhost$/i, @@ -94,7 +106,10 @@ interface FetchResult { reason?: string; } -async function fetchOne(url: URL): Promise { +/** Fetch raw HTML (byte-capped) for a single URL. Returns the decoded string + * or a failure reason. No extraction — callers run htmlToText / meta-refresh + * detection on the raw markup. */ +async function fetchRawHtml(url: URL): Promise<{ ok: true; raw: string } | { ok: false; reason: string }> { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); try { @@ -104,18 +119,13 @@ async function fetchOne(url: URL): Promise { signal: controller.signal, redirect: "follow", }); - if (!resp.ok) { - return { url: url.toString(), status: "failed", reason: `HTTP ${resp.status}` }; - } + if (!resp.ok) return { ok: false, reason: `HTTP ${resp.status}` }; const ct = resp.headers.get("content-type") ?? ""; - if (!/text\/html|application\/xhtml/i.test(ct)) { - return { url: url.toString(), status: "failed", reason: `non-html content-type: ${ct}` }; - } - // Cap bytes by reading until limit + if (!/text\/html|application\/xhtml/i.test(ct)) return { ok: false, reason: `non-html content-type: ${ct}` }; const reader = resp.body?.getReader(); if (!reader) { const txt = await resp.text(); - return { url: url.toString(), status: "ok", body: htmlToText(txt.slice(0, MAX_BYTES)) }; + return { ok: true, raw: txt.slice(0, MAX_BYTES) }; } const chunks: Uint8Array[] = []; let received = 0; @@ -130,16 +140,46 @@ async function fetchOne(url: URL): Promise { const merged = new Uint8Array(concatLen); let off = 0; for (const c of chunks) { merged.set(c, off); off += c.length; } - const raw = new TextDecoder("utf-8", { fatal: false }).decode(merged.slice(0, MAX_BYTES)); - return { url: url.toString(), status: "ok", body: htmlToText(raw) }; + return { ok: true, raw: new TextDecoder("utf-8", { fatal: false }).decode(merged.slice(0, MAX_BYTES)) }; } catch (e) { const reason = (e as Error).name === "AbortError" ? `timeout after ${FETCH_TIMEOUT_MS}ms` : (e as Error).message; - return { url: url.toString(), status: "failed", reason }; + return { ok: false, reason }; } finally { clearTimeout(timer); } } +async function fetchOne(url: URL): Promise { + const first = await fetchRawHtml(url); + if (!first.ok) return { url: url.toString(), status: "failed", reason: first.reason }; + let raw = first.raw; + + // Follow ONE meta-refresh hop (parked domains + some real sites use it; plain + // fetch() doesn't). Resolve the target relative to the current URL, validate + // it's a safe public host, and re-fetch. + const mr = raw.match(META_REFRESH_REGEX); + if (mr) { + try { + const target = new URL(mr[1].replace(/&/g, "&"), url); + if (isSafePublicUrl(target.toString())) { + const hop = await fetchRawHtml(target); + if (hop.ok) raw = hop.raw; + } + } catch { /* keep the first page */ } + } + + const body = htmlToText(raw); + // Strip the [META] ... prefix when measuring real body text. + const textOnly = body.replace(/^\[META\][^\n]*\n+/, "").trim(); + if (textOnly.length < MIN_CONTENT_CHARS) { + return { url: url.toString(), status: "failed", reason: "no readable content (site may be parked, empty, or JavaScript-rendered)" }; + } + if (PARKED_RE.test(body)) { + return { url: url.toString(), status: "failed", reason: "domain appears parked / not a live site" }; + } + return { url: url.toString(), status: "ok", body }; +} + /** Pre-fetch every public http(s) URL in `prompt` and return a new prompt * with FETCHED / FETCH FAILED markers injected before each URL's first * appearance. Idempotent — if the prompt already contains [FETCHED] markers