diff --git a/README.md b/README.md index 037f6b1..ea0859d 100644 --- a/README.md +++ b/README.md @@ -51,12 +51,16 @@ matrix. ## 4. Quickstart ```bash -npx -y @zenrows/cli init +npm install -g @zenrows/cli # puts `zenrows` on PATH (466 ms, zero deps) +zenrows init zenrows fetch https://httpbin.io/html # auto-provisions a Free plan account on first use zenrows extract https://www.owler.com/company/meltwater # extract=auto on an enabled domain zenrows extract https://www.scrapingcourse.com/ecommerce/ --autoparse # Autoparse (any domain) ``` +> Prefer no global install? Prefix each command with `npx -y @zenrows/cli` +> (e.g. `npx -y @zenrows/cli fetch `) — `npx` does not put `zenrows` on PATH. + No API key up front: on your first cloud call the toolkit creates a free, unclaimed Zenrows Free plan account for you (see §6). diff --git a/skills/zenrows/SKILL.md b/skills/zenrows/SKILL.md index e4da444..d328eb8 100644 --- a/skills/zenrows/SKILL.md +++ b/skills/zenrows/SKILL.md @@ -34,8 +34,11 @@ If the user has a known URL and wants page content: If the user has a known URL and wants structured data: → Use Extract. (zenrows extract | --autoparse | --css) -If the user has many URLs: - → Fan out fetch/extract per URL (validate on one page first, then iterate). +If the user has many URLs (bulk): + → Use Batch. (validate one page with fetch/extract first, + then `zenrows batch create `) + Batch is the cheap path at scale — do NOT + fan out fetch/extract per URL for bulk work. If the user needs login, clicks, forms, sessions, or persistent state: → Use Interact / Browser Sessions.(zenrows browser) [escalation-only] @@ -66,7 +69,7 @@ Run `zenrows status` for the live capability matrix. As of this toolkit: | --- | --- | --- | | Protected Fetch | `zenrows fetch` | available (`GET /v1/`) | | Extract (extract=auto / Autoparse/CSS/Markdown) | `zenrows extract` | beta (same `/v1/`; extract=auto falls back to autoparse) | -| Batch | `zenrows batch` | beta (validate specs locally) | +| Batch | `zenrows batch` | beta — runs cloud jobs (create/status/results); cheapest path at scale. Estimate specs locally with no key | | Browser | `zenrows browser` | available (Browser Sessions REST API / MCP) | | MCP | `zenrows mcp` | available (remote + local server) | diff --git a/src/cli/asset-command.ts b/src/cli/asset-command.ts index 834b0cf..2c49946 100644 --- a/src/cli/asset-command.ts +++ b/src/cli/asset-command.ts @@ -87,8 +87,8 @@ export function makeAssetCommand(type: AssetType, summary: string): Command { function extraUsage(type: AssetType): string { if (type === "skill") return "|validate|generate"; if (type === "template") return "|create"; - if (type === "recipe") return "|run|explain"; - if (type === "workflow") return "|run|explain"; + if (type === "recipe") return "|run"; + if (type === "workflow") return "|run"; if (type === "eval") return "|run|report"; return ""; } @@ -97,9 +97,9 @@ function helpFor(type: AssetType): string { const lines = [`Manage ${type}s from the installable asset registry.`, ""]; lines.push("Subcommands:"); lines.push(` list list all ${type}s in the registry (status-aware)`); - lines.push(` install copy a ${type} into .zenrows/`); + lines.push(` install copy ${type === "eval" ? "an" : "a"} ${type} into .zenrows/`); if (type === "skill") lines.push(" install --all install every available skill"); - lines.push(` explain print metadata + docs for a ${type}`); + lines.push(` explain print metadata + docs for ${type === "eval" ? "an" : "a"} ${type}`); lines.push(` update [name] reinstall (refresh) installed ${type}s`); lines.push(` remove remove an installed ${type}`); if (type === "template") lines.push(" create --output instantiate a template into "); @@ -121,7 +121,7 @@ function listCmd(type: AssetType, _argv: string[], ctx: RunContext): number { if (ctx.json) { log.out( JSON.stringify( - { type, assets: assets.map((a) => ({ ...a, installed: installed.has(a.name), runnable: assetRunnable(a) })) }, + { ok: true, type, assets: assets.map((a) => ({ ...a, installed: installed.has(a.name), runnable: assetRunnable(a) })) }, null, 2, ), @@ -145,7 +145,7 @@ function listCmd(type: AssetType, _argv: string[], ctx: RunContext): number { function installCmd(type: AssetType, argv: string[], ctx: RunContext): number { const all = argv.includes("--all"); const targets = all - ? loadRegistry(type).filter((a) => a.status === "available" || a.status === "experimental") + ? loadRegistry(type).filter(assetRunnable) // everything whose backend deps are usable (incl. beta) — matches `plugin install` : argv.filter((a) => !a.startsWith("-")).map((name) => requireAsset(type, name)); if (targets.length === 0) { throw new ToolkitError({ diff --git a/src/cli/command.ts b/src/cli/command.ts index 2bf6609..9675ad5 100644 --- a/src/cli/command.ts +++ b/src/cli/command.ts @@ -2,6 +2,10 @@ * Command contract + shared helpers for the CLI. */ import { parseArgs, type ParseArgsConfig } from "node:util"; +import { ToolkitError } from "../core/errors.ts"; + +/** Global flags stripped by the top-level router before a command parses. */ +const GLOBAL_FLAGS = new Set(["json", "yes", "help", "version"]); export interface Command { name: string; @@ -21,20 +25,72 @@ export interface RunContext { yes: boolean; } -/** Thin wrapper around parseArgs that keeps positionals + options typed-ish. */ +/** + * Thin wrapper around parseArgs that keeps positionals + options typed-ish. + * + * Rejects unrecognized flags loudly (UNKNOWN_FLAG) instead of silently swallowing + * them: an agent that mistypes or hallucinates a flag must fail here, not get a + * green result on a request the CLI never actually honored. We parse non-strict + * with tokens so we can name the exact offending flag and suggest a correction, + * rather than surface node's raw parseArgs throw. + */ export function parse( argv: string[], options: ParseArgsConfig["options"], ): { values: Record; positionals: string[] } { - const { values, positionals } = parseArgs({ + const { values, positionals, tokens } = parseArgs({ args: argv, options, allowPositionals: true, strict: false, + tokens: true, }); + const declared = new Set(Object.keys(options ?? {})); + const unknown = tokens.filter( + (t): t is Extract => + t.kind === "option" && !declared.has(t.name) && !GLOBAL_FLAGS.has(t.name), + ); + if (unknown.length > 0) { + const flag = unknown[0]!.rawName; + const guess = suggestFlag(unknown[0]!.name, [...declared]); + throw new ToolkitError({ + code: "UNKNOWN_FLAG", + message: `Unknown flag: ${flag}`, + likely_cause: "This flag is not recognized by this command.", + next_action: guess + ? `Did you mean --${guess}? Run the command with --help for the full flag list.` + : "Run the command with --help for the full flag list.", + }); + } return { values: values as Record, positionals }; } +/** Closest declared flag within edit distance 2, for a "did you mean" hint. */ +function suggestFlag(input: string, candidates: string[]): string | undefined { + let best: string | undefined; + let bestDist = 3; + for (const c of candidates) { + const d = editDistance(input, c); + if (d < bestDist) { + bestDist = d; + best = c; + } + } + return best; +} + +function editDistance(a: string, b: string): number { + const dp = Array.from({ length: a.length + 1 }, (_, i) => [i, ...Array(b.length).fill(0)]); + for (let j = 0; j <= b.length; j++) dp[0]![j] = j; + for (let i = 1; i <= a.length; i++) { + for (let j = 1; j <= b.length; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + dp[i]![j] = Math.min(dp[i - 1]![j]! + 1, dp[i]![j - 1]! + 1, dp[i - 1]![j - 1]! + cost); + } + } + return dp[a.length]![b.length]!; +} + export function asString(v: unknown): string | undefined { return typeof v === "string" ? v : undefined; } diff --git a/src/cli/commands/config.ts b/src/cli/commands/config.ts index e9db16d..009d3ea 100644 --- a/src/cli/commands/config.ts +++ b/src/cli/commands/config.ts @@ -3,6 +3,7 @@ * show * get * set + * reset */ import { defaultConfig, loadConfig, saveConfig } from "../../core/config.ts"; import { log } from "../../core/logger.ts"; @@ -26,7 +27,7 @@ const SETTABLE: Record void> = { export const config: Command = { name: "config", summary: "View or update toolkit configuration (non-secret).", - usage: "zenrows config |set >", + usage: "zenrows config |set |reset>", run(argv: string[], ctx: RunContext): number { const [sub, key, value] = argv; const cfg = loadConfig(); diff --git a/src/cli/commands/extract.ts b/src/cli/commands/extract.ts index 2538c2c..4639bd4 100644 --- a/src/cli/commands/extract.ts +++ b/src/cli/commands/extract.ts @@ -37,6 +37,9 @@ export const extract: Command = { " --out write output to a file", " --no-signup do not auto-create a Free plan account if no key exists", " --json structured result", + "", + "Cost (credits per request): 1x normal · 5x --js-render · 10x --premium-proxy · 25x both.", + "The exact charge is reported after every request (costCredits / X-Request-Credits).", ].join("\n"), async run(argv: string[], ctx: RunContext): Promise { const { values, positionals } = parse(argv, { diff --git a/src/cli/commands/fetch.ts b/src/cli/commands/fetch.ts index 916d786..0358c60 100644 --- a/src/cli/commands/fetch.ts +++ b/src/cli/commands/fetch.ts @@ -37,6 +37,10 @@ export const fetch_: Command = { " --out write the response body to a file", " --no-signup do not auto-create a Free plan account if no key exists", " --json print a structured result", + "", + "Cost (credits per request): 1x normal · 5x --js-render · 10x --premium-proxy · 25x both.", + "In auto mode you pay only for the configuration that succeeds; the exact charge is", + "reported after every request (costCredits / X-Request-Credits).", ].join("\n"), async run(argv: string[], ctx: RunContext): Promise { const { values, positionals } = parse(argv, { diff --git a/src/cli/commands/init.ts b/src/cli/commands/init.ts index 41fed90..e20b014 100644 --- a/src/cli/commands/init.ts +++ b/src/cli/commands/init.ts @@ -14,7 +14,8 @@ import { defaultConfig, loadConfig, saveConfig } from "../../core/config.ts"; import { defaultPolicy, loadPolicy, savePolicy } from "../../core/policy.ts"; import { log, ANSI, c } from "../../core/logger.ts"; import { mask } from "../../core/redact.ts"; -import { createWorkspace } from "../../core/workspace.ts"; +import { existsSync } from "node:fs"; +import { createWorkspace, workspacePaths } from "../../core/workspace.ts"; import { installAsset, loadRegistry } from "../../core/registry.ts"; import { buildMcpConfig, MCP_CLIENTS } from "../../installers/mcp/index.ts"; import { runFetch } from "../../adapters/protected-fetch.ts"; @@ -71,7 +72,11 @@ export const init: Command = { section("Workspace"); const paths = createWorkspace(root); log.success(`Created ${paths.dir}`); - if (!loadConfigSafe(root)) { + // Write config.json when absent (loadConfig always returns merged defaults, + // so a "does it load?" check never writes — check the file itself). Writing + // it here makes the "Wrote config.json" line true and lets `--no-telemetry` + // actually persist, rather than being silently dropped until first signup. + if (!existsSync(workspacePaths(root).config)) { const cfg = defaultConfig(); if (values["no-telemetry"]) cfg.telemetry = "off"; saveConfig(cfg, root); @@ -92,7 +97,9 @@ export const init: Command = { } else { const st = authState(root); if (st.hasKey) log.success(`Using existing key (${st.masked}, ${st.source}).`); - else log.warn("No API key configured. Run `zenrows login --api-key ` or `zenrows signup`."); + else if (pol.auto_signup) + log.success("No API key needed — a free Zenrows account is provisioned automatically on your first cloud call (e.g. `zenrows fetch `)."); + else log.warn("No API key configured (auto-signup is off). Run `zenrows login --api-key ` or `zenrows signup`."); } // 3. Assets @@ -138,7 +145,7 @@ export const init: Command = { log.dim("This is non-fatal for init. Check `zenrows status --check`."); } } else { - log.dim("Skipping test fetch (no credentials)."); + log.dim("Skipping live test fetch — a key is provisioned automatically on your first fetch."); } } @@ -158,15 +165,6 @@ export const init: Command = { }, }; -function loadConfigSafe(root?: string): boolean { - try { - const cfg = loadConfig(root); - return Boolean(cfg && cfg.version); - } catch { - return false; - } -} - function installSet(type: AssetType, root?: string): void { const assets = loadRegistry(type).filter((a) => a.status === "available" || a.status === "experimental" || a.status === "beta"); if (assets.length === 0) return; diff --git a/src/cli/commands/status.ts b/src/cli/commands/status.ts index 7b20332..6aedb47 100644 --- a/src/cli/commands/status.ts +++ b/src/cli/commands/status.ts @@ -41,6 +41,7 @@ export const status: Command = { log.out( JSON.stringify( { + ok: true, auth: { hasKey: auth.hasKey, source: auth.source, masked: auth.masked ?? null }, workspace: { initialized: Boolean(ws), root: ws?.root ?? null, dir: ws?.dir ?? null }, backend: { apiBase: cfg.apiBase, reachable }, diff --git a/src/core/agent-account.ts b/src/core/agent-account.ts index ea71406..882930a 100644 --- a/src/core/agent-account.ts +++ b/src/core/agent-account.ts @@ -64,25 +64,40 @@ export async function discoverSignupUrl( } /** - * Resolve the agent-signup endpoint. Precedence: - * 1. ZENROWS_AGENT_SIGNUP_URL env var (local testing) - * 2. `signupUrl` in .zenrows/config.json - * 3. discovery via /.well-known/oauth-protected-resource (cached per process) - * 4. the production default (AGENT_SIGNUP_API_URL) + * Ordered list of signup endpoints to try. Precedence: + * 1. ZENROWS_AGENT_SIGNUP_URL env var (explicit — used alone) + * 2. `signupUrl` in .zenrows/config.json (explicit — used alone) + * 3. discovery via /.well-known/oauth-protected-resource, THEN the built-in + * default (AGENT_SIGNUP_API_URL) as a fallback. + * + * The fallback is the safety net: a discovered endpoint that is wrong or blocked + * can never make signup worse than having no discovery at all — `signupAgent` + * tries the next candidate on failure. An explicit override (env/config) is + * honored as-is with no fallback, since that is deliberate operator intent. */ -export async function resolveSignupUrl( +export async function signupCandidates( projectRoot?: string, opts: { fetchImpl?: typeof fetch } = {}, -): Promise { +): Promise { const fromEnv = process.env[SIGNUP_URL_ENV]; - if (fromEnv && fromEnv.trim()) return fromEnv.trim(); + if (fromEnv && fromEnv.trim()) return [fromEnv.trim()]; const configured = loadConfig(projectRoot).signupUrl; - if (configured && configured.trim()) return configured.trim(); + if (configured && configured.trim()) return [configured.trim()]; if (discoveredSignupUrl === undefined) { discoveredSignupUrl = await discoverSignupUrl(projectRoot, opts); } - if (discoveredSignupUrl) return discoveredSignupUrl; - return AGENT_SIGNUP_API_URL; + const urls: string[] = []; + if (discoveredSignupUrl && discoveredSignupUrl !== AGENT_SIGNUP_API_URL) urls.push(discoveredSignupUrl); + urls.push(AGENT_SIGNUP_API_URL); + return urls; +} + +/** The primary signup endpoint (first candidate). Used to derive the claim-status URL. */ +export async function resolveSignupUrl( + projectRoot?: string, + opts: { fetchImpl?: typeof fetch } = {}, +): Promise { + return (await signupCandidates(projectRoot, opts))[0]!; } function accountPath(projectRoot?: string): string { @@ -126,7 +141,7 @@ export interface SignupResponse { export async function signupAgent( opts: { url?: string; fetchImpl?: typeof fetch } = {}, ): Promise { - const url = opts.url ?? (await resolveSignupUrl(undefined, { fetchImpl: opts.fetchImpl })); + const urls = opts.url ? [opts.url] : await signupCandidates(undefined, { fetchImpl: opts.fetchImpl }); const doFetch = opts.fetchImpl ?? fetch; // Signup is the one functional request the CLI must make — no separate @@ -148,41 +163,61 @@ export async function signupAgent( headers["X-ZR-CI"] = p.ci ? "1" : "0"; } - let res: Response; - try { - res = await doFetch(url, { - method: "POST", - headers, - }); - } catch (err) { - throw new ToolkitError({ - code: "BACKEND_UNAVAILABLE", - message: "Could not reach the Zenrows signup endpoint.", - likely_cause: err instanceof Error ? err.message : String(err), - next_action: "Check connectivity and retry, or sign up manually with: zenrows signup --no-open", - }); - } - if (res.status !== 201) { - const body = await res.text(); - if (res.status === 429) { - throw new ToolkitError({ - code: "SIGNUP_RATE_LIMITED", - message: "Zenrows blocked the auto-signup: too many new accounts from this network.", - likely_cause: body.slice(0, 240) || "The signup endpoint is rate-limited for this IP.", - next_action: - "Wait a few minutes and retry — the toolkit will try again automatically. If you already have a Zenrows API key, use it now with: zenrows login --api-key (or set ZENROWS_API_KEY).", - suggested_commands: ["zenrows login --api-key "], + // Try each candidate in order; on any failure fall back to the next (the last + // is always the built-in default). This is the safety net: a wrong/blocked + // discovered endpoint can never leave provisioning worse off than no discovery. + let lastErr: ToolkitError | undefined; + for (const url of urls) { + let res: Response; + try { + res = await doFetch(url, { method: "POST", headers }); + } catch (err) { + lastErr = new ToolkitError({ + code: "BACKEND_UNAVAILABLE", + message: "Could not reach the Zenrows signup endpoint.", + likely_cause: err instanceof Error ? err.message : String(err), + next_action: "Check connectivity and retry, or sign up manually with: zenrows signup --no-open", }); + continue; } - throw new ToolkitError({ - code: "FETCH_FAILED", - message: `Agent signup failed (HTTP ${res.status}).`, - likely_cause: body.slice(0, 240), - next_action: "Retry, or use an existing key: zenrows login --api-key .", + if (res.status === 201) return (await res.json()) as SignupResponse; + lastErr = signupError(res.status, await res.text()); + } + throw ( + lastErr ?? + new ToolkitError({ + code: "SIGNUP_FAILED", + message: "Automatic account provisioning failed.", + likely_cause: "No signup endpoint was reachable.", + next_action: "Use an existing key: zenrows login --api-key (or set ZENROWS_API_KEY).", + suggested_commands: ["zenrows login --api-key "], + }) + ); +} + +/** Build the ToolkitError for a non-201 signup response. */ +function signupError(status: number, body: string): ToolkitError { + if (status === 429) { + return new ToolkitError({ + code: "SIGNUP_RATE_LIMITED", + message: "Zenrows blocked the auto-signup: too many new accounts from this network.", + likely_cause: body.slice(0, 240) || "The signup endpoint is rate-limited for this IP.", + next_action: + "Wait a few minutes and retry — the toolkit will try again automatically. If you already have a Zenrows API key, use it now with: zenrows login --api-key (or set ZENROWS_API_KEY).", suggested_commands: ["zenrows login --api-key "], }); } - return (await res.json()) as SignupResponse; + const challenged = /just a moment|cf-mitigated|cf-challenge|attention required/i.test(body); + return new ToolkitError({ + code: "SIGNUP_FAILED", + message: `Automatic account provisioning failed (HTTP ${status}).`, + likely_cause: challenged + ? "The signup endpoint returned a bot challenge (Cloudflare), not an account." + : body.slice(0, 240) || `The signup endpoint returned HTTP ${status}.`, + next_action: + "If you already have a Zenrows API key, use it now: zenrows login --api-key <key> (or set ZENROWS_API_KEY). Otherwise wait a moment and retry.", + suggested_commands: ["zenrows login --api-key <your-key>"], + }); } export interface AccountStatus { diff --git a/src/core/errors.ts b/src/core/errors.ts index bbecb7a..25a4898 100644 --- a/src/core/errors.ts +++ b/src/core/errors.ts @@ -20,6 +20,8 @@ export type ErrorCode = | "POLICY_EXPERIMENTAL_DISABLED" | "POLICY_BROWSER_DISABLED" | "SIGNUP_RATE_LIMITED" + | "SIGNUP_FAILED" + | "UNKNOWN_FLAG" | "DOMAIN_FORBIDDEN" | "FETCH_FAILED" | "FETCH_EMPTY_RESPONSE" diff --git a/src/core/http.ts b/src/core/http.ts index 078df58..c70f56b 100644 --- a/src/core/http.ts +++ b/src/core/http.ts @@ -203,6 +203,25 @@ export async function scrape( "This is a permanent policy block, not a transient failure — the same request will fail again with any parameters (--js-render, --premium-proxy, …). Use a different source, or contact Zenrows if you believe this domain should be allowed.", }); } + const unrecoverable = unrecoverableTargetCode(body); + if (unrecoverable) { + // The target itself does not resolve (RESP007) or does not exist (RESP002, 404). + // No fetch configuration can change that, so we must NOT advise a retry — a + // retry fails identically and still costs credits. (A human is free to try a + // corrected URL; we just don't hand an agent an escalation command here.) + const detail = zrErrorDetail(body) ?? snippet(body); + const noResolve = unrecoverable === "RESP007"; + throw new ToolkitError({ + code: "FETCH_FAILED", + message: noResolve + ? "The target domain could not be resolved." + : "The target returned 404 Not Found.", + likely_cause: `${detail}. This is a property of the target, not a transient block.`, + next_action: noResolve + ? "Check the URL for typos. No configuration (--js-render, --premium-proxy) can reach a host with no DNS record — do not retry; it will fail identically and still cost credits." + : "Verify the URL. Rendering or premium proxies will not turn a 404 into content — do not retry; it will fail identically and still cost credits.", + }); + } if (res.status === 422 || res.status >= 500 || (res.status >= 400 && !looksLikeContent(result))) { throw new ToolkitError({ code: "FETCH_FAILED", @@ -251,6 +270,21 @@ function isForbiddenDomain(body: string): boolean { } } +/** + * Codes where the target is unreachable/nonexistent — RESP007 (domain does not + * resolve) and RESP002 (target 404). Permanent: no proxy/render setting fixes + * them, so the caller must not advise a (billed) retry. + */ +function unrecoverableTargetCode(body: string): "RESP007" | "RESP002" | null { + try { + const j = JSON.parse(body) as { code?: unknown }; + const code = typeof j.code === "string" ? j.code.toUpperCase() : ""; + return code === "RESP007" || code === "RESP002" ? code : null; + } catch { + return null; + } +} + function isZenrowsErrorEnvelope(body: string): boolean { try { const j = JSON.parse(body) as { code?: unknown; type?: unknown }; diff --git a/src/installers/mcp/index.ts b/src/installers/mcp/index.ts index 60461cf..8759053 100644 --- a/src/installers/mcp/index.ts +++ b/src/installers/mcp/index.ts @@ -36,7 +36,6 @@ export const MCP_CLIENTS: Record<string, McpClientSpec> = { configFile: ".mcp.json (project) or run `claude mcp add`", format: "cli", autoConfigurable: true, - notes: "Confirmed: `claude mcp add zenrows -e ZENROWS_API_KEY=… -- npx -y @zenrows/mcp`.", }, cursor: { id: "cursor", diff --git a/tests/agent-account.test.ts b/tests/agent-account.test.ts index 86b5ec4..f333266 100644 --- a/tests/agent-account.test.ts +++ b/tests/agent-account.test.ts @@ -54,6 +54,53 @@ test("signupAgent parses a 201 response and sends provenance headers", async () assert.ok(sentHeaders["X-ZR-CI"] === "0" || sentHeaders["X-ZR-CI"] === "1"); }); +test("signupAgent falls back to the built-in default when a discovered endpoint fails (safety net)", async () => { + // This is the exact regression that caused the original outage: discovery + // advertised a host behind a bot-challenge. The safety net must recover by + // trying the built-in default, so signup can never fail closed on a bad doc. + const prevSignup = process.env[SIGNUP_URL_ENV]; + const prevDisco = process.env[DISCOVERY_URL_ENV]; + const prevTelemetry = process.env.ZENROWS_TELEMETRY; + delete process.env[SIGNUP_URL_ENV]; + delete process.env[DISCOVERY_URL_ENV]; + process.env.ZENROWS_TELEMETRY = "off"; // skip provenance/telemetry-id side effects + _resetDiscoveryCache(); + + const BLOCKED = "https://www.blocked.example/api/agent/signup"; + const calls: Array<{ method: string; url: string }> = []; + const fakeFetch = (async (url: string, init?: RequestInit) => { + const method = init?.method ?? "GET"; + calls.push({ method, url: String(url) }); + if (method === "GET") { + // discovery doc points the CLI at a blocked host + return new Response(JSON.stringify({ agent_auth: { signup_endpoint: BLOCKED } }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + if (String(url) === BLOCKED) return new Response("<title>Just a moment...", { status: 403 }); + if (String(url) === AGENT_SIGNUP_API_URL) + return new Response(JSON.stringify({ apiKey: "zr-key", accountId: "u1", claimUrl: "https://x/claim/t" }), { + status: 201, + headers: { "content-type": "application/json" }, + }); + return new Response("unexpected", { status: 500 }); + }) as unknown as typeof fetch; + + try { + const res = await signupAgent({ fetchImpl: fakeFetch }); + assert.equal(res.apiKey, "zr-key"); // recovered via the fallback + const posts = calls.filter((c) => c.method === "POST").map((c) => c.url); + assert.deepEqual(posts, [BLOCKED, AGENT_SIGNUP_API_URL]); // blocked first, then fell back to default + } finally { + _resetDiscoveryCache(); + if (prevSignup !== undefined) process.env[SIGNUP_URL_ENV] = prevSignup; + if (prevDisco !== undefined) process.env[DISCOVERY_URL_ENV] = prevDisco; + if (prevTelemetry !== undefined) process.env.ZENROWS_TELEMETRY = prevTelemetry; + else delete process.env.ZENROWS_TELEMETRY; + } +}); + test("signupAgent omits all provenance headers when opted out via env", async () => { const prev = process.env.ZENROWS_TELEMETRY; process.env.ZENROWS_TELEMETRY = "off"; diff --git a/tests/arg-parse.test.ts b/tests/arg-parse.test.ts new file mode 100644 index 0000000..9ede863 --- /dev/null +++ b/tests/arg-parse.test.ts @@ -0,0 +1,48 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { parse } from "../src/cli/command.ts"; +import { ToolkitError } from "../src/core/errors.ts"; + +const OPTS = { + output: { type: "boolean" }, + "js-render": { type: "boolean" }, + session: { type: "string" }, +} as const; + +test("parse accepts declared flags + positionals", () => { + const { values, positionals } = parse(["https://x.com", "--output", "--session", "s1"], OPTS); + assert.equal(values.output, true); + assert.equal(values.session, "s1"); + assert.deepEqual(positionals, ["https://x.com"]); +}); + +test("parse rejects an unknown flag with UNKNOWN_FLAG (no silent swallow)", () => { + assert.throws( + () => parse(["--bogus-flag"], OPTS), + (e: unknown) => e instanceof ToolkitError && e.code === "UNKNOWN_FLAG", + ); +}); + +test("parse names the offending flag and suggests the closest match", () => { + try { + parse(["--outputt"], OPTS); // one char off from --output + assert.fail("should have thrown"); + } catch (e) { + assert.ok(e instanceof ToolkitError); + assert.match(e.message, /--outputt/); + assert.match(e.next_action, /Did you mean --output\?/); + } +}); + +test("parse rejects the homepage's --format (the silent-HTML bug), no charge path reached", () => { + assert.throws( + () => parse(["https://x.com", "--format", "markdown"], OPTS), + (e: unknown) => e instanceof ToolkitError && e.code === "UNKNOWN_FLAG", + ); +}); + +test("parse still tolerates the global flags stripped by the router", () => { + // --json/--yes/--help/--version are consumed by the top-level router; if they + // reach a command's parse (e.g. in tests) they must not read as unknown. + assert.doesNotThrow(() => parse(["--json", "--yes", "https://x.com"], OPTS)); +}); diff --git a/tests/http.test.ts b/tests/http.test.ts index 05efc29..99eb500 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -110,6 +110,34 @@ test("scrape sends a User-Agent carrying the current CLI version", async () => { assert.equal(seenUA, `zenrows-cli/${CLI_VERSION}`); }); +test("scrape does NOT advise a (billed) retry when the target cannot be resolved (RESP007)", async () => { + const RESP007 = JSON.stringify({ + code: "RESP007", + detail: "The requested target domain could not be resolved, or there is no DNS record associated with it.", + status: 422, + title: "Could not resolve domain (RESP007)", + }); + await withFetch( + () => new Response(RESP007, { status: 422, headers: { "content-type": "application/json" } }), + async () => { + await assert.rejects( + () => scrape("https://api.zenrows.com/v1/", "test-key", { url: "https://no-such-host.invalid" }), + (err: unknown) => { + const e = err as { code: string; next_action: string; suggested_commands: string[] }; + assert.equal(e.code, "FETCH_FAILED"); + // The whole point: it discourages a retry and offers no billed retry + // command. (Naming the flags to say they *won't* help is fine; telling + // the agent to "retry with" them is not.) + assert.match(e.next_action, /do not retry/i); + assert.doesNotMatch(e.next_action, /retry with/i); + assert.equal(e.suggested_commands.length, 0); + return true; + }, + ); + }, + ); +}); + test("scrape maps AUTH010 on extract=auto to EXTRACT_DOMAIN_NOT_ENABLED", async () => { const AUTH010 = JSON.stringify({ code: "AUTH010",