diff --git a/docs/docs/reference/telemetry.md b/docs/docs/reference/telemetry.md index 5d499d8e4d..c3b72cc6d4 100644 --- a/docs/docs/reference/telemetry.md +++ b/docs/docs/reference/telemetry.md @@ -11,9 +11,9 @@ We collect the following categories of events: | `session_start` | A new CLI session begins | | `session_end` | A CLI session ends (includes duration) | | `session_forked` | A session is forked from an existing one | -| `generation` | An AI model generation completes (model ID, token counts, duration, but no prompt content) | -| `tool_call` | A tool is invoked (tool name and category, but no arguments or output) | -| `bridge_call` | A native tool call completes (method name and duration, but no arguments) | +| `generation` | An AI model generation completes (model ID, token counts, duration — no prompt content) | +| `tool_call` | A tool is invoked (tool name and category — no arguments or output) | +| `native_call` | A native engine call completes (method name and duration — no arguments) | | `command` | A CLI command is executed (command name only) | | `error` | An unhandled error occurs (error type and truncated message, but no stack traces) | | `auth_login` | Authentication succeeds or fails (provider and method, but no credentials) | @@ -33,8 +33,11 @@ We collect the following categories of events: | `error_recovered` | Successful recovery from a transient error (error type, strategy, attempt count) | | `mcp_server_census` | MCP server capabilities after connect (tool and resource counts, but no tool names) | | `context_overflow_recovered` | Context overflow is handled (strategy) | +| `skill_used` | A skill is loaded (skill name and source — `builtin`, `global`, or `project` — no skill content) | +| `sql_execute_failure` | A SQL execution fails (warehouse type, query type, error message, PII-masked SQL — no raw values) | +| `core_failure` | An internal tool error occurs (tool name, category, error class, truncated error message, PII-safe input signature, and optionally masked arguments — no raw values or credentials) | -Each event includes a timestamp, anonymous session ID, and the CLI version. +Each event includes a timestamp, anonymous session ID, CLI version, and an anonymous machine ID (a random UUID stored in `~/.altimate/machine-id`, generated once and never tied to any personal information). ## Delivery & Reliability @@ -113,9 +116,9 @@ Event type names use **snake_case** with a `domain_action` pattern: ### Adding a New Event -1. **Define the type.** Add a new variant to the `Telemetry.Event` union in `packages/altimate-code/src/telemetry/index.ts` -2. **Emit the event.** Call `Telemetry.track()` at the appropriate location -3. **Update docs.** Add a row to the event table above +1. **Define the type** — Add a new variant to the `Telemetry.Event` union in `packages/opencode/src/altimate/telemetry/index.ts` +2. **Emit the event** — Call `Telemetry.track()` at the appropriate location +3. **Update docs** — Add a row to the event table above ### Privacy Checklist diff --git a/packages/drivers/src/sqlserver.ts b/packages/drivers/src/sqlserver.ts index 387a65cf73..2290887194 100644 --- a/packages/drivers/src/sqlserver.ts +++ b/packages/drivers/src/sqlserver.ts @@ -7,7 +7,6 @@ import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from export async function connect(config: ConnectionConfig): Promise { let mssql: any try { - // @ts-expect-error — optional dependency, loaded at runtime mssql = await import("mssql") mssql = mssql.default || mssql } catch { diff --git a/packages/opencode/src/altimate/native/connections/register.ts b/packages/opencode/src/altimate/native/connections/register.ts index 18d029197b..7267a142d0 100644 --- a/packages/opencode/src/altimate/native/connections/register.ts +++ b/packages/opencode/src/altimate/native/connections/register.ts @@ -228,6 +228,8 @@ register("sql.execute", async (params: SqlExecuteParams): Promise { auth_method: detectAuthMethod(config), success: false, duration_ms: Date.now() - startTime, - error: String(e).slice(0, 500), + error: Telemetry.maskString(String(e)).slice(0, 500), error_category: categorizeConnectionError(e), }) } catch {} diff --git a/packages/opencode/src/altimate/native/dispatcher.ts b/packages/opencode/src/altimate/native/dispatcher.ts index 346a77e2b4..944647630a 100644 --- a/packages/opencode/src/altimate/native/dispatcher.ts +++ b/packages/opencode/src/altimate/native/dispatcher.ts @@ -75,7 +75,7 @@ export async function call( method: method as string, status: "error", duration_ms: Date.now() - startTime, - error: String(e).slice(0, 500), + error: Telemetry.maskString(String(e)).slice(0, 500), }) } catch { // Telemetry must never prevent error propagation diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 16cd4e431b..e0f0130e44 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -2,7 +2,10 @@ import { Account } from "@/account" import { Config } from "@/config/config" import { Installation } from "@/installation" import { Log } from "@/util/log" -import { createHash } from "crypto" +import { createHash, randomUUID } from "crypto" +import fs from "fs" +import path from "path" +import os from "os" const log = Log.create({ service: "telemetry" }) @@ -63,6 +66,7 @@ export namespace Telemetry { duration_ms: number sequence_index: number previous_tool: string | null + input_signature?: string error?: string } | { @@ -331,6 +335,166 @@ export namespace Telemetry { has_ssh_tunnel: boolean has_keychain: boolean } + | { + type: "skill_used" + timestamp: number + session_id: string + message_id: string + skill_name: string + skill_source: "builtin" | "global" | "project" + duration_ms: number + } + | { + type: "sql_execute_failure" + timestamp: number + session_id: string + warehouse_type: string + query_type: string + error_message: string + masked_sql: string + duration_ms: number + } + | { + type: "core_failure" + timestamp: number + session_id: string + tool_name: string + tool_category: string + error_class: + | "parse_error" + | "connection" + | "timeout" + | "validation" + | "internal" + | "permission" + | "unknown" + error_message: string + input_signature: string + masked_args?: string + duration_ms: number + } + + const ERROR_PATTERNS: Array<{ + class: Telemetry.Event & { type: "core_failure" } extends { error_class: infer C } ? C : never + keywords: string[] + }> = [ + { class: "parse_error", keywords: ["parse", "syntax", "binder", "unexpected token", "sqlglot"] }, + { + class: "connection", + keywords: ["econnrefused", "connection", "socket", "enotfound", "econnreset"], + }, + { class: "timeout", keywords: ["timeout", "etimedout", "bridge timeout", "timed out"] }, + { class: "permission", keywords: ["permission", "denied", "unauthorized", "forbidden"] }, + { class: "validation", keywords: ["invalid params", "invalid", "missing", "required"] }, + { class: "internal", keywords: ["internal", "assertion"] }, + ] + + export function classifyError( + message: string, + ): Telemetry.Event & { type: "core_failure" } extends { error_class: infer C } ? C : never { + const lower = message.toLowerCase() + for (const { class: cls, keywords } of ERROR_PATTERNS) { + if (keywords.some((kw) => lower.includes(kw))) return cls + } + return "unknown" + } + + export function computeInputSignature(args: Record): string { + const sig: Record = {} + for (const [k, v] of Object.entries(args)) { + if (v === null || v === undefined) { + sig[k] = "null" + } else if (typeof v === "string") { + sig[k] = `string:${v.length}` + } else if (typeof v === "number") { + sig[k] = "number" + } else if (typeof v === "boolean") { + sig[k] = "boolean" + } else if (Array.isArray(v)) { + sig[k] = `array:${v.length}` + } else if (typeof v === "object") { + sig[k] = `object:${Object.keys(v).length}` + } else { + sig[k] = typeof v + } + } + const result = JSON.stringify(sig) + if (result.length <= 1000) return result + // Drop keys from the end until the JSON fits, preserving valid JSON structure + const keys = Object.keys(sig) + while (keys.length > 0) { + keys.pop() + const truncated: Record = {} + for (const k of keys) truncated[k] = sig[k] + truncated["..."] = `${Object.keys(sig).length - keys.length} more` + const out = JSON.stringify(truncated) + if (out.length <= 1000) return out + } + return JSON.stringify({ "...": `${Object.keys(sig).length} keys` }) + } + + // Mirrors altimate-sdk (Rust) SENSITIVE_KEYS — keep in sync. + const SENSITIVE_KEYS: string[] = [ + "key", "api_key", "apikey", "apiKey", "token", "access_token", "refresh_token", + "secret", "secret_key", "password", "passwd", "pwd", + "credential", "credentials", "authorization", "auth", + "signature", "sig", "private_key", "connection_string", + // camelCase variants not caught by prefix/suffix matching + "authtoken", "accesstoken", "refreshtoken", "bearertoken", "jwttoken", + "jwtsecret", "clientsecret", "appsecret", + ] + + function isSensitiveKey(key: string): boolean { + const lower = key.toLowerCase() + return SENSITIVE_KEYS.some( + (k) => lower === k || lower.endsWith(`_${k}`) || lower.startsWith(`${k}_`), + ) + } + + export function maskString(s: string): string { + return s + .replace(/'(?:[^'\\]|\\.)*'/g, "?") + .replace(/"(?:[^"\\]|\\.)*"/g, "?") + .replace(/\s+/g, " ") + .trim() + } + + function maskValue(value: unknown, key?: string): unknown { + if (key && isSensitiveKey(key)) return "****" + if (typeof value === "string") return maskString(value) + if (Array.isArray(value)) return value.map((v) => maskValue(v, key)) + if (value !== null && typeof value === "object") { + const masked: Record = {} + for (const [k, v] of Object.entries(value as Record)) { + masked[k] = maskValue(v, k) + } + return masked + } + return value + } + + /** PII-mask tool arguments for failure telemetry. + * Mirrors altimate-sdk mask_value: sensitive keys → "****", + * string literals in SQL → ?, whitespace collapsed. Truncates to 2000 chars. */ + export function maskArgs(args: Record): string { + const masked: Record = {} + for (const [k, v] of Object.entries(args)) { + masked[k] = maskValue(v, k) + } + const result = JSON.stringify(masked) + if (result.length <= 2000) return result + // Drop keys from the end until valid JSON fits, same approach as computeInputSignature + const keys = Object.keys(masked) + while (keys.length > 0) { + keys.pop() + const truncated: Record = {} + for (const k of keys) truncated[k] = masked[k] + truncated["..."] = `${Object.keys(masked).length - keys.length} more` + const out = JSON.stringify(truncated) + if (out.length <= 2000) return out + } + return JSON.stringify({ "...": `${Object.keys(masked).length} keys` }) + } const FILE_TOOLS = new Set(["read", "write", "edit", "glob", "grep", "bash"]) @@ -373,6 +537,7 @@ export namespace Telemetry { let buffer: Event[] = [] let flushTimer: ReturnType | undefined let userEmail = "" + let machineId = "" let sessionId = "" let projectId = "" let appInsights: AppInsightsConfig | undefined @@ -402,12 +567,13 @@ export namespace Telemetry { const properties: Record = { cli_version: Installation.VERSION, project_id: fields.project_id ?? projectId, + ...(machineId && { machine_id: machineId }), } const measurements: Record = {} // Flatten all fields — nested `tokens` object gets prefixed keys for (const [k, v] of Object.entries(fields)) { - if (k === "session_id" || k === "project_id") continue + if (k === "session_id" || k === "project_id" || k === "_retried") continue if (k === "tokens" && typeof v === "object" && v !== null) { for (const [tk, tv] of Object.entries(v as Record)) { if (typeof tv === "number") measurements[`tokens_${tk}`] = tv @@ -490,6 +656,18 @@ export namespace Telemetry { } catch { // Account unavailable — proceed without user ID } + try { + const machineIdPath = path.join(os.homedir(), ".altimate", "machine-id") + try { + machineId = fs.readFileSync(machineIdPath, "utf8").trim() + } catch { + machineId = randomUUID() + fs.mkdirSync(path.dirname(machineIdPath), { recursive: true }) + fs.writeFileSync(machineIdPath, machineId, "utf8") + } + } catch { + // Machine ID unavailable — proceed without it + } enabled = true log.info("telemetry initialized", { mode: "appinsights" }) const timer = setInterval(flush, FLUSH_INTERVAL_MS) @@ -591,6 +769,7 @@ export namespace Telemetry { droppedEvents = 0 sessionId = "" projectId = "" + machineId = "" initPromise = undefined initDone = false } diff --git a/packages/opencode/src/tool/skill.ts b/packages/opencode/src/tool/skill.ts index e0c033bdbf..b0754044be 100644 --- a/packages/opencode/src/tool/skill.ts +++ b/packages/opencode/src/tool/skill.ts @@ -9,8 +9,18 @@ import { iife } from "@/util/iife" import { Fingerprint } from "../altimate/fingerprint" import { Config } from "../config/config" import { selectSkillsWithLLM } from "../altimate/skill-selector" +import { Telemetry } from "../altimate/telemetry" +import os from "os" const MAX_DISPLAY_SKILLS = 50 + +// altimate_change start — classifySkillSource helper for skill telemetry +function classifySkillSource(location: string): "builtin" | "global" | "project" { + if (location.includes("node_modules") || location.includes(".altimate/builtin")) return "builtin" + if (location.startsWith(os.homedir())) return "global" + return "project" +} +// altimate_change end // altimate_change end export const SkillTool = Tool.define("skill", async (ctx) => { @@ -83,6 +93,9 @@ export const SkillTool = Tool.define("skill", async (ctx) => { description, parameters, async execute(params: z.infer, ctx) { + // altimate_change start — telemetry: startTime for skill_used duration + const startTime = Date.now() + // altimate_change end // altimate_change start - use upstream Skill.get() for exact name lookup const skill = await Skill.get(params.name) @@ -122,6 +135,22 @@ export const SkillTool = Tool.define("skill", async (ctx) => { return arr }).then((f) => f.map((file) => `${file}`).join("\n")) + // altimate_change start — telemetry instrumentation for skill loading + try { + Telemetry.track({ + type: "skill_used", + timestamp: Date.now(), + session_id: ctx.sessionID, + message_id: ctx.messageID, + skill_name: skill.name, + skill_source: classifySkillSource(skill.location), + duration_ms: Date.now() - startTime, + }) + } catch { + // Telemetry must never break skill loading + } + // altimate_change end + return { title: `Loaded skill: ${skill.name}`, output: [ diff --git a/packages/opencode/src/tool/tool.ts b/packages/opencode/src/tool/tool.ts index 8cc7b57d85..bbc4f8e49c 100644 --- a/packages/opencode/src/tool/tool.ts +++ b/packages/opencode/src/tool/tool.ts @@ -4,6 +4,9 @@ import type { Agent } from "../agent/agent" import type { PermissionNext } from "../permission/next" import type { SessionID, MessageID } from "../session/schema" import { Truncate } from "./truncation" +// altimate_change start — telemetry instrumentation for tool execution +import { Telemetry } from "../altimate/telemetry" +// altimate_change end export namespace Tool { interface Metadata { @@ -67,8 +70,98 @@ export namespace Tool { { cause: error }, ) } - const result = await execute(args, ctx) - // skip truncation for tools that handle it themselves + // altimate_change start — telemetry instrumentation for tool execution + const startTime = Date.now() + let result: Awaited> + try { + result = await execute(args, ctx) + } catch (error) { + // Telemetry must never prevent the original error from propagating + try { + const errorMsg = error instanceof Error ? error.message : String(error) + const maskedErrorMsg = Telemetry.maskString(errorMsg).slice(0, 500) + Telemetry.track({ + type: "tool_call", + timestamp: Date.now(), + session_id: ctx.sessionID, + message_id: ctx.messageID, + tool_name: id, + tool_type: "standard", + tool_category: Telemetry.categorizeToolName(id, "standard"), + status: "error", + duration_ms: Date.now() - startTime, + sequence_index: 0, + previous_tool: null, + input_signature: Telemetry.computeInputSignature(args as Record), + error: maskedErrorMsg, + }) + Telemetry.track({ + type: "core_failure", + timestamp: Date.now(), + session_id: ctx.sessionID, + tool_name: id, + tool_category: Telemetry.categorizeToolName(id, "standard"), + error_class: Telemetry.classifyError(errorMsg), + error_message: maskedErrorMsg, + input_signature: Telemetry.computeInputSignature(args as Record), + masked_args: Telemetry.maskArgs(args as Record), + duration_ms: Date.now() - startTime, + }) + } catch { + // Telemetry failure must never mask the original tool error + } + throw error + } + // Telemetry runs after execute() succeeds — wrapped so it never breaks the tool + try { + const isSoftFailure = result.metadata?.success === false + const durationMs = Date.now() - startTime + const toolCategory = Telemetry.categorizeToolName(id, "standard") + // Skip success tool_call for file tools (read/write/edit/glob/grep/bash) — high + // volume, low signal. Failures are still captured via core_failure below. + if (isSoftFailure || toolCategory !== "file") { + Telemetry.track({ + type: "tool_call", + timestamp: Date.now(), + session_id: ctx.sessionID, + message_id: ctx.messageID, + tool_name: id, + tool_type: "standard", + tool_category: toolCategory, + status: isSoftFailure ? "error" : "success", + duration_ms: durationMs, + sequence_index: 0, + previous_tool: null, + ...(isSoftFailure && { + input_signature: Telemetry.computeInputSignature(args as Record), + }), + }) + } + if (isSoftFailure) { + const errorMsg = + typeof result.metadata?.error === "string" + ? result.metadata.error + : "unknown error" + const maskedErrorMsg = Telemetry.maskString(errorMsg).slice(0, 500) + Telemetry.track({ + type: "core_failure", + timestamp: Date.now(), + session_id: ctx.sessionID, + tool_name: id, + tool_category: Telemetry.categorizeToolName(id, "standard"), + error_class: Telemetry.classifyError(errorMsg), + error_message: maskedErrorMsg, + input_signature: Telemetry.computeInputSignature(args as Record), + masked_args: Telemetry.maskArgs(args as Record), + duration_ms: durationMs, + }) + } + } catch { + // Telemetry must never break tool execution + } + // altimate_change end + // Truncation runs after telemetry so I/O errors from + // Truncate.output() are not misattributed as tool failures. if (result.metadata.truncated !== undefined) { return result } diff --git a/packages/opencode/test/telemetry/telemetry.test.ts b/packages/opencode/test/telemetry/telemetry.test.ts index ef905a9b62..c7c5757365 100644 --- a/packages/opencode/test/telemetry/telemetry.test.ts +++ b/packages/opencode/test/telemetry/telemetry.test.ts @@ -193,7 +193,7 @@ describe("telemetry.context", () => { }) // --------------------------------------------------------------------------- -// 5. Event type completeness — all 25 event types +// 5. Event type completeness — all 33 event types // --------------------------------------------------------------------------- describe("telemetry.event-types", () => { test("all event types are valid", () => { @@ -202,7 +202,7 @@ describe("telemetry.event-types", () => { "session_end", "generation", "tool_call", - "bridge_call", + "native_call", "error", "command", "context_overflow_recovered", @@ -223,8 +223,16 @@ describe("telemetry.event-types", () => { "agent_outcome", "error_recovered", "mcp_server_census", + "memory_operation", + "memory_injection", + "warehouse_connect", + "warehouse_query", + "warehouse_introspection", + "warehouse_discovery", + "warehouse_census", + "core_failure", ] - expect(eventTypes.length).toBe(25) + expect(eventTypes.length).toBe(33) }) }) @@ -315,7 +323,7 @@ describe("telemetry.naming-convention", () => { "session_end", "generation", "tool_call", - "bridge_call", + "native_call", "error", "command", "context_overflow_recovered", @@ -336,6 +344,14 @@ describe("telemetry.naming-convention", () => { "agent_outcome", "error_recovered", "mcp_server_census", + "memory_operation", + "memory_injection", + "warehouse_connect", + "warehouse_query", + "warehouse_introspection", + "warehouse_discovery", + "warehouse_census", + "core_failure", ] for (const t of types) { expect(t).toMatch(/^[a-z][a-z0-9_]*$/) @@ -1503,3 +1519,291 @@ describe("Telemetry.isEnabled()", () => { } }) }) + +// --------------------------------------------------------------------------- +// 15. classifyError +// --------------------------------------------------------------------------- +describe("telemetry.classifyError", () => { + test("classifies parse errors", () => { + expect(Telemetry.classifyError("ParseError: unexpected token")).toBe("parse_error") + expect(Telemetry.classifyError("SyntaxError in SQL")).toBe("parse_error") + expect(Telemetry.classifyError("binder error on column")).toBe("parse_error") + expect(Telemetry.classifyError("sqlglot transpilation failed")).toBe("parse_error") + }) + + test("classifies connection errors", () => { + expect(Telemetry.classifyError("ECONNREFUSED 127.0.0.1:5432")).toBe("connection") + expect(Telemetry.classifyError("Connection refused by host")).toBe("connection") + expect(Telemetry.classifyError("Socket hang up")).toBe("connection") + expect(Telemetry.classifyError("ENOTFOUND db.example.com")).toBe("connection") + expect(Telemetry.classifyError("ECONNRESET")).toBe("connection") + }) + + test("classifies timeout errors", () => { + expect(Telemetry.classifyError("Request timeout after 30s")).toBe("timeout") + expect(Telemetry.classifyError("ETIMEDOUT")).toBe("timeout") + expect(Telemetry.classifyError("Bridge timeout waiting for response")).toBe("timeout") + expect(Telemetry.classifyError("Operation timed out")).toBe("timeout") + }) + + test("classifies validation errors", () => { + expect(Telemetry.classifyError("Invalid params: missing 'sql'")).toBe("validation") + expect(Telemetry.classifyError("Invalid dialect specified")).toBe("validation") + expect(Telemetry.classifyError("Missing required field")).toBe("validation") + expect(Telemetry.classifyError("Required parameter 'query' not provided")).toBe("validation") + }) + + test("classifies permission errors", () => { + expect(Telemetry.classifyError("Permission denied on table")).toBe("permission") + expect(Telemetry.classifyError("Access denied for user")).toBe("permission") + expect(Telemetry.classifyError("Unauthorized access to resource")).toBe("permission") + expect(Telemetry.classifyError("403 Forbidden")).toBe("permission") + }) + + test("classifies internal errors", () => { + expect(Telemetry.classifyError("Internal server error")).toBe("internal") + expect(Telemetry.classifyError("Assertion failed: x > 0")).toBe("internal") + }) + + test("returns unknown for unrecognized errors", () => { + expect(Telemetry.classifyError("Something went wrong")).toBe("unknown") + expect(Telemetry.classifyError("")).toBe("unknown") + expect(Telemetry.classifyError("42")).toBe("unknown") + }) + + test("is case-insensitive", () => { + expect(Telemetry.classifyError("PARSEERROR")).toBe("parse_error") + expect(Telemetry.classifyError("CONNECTION REFUSED")).toBe("connection") + expect(Telemetry.classifyError("TIMEOUT")).toBe("timeout") + }) +}) + +// --------------------------------------------------------------------------- +// 16. computeInputSignature +// --------------------------------------------------------------------------- +describe("telemetry.computeInputSignature", () => { + test("records string values as type:length", () => { + const sig = Telemetry.computeInputSignature({ sql: "SELECT 1", dialect: "snowflake" }) + const parsed = JSON.parse(sig) + expect(parsed.sql).toBe("string:8") + expect(parsed.dialect).toBe("string:9") + }) + + test("records number values as 'number'", () => { + const sig = Telemetry.computeInputSignature({ limit: 100, offset: 0 }) + const parsed = JSON.parse(sig) + expect(parsed.limit).toBe("number") + expect(parsed.offset).toBe("number") + }) + + test("records boolean values as 'boolean'", () => { + const sig = Telemetry.computeInputSignature({ verbose: true, dry_run: false }) + const parsed = JSON.parse(sig) + expect(parsed.verbose).toBe("boolean") + expect(parsed.dry_run).toBe("boolean") + }) + + test("records array values as array:length", () => { + const sig = Telemetry.computeInputSignature({ tables: ["a", "b", "c"] }) + const parsed = JSON.parse(sig) + expect(parsed.tables).toBe("array:3") + }) + + test("records object values as object:keyCount", () => { + const sig = Telemetry.computeInputSignature({ config: { a: 1, b: 2 } }) + const parsed = JSON.parse(sig) + expect(parsed.config).toBe("object:2") + }) + + test("records null and undefined as 'null'", () => { + const sig = Telemetry.computeInputSignature({ a: null, b: undefined }) + const parsed = JSON.parse(sig) + expect(parsed.a).toBe("null") + expect(parsed.b).toBe("null") + }) + + test("never includes actual values — only key names and type descriptors", () => { + const sig = Telemetry.computeInputSignature({ + sql: "SELECT password FROM users WHERE email = 'admin@example.com'", + secret: "super-secret-123", + api_key: "sk-abc123", + }) + // Key names ARE included (that's by design), but actual values are NOT + expect(sig).not.toContain("super-secret") + expect(sig).not.toContain("sk-abc123") + expect(sig).not.toContain("SELECT") + expect(sig).not.toContain("admin@example.com") + // Only type descriptors appear as values + const parsed = JSON.parse(sig) + expect(parsed.sql).toBe("string:60") + expect(parsed.secret).toBe("string:16") + expect(parsed.api_key).toBe("string:9") + }) + + test("truncates output at 1000 chars with valid JSON", () => { + // Create args with many long key names to exceed 1000 chars + const args: Record = {} + for (let i = 0; i < 100; i++) { + args[`very_long_key_name_for_testing_truncation_${i}`] = "value" + } + const sig = Telemetry.computeInputSignature(args) + expect(sig.length).toBeLessThanOrEqual(1000) + // Must produce valid JSON even when truncated + const parsed = JSON.parse(sig) + expect(parsed["..."]).toBeDefined() + expect(typeof parsed["..."]).toBe("string") + }) + + test("returns valid JSON for empty input", () => { + const sig = Telemetry.computeInputSignature({}) + expect(JSON.parse(sig)).toEqual({}) + }) +}) + +// --------------------------------------------------------------------------- +// 17. core_failure event privacy +// --------------------------------------------------------------------------- +describe("telemetry.core_failure privacy", () => { + test("core_failure event contains no raw SQL or argument values", () => { + const event: Telemetry.Event = { + type: "core_failure", + timestamp: Date.now(), + session_id: "test", + tool_name: "sql_analyze", + tool_category: "sql", + error_class: "parse_error", + error_message: "ParseError: unexpected token near SELECT".slice(0, 500), + input_signature: Telemetry.computeInputSignature({ + sql: "SELECT * FROM users WHERE id = 1", + dialect: "snowflake", + }), + duration_ms: 42, + } + // input_signature has types/lengths, not values + expect(event.input_signature).not.toContain("SELECT") + expect(event.input_signature).not.toContain("users") + expect(event.input_signature).not.toContain("snowflake") + const parsed = JSON.parse(event.input_signature) + expect(parsed.sql).toBe("string:32") + expect(parsed.dialect).toBe("string:9") + }) + + test("error_message is capped at 500 chars", () => { + const longError = "x".repeat(1000) + const event: Telemetry.Event = { + type: "core_failure", + timestamp: Date.now(), + session_id: "test", + tool_name: "sql_analyze", + tool_category: "sql", + error_class: "parse_error", + error_message: longError.slice(0, 500), + input_signature: "{}", + duration_ms: 0, + } + expect(event.error_message.length).toBe(500) + }) + + test("core_failure event does NOT include raw arguments, outputs, or SQL", () => { + const event: Telemetry.Event = { + type: "core_failure", + timestamp: Date.now(), + session_id: "test", + tool_name: "sql_analyze", + tool_category: "sql", + error_class: "unknown", + error_message: "test error", + input_signature: '{"sql":"string:10"}', + duration_ms: 100, + } + expect("args" in event).toBe(false) + expect("input" in event).toBe(false) + expect("output" in event).toBe(false) + expect("sql" in event).toBe(false) + expect("query" in event).toBe(false) + }) +}) + +// --------------------------------------------------------------------------- +// 18. maskArgs +// --------------------------------------------------------------------------- +describe("telemetry.maskArgs", () => { + test("replaces string literals in SQL with ?", () => { + const masked = Telemetry.maskArgs({ + sql: "SELECT * FROM users WHERE name = 'John Doe' AND email = 'john@example.com'", + }) + const parsed = JSON.parse(masked) + expect(parsed.sql).toContain("SELECT * FROM users WHERE name = ") + expect(parsed.sql).not.toContain("John Doe") + expect(parsed.sql).not.toContain("john@example.com") + expect(parsed.sql).toContain("?") + }) + + test("redacts sensitive keys entirely", () => { + const masked = Telemetry.maskArgs({ + sql: "SELECT 1", + password: "super-secret-123", + api_key: "sk-abc123", + token: "eyJhbGciOi...", + secret: "my-secret", + }) + const parsed = JSON.parse(masked) + expect(parsed.password).toBe("****") + expect(parsed.api_key).toBe("****") + expect(parsed.token).toBe("****") + expect(parsed.secret).toBe("****") + expect(parsed.sql).toBe("SELECT 1") + }) + + test("preserves SQL structure while masking literals", () => { + const masked = Telemetry.maskArgs({ + sql: "INSERT INTO orders (name, total) VALUES ('Alice', 99.99)", + }) + const parsed = JSON.parse(masked) + expect(parsed.sql).toContain("INSERT INTO orders") + expect(parsed.sql).toContain("VALUES (") + expect(parsed.sql).not.toContain("Alice") + }) + + test("handles escaped quotes in SQL strings", () => { + const masked = Telemetry.maskArgs({ + sql: "SELECT * FROM t WHERE name = 'O\\'Brien'", + }) + const parsed = JSON.parse(masked) + expect(parsed.sql).not.toContain("Brien") + }) + + test("preserves non-string values as-is", () => { + const masked = Telemetry.maskArgs({ + dialect: "snowflake", + depth: "full", + limit: 100, + verbose: true, + }) + const parsed = JSON.parse(masked) + expect(parsed.dialect).toBe("snowflake") + expect(parsed.depth).toBe("full") + expect(parsed.limit).toBe(100) + expect(parsed.verbose).toBe(true) + }) + + test("truncates output at 2000 chars with valid JSON", () => { + const args: Record = {} + for (let i = 0; i < 200; i++) { + args[`long_key_name_for_testing_truncation_${i}`] = "SELECT " + "a".repeat(100) + } + const masked = Telemetry.maskArgs(args) + expect(masked.length).toBeLessThanOrEqual(2000) + // Must produce valid JSON even when truncated + const parsed = JSON.parse(masked) + expect(parsed["..."]).toBeDefined() + }) + + test("redacts connection_string key", () => { + const masked = Telemetry.maskArgs({ + connection_string: "Server=db.prod.internal;Password=hunter2", + }) + const parsed = JSON.parse(masked) + expect(parsed.connection_string).toBe("****") + }) +})