From afe04ad494e1cd090a09a1c33e6f3245b542dbde Mon Sep 17 00:00:00 2001 From: suryaiyer95 Date: Wed, 18 Mar 2026 10:12:25 -0700 Subject: [PATCH 01/13] feat: add `core_failure` telemetry with PII-safe masking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new `core_failure` event emitted on both soft failures (`metadata.success === false`) and uncaught tool exceptions, with privacy-preserving context for debugging: - `classifyError()` — keyword-based error classification (parse, connection, timeout, validation, permission, internal, unknown) - `computeInputSignature()` — records key names + value types/lengths, never actual values; truncates by dropping keys to preserve valid JSON - `maskArgs()` — PII masking aligned to Rust SDK: 19 sensitive keys redacted, string literals in SQL replaced with `?`, recursive object traversal Telemetry is fully isolated from tool execution — all tracking calls are wrapped in `try/catch` so telemetry failures never break tools. `Truncate.output()` runs outside the telemetry error boundary so I/O errors aren't misattributed as tool failures. Co-Authored-By: Claude Opus 4.6 --- docs/docs/configure/telemetry.md | 5 +- .../opencode/src/altimate/telemetry/index.ts | 136 ++++++++ packages/opencode/src/tool/tool.ts | 86 ++++- .../opencode/test/telemetry/telemetry.test.ts | 312 +++++++++++++++++- 4 files changed, 531 insertions(+), 8 deletions(-) diff --git a/docs/docs/configure/telemetry.md b/docs/docs/configure/telemetry.md index 84b9acaa0c..a94b321146 100644 --- a/docs/docs/configure/telemetry.md +++ b/docs/docs/configure/telemetry.md @@ -13,7 +13,7 @@ We collect the following categories of events: | `session_forked` | A session is forked from an existing one | | `generation` | An AI model generation completes (model ID, token counts, duration — no prompt content) | | `tool_call` | A tool is invoked (tool name and category — no arguments or output) | -| `bridge_call` | A Python engine RPC call completes (method name and duration — no arguments) | +| `native_call` | A native engine call completes (method name and duration — no arguments) | | `command` | A CLI command is executed (command name only) | | `error` | An unhandled error occurs (error type and truncated message — no stack traces) | | `auth_login` | Authentication succeeds or fails (provider and method — no credentials) | @@ -33,6 +33,7 @@ We collect the following categories of events: | `error_recovered` | Successful recovery from a transient error (error type, strategy, attempt count) | | `mcp_server_census` | MCP server capabilities after connect (tool and resource counts — no tool names) | | `context_overflow_recovered` | Context overflow is handled (strategy) | +| `core_failure` | A tool failure occurs — error category, error message (truncated to 500 chars), and PII-masked arguments (string literals in SQL replaced with `?`, sensitive keys like `password`/`token`/`secret` fully redacted) | Each event includes a timestamp, anonymous session ID, and the CLI version. @@ -113,7 +114,7 @@ Event type names use **snake_case** with a `domain_action` pattern: ### Adding a New Event -1. **Define the type** — Add a new variant to the `Telemetry.Event` union in `packages/altimate-code/src/telemetry/index.ts` +1. **Define the type** — Add a new variant to the `Telemetry.Event` union in `packages/opencode/src/altimate/telemetry/index.ts` 2. **Emit the event** — Call `Telemetry.track()` at the appropriate location 3. **Update docs** — Add a row to the event table above diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 16cd4e431b..00a0021129 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -63,6 +63,7 @@ export namespace Telemetry { duration_ms: number sequence_index: number previous_tool: string | null + input_signature?: string error?: string } | { @@ -331,6 +332,141 @@ export namespace Telemetry { has_ssh_tunnel: boolean has_keychain: boolean } + | { + type: "core_failure" + timestamp: number + session_id: string + tool_name: string + tool_category: string + error_class: + | "parse_error" + | "connection" + | "timeout" + | "validation" + | "internal" + | "permission" + | "unknown" + error_message: string + input_signature: string + masked_args?: string + duration_ms: number + } + + const ERROR_PATTERNS: Array<{ + class: Telemetry.Event & { type: "core_failure" } extends { error_class: infer C } ? C : never + keywords: string[] + }> = [ + { class: "parse_error", keywords: ["parse", "syntax", "binder", "unexpected token", "sqlglot"] }, + { + class: "connection", + keywords: ["econnrefused", "connection", "socket", "enotfound", "econnreset"], + }, + { class: "timeout", keywords: ["timeout", "etimedout", "bridge timeout", "timed out"] }, + { class: "validation", keywords: ["invalid params", "invalid", "missing", "required"] }, + { class: "permission", keywords: ["permission", "denied", "unauthorized", "forbidden"] }, + { class: "internal", keywords: ["internal", "assertion"] }, + ] + + export function classifyError( + message: string, + ): Telemetry.Event & { type: "core_failure" } extends { error_class: infer C } ? C : never { + const lower = message.toLowerCase() + for (const { class: cls, keywords } of ERROR_PATTERNS) { + if (keywords.some((kw) => lower.includes(kw))) return cls + } + return "unknown" + } + + export function computeInputSignature(args: Record): string { + const sig: Record = {} + for (const [k, v] of Object.entries(args)) { + if (v === null || v === undefined) { + sig[k] = "null" + } else if (typeof v === "string") { + sig[k] = `string:${v.length}` + } else if (typeof v === "number") { + sig[k] = "number" + } else if (typeof v === "boolean") { + sig[k] = "boolean" + } else if (Array.isArray(v)) { + sig[k] = `array:${v.length}` + } else if (typeof v === "object") { + sig[k] = `object:${Object.keys(v).length}` + } else { + sig[k] = typeof v + } + } + const result = JSON.stringify(sig) + if (result.length <= 1000) return result + // Drop keys from the end until the JSON fits, preserving valid JSON structure + const keys = Object.keys(sig) + while (keys.length > 0) { + keys.pop() + const truncated: Record = {} + for (const k of keys) truncated[k] = sig[k] + truncated["..."] = `${Object.keys(sig).length - keys.length} more` + const out = JSON.stringify(truncated) + if (out.length <= 1000) return out + } + return JSON.stringify({ "...": `${Object.keys(sig).length} keys` }) + } + + // Mirrors altimate-sdk (Rust) SENSITIVE_KEYS — keep in sync. + const SENSITIVE_KEYS: string[] = [ + "key", "api_key", "apikey", "token", "access_token", "refresh_token", + "secret", "secret_key", "password", "passwd", "pwd", + "credential", "credentials", "authorization", "auth", + "signature", "sig", "private_key", "connection_string", + ] + + function isSensitiveKey(key: string): boolean { + const lower = key.toLowerCase() + return SENSITIVE_KEYS.some( + (k) => lower === k || lower.endsWith(`_${k}`) || lower.startsWith(`${k}_`), + ) + } + + // Mirrors altimate-sdk mask_string: replace '...' → ?, collapse whitespace. + function maskString(s: string): string { + return s.replace(/'(?:[^'\\]|\\.)*'/g, "?").replace(/\s+/g, " ").trim() + } + + function maskValue(value: unknown, key?: string): unknown { + if (key && isSensitiveKey(key)) return "****" + if (typeof value === "string") return maskString(value) + if (Array.isArray(value)) return value.map((v) => maskValue(v, key)) + if (value !== null && typeof value === "object") { + const masked: Record = {} + for (const [k, v] of Object.entries(value as Record)) { + masked[k] = maskValue(v, k) + } + return masked + } + return value + } + + /** PII-mask tool arguments for failure telemetry. + * Mirrors altimate-sdk mask_value: sensitive keys → "****", + * string literals in SQL → ?, whitespace collapsed. Truncates to 2000 chars. */ + export function maskArgs(args: Record): string { + const masked: Record = {} + for (const [k, v] of Object.entries(args)) { + masked[k] = maskValue(v, k) + } + const result = JSON.stringify(masked) + if (result.length <= 2000) return result + // Drop keys from the end until valid JSON fits, same approach as computeInputSignature + const keys = Object.keys(masked) + while (keys.length > 0) { + keys.pop() + const truncated: Record = {} + for (const k of keys) truncated[k] = masked[k] + truncated["..."] = `${Object.keys(masked).length - keys.length} more` + const out = JSON.stringify(truncated) + if (out.length <= 2000) return out + } + return JSON.stringify({ "...": `${Object.keys(masked).length} keys` }) + } const FILE_TOOLS = new Set(["read", "write", "edit", "glob", "grep", "bash"]) diff --git a/packages/opencode/src/tool/tool.ts b/packages/opencode/src/tool/tool.ts index 8cc7b57d85..867882ab7d 100644 --- a/packages/opencode/src/tool/tool.ts +++ b/packages/opencode/src/tool/tool.ts @@ -4,6 +4,7 @@ import type { Agent } from "../agent/agent" import type { PermissionNext } from "../permission/next" import type { SessionID, MessageID } from "../session/schema" import { Truncate } from "./truncation" +import { Telemetry } from "../altimate/telemetry" export namespace Tool { interface Metadata { @@ -67,8 +68,89 @@ export namespace Tool { { cause: error }, ) } - const result = await execute(args, ctx) - // skip truncation for tools that handle it themselves + const startTime = Date.now() + let result: Awaited> + try { + result = await execute(args, ctx) + } catch (error) { + // Telemetry must never prevent the original error from propagating + try { + const errorMsg = error instanceof Error ? error.message : String(error) + Telemetry.track({ + type: "tool_call", + timestamp: Date.now(), + session_id: ctx.sessionID, + message_id: ctx.messageID, + tool_name: id, + tool_type: "standard", + tool_category: Telemetry.categorizeToolName(id, "standard"), + status: "error", + duration_ms: Date.now() - startTime, + sequence_index: 0, + previous_tool: null, + input_signature: Telemetry.computeInputSignature(args as Record), + error: errorMsg.slice(0, 500), + }) + Telemetry.track({ + type: "core_failure", + timestamp: Date.now(), + session_id: ctx.sessionID, + tool_name: id, + tool_category: Telemetry.categorizeToolName(id, "standard"), + error_class: Telemetry.classifyError(errorMsg), + error_message: errorMsg.slice(0, 500), + input_signature: Telemetry.computeInputSignature(args as Record), + masked_args: Telemetry.maskArgs(args as Record), + duration_ms: Date.now() - startTime, + }) + } catch { + // Telemetry failure must never mask the original tool error + } + throw error + } + // Telemetry runs after execute() succeeds — wrapped so it never breaks the tool + try { + const isSoftFailure = result.metadata?.success === false + const durationMs = Date.now() - startTime + Telemetry.track({ + type: "tool_call", + timestamp: Date.now(), + session_id: ctx.sessionID, + message_id: ctx.messageID, + tool_name: id, + tool_type: "standard", + tool_category: Telemetry.categorizeToolName(id, "standard"), + status: isSoftFailure ? "error" : "success", + duration_ms: durationMs, + sequence_index: 0, + previous_tool: null, + ...(isSoftFailure && { + input_signature: Telemetry.computeInputSignature(args as Record), + }), + }) + if (isSoftFailure) { + const errorMsg = + typeof result.metadata?.error === "string" + ? result.metadata.error + : result.output?.slice(0, 500) || "unknown error" + Telemetry.track({ + type: "core_failure", + timestamp: Date.now(), + session_id: ctx.sessionID, + tool_name: id, + tool_category: Telemetry.categorizeToolName(id, "standard"), + error_class: Telemetry.classifyError(errorMsg), + error_message: errorMsg.slice(0, 500), + input_signature: Telemetry.computeInputSignature(args as Record), + masked_args: Telemetry.maskArgs(args as Record), + duration_ms: durationMs, + }) + } + } catch { + // Telemetry must never break tool execution + } + // Truncation runs after telemetry so I/O errors from + // Truncate.output() are not misattributed as tool failures. if (result.metadata.truncated !== undefined) { return result } diff --git a/packages/opencode/test/telemetry/telemetry.test.ts b/packages/opencode/test/telemetry/telemetry.test.ts index ef905a9b62..c7c5757365 100644 --- a/packages/opencode/test/telemetry/telemetry.test.ts +++ b/packages/opencode/test/telemetry/telemetry.test.ts @@ -193,7 +193,7 @@ describe("telemetry.context", () => { }) // --------------------------------------------------------------------------- -// 5. Event type completeness — all 25 event types +// 5. Event type completeness — all 33 event types // --------------------------------------------------------------------------- describe("telemetry.event-types", () => { test("all event types are valid", () => { @@ -202,7 +202,7 @@ describe("telemetry.event-types", () => { "session_end", "generation", "tool_call", - "bridge_call", + "native_call", "error", "command", "context_overflow_recovered", @@ -223,8 +223,16 @@ describe("telemetry.event-types", () => { "agent_outcome", "error_recovered", "mcp_server_census", + "memory_operation", + "memory_injection", + "warehouse_connect", + "warehouse_query", + "warehouse_introspection", + "warehouse_discovery", + "warehouse_census", + "core_failure", ] - expect(eventTypes.length).toBe(25) + expect(eventTypes.length).toBe(33) }) }) @@ -315,7 +323,7 @@ describe("telemetry.naming-convention", () => { "session_end", "generation", "tool_call", - "bridge_call", + "native_call", "error", "command", "context_overflow_recovered", @@ -336,6 +344,14 @@ describe("telemetry.naming-convention", () => { "agent_outcome", "error_recovered", "mcp_server_census", + "memory_operation", + "memory_injection", + "warehouse_connect", + "warehouse_query", + "warehouse_introspection", + "warehouse_discovery", + "warehouse_census", + "core_failure", ] for (const t of types) { expect(t).toMatch(/^[a-z][a-z0-9_]*$/) @@ -1503,3 +1519,291 @@ describe("Telemetry.isEnabled()", () => { } }) }) + +// --------------------------------------------------------------------------- +// 15. classifyError +// --------------------------------------------------------------------------- +describe("telemetry.classifyError", () => { + test("classifies parse errors", () => { + expect(Telemetry.classifyError("ParseError: unexpected token")).toBe("parse_error") + expect(Telemetry.classifyError("SyntaxError in SQL")).toBe("parse_error") + expect(Telemetry.classifyError("binder error on column")).toBe("parse_error") + expect(Telemetry.classifyError("sqlglot transpilation failed")).toBe("parse_error") + }) + + test("classifies connection errors", () => { + expect(Telemetry.classifyError("ECONNREFUSED 127.0.0.1:5432")).toBe("connection") + expect(Telemetry.classifyError("Connection refused by host")).toBe("connection") + expect(Telemetry.classifyError("Socket hang up")).toBe("connection") + expect(Telemetry.classifyError("ENOTFOUND db.example.com")).toBe("connection") + expect(Telemetry.classifyError("ECONNRESET")).toBe("connection") + }) + + test("classifies timeout errors", () => { + expect(Telemetry.classifyError("Request timeout after 30s")).toBe("timeout") + expect(Telemetry.classifyError("ETIMEDOUT")).toBe("timeout") + expect(Telemetry.classifyError("Bridge timeout waiting for response")).toBe("timeout") + expect(Telemetry.classifyError("Operation timed out")).toBe("timeout") + }) + + test("classifies validation errors", () => { + expect(Telemetry.classifyError("Invalid params: missing 'sql'")).toBe("validation") + expect(Telemetry.classifyError("Invalid dialect specified")).toBe("validation") + expect(Telemetry.classifyError("Missing required field")).toBe("validation") + expect(Telemetry.classifyError("Required parameter 'query' not provided")).toBe("validation") + }) + + test("classifies permission errors", () => { + expect(Telemetry.classifyError("Permission denied on table")).toBe("permission") + expect(Telemetry.classifyError("Access denied for user")).toBe("permission") + expect(Telemetry.classifyError("Unauthorized access to resource")).toBe("permission") + expect(Telemetry.classifyError("403 Forbidden")).toBe("permission") + }) + + test("classifies internal errors", () => { + expect(Telemetry.classifyError("Internal server error")).toBe("internal") + expect(Telemetry.classifyError("Assertion failed: x > 0")).toBe("internal") + }) + + test("returns unknown for unrecognized errors", () => { + expect(Telemetry.classifyError("Something went wrong")).toBe("unknown") + expect(Telemetry.classifyError("")).toBe("unknown") + expect(Telemetry.classifyError("42")).toBe("unknown") + }) + + test("is case-insensitive", () => { + expect(Telemetry.classifyError("PARSEERROR")).toBe("parse_error") + expect(Telemetry.classifyError("CONNECTION REFUSED")).toBe("connection") + expect(Telemetry.classifyError("TIMEOUT")).toBe("timeout") + }) +}) + +// --------------------------------------------------------------------------- +// 16. computeInputSignature +// --------------------------------------------------------------------------- +describe("telemetry.computeInputSignature", () => { + test("records string values as type:length", () => { + const sig = Telemetry.computeInputSignature({ sql: "SELECT 1", dialect: "snowflake" }) + const parsed = JSON.parse(sig) + expect(parsed.sql).toBe("string:8") + expect(parsed.dialect).toBe("string:9") + }) + + test("records number values as 'number'", () => { + const sig = Telemetry.computeInputSignature({ limit: 100, offset: 0 }) + const parsed = JSON.parse(sig) + expect(parsed.limit).toBe("number") + expect(parsed.offset).toBe("number") + }) + + test("records boolean values as 'boolean'", () => { + const sig = Telemetry.computeInputSignature({ verbose: true, dry_run: false }) + const parsed = JSON.parse(sig) + expect(parsed.verbose).toBe("boolean") + expect(parsed.dry_run).toBe("boolean") + }) + + test("records array values as array:length", () => { + const sig = Telemetry.computeInputSignature({ tables: ["a", "b", "c"] }) + const parsed = JSON.parse(sig) + expect(parsed.tables).toBe("array:3") + }) + + test("records object values as object:keyCount", () => { + const sig = Telemetry.computeInputSignature({ config: { a: 1, b: 2 } }) + const parsed = JSON.parse(sig) + expect(parsed.config).toBe("object:2") + }) + + test("records null and undefined as 'null'", () => { + const sig = Telemetry.computeInputSignature({ a: null, b: undefined }) + const parsed = JSON.parse(sig) + expect(parsed.a).toBe("null") + expect(parsed.b).toBe("null") + }) + + test("never includes actual values — only key names and type descriptors", () => { + const sig = Telemetry.computeInputSignature({ + sql: "SELECT password FROM users WHERE email = 'admin@example.com'", + secret: "super-secret-123", + api_key: "sk-abc123", + }) + // Key names ARE included (that's by design), but actual values are NOT + expect(sig).not.toContain("super-secret") + expect(sig).not.toContain("sk-abc123") + expect(sig).not.toContain("SELECT") + expect(sig).not.toContain("admin@example.com") + // Only type descriptors appear as values + const parsed = JSON.parse(sig) + expect(parsed.sql).toBe("string:60") + expect(parsed.secret).toBe("string:16") + expect(parsed.api_key).toBe("string:9") + }) + + test("truncates output at 1000 chars with valid JSON", () => { + // Create args with many long key names to exceed 1000 chars + const args: Record = {} + for (let i = 0; i < 100; i++) { + args[`very_long_key_name_for_testing_truncation_${i}`] = "value" + } + const sig = Telemetry.computeInputSignature(args) + expect(sig.length).toBeLessThanOrEqual(1000) + // Must produce valid JSON even when truncated + const parsed = JSON.parse(sig) + expect(parsed["..."]).toBeDefined() + expect(typeof parsed["..."]).toBe("string") + }) + + test("returns valid JSON for empty input", () => { + const sig = Telemetry.computeInputSignature({}) + expect(JSON.parse(sig)).toEqual({}) + }) +}) + +// --------------------------------------------------------------------------- +// 17. core_failure event privacy +// --------------------------------------------------------------------------- +describe("telemetry.core_failure privacy", () => { + test("core_failure event contains no raw SQL or argument values", () => { + const event: Telemetry.Event = { + type: "core_failure", + timestamp: Date.now(), + session_id: "test", + tool_name: "sql_analyze", + tool_category: "sql", + error_class: "parse_error", + error_message: "ParseError: unexpected token near SELECT".slice(0, 500), + input_signature: Telemetry.computeInputSignature({ + sql: "SELECT * FROM users WHERE id = 1", + dialect: "snowflake", + }), + duration_ms: 42, + } + // input_signature has types/lengths, not values + expect(event.input_signature).not.toContain("SELECT") + expect(event.input_signature).not.toContain("users") + expect(event.input_signature).not.toContain("snowflake") + const parsed = JSON.parse(event.input_signature) + expect(parsed.sql).toBe("string:32") + expect(parsed.dialect).toBe("string:9") + }) + + test("error_message is capped at 500 chars", () => { + const longError = "x".repeat(1000) + const event: Telemetry.Event = { + type: "core_failure", + timestamp: Date.now(), + session_id: "test", + tool_name: "sql_analyze", + tool_category: "sql", + error_class: "parse_error", + error_message: longError.slice(0, 500), + input_signature: "{}", + duration_ms: 0, + } + expect(event.error_message.length).toBe(500) + }) + + test("core_failure event does NOT include raw arguments, outputs, or SQL", () => { + const event: Telemetry.Event = { + type: "core_failure", + timestamp: Date.now(), + session_id: "test", + tool_name: "sql_analyze", + tool_category: "sql", + error_class: "unknown", + error_message: "test error", + input_signature: '{"sql":"string:10"}', + duration_ms: 100, + } + expect("args" in event).toBe(false) + expect("input" in event).toBe(false) + expect("output" in event).toBe(false) + expect("sql" in event).toBe(false) + expect("query" in event).toBe(false) + }) +}) + +// --------------------------------------------------------------------------- +// 18. maskArgs +// --------------------------------------------------------------------------- +describe("telemetry.maskArgs", () => { + test("replaces string literals in SQL with ?", () => { + const masked = Telemetry.maskArgs({ + sql: "SELECT * FROM users WHERE name = 'John Doe' AND email = 'john@example.com'", + }) + const parsed = JSON.parse(masked) + expect(parsed.sql).toContain("SELECT * FROM users WHERE name = ") + expect(parsed.sql).not.toContain("John Doe") + expect(parsed.sql).not.toContain("john@example.com") + expect(parsed.sql).toContain("?") + }) + + test("redacts sensitive keys entirely", () => { + const masked = Telemetry.maskArgs({ + sql: "SELECT 1", + password: "super-secret-123", + api_key: "sk-abc123", + token: "eyJhbGciOi...", + secret: "my-secret", + }) + const parsed = JSON.parse(masked) + expect(parsed.password).toBe("****") + expect(parsed.api_key).toBe("****") + expect(parsed.token).toBe("****") + expect(parsed.secret).toBe("****") + expect(parsed.sql).toBe("SELECT 1") + }) + + test("preserves SQL structure while masking literals", () => { + const masked = Telemetry.maskArgs({ + sql: "INSERT INTO orders (name, total) VALUES ('Alice', 99.99)", + }) + const parsed = JSON.parse(masked) + expect(parsed.sql).toContain("INSERT INTO orders") + expect(parsed.sql).toContain("VALUES (") + expect(parsed.sql).not.toContain("Alice") + }) + + test("handles escaped quotes in SQL strings", () => { + const masked = Telemetry.maskArgs({ + sql: "SELECT * FROM t WHERE name = 'O\\'Brien'", + }) + const parsed = JSON.parse(masked) + expect(parsed.sql).not.toContain("Brien") + }) + + test("preserves non-string values as-is", () => { + const masked = Telemetry.maskArgs({ + dialect: "snowflake", + depth: "full", + limit: 100, + verbose: true, + }) + const parsed = JSON.parse(masked) + expect(parsed.dialect).toBe("snowflake") + expect(parsed.depth).toBe("full") + expect(parsed.limit).toBe(100) + expect(parsed.verbose).toBe(true) + }) + + test("truncates output at 2000 chars with valid JSON", () => { + const args: Record = {} + for (let i = 0; i < 200; i++) { + args[`long_key_name_for_testing_truncation_${i}`] = "SELECT " + "a".repeat(100) + } + const masked = Telemetry.maskArgs(args) + expect(masked.length).toBeLessThanOrEqual(2000) + // Must produce valid JSON even when truncated + const parsed = JSON.parse(masked) + expect(parsed["..."]).toBeDefined() + }) + + test("redacts connection_string key", () => { + const masked = Telemetry.maskArgs({ + connection_string: "Server=db.prod.internal;Password=hunter2", + }) + const parsed = JSON.parse(masked) + expect(parsed.connection_string).toBe("****") + }) +}) From 2e73f2e830033693552bb6ece7412070d7903813 Mon Sep 17 00:00:00 2001 From: suryaiyer95 Date: Wed, 18 Mar 2026 16:44:09 -0700 Subject: [PATCH 02/13] feat: add `skill_used` telemetry event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tracks which skill is loaded and where it came from (`builtin`, `global`, or `project`) with duration. Wrapped in try/catch — cannot break skill loading. Docs table updated. Co-Authored-By: Claude Sonnet 4.6 --- docs/docs/configure/telemetry.md | 1 + packages/drivers/src/sqlserver.ts | 1 - .../opencode/src/altimate/telemetry/index.ts | 9 ++++++++ packages/opencode/src/tool/skill.ts | 23 +++++++++++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/docs/configure/telemetry.md b/docs/docs/configure/telemetry.md index a94b321146..f53c5f02dc 100644 --- a/docs/docs/configure/telemetry.md +++ b/docs/docs/configure/telemetry.md @@ -33,6 +33,7 @@ We collect the following categories of events: | `error_recovered` | Successful recovery from a transient error (error type, strategy, attempt count) | | `mcp_server_census` | MCP server capabilities after connect (tool and resource counts — no tool names) | | `context_overflow_recovered` | Context overflow is handled (strategy) | +| `skill_used` | A skill is loaded (skill name and source — `builtin`, `global`, or `project` — no skill content) | | `core_failure` | A tool failure occurs — error category, error message (truncated to 500 chars), and PII-masked arguments (string literals in SQL replaced with `?`, sensitive keys like `password`/`token`/`secret` fully redacted) | Each event includes a timestamp, anonymous session ID, and the CLI version. diff --git a/packages/drivers/src/sqlserver.ts b/packages/drivers/src/sqlserver.ts index 387a65cf73..2290887194 100644 --- a/packages/drivers/src/sqlserver.ts +++ b/packages/drivers/src/sqlserver.ts @@ -7,7 +7,6 @@ import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from export async function connect(config: ConnectionConfig): Promise { let mssql: any try { - // @ts-expect-error — optional dependency, loaded at runtime mssql = await import("mssql") mssql = mssql.default || mssql } catch { diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 00a0021129..a205d25846 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -332,6 +332,15 @@ export namespace Telemetry { has_ssh_tunnel: boolean has_keychain: boolean } + | { + type: "skill_used" + timestamp: number + session_id: string + message_id: string + skill_name: string + skill_source: "builtin" | "global" | "project" + duration_ms: number + } | { type: "core_failure" timestamp: number diff --git a/packages/opencode/src/tool/skill.ts b/packages/opencode/src/tool/skill.ts index e0c033bdbf..781e4bcd31 100644 --- a/packages/opencode/src/tool/skill.ts +++ b/packages/opencode/src/tool/skill.ts @@ -9,8 +9,16 @@ import { iife } from "@/util/iife" import { Fingerprint } from "../altimate/fingerprint" import { Config } from "../config/config" import { selectSkillsWithLLM } from "../altimate/skill-selector" +import { Telemetry } from "../altimate/telemetry" +import os from "os" const MAX_DISPLAY_SKILLS = 50 + +function classifySkillSource(location: string): "builtin" | "global" | "project" { + if (location.includes("node_modules") || location.includes(".altimate/builtin")) return "builtin" + if (location.startsWith(os.homedir())) return "global" + return "project" +} // altimate_change end export const SkillTool = Tool.define("skill", async (ctx) => { @@ -83,6 +91,7 @@ export const SkillTool = Tool.define("skill", async (ctx) => { description, parameters, async execute(params: z.infer, ctx) { + const startTime = Date.now() // altimate_change start - use upstream Skill.get() for exact name lookup const skill = await Skill.get(params.name) @@ -122,6 +131,20 @@ export const SkillTool = Tool.define("skill", async (ctx) => { return arr }).then((f) => f.map((file) => `${file}`).join("\n")) + try { + Telemetry.track({ + type: "skill_used", + timestamp: Date.now(), + session_id: ctx.sessionID, + message_id: ctx.messageID, + skill_name: skill.name, + skill_source: classifySkillSource(skill.location), + duration_ms: Date.now() - startTime, + }) + } catch { + // Telemetry must never break skill loading + } + return { title: `Loaded skill: ${skill.name}`, output: [ From ef1d1a7f52125325de8499455d1053c7c993be64 Mon Sep 17 00:00:00 2001 From: suryaiyer95 Date: Wed, 18 Mar 2026 17:04:14 -0700 Subject: [PATCH 03/13] feat: add \`sql_execute_failure\` telemetry for SQL execution errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit \`core_failure\` is for internal tool failures. SQL execution via the dispatcher is a separate concern — soft errors are returned as results (not thrown), so \`core_failure\` never fires for them. New \`sql_execute_failure\` event captures: warehouse type, query type, error message (truncated to 500 chars), and PII-masked SQL. Fires from the \`sql.execute\` handler catch path. Co-Authored-By: Claude Sonnet 4.6 --- docs/docs/configure/telemetry.md | 1 + .../src/altimate/native/connections/register.ts | 15 +++++++++++++-- packages/opencode/src/altimate/telemetry/index.ts | 10 ++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/docs/docs/configure/telemetry.md b/docs/docs/configure/telemetry.md index f53c5f02dc..0bd39bcc02 100644 --- a/docs/docs/configure/telemetry.md +++ b/docs/docs/configure/telemetry.md @@ -34,6 +34,7 @@ We collect the following categories of events: | `mcp_server_census` | MCP server capabilities after connect (tool and resource counts — no tool names) | | `context_overflow_recovered` | Context overflow is handled (strategy) | | `skill_used` | A skill is loaded (skill name and source — `builtin`, `global`, or `project` — no skill content) | +| `sql_execute_failure` | A SQL execution fails (warehouse type, query type, error message, PII-masked SQL — no raw values) | | `core_failure` | A tool failure occurs — error category, error message (truncated to 500 chars), and PII-masked arguments (string literals in SQL replaced with `?`, sensitive keys like `password`/`token`/`secret` fully redacted) | Each event includes a timestamp, anonymous session ID, and the CLI version. diff --git a/packages/opencode/src/altimate/native/connections/register.ts b/packages/opencode/src/altimate/native/connections/register.ts index 18d029197b..cf71496655 100644 --- a/packages/opencode/src/altimate/native/connections/register.ts +++ b/packages/opencode/src/altimate/native/connections/register.ts @@ -228,6 +228,7 @@ register("sql.execute", async (params: SqlExecuteParams): Promise Date: Wed, 18 Mar 2026 17:34:41 -0700 Subject: [PATCH 04/13] feat: add persistent machine ID from \`~/.altimate/machine-id\` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generated once as a random UUID and stored at \`~/.altimate/machine-id\` (alongside \`altimate.json\`, \`connections.json\`, etc.). Sent as \`machine_id\` in \`customDimensions\` on every App Insights event. No PII — pure random UUID, never tied to user identity. Co-Authored-By: Claude Sonnet 4.6 --- .../opencode/src/altimate/telemetry/index.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 1f806b689f..b054e7198e 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -2,7 +2,10 @@ import { Account } from "@/account" import { Config } from "@/config/config" import { Installation } from "@/installation" import { Log } from "@/util/log" -import { createHash } from "crypto" +import { createHash, randomUUID } from "crypto" +import fs from "fs" +import path from "path" +import os from "os" const log = Log.create({ service: "telemetry" }) @@ -528,6 +531,7 @@ export namespace Telemetry { let buffer: Event[] = [] let flushTimer: ReturnType | undefined let userEmail = "" + let machineId = "" let sessionId = "" let projectId = "" let appInsights: AppInsightsConfig | undefined @@ -557,6 +561,7 @@ export namespace Telemetry { const properties: Record = { cli_version: Installation.VERSION, project_id: fields.project_id ?? projectId, + ...(machineId && { machine_id: machineId }), } const measurements: Record = {} @@ -645,6 +650,18 @@ export namespace Telemetry { } catch { // Account unavailable — proceed without user ID } + try { + const machineIdPath = path.join(os.homedir(), ".altimate", "machine-id") + try { + machineId = fs.readFileSync(machineIdPath, "utf8").trim() + } catch { + machineId = randomUUID() + fs.mkdirSync(path.dirname(machineIdPath), { recursive: true }) + fs.writeFileSync(machineIdPath, machineId, "utf8") + } + } catch { + // Machine ID unavailable — proceed without it + } enabled = true log.info("telemetry initialized", { mode: "appinsights" }) const timer = setInterval(flush, FLUSH_INTERVAL_MS) @@ -746,6 +763,7 @@ export namespace Telemetry { droppedEvents = 0 sessionId = "" projectId = "" + machineId = "" initPromise = undefined initDone = false } From eb55714e1a4a14eb3b94a2155fc4b72f164919ee Mon Sep 17 00:00:00 2001 From: suryaiyer95 Date: Wed, 18 Mar 2026 18:10:48 -0700 Subject: [PATCH 05/13] fix: correct `masked_sql` field and `ERROR_PATTERNS` ordering in telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `sql_execute_failure`: use `Telemetry.maskString(params.sql)` instead of `Telemetry.maskArgs({ sql: params.sql })` — the latter serializes a JSON object string `{"sql":"..."}` rather than the raw masked SQL - `ERROR_PATTERNS`: move `permission` before `validation` so errors like "Invalid permission denied" are not misclassified as `validation_error` Co-Authored-By: Claude Sonnet 4.6 --- packages/opencode/src/altimate/native/connections/register.ts | 2 +- packages/opencode/src/altimate/telemetry/index.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/altimate/native/connections/register.ts b/packages/opencode/src/altimate/native/connections/register.ts index cf71496655..e524e427ee 100644 --- a/packages/opencode/src/altimate/native/connections/register.ts +++ b/packages/opencode/src/altimate/native/connections/register.ts @@ -250,7 +250,7 @@ register("sql.execute", async (params: SqlExecuteParams): Promise Date: Wed, 18 Mar 2026 19:42:52 -0700 Subject: [PATCH 06/13] perf: skip success \`tool_call\` telemetry for file tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read/write/edit/glob/grep/bash succeed constantly in normal operation — tracking every success is high-volume noise with no actionable signal. Failures (hard throws and soft failures) are still fully captured via \`tool_call\` (status=error) and \`core_failure\`. Co-Authored-By: Claude Sonnet 4.6 --- packages/opencode/src/tool/tool.ts | 37 +++++++++++++++++------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/packages/opencode/src/tool/tool.ts b/packages/opencode/src/tool/tool.ts index 867882ab7d..bfd7744e5d 100644 --- a/packages/opencode/src/tool/tool.ts +++ b/packages/opencode/src/tool/tool.ts @@ -112,22 +112,27 @@ export namespace Tool { try { const isSoftFailure = result.metadata?.success === false const durationMs = Date.now() - startTime - Telemetry.track({ - type: "tool_call", - timestamp: Date.now(), - session_id: ctx.sessionID, - message_id: ctx.messageID, - tool_name: id, - tool_type: "standard", - tool_category: Telemetry.categorizeToolName(id, "standard"), - status: isSoftFailure ? "error" : "success", - duration_ms: durationMs, - sequence_index: 0, - previous_tool: null, - ...(isSoftFailure && { - input_signature: Telemetry.computeInputSignature(args as Record), - }), - }) + const toolCategory = Telemetry.categorizeToolName(id, "standard") + // Skip success tool_call for file tools (read/write/edit/glob/grep/bash) — high + // volume, low signal. Failures are still captured via core_failure below. + if (isSoftFailure || toolCategory !== "file") { + Telemetry.track({ + type: "tool_call", + timestamp: Date.now(), + session_id: ctx.sessionID, + message_id: ctx.messageID, + tool_name: id, + tool_type: "standard", + tool_category: toolCategory, + status: isSoftFailure ? "error" : "success", + duration_ms: durationMs, + sequence_index: 0, + previous_tool: null, + ...(isSoftFailure && { + input_signature: Telemetry.computeInputSignature(args as Record), + }), + }) + } if (isSoftFailure) { const errorMsg = typeof result.metadata?.error === "string" From 025515705ec0f2841850f745e17bc215e7a60e91 Mon Sep 17 00:00:00 2001 From: suryaiyer95 Date: Wed, 18 Mar 2026 20:06:32 -0700 Subject: [PATCH 07/13] docs: clarify `core_failure` event description in telemetry docs Co-Authored-By: Claude Sonnet 4.6 --- docs/docs/configure/telemetry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/configure/telemetry.md b/docs/docs/configure/telemetry.md index 0bd39bcc02..1b75da36bc 100644 --- a/docs/docs/configure/telemetry.md +++ b/docs/docs/configure/telemetry.md @@ -35,7 +35,7 @@ We collect the following categories of events: | `context_overflow_recovered` | Context overflow is handled (strategy) | | `skill_used` | A skill is loaded (skill name and source — `builtin`, `global`, or `project` — no skill content) | | `sql_execute_failure` | A SQL execution fails (warehouse type, query type, error message, PII-masked SQL — no raw values) | -| `core_failure` | A tool failure occurs — error category, error message (truncated to 500 chars), and PII-masked arguments (string literals in SQL replaced with `?`, sensitive keys like `password`/`token`/`secret` fully redacted) | +| `core_failure` | An internal library error occurs — function name, error category, error message (truncated to 500 chars), and PII-masked arguments (string literals replaced with `?`, sensitive keys fully redacted) | Each event includes a timestamp, anonymous session ID, and the CLI version. From 94ac4ad1facba65c85c71a718e6c9df0bba9adf2 Mon Sep 17 00:00:00 2001 From: suryaiyer95 Date: Wed, 18 Mar 2026 20:06:54 -0700 Subject: [PATCH 08/13] docs: simplify `core_failure` description Co-Authored-By: Claude Sonnet 4.6 --- docs/docs/configure/telemetry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/configure/telemetry.md b/docs/docs/configure/telemetry.md index 1b75da36bc..66438c98bc 100644 --- a/docs/docs/configure/telemetry.md +++ b/docs/docs/configure/telemetry.md @@ -35,7 +35,7 @@ We collect the following categories of events: | `context_overflow_recovered` | Context overflow is handled (strategy) | | `skill_used` | A skill is loaded (skill name and source — `builtin`, `global`, or `project` — no skill content) | | `sql_execute_failure` | A SQL execution fails (warehouse type, query type, error message, PII-masked SQL — no raw values) | -| `core_failure` | An internal library error occurs — function name, error category, error message (truncated to 500 chars), and PII-masked arguments (string literals replaced with `?`, sensitive keys fully redacted) | +| `core_failure` | An internal library error occurs — function name, error category, error message (truncated to 500 chars), and arguments with PII and sensitive fields masked | Each event includes a timestamp, anonymous session ID, and the CLI version. From 2625f42a2ec5e3b48fc90e43392b1f4419d07b4e Mon Sep 17 00:00:00 2001 From: suryaiyer95 Date: Wed, 18 Mar 2026 20:12:46 -0700 Subject: [PATCH 09/13] fix: mask error messages before sending to telemetry Error messages from SQL engines can embed data values (e.g. "Value 'john@email.com' does not match type INTEGER"). Apply maskString() to all error_message fields before transmission, consistent with how args are already masked. Affects: core_failure (tool.ts), sql_execute_failure (register.ts) Co-Authored-By: Claude Sonnet 4.6 --- docs/docs/configure/telemetry.md | 2 +- .../src/altimate/native/connections/register.ts | 5 +++-- packages/opencode/src/tool/tool.ts | 10 ++++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/docs/configure/telemetry.md b/docs/docs/configure/telemetry.md index 66438c98bc..6df5a05f81 100644 --- a/docs/docs/configure/telemetry.md +++ b/docs/docs/configure/telemetry.md @@ -35,7 +35,7 @@ We collect the following categories of events: | `context_overflow_recovered` | Context overflow is handled (strategy) | | `skill_used` | A skill is loaded (skill name and source — `builtin`, `global`, or `project` — no skill content) | | `sql_execute_failure` | A SQL execution fails (warehouse type, query type, error message, PII-masked SQL — no raw values) | -| `core_failure` | An internal library error occurs — function name, error category, error message (truncated to 500 chars), and arguments with PII and sensitive fields masked | +| `core_failure` | An internal library error occurs — function name, error category, and error message and arguments with PII and sensitive fields masked | Each event includes a timestamp, anonymous session ID, and the CLI version. diff --git a/packages/opencode/src/altimate/native/connections/register.ts b/packages/opencode/src/altimate/native/connections/register.ts index e524e427ee..e960ff0f3e 100644 --- a/packages/opencode/src/altimate/native/connections/register.ts +++ b/packages/opencode/src/altimate/native/connections/register.ts @@ -229,6 +229,7 @@ register("sql.execute", async (params: SqlExecuteParams): Promise), - error: errorMsg.slice(0, 500), + error: maskedErrorMsg, }) Telemetry.track({ type: "core_failure", @@ -98,7 +99,7 @@ export namespace Tool { tool_name: id, tool_category: Telemetry.categorizeToolName(id, "standard"), error_class: Telemetry.classifyError(errorMsg), - error_message: errorMsg.slice(0, 500), + error_message: maskedErrorMsg, input_signature: Telemetry.computeInputSignature(args as Record), masked_args: Telemetry.maskArgs(args as Record), duration_ms: Date.now() - startTime, @@ -137,7 +138,8 @@ export namespace Tool { const errorMsg = typeof result.metadata?.error === "string" ? result.metadata.error - : result.output?.slice(0, 500) || "unknown error" + : result.output || "unknown error" + const maskedErrorMsg = Telemetry.maskString(errorMsg).slice(0, 500) Telemetry.track({ type: "core_failure", timestamp: Date.now(), @@ -145,7 +147,7 @@ export namespace Tool { tool_name: id, tool_category: Telemetry.categorizeToolName(id, "standard"), error_class: Telemetry.classifyError(errorMsg), - error_message: errorMsg.slice(0, 500), + error_message: maskedErrorMsg, input_signature: Telemetry.computeInputSignature(args as Record), masked_args: Telemetry.maskArgs(args as Record), duration_ms: durationMs, From c95f7b314ea8f2662e7c9471dc561f5c21fcc328 Mon Sep 17 00:00:00 2001 From: suryaiyer95 Date: Wed, 18 Mar 2026 20:38:24 -0700 Subject: [PATCH 10/13] fix: security hardening for telemetry PII safety MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Mask error messages in `native_call` (dispatcher.ts) and `warehouse_connect` (registry.ts) — these were sending raw error strings that could embed credentials or query fragments - Fix soft-failure `error_message` fallback: drop `result.output` as a source (raw tool output could contain file contents or secrets); fall back to `"unknown error"` instead - Strip `_retried` internal flag from App Insights payload — was leaking into `properties` on retried events - Add camelCase variants to `SENSITIVE_KEYS` (`authToken`, `bearerToken`, `jwtSecret`, etc.) — underscore prefix/suffix matching missed these - Document `machine_id` in telemetry privacy docs Co-Authored-By: Claude Sonnet 4.6 --- docs/docs/configure/telemetry.md | 2 +- .../opencode/src/altimate/native/connections/registry.ts | 2 +- packages/opencode/src/altimate/native/dispatcher.ts | 2 +- packages/opencode/src/altimate/telemetry/index.ts | 7 +++++-- packages/opencode/src/tool/tool.ts | 2 +- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/docs/configure/telemetry.md b/docs/docs/configure/telemetry.md index 6df5a05f81..e69abbff45 100644 --- a/docs/docs/configure/telemetry.md +++ b/docs/docs/configure/telemetry.md @@ -37,7 +37,7 @@ We collect the following categories of events: | `sql_execute_failure` | A SQL execution fails (warehouse type, query type, error message, PII-masked SQL — no raw values) | | `core_failure` | An internal library error occurs — function name, error category, and error message and arguments with PII and sensitive fields masked | -Each event includes a timestamp, anonymous session ID, and the CLI version. +Each event includes a timestamp, anonymous session ID, CLI version, and an anonymous machine ID (a random UUID stored in `~/.altimate/machine-id`, generated once and never tied to any personal information). ## Delivery & Reliability diff --git a/packages/opencode/src/altimate/native/connections/registry.ts b/packages/opencode/src/altimate/native/connections/registry.ts index f0826d0576..67fb35f271 100644 --- a/packages/opencode/src/altimate/native/connections/registry.ts +++ b/packages/opencode/src/altimate/native/connections/registry.ts @@ -291,7 +291,7 @@ export async function get(name: string): Promise { auth_method: detectAuthMethod(config), success: false, duration_ms: Date.now() - startTime, - error: String(e).slice(0, 500), + error: Telemetry.maskString(String(e)).slice(0, 500), error_category: categorizeConnectionError(e), }) } catch {} diff --git a/packages/opencode/src/altimate/native/dispatcher.ts b/packages/opencode/src/altimate/native/dispatcher.ts index 346a77e2b4..944647630a 100644 --- a/packages/opencode/src/altimate/native/dispatcher.ts +++ b/packages/opencode/src/altimate/native/dispatcher.ts @@ -75,7 +75,7 @@ export async function call( method: method as string, status: "error", duration_ms: Date.now() - startTime, - error: String(e).slice(0, 500), + error: Telemetry.maskString(String(e)).slice(0, 500), }) } catch { // Telemetry must never prevent error propagation diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 498aadea03..5ccb33685b 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -435,10 +435,13 @@ export namespace Telemetry { // Mirrors altimate-sdk (Rust) SENSITIVE_KEYS — keep in sync. const SENSITIVE_KEYS: string[] = [ - "key", "api_key", "apikey", "token", "access_token", "refresh_token", + "key", "api_key", "apikey", "apiKey", "token", "access_token", "refresh_token", "secret", "secret_key", "password", "passwd", "pwd", "credential", "credentials", "authorization", "auth", "signature", "sig", "private_key", "connection_string", + // camelCase variants not caught by prefix/suffix matching + "authtoken", "accesstoken", "refreshtoken", "bearertoken", "jwttoken", + "jwtsecret", "clientsecret", "appsecret", ] function isSensitiveKey(key: string): boolean { @@ -567,7 +570,7 @@ export namespace Telemetry { // Flatten all fields — nested `tokens` object gets prefixed keys for (const [k, v] of Object.entries(fields)) { - if (k === "session_id" || k === "project_id") continue + if (k === "session_id" || k === "project_id" || k === "_retried") continue if (k === "tokens" && typeof v === "object" && v !== null) { for (const [tk, tv] of Object.entries(v as Record)) { if (typeof tv === "number") measurements[`tokens_${tk}`] = tv diff --git a/packages/opencode/src/tool/tool.ts b/packages/opencode/src/tool/tool.ts index 26af6a3aab..eb527c3a0d 100644 --- a/packages/opencode/src/tool/tool.ts +++ b/packages/opencode/src/tool/tool.ts @@ -138,7 +138,7 @@ export namespace Tool { const errorMsg = typeof result.metadata?.error === "string" ? result.metadata.error - : result.output || "unknown error" + : "unknown error" const maskedErrorMsg = Telemetry.maskString(errorMsg).slice(0, 500) Telemetry.track({ type: "core_failure", From 0a9c4339eff9526cd4f0ed01c82eafb8828558cf Mon Sep 17 00:00:00 2001 From: suryaiyer95 Date: Wed, 18 Mar 2026 21:22:32 -0700 Subject: [PATCH 11/13] fix: address major review findings in telemetry PII masking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extend `maskString` to also mask double-quoted strings (`"John"`, `$$secret$$`-adjacent) — single-quoted-only regex was flagged as PII leak - Keep `connection` in `ERROR_PATTERNS` keywords (broad but intentional) - Truncate `masked_sql` to 2000 chars before sending — was unbounded unlike `error_message` (500) and `masked_args` (2000) Co-Authored-By: Claude Sonnet 4.6 --- .../opencode/src/altimate/native/connections/register.ts | 2 +- packages/opencode/src/altimate/telemetry/index.ts | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/altimate/native/connections/register.ts b/packages/opencode/src/altimate/native/connections/register.ts index e960ff0f3e..7267a142d0 100644 --- a/packages/opencode/src/altimate/native/connections/register.ts +++ b/packages/opencode/src/altimate/native/connections/register.ts @@ -251,7 +251,7 @@ register("sql.execute", async (params: SqlExecuteParams): Promise Date: Wed, 18 Mar 2026 21:26:22 -0700 Subject: [PATCH 12/13] docs: update `core_failure` event description in telemetry reference Co-Authored-By: Claude Sonnet 4.6 --- docs/docs/reference/telemetry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/reference/telemetry.md b/docs/docs/reference/telemetry.md index d878c7696e..c3b72cc6d4 100644 --- a/docs/docs/reference/telemetry.md +++ b/docs/docs/reference/telemetry.md @@ -35,7 +35,7 @@ We collect the following categories of events: | `context_overflow_recovered` | Context overflow is handled (strategy) | | `skill_used` | A skill is loaded (skill name and source — `builtin`, `global`, or `project` — no skill content) | | `sql_execute_failure` | A SQL execution fails (warehouse type, query type, error message, PII-masked SQL — no raw values) | -| `core_failure` | An internal library error occurs — function name, error category, and error message and arguments with PII and sensitive fields masked | +| `core_failure` | An internal tool error occurs (tool name, category, error class, truncated error message, PII-safe input signature, and optionally masked arguments — no raw values or credentials) | Each event includes a timestamp, anonymous session ID, CLI version, and an anonymous machine ID (a random UUID stored in `~/.altimate/machine-id`, generated once and never tied to any personal information). From 39462c0b966e57c300572a3a60899ef4b5476a03 Mon Sep 17 00:00:00 2001 From: suryaiyer95 Date: Wed, 18 Mar 2026 21:50:34 -0700 Subject: [PATCH 13/13] chore: add altimate_change markers to upstream-shared tool files Wrap all telemetry additions in `packages/opencode/src/tool/tool.ts` and `packages/opencode/src/tool/skill.ts` with `// altimate_change start/end` markers so the upstream marker-guard CI passes. - `tool.ts`: markers around `import { Telemetry }` and the full telemetry instrumentation block (startTime through soft-failure core_failure emission) - `skill.ts`: markers around `classifySkillSource` helper, `startTime` declaration, and the `Telemetry.track` try-catch for `skill_used` Co-Authored-By: Claude Sonnet 4.6 --- packages/opencode/src/tool/skill.ts | 6 ++++++ packages/opencode/src/tool/tool.ts | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/packages/opencode/src/tool/skill.ts b/packages/opencode/src/tool/skill.ts index 781e4bcd31..b0754044be 100644 --- a/packages/opencode/src/tool/skill.ts +++ b/packages/opencode/src/tool/skill.ts @@ -14,12 +14,14 @@ import os from "os" const MAX_DISPLAY_SKILLS = 50 +// altimate_change start — classifySkillSource helper for skill telemetry function classifySkillSource(location: string): "builtin" | "global" | "project" { if (location.includes("node_modules") || location.includes(".altimate/builtin")) return "builtin" if (location.startsWith(os.homedir())) return "global" return "project" } // altimate_change end +// altimate_change end export const SkillTool = Tool.define("skill", async (ctx) => { const list = await Skill.available(ctx?.agent) @@ -91,7 +93,9 @@ export const SkillTool = Tool.define("skill", async (ctx) => { description, parameters, async execute(params: z.infer, ctx) { + // altimate_change start — telemetry: startTime for skill_used duration const startTime = Date.now() + // altimate_change end // altimate_change start - use upstream Skill.get() for exact name lookup const skill = await Skill.get(params.name) @@ -131,6 +135,7 @@ export const SkillTool = Tool.define("skill", async (ctx) => { return arr }).then((f) => f.map((file) => `${file}`).join("\n")) + // altimate_change start — telemetry instrumentation for skill loading try { Telemetry.track({ type: "skill_used", @@ -144,6 +149,7 @@ export const SkillTool = Tool.define("skill", async (ctx) => { } catch { // Telemetry must never break skill loading } + // altimate_change end return { title: `Loaded skill: ${skill.name}`, diff --git a/packages/opencode/src/tool/tool.ts b/packages/opencode/src/tool/tool.ts index eb527c3a0d..bbc4f8e49c 100644 --- a/packages/opencode/src/tool/tool.ts +++ b/packages/opencode/src/tool/tool.ts @@ -4,7 +4,9 @@ import type { Agent } from "../agent/agent" import type { PermissionNext } from "../permission/next" import type { SessionID, MessageID } from "../session/schema" import { Truncate } from "./truncation" +// altimate_change start — telemetry instrumentation for tool execution import { Telemetry } from "../altimate/telemetry" +// altimate_change end export namespace Tool { interface Metadata { @@ -68,6 +70,7 @@ export namespace Tool { { cause: error }, ) } + // altimate_change start — telemetry instrumentation for tool execution const startTime = Date.now() let result: Awaited> try { @@ -156,6 +159,7 @@ export namespace Tool { } catch { // Telemetry must never break tool execution } + // altimate_change end // Truncation runs after telemetry so I/O errors from // Truncate.output() are not misattributed as tool failures. if (result.metadata.truncated !== undefined) {