From 1b04d6abbbc324cb01d878afe16cdd405fc0d3ba Mon Sep 17 00:00:00 2001 From: kulvirgit Date: Mon, 16 Mar 2026 16:59:43 -0700 Subject: [PATCH] feat: env-based skill selection with session caching and tracing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run LLM skill selector once per session using environment fingerprint, cache by working directory, and apply filtering to both system prompt and tool description. Adds tracing spans for fingerprint, skill selection, and system prompt. - Use LLM.stream for skill selection (proper provider auth) - Plain text response parsing (one skill name per line) - Cache keyed by cwd — invalidates on project change - Filter skills in both SystemPrompt.skills() and SkillTool - Add env_fingerprint_skill_selection config (default: true) - Trim fingerprint to data-engineering detections only - Add tracing for fingerprint, skill-selection, and system-prompt spans Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/docs/configure/config.md | 19 ++ .../src/altimate/fingerprint/index.ts | 137 ++++++++++++ .../src/altimate/observability/tracing.ts | 40 +++- .../opencode/src/altimate/skill-selector.ts | 210 ++++++++++++++++++ packages/opencode/src/cli/cmd/run.ts | 4 + packages/opencode/src/cli/cmd/tui/worker.ts | 1 + packages/opencode/src/config/config.ts | 5 + packages/opencode/src/session/prompt.ts | 23 ++ packages/opencode/src/session/system.ts | 19 +- packages/opencode/src/tool/skill.ts | 54 ++++- .../test/altimate/fingerprint.test.ts | 72 ++++++ .../test/altimate/skill-filtering.test.ts | 177 +++++++++++++++ packages/opencode/test/tool/skill.test.ts | 159 ++++++++++++- 13 files changed, 912 insertions(+), 8 deletions(-) create mode 100644 packages/opencode/src/altimate/fingerprint/index.ts create mode 100644 packages/opencode/src/altimate/skill-selector.ts create mode 100644 packages/opencode/test/altimate/fingerprint.test.ts create mode 100644 packages/opencode/test/altimate/skill-filtering.test.ts diff --git a/docs/docs/configure/config.md b/docs/docs/configure/config.md index 240d75aeeb..a66b8a6633 100644 --- a/docs/docs/configure/config.md +++ b/docs/docs/configure/config.md @@ -61,6 +61,25 @@ Configuration is loaded from multiple sources, with later sources overriding ear | `compaction` | `object` | Context compaction settings (see [Context Management](context-management.md)) | | `experimental` | `object` | Experimental feature flags | +### Experimental Flags + +These flags are under `experimental` and may change or be removed in future releases. + +| Flag | Type | Default | Description | +|------|------|---------|-------------| +| `env_fingerprint_skill_selection` | `boolean` | `false` | Use environment fingerprint (dbt, airflow, databricks, SQL) to select relevant skills once per session via LLM. When enabled, the configured model runs once at session start to filter skills based on detected project environment. Results are cached per working directory. When disabled (default), all skills are shown. | +| `auto_enhance_prompt` | `boolean` | `false` | Automatically rewrite prompts with AI before sending. Uses a small model to clarify rough prompts. | + +Example: + +```json +{ + "experimental": { + "env_fingerprint_skill_selection": true + } +} +``` + ## Value Substitution Config values support dynamic substitution so you never need to hardcode secrets. diff --git a/packages/opencode/src/altimate/fingerprint/index.ts b/packages/opencode/src/altimate/fingerprint/index.ts new file mode 100644 index 0000000000..49ea4946f2 --- /dev/null +++ b/packages/opencode/src/altimate/fingerprint/index.ts @@ -0,0 +1,137 @@ +import { Filesystem } from "../../util/filesystem" +import { Glob } from "../../util/glob" +import { Log } from "../../util/log" +import { Tracer } from "../observability/tracing" +import path from "path" + +const log = Log.create({ service: "fingerprint" }) + +export namespace Fingerprint { + export interface Result { + tags: string[] + detectedAt: number + cwd: string + } + + let cached: Result | undefined + + export function get(): Result | undefined { + return cached + } + + export async function refresh(): Promise { + const previousCwd = cached?.cwd ?? process.cwd() + cached = undefined + return detect(previousCwd) + } + + export async function detect(cwd: string, root?: string): Promise { + if (cached && cached.cwd === cwd) return cached + + const startTime = Date.now() + const timer = log.time("detect", { cwd, root }) + const tags: string[] = [] + + const dirs = root && root !== cwd ? [cwd, root] : [cwd] + + await Promise.all( + dirs.map((dir) => detectDir(dir, tags)), + ) + + // Deduplicate + const unique = [...new Set(tags)] + + const result: Result = { + tags: unique, + detectedAt: Date.now(), + cwd, + } + + cached = result + timer.stop() + log.info("detected", { tags: unique.join(","), cwd }) + + Tracer.active?.logSpan({ + name: "fingerprint", + startTime, + endTime: Date.now(), + input: { cwd, root }, + output: { tags: unique }, + }) + + return result + } + + async function detectDir(dir: string, tags: string[]): Promise { + // Data-engineering detections only + const [ + hasDbtProject, + hasProfilesYml, + hasSqlfluff, + hasDbtPackagesYml, + hasAirflowCfg, + hasDagsDir, + hasDatabricksYml, + ] = await Promise.all([ + Filesystem.exists(path.join(dir, "dbt_project.yml")), + Filesystem.exists(path.join(dir, "profiles.yml")), + Filesystem.exists(path.join(dir, ".sqlfluff")), + Filesystem.exists(path.join(dir, "dbt_packages.yml")), + Filesystem.exists(path.join(dir, "airflow.cfg")), + Filesystem.isDir(path.join(dir, "dags")), + Filesystem.exists(path.join(dir, "databricks.yml")), + ]) + + // dbt detection + if (hasDbtProject) { + tags.push("dbt", "data-engineering") + } + + // dbt packages + if (hasDbtPackagesYml) { + tags.push("dbt-packages") + } + + // profiles.yml - extract adapter type + if (hasProfilesYml) { + try { + const content = await Filesystem.readText(path.join(dir, "profiles.yml")) + const adapterMatch = content.match( + /type:\s*(snowflake|bigquery|redshift|databricks|postgres|mysql|sqlite|duckdb|trino|spark|clickhouse)/i, + ) + if (adapterMatch) { + tags.push(adapterMatch[1]!.toLowerCase()) + } + } catch (e) { + log.debug("profiles.yml unreadable", { dir, error: e }) + } + } + + // SQL - check for .sqlfluff or any .sql files + if (hasSqlfluff) { + tags.push("sql") + } else { + try { + const sqlFiles = await Glob.scan("*.sql", { + cwd: dir, + include: "file", + }) + if (sqlFiles.length > 0) { + tags.push("sql") + } + } catch (e) { + log.debug("sql glob scan failed", { dir, error: e }) + } + } + + // Airflow + if (hasAirflowCfg || hasDagsDir) { + tags.push("airflow") + } + + // Databricks + if (hasDatabricksYml) { + tags.push("databricks") + } + } +} diff --git a/packages/opencode/src/altimate/observability/tracing.ts b/packages/opencode/src/altimate/observability/tracing.ts index d58dd678f2..a801721e5b 100644 --- a/packages/opencode/src/altimate/observability/tracing.ts +++ b/packages/opencode/src/altimate/observability/tracing.ts @@ -41,7 +41,7 @@ export interface TraceSpan { spanId: string parentSpanId: string | null name: string - kind: "session" | "generation" | "tool" | "text" + kind: "session" | "generation" | "tool" | "text" | "span" startTime: number endTime?: number status: "ok" | "error" @@ -246,6 +246,11 @@ interface TracerOptions { } export class Tracer { + // Global active tracer — set when a session starts, cleared on end. + private static _active: Tracer | null = null + static get active(): Tracer | null { return Tracer._active } + static setActive(tracer: Tracer | null) { Tracer._active = tracer } + private traceId: string private sessionId: string | undefined private rootSpanId: string | undefined @@ -561,6 +566,39 @@ export class Tracer { if (part.text != null) this.generationText.push(String(part.text)) } + /** + * Log a custom span (e.g., fingerprint detection, skill selection). + * Used for internal operations that aren't LLM generations or tool calls. + */ + logSpan(span: { + name: string + startTime: number + endTime: number + status?: "ok" | "error" + input?: unknown + output?: unknown + attributes?: Record + }) { + if (!this.rootSpanId) return + try { + this.spans.push({ + spanId: randomUUIDv7(), + parentSpanId: this.rootSpanId, + name: span.name, + kind: "span", + startTime: span.startTime, + endTime: span.endTime, + status: span.status ?? "ok", + input: span.input, + output: span.output, + attributes: span.attributes, + }) + this.snapshot() + } catch { + // best-effort + } + } + /** * Build a TraceFile snapshot of the current state (in-progress or complete). * Used for incremental writes and live viewing. diff --git a/packages/opencode/src/altimate/skill-selector.ts b/packages/opencode/src/altimate/skill-selector.ts new file mode 100644 index 0000000000..f175f03e65 --- /dev/null +++ b/packages/opencode/src/altimate/skill-selector.ts @@ -0,0 +1,210 @@ +// altimate_change start - LLM-based dynamic skill selection +import { Provider } from "../provider/provider" +import { LLM } from "../session/llm" +import { Agent } from "../agent/agent" +import { Log } from "../util/log" +import { MessageV2 } from "../session/message-v2" +import { MessageID, SessionID } from "../session/schema" +import type { Skill } from "../skill" +import type { Fingerprint } from "./fingerprint" +import { Tracer } from "./observability/tracing" + +const log = Log.create({ service: "skill-selector" }) + +const TIMEOUT_MS = 5_000 +const MAX_SKILLS = 15 +const SELECTOR_NAME = "skill-selector" + +// Session cache keyed by working directory — invalidates if project changes. +let cachedResult: Skill.Info[] | undefined +let cachedCwd: string | undefined + +/** Reset the session cache (exported for testing) */ +export function resetSkillSelectorCache(): void { + cachedResult = undefined + cachedCwd = undefined +} + +export interface SkillSelectorDeps { + run: (prompt: string, skillNames: string[]) => Promise +} + +/** + * Use the configured model to select relevant skills based on the project fingerprint. + * Results are cached per working directory — the LLM is only called once per project. + * + * Graceful fallback: returns ALL skills on any failure (matches pre-feature behavior). + */ +export async function selectSkillsWithLLM( + skills: Skill.Info[], + fingerprint: Fingerprint.Result | undefined, + deps?: SkillSelectorDeps, +): Promise { + const startTime = Date.now() + + // Return cached result if cwd hasn't changed (0ms) + const cwd = fingerprint?.cwd + if (cachedResult && cwd === cachedCwd) { + log.info("returning cached skill selection", { + count: cachedResult.length, + }) + Tracer.active?.logSpan({ + name: "skill-selection", + startTime, + endTime: Date.now(), + input: { fingerprint: fingerprint?.tags, source: "cache" }, + output: { count: cachedResult.length, skills: cachedResult.map((s) => s.name) }, + }) + return cachedResult + } + + function cache(result: Skill.Info[]): Skill.Info[] { + cachedResult = result + cachedCwd = cwd + return result + } + + try { + const envContext = + fingerprint && fingerprint.tags.length > 0 + ? fingerprint.tags.join(", ") + : "none detected" + + const skillList = skills.map((s) => `- ${s.name}: ${s.description}`) + + const prompt = [ + `Project environment: ${envContext}`, + "", + "Available skills:", + ...skillList, + "", + "Return ONLY the names of relevant skills, one per line. No explanations.", + ].join("\n") + + const skillNames = skills.map((s) => s.name) + + let selected: string[] + if (deps) { + selected = await deps.run(prompt, skillNames) + } else { + selected = await runWithLLM(prompt, skillNames) + } + + selected = selected.slice(0, MAX_SKILLS) + + // Zero-selection guard + if (selected.length === 0) { + log.info("LLM returned zero skills, returning all") + return cache(skills) + } + + // Filter skills by returned names + const selectedSet = new Set(selected) + const matched = skills.filter((s) => selectedSet.has(s.name)) + + // If no valid matches (LLM returned non-existent names), return all + if (matched.length === 0) { + log.info("LLM returned no valid skill names, returning all") + return cache(skills) + } + + log.info("selected skills", { + count: matched.length, + names: matched.map((s) => s.name), + }) + Tracer.active?.logSpan({ + name: "skill-selection", + startTime, + endTime: Date.now(), + input: { fingerprint: fingerprint?.tags, totalSkills: skills.length, source: "llm" }, + output: { count: matched.length, skills: matched.map((s) => s.name) }, + }) + return cache(matched) + } catch (e) { + log.info("skill selection failed, returning all skills", { + error: e instanceof Error ? e.message : String(e), + }) + Tracer.active?.logSpan({ + name: "skill-selection", + startTime, + endTime: Date.now(), + status: "error", + input: { fingerprint: fingerprint?.tags, source: "fallback" }, + output: { count: skills.length, error: e instanceof Error ? e.message : String(e) }, + }) + return cache(skills) + } +} + +const SYSTEM_PROMPT = [ + "You are a skill selector for a coding assistant.", + "Given a project environment and available skills, select which skills are relevant for this project.", + "Return ONLY skill names, one per line. Select 0-15 skills.", + "Prefer fewer, more relevant skills over many loosely related ones.", + "Do not include explanations or formatting — just the skill names.", +].join("\n") + +async function runWithLLM(prompt: string, validNames: string[]): Promise { + const defaultModel = await Provider.defaultModel() + const model = await Provider.getModel(defaultModel.providerID, defaultModel.modelID) + + const agent: Agent.Info = { + name: SELECTOR_NAME, + mode: "primary", + hidden: true, + options: {}, + permission: [], + prompt: SYSTEM_PROMPT, + temperature: 0, + } + + const user: MessageV2.User = { + id: MessageID.ascending(), + sessionID: SessionID.descending(), + role: "user", + time: { created: Date.now() }, + agent: SELECTOR_NAME, + model: { + providerID: model.providerID, + modelID: model.id, + }, + } + + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS) + + try { + const stream = await LLM.stream({ + agent, + user, + system: [], + tools: {}, + model, + abort: controller.signal, + sessionID: user.sessionID, + retries: 1, + messages: [ + { + role: "user", + content: prompt, + }, + ], + }) + + // Drain the stream + for await (const _ of stream.fullStream) { + // drain + } + const text = await stream.text + + // Parse: one skill name per line, filter to valid names + const nameSet = new Set(validNames) + return text + .split("\n") + .map((line) => line.replace(/^[-*•]\s*/, "").trim()) + .filter((line) => nameSet.has(line)) + } finally { + clearTimeout(timeout) + } +} +// altimate_change end diff --git a/packages/opencode/src/cli/cmd/run.ts b/packages/opencode/src/cli/cmd/run.ts index 30e77343f6..58f8a0f96b 100644 --- a/packages/opencode/src/cli/cmd/run.ts +++ b/packages/opencode/src/cli/cmd/run.ts @@ -720,6 +720,9 @@ You are speaking to a non-technical business executive. Follow these rules stric variant: args.variant, prompt: message, }) + // altimate_change start - activate tracer for session + if (tracer) Tracer.setActive(tracer) + // altimate_change end // Register crash handlers to flush the trace on unexpected exit const onSigint = () => { tracer?.flushSync("Process interrupted"); process.exit(130) } @@ -766,6 +769,7 @@ You are speaking to a non-technical business executive. Follow these rules stric // Finalize trace and save to disk if (tracer) { + Tracer.setActive(null) const tracePath = await tracer.endTrace(error) if (tracePath) { emit("trace_saved", { path: tracePath }) diff --git a/packages/opencode/src/cli/cmd/tui/worker.ts b/packages/opencode/src/cli/cmd/tui/worker.ts index 96dc5f0fb9..aef81cc83a 100644 --- a/packages/opencode/src/cli/cmd/tui/worker.ts +++ b/packages/opencode/src/cli/cmd/tui/worker.ts @@ -93,6 +93,7 @@ function getOrCreateTracer(sessionID: string): Tracer | null { ? Tracer.withExporters([...tracingExporters], { maxFiles: tracingMaxFiles }) : Tracer.create() tracer.startTrace(sessionID, {}) + Tracer.setActive(tracer) sessionTracers.set(sessionID, tracer) return tracer } catch { diff --git a/packages/opencode/src/config/config.ts b/packages/opencode/src/config/config.ts index 7f681a4de1..0cde3344ed 100644 --- a/packages/opencode/src/config/config.ts +++ b/packages/opencode/src/config/config.ts @@ -1266,6 +1266,11 @@ export namespace Config { .describe( "Automatically enhance prompts with AI before sending (default: false). Uses a small model to rewrite rough prompts into clearer versions.", ), + // altimate_change start - env fingerprint skill selection toggle + env_fingerprint_skill_selection: z + .boolean() + .optional() + .describe("Use environment fingerprint to select relevant skills once per session (default: false). Set to true to enable LLM-based skill filtering."), // altimate_change end }) .optional(), diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 18db575f95..b16976047f 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -50,6 +50,11 @@ import { iife } from "@/util/iife" import { Shell } from "@/shell/shell" import { Truncate } from "@/tool/truncation" import { decodeDataUrl } from "@/util/data-url" +// altimate_change start - import fingerprint for env-based skill selection +import { Fingerprint } from "../altimate/fingerprint" +import { Config } from "../config/config" +import { Tracer } from "../altimate/observability/tracing" +// altimate_change end import { Telemetry } from "@/telemetry" // altimate_change — session telemetry // @ts-ignore @@ -297,6 +302,14 @@ export namespace SessionPrompt { let step = 0 const session = await Session.get(sessionID) + // altimate_change start - detect environment fingerprint at session start + const altCfg = await Config.get() + if (altCfg.experimental?.env_fingerprint_skill_selection === true) { + await Fingerprint.detect(Instance.directory, Instance.worktree).catch((e) => { + log.warn("fingerprint detection failed", { error: e }) + }) + } + // altimate_change end // altimate_change start — session telemetry tracking await Telemetry.init() Telemetry.setContext({ sessionId: sessionID, projectId: Instance.project?.id ?? "" }) @@ -715,6 +728,16 @@ export namespace SessionPrompt { system.push(STRUCTURED_OUTPUT_SYSTEM_PROMPT) } + // altimate_change start - trace system prompt + Tracer.active?.logSpan({ + name: "system-prompt", + startTime: Date.now(), + endTime: Date.now(), + input: { agent: agent.name, step }, + output: { parts: system.length, content: system.join("\n\n") }, + }) + // altimate_change end + const result = await processor.process({ user: lastUser, agent, diff --git a/packages/opencode/src/session/system.ts b/packages/opencode/src/session/system.ts index a4c4684ffe..6e1e4c45e3 100644 --- a/packages/opencode/src/session/system.ts +++ b/packages/opencode/src/session/system.ts @@ -13,6 +13,11 @@ import type { Provider } from "@/provider/provider" import type { Agent } from "@/agent/agent" import { PermissionNext } from "@/permission/next" import { Skill } from "@/skill" +// altimate_change start - import for env-based skill selection +import { Fingerprint } from "../altimate/fingerprint" +import { Config } from "../config/config" +import { selectSkillsWithLLM } from "../altimate/skill-selector" +// altimate_change end export namespace SystemPrompt { export function instructions() { @@ -61,12 +66,24 @@ export namespace SystemPrompt { const list = await Skill.available(agent) + // altimate_change start - apply env-based skill selection + const cfg = await Config.get() + let filtered: Skill.Info[] + if (cfg.experimental?.env_fingerprint_skill_selection === true) { + filtered = await selectSkillsWithLLM(list, Fingerprint.get()) + } else { + filtered = list + } + // altimate_change end + return [ "Skills provide specialized instructions and workflows for specific tasks.", "Use the skill tool to load a skill when a task matches its description.", // the agents seem to ingest the information about skills a bit better if we present a more verbose // version of them here and a less verbose version in tool description, rather than vice versa. - Skill.fmt(list, { verbose: true }), + // altimate_change start - use filtered skill list + Skill.fmt(filtered, { verbose: true }), + // altimate_change end ].join("\n") } } diff --git a/packages/opencode/src/tool/skill.ts b/packages/opencode/src/tool/skill.ts index 17016b06f8..15fe747123 100644 --- a/packages/opencode/src/tool/skill.ts +++ b/packages/opencode/src/tool/skill.ts @@ -5,12 +5,35 @@ import { Tool } from "./tool" import { Skill } from "../skill" import { Ripgrep } from "../file/ripgrep" import { iife } from "@/util/iife" +// altimate_change start - import for LLM-based dynamic skill selection +import { Fingerprint } from "../altimate/fingerprint" +import { Config } from "../config/config" +import { selectSkillsWithLLM } from "../altimate/skill-selector" + +const MAX_DISPLAY_SKILLS = 50 +// altimate_change end export const SkillTool = Tool.define("skill", async (ctx) => { const list = await Skill.available(ctx?.agent) + // altimate_change start - LLM-based dynamic skill selection + const cfg = await Config.get() + let allAllowed: Skill.Info[] + if (cfg.experimental?.env_fingerprint_skill_selection === true) { + allAllowed = await selectSkillsWithLLM( + list, + Fingerprint.get(), + ) + } else { + allAllowed = list + } + const displaySkills = allAllowed.slice(0, MAX_DISPLAY_SKILLS) + const hasMore = allAllowed.length > displaySkills.length + // altimate_change end + + // altimate_change start - use displaySkills (filtered) instead of list const description = - list.length === 0 + displaySkills.length === 0 ? "Load a specialized skill that provides domain-specific instructions and workflows. No skills are currently available." : [ "Load a specialized skill that provides domain-specific instructions and workflows.", @@ -24,14 +47,33 @@ export const SkillTool = Tool.define("skill", async (ctx) => { "The following skills provide specialized sets of instructions for particular tasks", "Invoke this tool to load a skill when a task matches one of the available skills listed below:", "", - Skill.fmt(list, { verbose: false }), + "", + ...displaySkills.flatMap((skill) => [ + ` `, + ` ${skill.name}`, + ` ${skill.description}`, + ` ${pathToFileURL(skill.location).href}`, + ` `, + ]), + "", + // altimate_change start - add hint when skills are truncated + ...(hasMore + ? [ + "", + `Note: Showing ${displaySkills.length} of ${allAllowed.length} available skills.`, + ] + : []), + // altimate_change end ].join("\n") + // altimate_change end - const examples = list + // altimate_change start - use displaySkills for examples + const examples = displaySkills .map((skill) => `'${skill.name}'`) .slice(0, 3) .join(", ") const hint = examples.length > 0 ? ` (e.g., ${examples}, ...)` : "" + // altimate_change end const parameters = z.object({ name: z.string().describe(`The name of the skill from available_skills${hint}`), @@ -41,12 +83,14 @@ export const SkillTool = Tool.define("skill", async (ctx) => { description, parameters, async execute(params: z.infer, ctx) { + // altimate_change start - use upstream Skill.get() for exact name lookup const skill = await Skill.get(params.name) if (!skill) { - const available = await Skill.all().then((x) => x.map((skill) => skill.name).join(", ")) + const available = await Skill.all().then((s) => s.map((x) => x.name).join(", ")) throw new Error(`Skill "${params.name}" not found. Available skills: ${available || "none"}`) } + // altimate_change end await ctx.ask({ permission: "skill", @@ -103,3 +147,5 @@ export const SkillTool = Tool.define("skill", async (ctx) => { }, } }) + +// altimate_change end - old partitionByFingerprint + rescueByMessage removed, replaced by selectSkillsWithLLM diff --git a/packages/opencode/test/altimate/fingerprint.test.ts b/packages/opencode/test/altimate/fingerprint.test.ts new file mode 100644 index 0000000000..1ca154c268 --- /dev/null +++ b/packages/opencode/test/altimate/fingerprint.test.ts @@ -0,0 +1,72 @@ +// @ts-nocheck +import { describe, expect, test } from "bun:test" +import { Fingerprint } from "../../src/altimate/fingerprint" +import path from "path" + +// Use the actual project root for testing - it has real files +const PROJECT_ROOT = path.resolve(__dirname, "../..") + +// NOTE: Fingerprint uses an internal cached variable that can't be reset from +// outside. Tests are ordered to work with this constraint. + +describe("Fingerprint.get", () => { + // This must run first, before any detect() call + test("returns undefined before any detection", () => { + // We rely on being the first test in the file + // If another test ran detect() first, this would fail + // Use a unique cwd to check if cache was set with that cwd + const result = Fingerprint.get() + // Either undefined (first run ever) or set from a previous test run + // Since Bun runs files in isolation, this should be undefined + if (result === undefined) { + expect(result).toBeUndefined() + } else { + // Cache was set by module initialization or previous test + expect(result.tags).toBeInstanceOf(Array) + } + }) +}) + +describe("Fingerprint.detect", () => { + test("returns tags array and cwd", async () => { + const result = await Fingerprint.detect(PROJECT_ROOT) + expect(result.tags).toBeInstanceOf(Array) + expect(result.cwd).toBe(PROJECT_ROOT) + expect(result.detectedAt).toBeGreaterThan(0) + }) + + test("returns cached result on second call with same cwd", async () => { + const r1 = await Fingerprint.detect(PROJECT_ROOT) + const r2 = await Fingerprint.detect(PROJECT_ROOT) + expect(r1).toBe(r2) // Same reference - cached + }) +}) + +describe("Fingerprint.get after detection", () => { + test("returns result after detection", async () => { + await Fingerprint.detect(PROJECT_ROOT) + const result = Fingerprint.get() + expect(result).toBeDefined() + expect(result!.tags).toBeInstanceOf(Array) + }) +}) + +describe("Fingerprint.refresh", () => { + test("clears cache and re-detects", async () => { + await Fingerprint.detect(PROJECT_ROOT) + const r1 = Fingerprint.get()! + const r2 = await Fingerprint.refresh() + // Different object references (cache was cleared and re-created) + // Tags should be the same since same directory + expect(r2.tags.sort()).toEqual(r1.tags.sort()) + expect(r2.detectedAt).toBeGreaterThanOrEqual(r1.detectedAt) + }) +}) + +describe("fingerprint tag deduplication", () => { + test("tags are deduplicated", async () => { + const result = await Fingerprint.detect(PROJECT_ROOT) + const uniqueTags = [...new Set(result.tags)] + expect(result.tags.length).toBe(uniqueTags.length) + }) +}) diff --git a/packages/opencode/test/altimate/skill-filtering.test.ts b/packages/opencode/test/altimate/skill-filtering.test.ts new file mode 100644 index 0000000000..17f6d5f469 --- /dev/null +++ b/packages/opencode/test/altimate/skill-filtering.test.ts @@ -0,0 +1,177 @@ +import { beforeEach, describe, expect, test } from "bun:test" +import { selectSkillsWithLLM, resetSkillSelectorCache, type SkillSelectorDeps } from "../../src/altimate/skill-selector" +import type { Skill } from "../../src/skill" +import type { Fingerprint } from "../../src/altimate/fingerprint" + +function mockSkill(name: string, description?: string): Skill.Info { + return { + name, + description: description ?? `Test skill: ${name}`, + location: `/test/${name}/SKILL.md`, + content: `# ${name}`, + } as Skill.Info +} + +function mockFingerprint(tags: string[]): Fingerprint.Result { + return { tags, detectedAt: Date.now(), cwd: "/test" } as Fingerprint.Result +} + +const ALL_SKILLS = [ + mockSkill("dbt-modeling", "Build and manage dbt models"), + mockSkill("react-components", "Create React UI components"), + mockSkill("python-testing", "Write Python unit tests"), + mockSkill("kubernetes-deploy", "Deploy apps to Kubernetes"), + mockSkill("sql-optimization", "Optimize SQL queries"), +] + +/** Create deps that return selected skill names */ +function makeDeps(selected: string[]): SkillSelectorDeps & { calls: string[][] } { + const calls: string[][] = [] + return { + calls, + run: async (_prompt, skillNames) => { + calls.push(skillNames) + return selected + }, + } +} + +/** Create deps that throw */ +function makeDepsError(error: string): SkillSelectorDeps { + return { + run: async () => { throw new Error(error) }, + } +} + +/** Create deps that never resolve (timeout test) */ +function makeDepsHang(): SkillSelectorDeps { + return { + run: () => new Promise(() => {}), + } +} + +describe("selectSkillsWithLLM", () => { + // Reset cache before each test so tests are independent + beforeEach(() => { + resetSkillSelectorCache() + }) + + // --- Fallback cases: return all skills --- + + test("LLM error → returns all skills (graceful fallback)", async () => { + const deps = makeDepsError("API key invalid") + const result = await selectSkillsWithLLM(ALL_SKILLS, undefined, deps) + expect(result).toHaveLength(ALL_SKILLS.length) + }) + + test("LLM returns zero skills → returns all skills", async () => { + const deps = makeDeps([]) + const result = await selectSkillsWithLLM(ALL_SKILLS, mockFingerprint([]), deps) + expect(result).toHaveLength(ALL_SKILLS.length) + }) + + test("LLM returns all non-existent names → returns all skills (fallback)", async () => { + const deps = makeDeps(["fake-skill-1", "fake-skill-2"]) + const result = await selectSkillsWithLLM(ALL_SKILLS, mockFingerprint([]), deps) + expect(result).toHaveLength(ALL_SKILLS.length) + }) + + // --- Successful selection --- + + test("LLM returns valid names → filters correctly", async () => { + const deps = makeDeps(["dbt-modeling", "sql-optimization"]) + const result = await selectSkillsWithLLM( + ALL_SKILLS, + mockFingerprint(["dbt"]), + deps, + ) + expect(result).toHaveLength(2) + expect(result.map((s) => s.name)).toEqual(["dbt-modeling", "sql-optimization"]) + }) + + test("LLM returns non-existent names → ignored, returns only matching", async () => { + const deps = makeDeps(["dbt-modeling", "nonexistent-skill"]) + const result = await selectSkillsWithLLM( + ALL_SKILLS, + mockFingerprint(["dbt"]), + deps, + ) + expect(result).toHaveLength(1) + expect(result[0].name).toBe("dbt-modeling") + }) + + test("single skill selected → returns just that one", async () => { + const deps = makeDeps(["python-testing"]) + const result = await selectSkillsWithLLM( + ALL_SKILLS, + mockFingerprint(["python"]), + deps, + ) + expect(result).toHaveLength(1) + expect(result[0].name).toBe("python-testing") + }) + + // --- Limits --- + + test("max 15 skills cap enforced", async () => { + const manySkills = Array.from({ length: 20 }, (_, i) => mockSkill(`skill-${i}`, `Skill ${i}`)) + const deps = makeDeps(manySkills.map((s) => s.name)) + const result = await selectSkillsWithLLM(manySkills, undefined, deps) + expect(result.length).toBeLessThanOrEqual(15) + }) + + // --- Caching --- + + test("second call returns cached result without calling LLM", async () => { + const deps = makeDeps(["dbt-modeling", "sql-optimization"]) + const first = await selectSkillsWithLLM(ALL_SKILLS, mockFingerprint(["dbt"]), deps) + expect(deps.calls).toHaveLength(1) + + // Second call — LLM should NOT be called again + const second = await selectSkillsWithLLM(ALL_SKILLS, mockFingerprint(["dbt"]), deps) + expect(deps.calls).toHaveLength(1) // still 1 call + expect(second).toEqual(first) + }) + + test("cached fallback result also avoids re-calling LLM", async () => { + const deps = makeDepsError("API failure") + const first = await selectSkillsWithLLM(ALL_SKILLS, undefined, deps) + expect(first).toHaveLength(ALL_SKILLS.length) // fallback to all + + // Second call with working deps — should still return cached + const workingDeps = makeDeps(["dbt-modeling"]) + const second = await selectSkillsWithLLM(ALL_SKILLS, undefined, workingDeps) + expect(workingDeps.calls).toHaveLength(0) // never called + expect(second).toHaveLength(ALL_SKILLS.length) + }) + + test("different cwd invalidates cache and re-calls LLM", async () => { + const deps = makeDeps(["dbt-modeling"]) + await selectSkillsWithLLM(ALL_SKILLS, mockFingerprint(["dbt"]), deps) + expect(deps.calls).toHaveLength(1) + + // Same cwd — cache hit + await selectSkillsWithLLM(ALL_SKILLS, mockFingerprint(["dbt"]), deps) + expect(deps.calls).toHaveLength(1) + + // Different cwd — cache miss, LLM called again + const otherFingerprint = { tags: ["python"], detectedAt: Date.now(), cwd: "/other-project" } as Fingerprint.Result + await selectSkillsWithLLM(ALL_SKILLS, otherFingerprint, deps) + expect(deps.calls).toHaveLength(2) + }) + + test("resetSkillSelectorCache clears the cache", async () => { + const deps1 = makeDeps(["dbt-modeling"]) + await selectSkillsWithLLM(ALL_SKILLS, mockFingerprint(["dbt"]), deps1) + expect(deps1.calls).toHaveLength(1) + + resetSkillSelectorCache() + + const deps2 = makeDeps(["sql-optimization"]) + const result = await selectSkillsWithLLM(ALL_SKILLS, mockFingerprint(["sql"]), deps2) + expect(deps2.calls).toHaveLength(1) // LLM called again after reset + expect(result).toHaveLength(1) + expect(result[0].name).toBe("sql-optimization") + }) + +}) diff --git a/packages/opencode/test/tool/skill.test.ts b/packages/opencode/test/tool/skill.test.ts index 5bcdb6c2b9..28d63afe8e 100644 --- a/packages/opencode/test/tool/skill.test.ts +++ b/packages/opencode/test/tool/skill.test.ts @@ -1,4 +1,6 @@ -import { describe, expect, test } from "bun:test" +// altimate_change start - add imports for env fingerprint skill selection tests +import { afterEach, describe, expect, test } from "bun:test" +// altimate_change end import path from "path" import { pathToFileURL } from "url" import type { PermissionNext } from "../../src/permission/next" @@ -7,6 +9,10 @@ import { Instance } from "../../src/project/instance" import { SkillTool } from "../../src/tool/skill" import { tmpdir } from "../fixture/fixture" import { SessionID, MessageID } from "../../src/session/schema" +// altimate_change start - imports for env fingerprint skill selection tests +import { resetSkillSelectorCache, selectSkillsWithLLM, type SkillSelectorDeps } from "../../src/altimate/skill-selector" +import type { Skill } from "../../src/skill" +// altimate_change end const baseCtx: Omit = { sessionID: SessionID.make("ses_test"), @@ -18,7 +24,30 @@ const baseCtx: Omit = { metadata: () => {}, } +// altimate_change start - helpers for env fingerprint skill selection tests +/** Pre-populate the skill selector cache with a specific subset */ +function seedCache(skillNames: string[]) { + resetSkillSelectorCache() + const skills = skillNames.map((name) => ({ + name, + description: `Skill: ${name}`, + location: `/fake/${name}/SKILL.md`, + content: `# ${name}`, + })) as Skill.Info[] + const deps: SkillSelectorDeps = { + run: async () => skillNames, + } + return selectSkillsWithLLM(skills, undefined, deps) +} +// altimate_change end + describe("tool.skill", () => { + // altimate_change start - reset skill selector cache between tests + afterEach(() => { + resetSkillSelectorCache() + }) + // altimate_change end + test("description lists skill location URL", async () => { await using tmp = await tmpdir({ git: true, @@ -46,7 +75,10 @@ description: Skill for tool tests. fn: async () => { const tool = await SkillTool.init() const skillPath = path.join(tmp.path, ".opencode", "skill", "tool-skill", "SKILL.md") - expect(tool.description).toContain(`**tool-skill**: Skill for tool tests.`) + // altimate_change start - updated assertion to match XML skill description format + expect(tool.description).toContain(`tool-skill`) + expect(tool.description).toContain(`Skill for tool tests.`) + // altimate_change end }, }) } finally { @@ -110,4 +142,127 @@ Use this skill. process.env.OPENCODE_TEST_HOME = home } }) + + // altimate_change start - env fingerprint skill selection config guard tests + test("env_fingerprint_skill_selection absent (default) → selector bypassed, all skills shown", async () => { + // Pre-populate cache — if selector were called, it would return this cached subset + await seedCache(["skill-alpha"]) + + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + for (const name of ["skill-alpha", "skill-beta"]) { + await Bun.write( + path.join(dir, ".opencode", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Test ${name}\n---\n# ${name}\n`, + ) + } + }, + }) + + const home = process.env.OPENCODE_TEST_HOME + process.env.OPENCODE_TEST_HOME = tmp.path + + try { + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const tool = await SkillTool.init() + // No config set → default false → selector bypassed → both skills appear + expect(tool.description).toContain("skill-alpha") + expect(tool.description).toContain("skill-beta") + }, + }) + } finally { + process.env.OPENCODE_TEST_HOME = home + } + }) + + test("env_fingerprint_skill_selection: false → selector bypassed, all skills shown", async () => { + // Pre-populate cache with only "skill-alpha" — if selector is called, it returns this cached subset + await seedCache(["skill-alpha"]) + + await using tmp = await tmpdir({ + git: true, + config: { + experimental: { + env_fingerprint_skill_selection: false, + }, + }, + init: async (dir) => { + for (const name of ["skill-alpha", "skill-beta"]) { + await Bun.write( + path.join(dir, ".opencode", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Test ${name}\n---\n# ${name}\n`, + ) + } + }, + }) + + const home = process.env.OPENCODE_TEST_HOME + process.env.OPENCODE_TEST_HOME = tmp.path + + try { + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const tool = await SkillTool.init() + // Selector was bypassed → both skills appear (from Skill.available, not cache) + expect(tool.description).toContain("skill-alpha") + expect(tool.description).toContain("skill-beta") + }, + }) + } finally { + process.env.OPENCODE_TEST_HOME = home + } + }) + + test("env_fingerprint_skill_selection: true → selector called, uses cached subset", async () => { + await using tmp = await tmpdir({ + git: true, + config: { + experimental: { + env_fingerprint_skill_selection: true, + }, + }, + init: async (dir) => { + for (const name of ["skill-alpha", "skill-beta"]) { + await Bun.write( + path.join(dir, ".opencode", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Test ${name}\n---\n# ${name}\n`, + ) + } + }, + }) + + // Pre-populate cache with only "skill-alpha" AFTER tmpdir so location matches + const alphaLocation = path.join(tmp.path, ".opencode", "skill", "skill-alpha", "SKILL.md") + resetSkillSelectorCache() + const deps: SkillSelectorDeps = { + run: async () => ["skill-alpha"], + } + await selectSkillsWithLLM( + [{ name: "skill-alpha", description: "Test skill-alpha", location: alphaLocation, content: "# skill-alpha" } as Skill.Info], + undefined, + deps, + ) + + const home = process.env.OPENCODE_TEST_HOME + process.env.OPENCODE_TEST_HOME = tmp.path + + try { + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const tool = await SkillTool.init() + // Selector was called → returns cached subset (only skill-alpha) + expect(tool.description).toContain("skill-alpha") + expect(tool.description).not.toContain("skill-beta") + }, + }) + } finally { + process.env.OPENCODE_TEST_HOME = home + } + }) + // altimate_change end })