Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions apps/api/src/resolve-routing-context.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
import type { ProtocolRunState, RoutingContext } from "@datafoundry/agent-runtime";
import type { MetadataStore } from "@datafoundry/metadata";

import { MetadataProtocolStateStore } from "./protocol-state-store.js";

type ResolveRoutingContextInput = {
metadataStore: MetadataStore;
/** Current run id (excluded when looking up the previous run). */
runId: string;
selectedDatasourceId?: string;
/** Skill IDs active this run, if any. */
selectedSkillIds?: string[];
sessionId: string;
userId: string;
};

/**
* Resolve the compact routing context for the protocol classifier from the
* previous run in the same session. Lets short follow-ups such as "再次尝试"
* ("try again") inherit the prior data-analysis intent instead of being routed
* to general-task on the ambiguous text alone.
*
* Returns undefined when there is no prior run in the session (first turn),
* so callers without history are unaffected.
*/
export const resolveRoutingContext = (input: ResolveRoutingContextInput): RoutingContext | undefined => {
const previousRun = input.metadataStore.runs.findPreviousRunBySession({

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

正确性风险 #1 — "最近的终态 run" ≠ "用户正在延续的 run"

findPreviousRunBySession 返回最近的终态 run。考虑以下序列:

  1. 用户:“分析订单数据” → data-analysis, completed
  2. 用户:“谢谢” → general-task, completed
  3. 用户:“再试一次” → 应当继承第 1 轮的 data-analysis,但 resolveRoutingContext 把第 2 轮视作 previousProtocol = general-task 喂给分类器。分类器看到 previousProtocol=general-task 很可能维持 general-task —— 这加剧了该 bug,而非修复。

设计把"最近的终态 run"与"承载用户正在延续意图的 run"等同起来,但二者并不总是相同。测试套件只覆盖了前一轮是 data-analysis 的情况,没有覆盖这条间隙的 general-task 场景。

建议: 要么向后遍历终态 run 直到遇到非默认协议,要么在 session 元数据里持久化一个粘性的 lastNonDefaultProtocol,让后续提示能可靠继承。并补一个测试:前一轮是 general-task 而其祖先是 data-analysis

user_id: input.userId,
session_id: input.sessionId,
exclude_run_id: input.runId
});
if (!previousRun) {
return undefined;
}
const protocolStateStore = new MetadataProtocolStateStore(input.metadataStore, input.userId);
const previousProtocolState = protocolStateStore.find(previousRun.id);
const previousQuery = previousRun.user_input.trim() ? previousRun.user_input : null;
const previousProtocol = previousProtocolState
? buildPreviousProtocol(previousProtocolState)
: null;
const selectedSkillIds = input.selectedSkillIds?.length ? [...input.selectedSkillIds] : null;
const selectedDatasourceId = input.selectedDatasourceId ?? null;
if (!previousQuery && !previousProtocol && !selectedSkillIds && !selectedDatasourceId) {
return undefined;
}
const routingContext: RoutingContext = {
...(previousQuery ? { previousQuery } : {}),
...(previousProtocol ? { previousProtocol } : {}),
...(selectedSkillIds ? { selectedSkillIds } : {}),
...(selectedDatasourceId ? { selectedDatasourceId } : {})
};
return routingContext;
};

const buildPreviousProtocol = (state: ProtocolRunState): {
protocolId: string;
protocolVersion: string;
terminalStatus?: string;
} => {
const terminalStatus = state.status === "terminal"

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

正确性风险 #2buildPreviousProtocol 喂给分类器的词汇表不一致

const terminalStatus = state.status === "terminal"
  ? state.terminalDecision?.status
  : state.status;

三个问题:

  • state.status === "terminal"terminalDecisionundefined 时,terminalStatusundefined → 被整个丢弃。分类器看到 上一轮协议: data-analysis无终态,丢失信号。
  • state.status 不是 "terminal"(例如 "active""waiting""handed_off")时,回落到字面状态字符串 —— 分类器会看到 终态=active终态=handed_off,而对终态 run 看到的是 终态=completed/degraded/partial/continue/failed。这是两套互不兼容的词汇表被喂给同一个 LLM。
  • terminalDecision?.status 可以是 "continue"ProtocolCompletionDecision 的一个变体),不是终态,却被呈现为 终态=continue

字段叫 terminalStatus 但可能携带 "active" —— 有误导性,且 LLM 无法获知这些状态的含义。

建议: 规范化到一套一致的词汇表。要么始终映射到完成决策的状态(非终态 run 则省略该字段),要么在 prompt 里描述每个状态的含义。不要向分类器展示 终态=active

? state.terminalDecision?.status
: state.status;
return {
protocolId: state.protocolId,
protocolVersion: state.protocolVersion,
...(terminalStatus ? { terminalStatus } : {})
};
};
6 changes: 6 additions & 0 deletions apps/api/src/run-agent-assembly.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
type RunProtocolBoundary,
type ContextPackageRef,
type ProtocolStateStore,
type RoutingContext,
type TaskStateRuntime,
type WorkspaceAttachment
} from "@datafoundry/agent-runtime";
Expand Down Expand Up @@ -84,6 +85,10 @@ type CreateRunAgentAssemblyInput = {
userId: string;
workspaceId: string;
workspaceRoot: string;
/** Compact routing context resolved by the caller from the previous run's
* protocol snapshot + selected resources. Lets follow-up intents inherit the
* prior protocol so short prompts route correctly. */
routingContext?: RoutingContext;
};

/** Create the canonical agent run context used by Mastra tools, projections, and metadata. */
Expand Down Expand Up @@ -174,6 +179,7 @@ export const createRunAgentAssembly = async (
selectedSkills: input.selectedSkills,
skillSelection: input.skillSelection,
taskStateRuntime: input.taskStateRuntime,
...(input.routingContext ? { routingContext: input.routingContext } : {}),
...(!input.interactionResume && input.goal ? { goal: input.goal } : {}),
...(input.effectiveRunConfig.fileIds.length > 0
? { workspaceAttachments: resolveWorkspaceAttachments(input) }
Expand Down
10 changes: 10 additions & 0 deletions apps/api/src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ import { replayPendingProtocolEvents } from "./protocol-event-recovery.js";
import { assistantMessageIdFromEvent, completeProtocolRun } from "./protocol-run-completion.js";
import { persistCurrentUserMessage } from "./conversation-memory.js";
import { resolveEvidenceReferenceContext } from "./evidence-reference-context.js";
import { resolveRoutingContext } from "./resolve-routing-context.js";
import { createRunAgentAssembly, createRunAgentContext } from "./run-agent-assembly.js";
import { RunCheckpointProjector } from "./run-checkpoint-projector.js";
import { TraceSectionCoordinator } from "./trace-section-coordinator.js";
Expand Down Expand Up @@ -643,6 +644,14 @@ class DataFoundryAgUiAgent extends AbstractAgent {
eventPipeline.emit(event);
};
replayPendingProtocolEvents({ runId, stateStore: protocolStateStore, emit });
const routingContext = resolveRoutingContext({
metadataStore: this.input.metadataStore,
runId,
...(effectiveRunConfig.skillIds.length > 0 ? { selectedSkillIds: effectiveRunConfig.skillIds } : {}),
...(selectedDatasourceId ? { selectedDatasourceId } : {}),
sessionId,
userId: this.input.user.id
});
const agentAssembly = await createRunAgentAssembly({
abortSignal: runAbortController.signal,
contextPackageRecorder,
Expand Down Expand Up @@ -675,6 +684,7 @@ class DataFoundryAgUiAgent extends AbstractAgent {
selectedSkills,
skillSelection,
taskStateRuntime: this.input.taskStateRuntime,
...(routingContext ? { routingContext } : {}),
userId: this.input.user.id,
workspaceId: this.input.workspaceId,
workspaceRoot: this.input.workspaceRoot
Expand Down
9 changes: 8 additions & 1 deletion packages/agent-runtime/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ import { createTool, type ToolAction } from "@mastra/core/tools";
import { z } from "zod";
import {
createRunProtocolBoundary,
type RunProtocolBoundary
type RunProtocolBoundary,
type RoutingContext
} from "./protocol/run-protocol-boundary.js";
import type { ProtocolClassifier, ProtocolIdentity } from "./protocol/protocol-router.js";
import { createModelProtocolClassifier } from "./protocol/model-protocol-classifier.js";
Expand Down Expand Up @@ -253,6 +254,10 @@ export type CreateDataFoundryInput = {
protocolClassifier?: ProtocolClassifier;
analysisRequirementExtractor?: AnalysisRequirementExtractor;
analysisContractGrounder?: AnalysisContractGrounder;
/** Compact routing context for follow-up intents, sourced by the caller from the
* previous run's protocol snapshot + selected resources. Lets the protocol
* classifier see prior intent so short follow-ups inherit the right protocol. */
routingContext?: RoutingContext;
onProtocolEvent?(event: ProtocolEvent): void;
protocolStateStore?: ProtocolStateStore;
resourceRevisions?: Record<string, number>;
Expand Down Expand Up @@ -504,6 +509,7 @@ export const createDataFoundry = async (
}
: {}),
...(input.explicitProtocol ? { explicitProtocol: input.explicitProtocol } : {}),
...(input.routingContext ? { routingContext: input.routingContext } : {}),
classifier: input.protocolClassifier ?? createModelProtocolClassifier(input.modelProvider),
requirementExtractor: input.analysisRequirementExtractor
?? createModelAnalysisRequirementExtractor(input.modelProvider),
Expand Down Expand Up @@ -1486,6 +1492,7 @@ export { ProtocolRouter } from "./protocol/protocol-router.js";
export { ProtocolRuntime } from "./protocol/protocol-runtime.js";
export { createModelProtocolClassifier } from "./protocol/model-protocol-classifier.js";
export { createRunProtocolBoundary } from "./protocol/run-protocol-boundary.js";
export type { RoutingContext } from "./protocol/run-protocol-boundary.js";
export type * from "./protocol/run-protocol-boundary.js";
export { createGeneralTaskProtocol } from "./protocol/protocols/general-task.js";
export { createDataAnalysisProtocol } from "./protocol/protocols/data-analysis.js";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,31 @@ describe("createProtocolClassificationPrompt", () => {
expect(prompt).toContain("data-analysis@1");
expect(prompt).toContain("比较订单趋势");
expect(prompt).toContain("只能选择候选集合中的协议");
expect(prompt).toContain('"reasonCodes":["ANALYTIC_INTENT"]');
expect(prompt).toContain("INHERITED_PRIOR_PROTOCOL");
});

it("surfaces routing context fields so the classifier can inherit prior intent", () => {
const prompt = createProtocolClassificationPrompt({
candidates: [
{ protocolId: "general-task", protocolVersion: "1" },
{ protocolId: "data-analysis", protocolVersion: "1" }
],
value: {
userText: "再次尝试",
previousQuery: "帮我分析当前数据",
previousProtocol: { protocolId: "data-analysis", protocolVersion: "1", terminalStatus: "completed" },
selectedSkillIds: ["data-analysis"],
selectedDatasourceId: "orders-db"
}
});

expect(prompt).toContain("当前用户查询: 再次尝试");
expect(prompt).toContain("上一轮用户查询: 帮我分析当前数据");
expect(prompt).toContain("上一轮协议: data-analysis 终态=completed");
expect(prompt).toContain("已选 skill: data-analysis");
expect(prompt).toContain("已选数据源: orders-db");
expect(prompt).toContain("弱后续");
expect(prompt).toContain("延续 data-analysis");
});

it("strictly parses fenced JSON returned by compatible models", () => {
Expand Down
50 changes: 39 additions & 11 deletions packages/agent-runtime/src/protocol/model-protocol-classifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,45 @@ const classificationSchema = z.object({
export const createProtocolClassificationPrompt = (input: {
candidates: ProtocolIdentity[];
value: unknown;
}): string => [
"你是协议路由分类器,不是执行任务的 Agent。",
"只能选择候选集合中的协议,不得发明协议或调用工具。",
"data-analysis 用于需要数据源、schema、SQL、指标、统计或数据结论的任务。",
"general-task 用于日常问答、解释、总结、文件、知识检索和普通协作任务。",
`候选集合: ${input.candidates.map((item) => `${item.protocolId}@${item.protocolVersion}`).join(", ")}`,
`分类输入: ${JSON.stringify(input.value)}`,
"只返回一个 JSON 对象,不要 Markdown。字段为 protocolId、protocolVersion、confidence、reasonCodes。",
'格式示例: {"protocolId":"data-analysis","protocolVersion":"1","confidence":0.91,"reasonCodes":["ANALYTIC_INTENT"]}',
"reasonCodes 只能使用大写英文与下划线。"
].join("\n");
}): string => {
const value = isRecord(input.value) ? input.value : {};
const fields: string[] = [];
if (typeof value.userText === "string") {
fields.push(`当前用户查询: ${value.userText}`);
}
if (typeof value.previousQuery === "string") {
fields.push(`上一轮用户查询: ${value.previousQuery}`);
}
if (isRecord(value.previousProtocol)) {
const prev = value.previousProtocol;
const parts: string[] = [];
if (typeof prev.protocolId === "string") parts.push(prev.protocolId);
if (typeof prev.terminalStatus === "string") parts.push(`终态=${prev.terminalStatus}`);
if (parts.length > 0) fields.push(`上一轮协议: ${parts.join(" ")}`);
}
if (Array.isArray(value.selectedSkillIds) && value.selectedSkillIds.length > 0) {
fields.push(`已选 skill: ${value.selectedSkillIds.join(", ")}`);
}
if (typeof value.selectedDatasourceId === "string") {
fields.push(`已选数据源: ${value.selectedDatasourceId}`);
}
return [
"你是协议路由分类器,不是执行任务的 Agent。",
"只能选择候选集合中的协议,不得发明协议或调用工具。",
"data-analysis 用于需要数据源、schema、SQL、指标、统计或数据结论的任务。",
"general-task 用于日常问答、解释、总结、文件、知识检索和普通协作任务。",
"当当前查询是弱后续(如 继续、重试、再试、再来一次、try again 等)且上一轮使用的是 data-analysis 时,",
"应倾向于延续 data-analysis,除非用户明确切换了任务主题。",
`候选集合: ${input.candidates.map((item) => `${item.protocolId}@${item.protocolVersion}`).join(", ")}`,
...(fields.length > 0 ? ["分类输入:", ...fields] : [`分类输入: ${JSON.stringify(input.value)}`]),
"只返回一个 JSON 对象,不要 Markdown。字段为 protocolId、protocolVersion、confidence、reasonCodes。",
'格式示例: {"protocolId":"data-analysis","protocolVersion":"1","confidence":0.91,"reasonCodes":["INHERITED_PRIOR_PROTOCOL"]}',

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

正确性风险 #5INHERITED_PRIOR_PROTOCOL 不是可强制契约

prompt 示例建议用该 code,stub 分类器也返回了它,测试 expect(boundary.route.reasonCodes).toEqual(["INHERITED_PRIOR_PROTOCOL"]) 对 stub 通过。但真实 LLM 可返回任意 [A-Z_]+ code —— schema 允许。因此 reasonCodes 在生产中不是路由来源的可靠信号;它只是信息性的。

这没问题,但测试给人一种错误的确切感。建议: 要么在文档中注明 reasonCodes 是建议性而非可强制,要么从固定集合中约束它。

"reasonCodes 只能使用大写英文与下划线。"
].join("\n");
};

const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value);

/** Parse model text into the strict classifier contract without trusting provider-specific JSON modes. */
export const parseProtocolClassificationText = (text: string): z.infer<typeof classificationSchema> => {
Expand Down
102 changes: 102 additions & 0 deletions packages/agent-runtime/src/protocol/run-protocol-boundary.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1137,6 +1137,108 @@ describe("createRunProtocolBoundary", () => {
});
expect(boundary.protocolRuntime.getState("run-agent-handoff").phase).toBe("query_planning");
});

it("routes a weak follow-up to data-analysis when routingContext carries a prior data-analysis protocol", async () => {
const classificationInputs: unknown[] = [];
const boundary = await createRunProtocolBoundary({
runId: "run-follow-up-inherit",
userInput: "再次尝试",
authorizedProtocolIds: ["general-task", "data-analysis"],
initialContextPackageRef: { packageId: "context-follow-up", revision: 0 },
tools: {},
routingContext: {
previousQuery: "帮我分析当前数据",
previousProtocol: { protocolId: "data-analysis", protocolVersion: "1", terminalStatus: "completed" },
selectedSkillIds: ["data-analysis"],
selectedDatasourceId: "orders-db"
},
// Stub classifier simulates the LLM picking data-analysis when it sees the
// prior protocol in classificationInput. A real LLM is expected to do the same.
classifier: async ({ value }) => {
classificationInputs.push(value);
const record = value as Record<string, unknown>;
const previous = record.previousProtocol as { protocolId: string } | undefined;
return {
protocolId: previous?.protocolId === "data-analysis" ? "data-analysis" : "general-task",
protocolVersion: "1",
confidence: previous?.protocolId === "data-analysis" ? 0.9 : 0.4,
reasonCodes: previous?.protocolId === "data-analysis" ? ["INHERITED_PRIOR_PROTOCOL"] : ["WEAK_INTENT"]
};
},
projectContext: () => ({ packageId: "context-follow-up", revision: 0 })
});

expect(boundary.route.definition.id).toBe("data-analysis");
expect(boundary.route.source).toBe("classifier");
expect(boundary.route.reasonCodes).toEqual(["INHERITED_PRIOR_PROTOCOL"]);
// The classifier must receive the routing context fields, not just the bare follow-up text.
expect(classificationInputs).toEqual([expect.objectContaining({
userText: "再次尝试",
previousQuery: "帮我分析当前数据",
previousProtocol: expect.objectContaining({ protocolId: "data-analysis", terminalStatus: "completed" }),
selectedSkillIds: ["data-analysis"],
selectedDatasourceId: "orders-db"
})]);
});

it("falls back to general-task for a weak follow-up when no routingContext is provided", async () => {
const boundary = await createRunProtocolBoundary({
runId: "run-follow-up-no-context",
userInput: "再次尝试",
authorizedProtocolIds: ["general-task", "data-analysis"],
initialContextPackageRef: { packageId: "context-follow-up-none", revision: 0 },
tools: {},
// Without session context the classifier sees only the ambiguous text → low confidence.
classifier: async () => ({
protocolId: "data-analysis",
protocolVersion: "1",
confidence: 0.4,
reasonCodes: ["WEAK_INTENT"]
}),
projectContext: () => ({ packageId: "context-follow-up-none", revision: 0 })
});

expect(boundary.route.definition.id).toBe("general-task");
expect(boundary.route.source).toBe("default");
expect(boundary.route.warnings).toEqual(["PROTOCOL_CLASSIFICATION_LOW_CONFIDENCE"]);
});

it("emits a consistency warning when general-task is selected but data tools are exposed", async () => {
const events: ProtocolEvent[] = [];
await createRunProtocolBoundary({
runId: "run-consistency-mismatch",
userInput: "再次尝试",
authorizedProtocolIds: ["general-task", "data-analysis"],
explicitProtocol: { protocolId: "general-task", protocolVersion: "1" },
initialContextPackageRef: { packageId: "context-consistency", revision: 0 },
tools: { inspect_schema: { execute: async () => ({}) } },
routingContext: { selectedDatasourceId: "orders-db" },
runtimeOptions: { onEvent: (event) => events.push(event) },
projectContext: () => ({ packageId: "context-consistency", revision: 0 })
});

expect(events).toContainEqual(expect.objectContaining({
type: "protocol.route.consistency.warning",
payload: { code: "PROTOCOL_TOOL_POLICY_MISMATCH", reason: expect.stringContaining("general-task rejects every data action") }
}));
});

it("does not emit a consistency warning when no datasource is selected", async () => {
const events: ProtocolEvent[] = [];
const boundary = await createRunProtocolBoundary({
runId: "run-consistency-no-datasource",
userInput: "再次尝试",
authorizedProtocolIds: ["general-task", "data-analysis"],
explicitProtocol: { protocolId: "general-task", protocolVersion: "1" },
initialContextPackageRef: { packageId: "context-consistency-none", revision: 0 },
tools: { inspect_schema: { execute: async () => ({}) } },
runtimeOptions: { onEvent: (event) => events.push(event) },
projectContext: () => ({ packageId: "context-consistency-none", revision: 0 })
});

expect(events.some((event) => event.type === "protocol.route.consistency.warning")).toBe(false);
expect(boundary.route.warnings).not.toContain("PROTOCOL_TOOL_POLICY_MISMATCH");
});
});

const liveSemanticProvider = (): { resolve(request: SemanticRequest): Promise<SemanticResolution> } => ({
Expand Down
Loading