diff --git a/apps/backend/.env.example b/apps/backend/.env.example index ea00fc7..2165d58 100644 --- a/apps/backend/.env.example +++ b/apps/backend/.env.example @@ -1,6 +1,7 @@ PORT=3002 NODE_ENV=development SQLITE_PATH=../../data/sqlite/text2sql.db +SQLITE_ALLOWED_DIRS=../../data/sqlite,../../data/uploads/datasources CORS_ALLOWED_ORIGINS=http://localhost:3000 DATASOURCE_UPLOAD_DIR=../../data/uploads/datasources DATASOURCE_UPLOAD_MAX_BYTES=10485760 @@ -23,9 +24,10 @@ LLM_MODEL=doubao-1.5-pro-32k LLM_API_KEY= LLM_BASE_URL= LLM_TIMEOUT_MS=30000 +LLM_STREAM_TIMEOUT_MS=90000 LLM_MOCK_MODE=false LLM_FALLBACK_PROVIDERS=siliconflow,minimax -AGENT_PLANNING_SCAFFOLD_ENABLED=false +AGENT_PLANNING_SCAFFOLD_ENABLED=true # Clarification hybrid gate defaults: # - enabled by default diff --git a/apps/backend/src/modules/chat/chat.controller.ts b/apps/backend/src/modules/chat/chat.controller.ts index 3aa72af..9fdc7f1 100644 --- a/apps/backend/src/modules/chat/chat.controller.ts +++ b/apps/backend/src/modules/chat/chat.controller.ts @@ -133,7 +133,9 @@ export class ChatController { const run = await this.chatService.sendMessage( sessionId, body.message, - req.requestId + req.requestId, + undefined, + req.actor ); return ok(req.requestId, { kind: "agent-run", @@ -195,7 +197,9 @@ export class ChatController { req.requestId, async (event) => { sendEvent(event.type, event); - } + }, + undefined, + req.actor ); } catch (error) { const fallbackEvent: ChatStreamEvent = { diff --git a/apps/backend/src/modules/config/app-config.service.ts b/apps/backend/src/modules/config/app-config.service.ts index 907dda8..b906bf3 100644 --- a/apps/backend/src/modules/config/app-config.service.ts +++ b/apps/backend/src/modules/config/app-config.service.ts @@ -30,15 +30,20 @@ export class AppConfigService { get sqlitePath(): string { const raw = this.config.get("SQLITE_PATH", "data/sqlite/text2sql.db"); - if (isAbsolute(raw)) { - return raw; - } - const direct = resolve(process.cwd(), raw); - if (existsSync(direct)) { - return direct; - } - const fallback = resolve(process.cwd(), "../../", raw); - return fallback; + return this.resolveConfiguredPath(raw); + } + + get sqliteAllowedDirs(): string[] { + const raw = this.config.get( + "SQLITE_ALLOWED_DIRS", + "data/sqlite,data/uploads/datasources" + ); + const parsed = raw + .split(",") + .map((item) => item.trim()) + .filter(Boolean) + .map((item) => this.resolveConfiguredPath(item)); + return Array.from(new Set(parsed)); } get redisUrl(): string { @@ -50,14 +55,7 @@ export class AppConfigService { "DATASOURCE_UPLOAD_DIR", "data/uploads/datasources" ); - if (isAbsolute(raw)) { - return raw; - } - const direct = resolve(process.cwd(), raw); - if (existsSync(direct)) { - return direct; - } - return resolve(process.cwd(), "../../", raw); + return this.resolveConfiguredPath(raw); } get datasourceUploadMaxBytes(): number { @@ -125,6 +123,19 @@ export class AppConfigService { return Number(this.config.get("LLM_TIMEOUT_MS", "30000")); } + get llmStreamTimeoutMs(): number { + const fallback = this.llmTimeoutMs; + const raw = this.config.get("LLM_STREAM_TIMEOUT_MS", String(fallback)); + const parsed = Number(raw); + if (Number.isFinite(parsed) && parsed > 0) { + return Math.floor(parsed); + } + this.logger.warn( + `LLM_STREAM_TIMEOUT_MS 配置无效（${raw}），已回退到 LLM_TIMEOUT_MS=${fallback}。` + ); + return fallback; + } + get llmMockMode(): boolean { return this.config.get("LLM_MOCK_MODE", "false") === "true"; } @@ -275,4 +286,17 @@ export class AppConfigService { this.logger.warn(`${key} 配置无效（${raw}），已回退默认值 ${defaultValue}。`); return defaultValue; } + + private resolveConfiguredPath(raw: string): string { + if (isAbsolute(raw)) { + return resolve(raw); + } + + const direct = resolve(process.cwd(), raw); + const fallback = resolve(process.cwd(), "../../", raw); + if (existsSync(direct) || !existsSync(fallback)) { + return direct; + } + return fallback; + } } diff --git a/apps/backend/src/modules/conversation/agent/agent.module.ts b/apps/backend/src/modules/conversation/agent/agent.module.ts index 7f01026..e9286d4 100644 --- a/apps/backend/src/modules/conversation/agent/agent.module.ts +++ b/apps/backend/src/modules/conversation/agent/agent.module.ts @@ -24,6 +24,7 @@ import { BuildSemanticQueryNode } from "./nodes/build-semantic-query.node"; import { GraphBuilderService } from "./graph/graph.builder"; import { LangGraphRuntimeService } from "./graph/langgraph.runtime"; import { GenerateSqlNode } from "./nodes/generate-sql.node"; +import { ResolveSavedPriorSqlNode } from "./nodes/resolve-saved-prior-sql.node"; import { SafetyCheckNode } from "./nodes/safety-check.node"; import { ExecuteSqlNode } from "./nodes/execute-sql.node"; import { SqlGenerationService } from "./sql/sql-generation.service"; @@ -70,6 +71,7 @@ import { PlannerCacheService } from "./planner/planner-cache.service"; PlannerCacheService, BuildSemanticQueryNode, BuildPhysicalPlanNode, + ResolveSavedPriorSqlNode, GenerateSqlNode, SafetyCheckNode, ExecuteSqlNode, diff --git a/apps/backend/src/modules/conversation/agent/graph/agent-step-metadata.ts b/apps/backend/src/modules/conversation/agent/graph/agent-step-metadata.ts index 81abd28..45f62c2 100644 --- a/apps/backend/src/modules/conversation/agent/graph/agent-step-metadata.ts +++ b/apps/backend/src/modules/conversation/agent/graph/agent-step-metadata.ts @@ -8,6 +8,7 @@ const AGENT_STEP_STAGE_MAP: Record = { "build-intent-plan": "analysis", "build-semantic-query": "analysis", "build-physical-plan": "analysis", + "resolve-saved-prior-sql": "analysis", "generate-sql": "generation", "safety-check": "validation", "execute-sql": "execution", @@ -20,6 +21,7 @@ const AGENT_STEP_TITLE_MAP: Record = { "build-intent-plan": "意图规划", "build-semantic-query": "语义规划", "build-physical-plan": "物理规划", + "resolve-saved-prior-sql": "Prior SQL 复用判断", "generate-sql": "生成 SQL", "safety-check": "安全校验", "execute-sql": "执行查询", diff --git a/apps/backend/src/modules/conversation/agent/graph/langgraph.node-handlers.ts b/apps/backend/src/modules/conversation/agent/graph/langgraph.node-handlers.ts index 89113e5..f4f19d2 100644 --- a/apps/backend/src/modules/conversation/agent/graph/langgraph.node-handlers.ts +++ b/apps/backend/src/modules/conversation/agent/graph/langgraph.node-handlers.ts @@ -8,6 +8,7 @@ import { ClarifyNode } from "../nodes/clarify.node"; import { ExecuteSqlNode } from "../nodes/execute-sql.node"; import { FormatAnswerNode } from "../nodes/format-answer.node"; import { GenerateSqlNode } from "../nodes/generate-sql.node"; +import { ResolveSavedPriorSqlNode } from "../nodes/resolve-saved-prior-sql.node"; import { RetrieveKnowledgeNode } from "../nodes/retrieve-knowledge.node"; import { SafetyCheckNode } from "../nodes/safety-check.node"; import type { @@ -198,6 +199,7 @@ export interface LangGraphNodeDependencies { buildIntentPlanNode: Pick; buildSemanticQueryNode: Pick; buildPhysicalPlanNode: Pick; + resolveSavedPriorSqlNode: Pick; generateSqlNode: Pick; safetyNode: Pick; executeNode: Pick; @@ -835,6 +837,7 @@ export const createLangGraphNodeHandlers = (deps: LangGraphNodeDependencies) => }, sql: generated.sql, explanation: generated.explanation, + savedPriorSqlFallbackRequested: false, error: undefined, fatalError: undefined }; @@ -920,6 +923,9 @@ export const createLangGraphNodeHandlers = (deps: LangGraphNodeDependencies) => workspaceId: state.accessContext?.workspaceId }; if (!safety.allowed) { + const shouldFallbackToGeneration = + state.savedPriorSqlResolution?.status === "hit" && + state.savedPriorSqlResolution.fallbackToGeneration === false; const trace = appendStep(state, { step: { node: "safety-check", @@ -939,10 +945,28 @@ export const createLangGraphNodeHandlers = (deps: LangGraphNodeDependencies) => error: safety.reason }); await runtime.emitStep(latestStep(trace)); + if (shouldFallbackToGeneration) { + return { + ...trace, + error: safety.reason, + safetyDecision: safety, + savedPriorSqlResolution: { + ...state.savedPriorSqlResolution, + safetyRejected: true, + fallbackToGeneration: true, + reused: true + }, + savedPriorSqlFallbackRequested: true, + sql: undefined, + explanation: undefined, + terminalStatus: undefined + }; + } return { ...trace, error: safety.reason, safetyDecision: safety, + savedPriorSqlFallbackRequested: false, terminalStatus: "rejected" }; } @@ -973,7 +997,82 @@ export const createLangGraphNodeHandlers = (deps: LangGraphNodeDependencies) => await runtime.emitStep(latestStep(trace)); return { ...trace, - safetyDecision: safety + safetyDecision: safety, + savedPriorSqlFallbackRequested: false + }; + }; + + const resolveSavedPriorSql = async ( + state: LangGraphState, + config?: LangGraphRunnableConfig + ) => { + const startedAt = new Date().toISOString(); + const runtime = createNodeRuntime(config); + await runtime.emitStep( + buildRunningStep( + state, + "resolve-saved-prior-sql", + startedAt, + "正在评估是否复用已保存 SQL" + ) + ); + const resolution = deps.resolveSavedPriorSqlNode.run({ + retrievalBundle: state.retrievalBundle, + question: state.question + }); + const endedAt = new Date().toISOString(); + const detail = + resolution.status === "hit" + ? "命中已保存 SQL，跳过 LLM SQL 生成。" + : `未命中 saved prior shortcut，状态=${resolution.status}。`; + const trace = appendStep(state, { + step: { + node: "resolve-saved-prior-sql", + status: resolution.status === "hit" ? "success" : "skipped", + detail, + ...withTiming(startedAt, endedAt), + outputSummary: summarize({ + status: resolution.status, + reasonCodes: resolution.reasonCodes, + selectedChunkId: resolution.selectedChunkId, + selectedViewId: resolution.selectedViewId, + selectedSourceRunId: resolution.selectedSourceRunId + }) + }, + outputs: { + status: resolution.status, + reasonCodes: resolution.reasonCodes, + selectedChunkId: resolution.selectedChunkId, + selectedViewId: resolution.selectedViewId, + selectedSourceRunId: resolution.selectedSourceRunId + } + }); + await runtime.emitStep(latestStep(trace)); + if (resolution.status !== "hit" || !resolution.sql) { + return { + ...trace, + savedPriorSqlResolution: { + ...resolution, + reused: false, + safetyRejected: false, + fallbackToGeneration: false + }, + savedPriorSqlFallbackRequested: false + }; + } + return { + ...trace, + model: "saved-prior-sql", + sql: resolution.sql, + explanation: resolution.explanation, + error: undefined, + savedPriorSqlResolution: { + ...resolution, + reused: true, + safetyRejected: false, + fallbackToGeneration: false + }, + savedPriorSqlFallbackRequested: false }; }; @@ -1144,6 +1243,7 @@ export const createLangGraphNodeHandlers = (deps: LangGraphNodeDependencies) => buildIntentPlan, buildSemanticQuery, buildPhysicalPlan, + resolveSavedPriorSql, generateSql, safetyCheck, executeSql, diff --git a/apps/backend/src/modules/conversation/agent/graph/langgraph.runtime.ts b/apps/backend/src/modules/conversation/agent/graph/langgraph.runtime.ts index 388c72e..e94a041 100644 --- a/apps/backend/src/modules/conversation/agent/graph/langgraph.runtime.ts +++ b/apps/backend/src/modules/conversation/agent/graph/langgraph.runtime.ts @@ -14,6 +14,7 @@ import { ClarifyNode } from "../nodes/clarify.node"; import { ExecuteSqlNode } from "../nodes/execute-sql.node"; import { FormatAnswerNode } from "../nodes/format-answer.node"; import { GenerateSqlNode } from "../nodes/generate-sql.node"; +import { ResolveSavedPriorSqlNode } from "../nodes/resolve-saved-prior-sql.node"; import { RetrieveKnowledgeNode } from "../nodes/retrieve-knowledge.node"; import { SafetyCheckNode } from "../nodes/safety-check.node"; import { @@ -46,6 +47,8 @@ const LangGraphStateAnnotation = Annotation.Root({ physicalPlan: Annotation(), planningStatus: Annotation(), planningWarnings: Annotation(), + savedPriorSqlResolution: Annotation(), + savedPriorSqlFallbackRequested: Annotation(), safetyDecision: Annotation(), sql: Annotation(), explanation: Annotation(), @@ -69,6 +72,7 @@ export const createLangGraphRuntime = (deps: LangGraphNodeDependencies) => { .addNode("build-intent-plan", handlers.buildIntentPlan) .addNode("build-semantic-query", handlers.buildSemanticQuery) .addNode("build-physical-plan", handlers.buildPhysicalPlan) + .addNode("resolve-saved-prior-sql", handlers.resolveSavedPriorSql) .addNode("generate-sql", handlers.generateSql) .addNode("safety-check", handlers.safetyCheck) .addNode("execute-sql", handlers.executeSql) @@ -86,7 +90,16 @@ export const createLangGraphRuntime = (deps: LangGraphNodeDependencies) => { .addEdge("retrieve-knowledge", "build-intent-plan") .addEdge("build-intent-plan", "build-semantic-query") .addEdge("build-semantic-query", "build-physical-plan") - .addEdge("build-physical-plan", "generate-sql") + .addEdge("build-physical-plan", "resolve-saved-prior-sql") + .addConditionalEdges( + "resolve-saved-prior-sql", + (state) => + state.savedPriorSqlResolution?.status === "hit" ? "shortcut" : "continue", + { + shortcut: "safety-check", + continue: "generate-sql" + } + ) .addConditionalEdges( "generate-sql", (state) => (state.fatalError ? "fatal" : "continue"), @@ -97,8 +110,14 @@ export const createLangGraphRuntime = (deps: LangGraphNodeDependencies) => { ) .addConditionalEdges( "safety-check", - (state) => (state.terminalStatus === "rejected" ? "rejected" : "continue"), + (state) => { + if (state.savedPriorSqlFallbackRequested) { + return "fallback-generate"; + } + return state.terminalStatus === "rejected" ? "rejected" : "continue"; + }, { + "fallback-generate": "generate-sql", rejected: END, continue: "execute-sql" } @@ -142,6 +161,7 @@ export class LangGraphRuntimeService { private readonly buildIntentPlanNode: BuildIntentPlanNode, private readonly buildSemanticQueryNode: BuildSemanticQueryNode, private readonly buildPhysicalPlanNode: BuildPhysicalPlanNode, + private readonly resolveSavedPriorSqlNode: ResolveSavedPriorSqlNode, private readonly generateSqlNode: GenerateSqlNode, private readonly safetyNode: SafetyCheckNode, private readonly executeNode: ExecuteSqlNode, @@ -153,6 +173,7 @@ export class LangGraphRuntimeService { buildIntentPlanNode: this.buildIntentPlanNode, buildSemanticQueryNode: this.buildSemanticQueryNode, buildPhysicalPlanNode: this.buildPhysicalPlanNode, + resolveSavedPriorSqlNode: this.resolveSavedPriorSqlNode, generateSqlNode: this.generateSqlNode, safetyNode: this.safetyNode, executeNode: this.executeNode, diff --git a/apps/backend/src/modules/conversation/agent/graph/langgraph.state.ts b/apps/backend/src/modules/conversation/agent/graph/langgraph.state.ts index f04e254..0170cc2 100644 --- a/apps/backend/src/modules/conversation/agent/graph/langgraph.state.ts +++ b/apps/backend/src/modules/conversation/agent/graph/langgraph.state.ts @@ -11,6 +11,7 @@ import type { IntentPlan } from "../nodes/build-intent-plan.node"; import type { SemanticQueryPlan } from "../nodes/build-semantic-query.node"; import type { PhysicalPlan } from "../nodes/build-physical-plan.node"; import type { SqlSafetyDecision } from "../sql/tools/sql-safety.guard"; +import type { SavedPriorSqlResolution } from "../nodes/resolve-saved-prior-sql.node"; import type { RagContextPack, RagRetrievalBundle @@ -37,6 +38,12 @@ export interface LangGraphState extends GraphInput { physicalPlan?: PhysicalPlan; planningStatus?: "legacy" | "ready" | "degraded"; planningWarnings?: string[]; + savedPriorSqlResolution?: SavedPriorSqlResolution & { + reused: boolean; + safetyRejected: boolean; + fallbackToGeneration: boolean; + }; + savedPriorSqlFallbackRequested?: boolean; safetyDecision?: SqlSafetyDecision; sql?: string; explanation?: string; diff --git a/apps/backend/src/modules/conversation/agent/nodes/clarification-semantic-evaluator.service.ts b/apps/backend/src/modules/conversation/agent/nodes/clarification-semantic-evaluator.service.ts index 6458812..33c1e94 100644 --- a/apps/backend/src/modules/conversation/agent/nodes/clarification-semantic-evaluator.service.ts +++ b/apps/backend/src/modules/conversation/agent/nodes/clarification-semantic-evaluator.service.ts @@ -9,8 +9,9 @@ import { AppConfigService } from "../../../config/app-config.service"; const METRIC_HINT_REGEX = /(总数|数量|计数|count|金额|交易额|销售额|gmv|平均|均值|占比|比例|转化率|留存|退款率|分布|top|排行|环比|同比)/i; +const COUNT_METRIC_HINT_REGEX = /(有多少|多少|几(个|笔|单|条|次|家|人|位))/i; const SUBJECT_HINT_REGEX = - /(订单|用户|客户|商品|sku|门店|支付|退款|交易|会话|工单|发票|供应商|渠道|地区|市场|销售|商家|账单)/i; + /(订单|用户|客户|顾客|消费者|商品|sku|门店|支付|退款|交易|会话|工单|发票|供应商|渠道|地区|市场|销售|商家|账单)/i; const TIME_HINT_REGEX = /(今天|昨日|昨天|本周|上周|本月|上月|本季度|上季度|本年|去年|近\d+\s*(天|周|月|年)|最近|过去|between|from|to|日期|时间)/i; const TREND_HINT_REGEX = /(趋势|变化|对比|同比|环比|走势)/i; @@ -208,11 +209,14 @@ export class ClarificationSemanticEvaluatorService { private collectMissingCriticalSlots(question: string): ClarificationSlotKey[] { const missing = new Set(); + const subjectDetected = SUBJECT_HINT_REGEX.test(question); + const metricDetected = + METRIC_HINT_REGEX.test(question) || COUNT_METRIC_HINT_REGEX.test(question); - if (!SUBJECT_HINT_REGEX.test(question)) { + if (!subjectDetected) { missing.add("subject"); } - if (!METRIC_HINT_REGEX.test(question)) { + if (!metricDetected) { missing.add("metric"); } if (TREND_HINT_REGEX.test(question) && !TIME_HINT_REGEX.test(question)) { diff --git a/apps/backend/src/modules/conversation/agent/nodes/format-answer.node.ts b/apps/backend/src/modules/conversation/agent/nodes/format-answer.node.ts index 0bfc4c0..1eabb7a 100644 --- a/apps/backend/src/modules/conversation/agent/nodes/format-answer.node.ts +++ b/apps/backend/src/modules/conversation/agent/nodes/format-answer.node.ts @@ -10,11 +10,84 @@ export class FormatAnswerNode { if (rows.length === 0) { return "查询已执行，但没有匹配数据。你可以补充时间范围或筛选条件。"; } - const preview = rows.slice(0, 3); - return [ - `已完成查询：${question}`, - `返回 ${rows.length} 行，字段为：${columns.join(", ") || "无"}`, - `样例结果：${JSON.stringify(preview)}` - ].join("\n"); + + const normalizedColumns = this.normalizeColumns(columns, rows); + const numericColumns = normalizedColumns.filter((column) => + this.isNumeric(rows[0]?.[column]) + ); + const lead = `已完成分析：${question}`; + const coverage = `共返回 ${rows.length} 条记录（字段：${normalizedColumns.slice(0, 4).join("、") || "未命名字段"}${ + normalizedColumns.length > 4 ? " 等" : "" + }）。`; + const metricSummary = this.buildMetricSummary(rows, numericColumns); + + return [lead, coverage, metricSummary, "如需核验明细，可展开图表、表格或 SQL 证据。"] + .filter((line) => line.length > 0) + .join("\n"); + } + + private buildMetricSummary( + rows: Array>, + numericColumns: string[] + ): string { + if (rows.length === 0 || numericColumns.length === 0) { + return "当前结果以明细数据为主，已生成可读预览。"; + } + + const firstRow = rows[0] ?? {}; + const fragments = numericColumns.slice(0, 2).map((column) => { + const value = firstRow[column]; + return `${column}=${this.formatValue(value)}`; + }); + if (fragments.length === 0) { + return "当前结果以明细数据为主，已生成可读预览。"; + } + return `核心指标摘要：${fragments.join("，")}。`; + } + + private normalizeColumns( + columns: string[], + rows: Array> + ): string[] { + const candidates = + columns.length > 0 + ? columns + : rows.length > 0 + ? Object.keys(rows[0] ?? {}) + : []; + const deduped: string[] = []; + const seen = new Set(); + for (const value of candidates) { + const trimmed = value.trim(); + if (!trimmed || seen.has(trimmed)) { + continue; + } + seen.add(trimmed); + deduped.push(trimmed); + } + return deduped; + } + + private isNumeric(value: unknown): boolean { + if (typeof value === "number" && Number.isFinite(value)) { + return true; + } + if (typeof value === "string" && value.trim().length > 0) { + return Number.isFinite(Number(value)); + } + return false; + } + + private formatValue(value: unknown): string { + if (typeof value === "number" && Number.isFinite(value)) { + return Number.isInteger(value) ? String(value) : value.toFixed(2); + } + if (typeof value === "string" && value.trim().length > 0) { + return value.trim(); + } + if (value === null || value === undefined) { + return "n/a"; + } + return String(value); } } diff --git a/apps/backend/src/modules/conversation/agent/nodes/resolve-saved-prior-sql.node.ts b/apps/backend/src/modules/conversation/agent/nodes/resolve-saved-prior-sql.node.ts new file mode 100644 index 0000000..2e89bca --- /dev/null +++ b/apps/backend/src/modules/conversation/agent/nodes/resolve-saved-prior-sql.node.ts @@ -0,0 +1,157 @@ +import { Injectable } from "@nestjs/common"; +import type { + RagPriorSqlLaneEvidence, + RagPriorSqlShortcutDecision, + RagRetrievalBundle, + RagRetrievalChunkPayload +} from "../../../knowledge/rag/retrieval/rag-retrieval.types"; + +export interface SavedPriorSqlResolution { + status: "hit" | "miss" | "filtered" | "stale" | "ambiguous"; + reasonCodes: string[]; + selectedChunkId?: string; + selectedViewId?: string; + selectedSourceRunId?: string; + sql?: string; + explanation?: string; +} + +@Injectable() +export class ResolveSavedPriorSqlNode { + run(input: { + retrievalBundle?: RagRetrievalBundle; + question: string; + }): SavedPriorSqlResolution { + const lane = this.readPriorSqlLane(input.retrievalBundle); + if (!lane) { + return { + status: "miss", + reasonCodes: ["prior_sql_lane_missing"] + }; + } + const decision = this.readShortcutDecision(lane); + const status = decision?.status ?? lane.status; + const reasonCodes = this.readReasonCodes(decision, lane); + + if (status !== "hit") { + return { + status, + reasonCodes, + selectedChunkId: decision?.selected_chunk_id ?? decision?.selectedChunkId, + selectedViewId: decision?.selected_view_id ?? decision?.selectedViewId, + selectedSourceRunId: + decision?.selected_source_run_id ?? decision?.selectedSourceRunId + }; + } + + const selectedChunk = this.pickSelectedChunk(input.retrievalBundle, decision); + const sql = this.extractSql(selectedChunk); + if (!sql) { + return { + status: "miss", + reasonCodes: ["prior_sql_shortcut_missing_sql"], + selectedChunkId: selectedChunk?.chunk_id, + selectedViewId: decision?.selected_view_id ?? decision?.selectedViewId, + selectedSourceRunId: + decision?.selected_source_run_id ?? decision?.selectedSourceRunId + }; + } + + return { + status: "hit", + reasonCodes, + selectedChunkId: selectedChunk?.chunk_id, + selectedViewId: this.readString( + selectedChunk?.metadata.sourceMetadata?.viewId ?? + selectedChunk?.metadata.sourceMetadata?.view_id ?? + decision?.selected_view_id ?? + decision?.selectedViewId + ), + selectedSourceRunId: this.readString( + selectedChunk?.metadata.sourceMetadata?.sourceRunId ?? + selectedChunk?.metadata.sourceMetadata?.source_run_id ?? + decision?.selected_source_run_id ?? + decision?.selectedSourceRunId + ), + sql, + explanation: `复用已保存 SQL（问题：${input.question}）。` + }; + } + + private readPriorSqlLane( + bundle: RagRetrievalBundle | undefined + ): RagPriorSqlLaneEvidence | undefined { + if (!bundle) { + return undefined; + } + return bundle.prior_sql_lane ?? bundle.priorSqlLane; + } + + private readShortcutDecision( + lane: RagPriorSqlLaneEvidence + ): RagPriorSqlShortcutDecision | undefined { + return lane.shortcut ?? lane.shortcutDecision; + } + + private readReasonCodes( + decision: RagPriorSqlShortcutDecision | undefined, + lane: RagPriorSqlLaneEvidence + ): string[] { + const reasonCodes = decision?.reason_codes ?? decision?.reasonCodes ?? []; + if (reasonCodes.length > 0) { + return reasonCodes; + } + return lane.degrade_reasons ?? lane.degradeReasons ?? []; + } + + private pickSelectedChunk( + bundle: RagRetrievalBundle | undefined, + decision?: RagPriorSqlShortcutDecision + ): RagRetrievalChunkPayload | undefined { + if (!bundle) { + return undefined; + } + const selectedChunkId = decision?.selected_chunk_id ?? decision?.selectedChunkId; + const selectedFromCandidates = bundle.candidates.find( + (candidate) => candidate.chunk_id === selectedChunkId + )?.chunk; + if (selectedFromCandidates) { + return selectedFromCandidates; + } + if (selectedChunkId) { + return bundle.selected_context?.find((chunk) => chunk.chunk_id === selectedChunkId); + } + return bundle.candidates.find((candidate) => + candidate.evidence.includes("prior_sql:trusted") + )?.chunk; + } + + private extractSql(chunk: RagRetrievalChunkPayload | undefined): string | undefined { + if (!chunk) { + return undefined; + } + const sourceMetadata = chunk.metadata.sourceMetadata; + const directSql = + this.readString(sourceMetadata?.sql) ?? + this.readString(sourceMetadata?.viewSql) ?? + this.readString(sourceMetadata?.view_sql); + if (directSql) { + return directSql; + } + + const fromContent = /\bSQL\s*:\s*([\s\S]+)/i.exec(chunk.content)?.[1]; + const normalized = this.readString(fromContent); + if (normalized) { + return normalized; + } + return this.readString(chunk.content); + } + + private readString(value: unknown): string | undefined { + if (typeof value !== "string") { + return undefined; + } + const normalized = value.trim(); + return normalized.length > 0 ? normalized : undefined; + } +} diff --git a/apps/backend/src/modules/conversation/agent/nodes/slot-filling-context.ts b/apps/backend/src/modules/conversation/agent/nodes/slot-filling-context.ts index ca1d075..c9151e3 100644 --- a/apps/backend/src/modules/conversation/agent/nodes/slot-filling-context.ts +++ b/apps/backend/src/modules/conversation/agent/nodes/slot-filling-context.ts @@ -133,8 +133,9 @@ function resolveConfidence(missingSlots: SlotKey[]): SlotFillingConfidence { const METRIC_HINT_REGEX = /(总数|数量|计数|count|金额|交易额|销售额|gmv|平均|均值|占比|比例|转化率|留存|退款率|分布|top|排行|环比|同比)/i; +const COUNT_METRIC_HINT_REGEX = /(有多少|多少|几(个|笔|单|条|次|家|人|位))/i; const SUBJECT_HINT_REGEX = - /(订单|用户|客户|商品|sku|门店|支付|退款|交易|会话|工单|发票|供应商|渠道|地区|市场|销售|商家|账单)/i; + /(订单|用户|客户|顾客|消费者|商品|sku|门店|支付|退款|交易|会话|工单|发票|供应商|渠道|地区|市场|销售|商家|账单)/i; const TIME_HINT_REGEX = /(今天|昨日|昨天|本周|上周|本月|上月|本季度|上季度|本年|去年|近\d+\s*(天|周|月|年)|最近|过去|between|from|to|日期|时间)/i; const DIMENSION_HINT_REGEX = @@ -185,8 +186,9 @@ function resolveSlots(question: string, contextEnvelope?: ContextEnvelope): Slot hasStringArray(contextEnvelope?.mustExcludeTables); const subjectFromEnvelope = dimensionFromEnvelope; - const metricFromQuestion = METRIC_HINT_REGEX.test(trimmedQuestion); const subjectFromQuestion = SUBJECT_HINT_REGEX.test(trimmedQuestion); + const metricFromQuestion = + METRIC_HINT_REGEX.test(trimmedQuestion) || COUNT_METRIC_HINT_REGEX.test(trimmedQuestion); const timeFromQuestion = TIME_HINT_REGEX.test(trimmedQuestion); const dimensionFromQuestion = DIMENSION_HINT_REGEX.test(trimmedQuestion); const filterFromQuestion = FILTER_HINT_REGEX.test(trimmedQuestion); diff --git a/apps/backend/src/modules/conversation/agent/sql/sql-generation.service.ts b/apps/backend/src/modules/conversation/agent/sql/sql-generation.service.ts index f65c5ff..1957980 100644 --- a/apps/backend/src/modules/conversation/agent/sql/sql-generation.service.ts +++ b/apps/backend/src/modules/conversation/agent/sql/sql-generation.service.ts @@ -165,7 +165,8 @@ export class SqlGenerationService { } return ( error.code === "LLM_TOOL_CALL_ONLY_RESPONSE" || - error.code === "LLM_SQL_EXTRACT_FAILED" + error.code === "LLM_SQL_EXTRACT_FAILED" || + error.code === "LLM_TOOL_CALL_EXECUTION_FAILED" ); } diff --git a/apps/backend/src/modules/conversation/agent/sql/sql-prompt.builder.ts b/apps/backend/src/modules/conversation/agent/sql/sql-prompt.builder.ts index a2a4777..79b9dbc 100644 --- a/apps/backend/src/modules/conversation/agent/sql/sql-prompt.builder.ts +++ b/apps/backend/src/modules/conversation/agent/sql/sql-prompt.builder.ts @@ -98,7 +98,9 @@ export class SqlPromptBuilder { return [ "Semantic guardrail: this is a business count-intent query.", "The final SQL must contain COUNT(...) aggregation over business data.", - "Do not return schema/metadata introspection SQL." + "Do not return schema/metadata introspection SQL.", + "Do not query schema system tables via tools", + "(sqlite_master/sqlite_schema/information_schema/pg_catalog/pragma) unless the user explicitly asks metadata." ].join(" "); } if (intent === "metadata") { @@ -109,7 +111,11 @@ export class SqlPromptBuilder { "Do not return business row counting SQL." ].join(" "); } - return "Semantic guardrail: ensure SQL semantics strictly match the user question intent."; + return [ + "Semantic guardrail: ensure SQL semantics strictly match the user question intent.", + "Do not query schema system tables via tools", + "(sqlite_master/sqlite_schema/information_schema/pg_catalog/pragma) unless the user explicitly asks metadata." + ].join(" "); } private buildRepairHintBlock(retryReason?: string): string { diff --git a/apps/backend/src/modules/conversation/chat/application/execute-message.usecase.ts b/apps/backend/src/modules/conversation/chat/application/execute-message.usecase.ts index 116956d..21b1103 100644 --- a/apps/backend/src/modules/conversation/chat/application/execute-message.usecase.ts +++ b/apps/backend/src/modules/conversation/chat/application/execute-message.usecase.ts @@ -7,7 +7,10 @@ import { DatasourceService } from "../../../governance/datasource/datasource.ser import { RedisBufferService } from "../../../platform/data/cache/index"; import { ChatRepository } from "../../../platform/data/persistence/index"; import { ChatDeliveryEnrichmentService } from "./shared/chat-delivery-enrichment.service"; -import { ChatPolicyGuardService } from "./shared/chat-policy-guard.service"; +import { + ChatPolicyGuardService, + type ChatPolicyActorInput +} from "./shared/chat-policy-guard.service"; import { ChatPostRunHooksService } from "./shared/chat-post-run-hooks.service"; import { ChatRunPersistenceService } from "./shared/chat-run-persistence.service"; import { DomainError } from "../../../../common/domain-error"; @@ -17,6 +20,7 @@ export interface ExecuteMessageInput { message: string; requestId?: string; contextEnvelope?: ContextEnvelope; + actor?: ChatPolicyActorInput; } @Injectable() @@ -45,7 +49,8 @@ export class ExecuteMessageUsecase { session.datasource ); const sqlAccessContext = await this.chatPolicyGuardService.resolveSqlAccessContext( - session + session, + input.actor ); const userMessage: ChatMessage = { diff --git a/apps/backend/src/modules/conversation/chat/application/save-view-from-run.usecase.ts b/apps/backend/src/modules/conversation/chat/application/save-view-from-run.usecase.ts index fe2ccff..bee68e7 100644 --- a/apps/backend/src/modules/conversation/chat/application/save-view-from-run.usecase.ts +++ b/apps/backend/src/modules/conversation/chat/application/save-view-from-run.usecase.ts @@ -1,13 +1,23 @@ import { createHash } from "node:crypto"; -import { Injectable } from "@nestjs/common"; +import { Inject, Injectable, Optional } from "@nestjs/common"; import { DomainError } from "../../../../common/domain-error"; import { ChatRepository } from "../../../platform/data/persistence"; import { ModelingGraphRepository } from "../../../platform/data/persistence/modeling-graph.repository"; import { ModelingGraphValidator } from "../../../platform/data/persistence/modeling-graph.validator"; import type { ModelingGraphPayload, ModelingGraphView } from "../../../platform/data/persistence/modeling-graph.types"; +import { + KNOWLEDGE_MEMORY_CONTRACT, + type KnowledgeMemoryContract, + type SavedPriorSqlCaptureResult +} from "../../../knowledge/contracts/knowledge-memory.contract"; const VIEW_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; +export interface SaveViewFromRunSavedPriorSqlDiagnostics + extends SavedPriorSqlCaptureResult { + message?: string; +} + export interface SaveViewFromRunInput { runId: string; name: string; @@ -25,6 +35,7 @@ export interface SaveViewFromRunResult { activeRevision?: number; draftRevision: number; view: ModelingGraphView; + savedPriorSql?: SaveViewFromRunSavedPriorSqlDiagnostics; } @Injectable() @@ -32,7 +43,10 @@ export class SaveViewFromRunUsecase { constructor( private readonly chatRepository: ChatRepository, private readonly modelingGraphRepository: ModelingGraphRepository, - private readonly modelingGraphValidator: ModelingGraphValidator + private readonly modelingGraphValidator: ModelingGraphValidator, + @Optional() + @Inject(KNOWLEDGE_MEMORY_CONTRACT) + private readonly knowledgeMemoryContract?: KnowledgeMemoryContract ) {} async execute(input: SaveViewFromRunInput): Promise { @@ -79,6 +93,20 @@ export class SaveViewFromRunUsecase { const viewId = this.buildViewId(runId); const existingById = basePayload.views.find((item) => item.id === viewId); if (existingById) { + const savedPriorSql = await this.captureSavedPriorSql({ + workspaceId, + datasourceId, + run: { + runId: run.runId, + status: run.status, + createdAt: run.createdAt, + question: run.question, + sql, + columns: run.columns + }, + view: existingById, + replayed: true + }); return { stage: "chat_run_view_saved", workspaceId, @@ -87,7 +115,8 @@ export class SaveViewFromRunUsecase { replayed: true, activeRevision: current.activeRevision, draftRevision: current.draft?.revision ?? 0, - view: existingById + view: existingById, + savedPriorSql }; } @@ -135,6 +164,20 @@ export class SaveViewFromRunUsecase { graphPayload: nextPayload, actorId: this.normalizeOptional(input.actorId) ?? undefined }); + const savedPriorSql = await this.captureSavedPriorSql({ + workspaceId, + datasourceId, + run: { + runId: run.runId, + status: run.status, + createdAt: run.createdAt, + question: run.question, + sql, + columns: run.columns + }, + view: normalizedView, + replayed: false + }); return { stage: "chat_run_view_saved", @@ -144,8 +187,80 @@ export class SaveViewFromRunUsecase { replayed: false, activeRevision: current.activeRevision, draftRevision: nextRevision.revision, - view: normalizedView + view: normalizedView, + savedPriorSql + }; + } + + private async captureSavedPriorSql(input: { + workspaceId: string; + datasourceId: string; + run: { + runId: string; + status: string; + createdAt: string; + question?: string; + sql: string; + columns?: string[]; }; + view: Pick; + replayed: boolean; + }): Promise { + const service = this.knowledgeMemoryContract?.savedPriorSql; + if (!service) { + return undefined; + } + try { + return await service.captureFromSavedView({ + workspaceId: input.workspaceId, + datasourceId: input.datasourceId, + sourceRunId: input.run.runId, + sourceRunStatus: input.run.status, + sourceRunCreatedAt: input.run.createdAt, + question: this.normalizeOptional(input.run.question) ?? undefined, + sql: input.run.sql, + viewId: input.view.id, + viewName: input.view.name, + viewSql: input.view.sql, + tableNames: [], + columnNames: Array.isArray(input.run.columns) + ? input.run.columns.filter( + (column): column is string => + typeof column === "string" && column.trim().length > 0 + ) + : [], + replayed: input.replayed, + savedAt: new Date().toISOString() + }); + } catch (error) { + return { + outcome: "capture_failed", + priorId: this.safeBuildPriorId(input), + replayKey: "saved_prior_sql:capture_failed", + reason: "saved_prior_sql_capture_exception", + message: error instanceof Error ? error.message : String(error) + }; + } + } + + private safeBuildPriorId(input: { + workspaceId: string; + datasourceId: string; + view: Pick; + }): string { + const service = this.knowledgeMemoryContract?.savedPriorSql; + if (!service) { + return "saved_prior_sql.unavailable"; + } + try { + return service.buildPriorId({ + workspaceId: input.workspaceId, + datasourceId: input.datasourceId, + viewId: input.view.id + }); + } catch { + return "saved_prior_sql.unresolved"; + } } private createEmptyPayload(): ModelingGraphPayload { diff --git a/apps/backend/src/modules/conversation/chat/application/shared/chat-delivery-enrichment.service.ts b/apps/backend/src/modules/conversation/chat/application/shared/chat-delivery-enrichment.service.ts index f7a2f8f..7a235a5 100644 --- a/apps/backend/src/modules/conversation/chat/application/shared/chat-delivery-enrichment.service.ts +++ b/apps/backend/src/modules/conversation/chat/application/shared/chat-delivery-enrichment.service.ts @@ -1,9 +1,14 @@ import { Inject, Injectable } from "@nestjs/common"; -import type { PromptTemplateTraceEvidenceCompat, SqlRun } from "@text2sql/shared-types"; +import type { + DeliveryContract, + PromptTemplateTraceEvidenceCompat, + SqlRun +} from "@text2sql/shared-types"; import { DeliveryContractMapper, type DeliveryReplayRecordInput } from "../../../delivery/delivery-contract.mapper"; +import { ChartBiArtifactService } from "../../../delivery/chartbi/chartbi-artifact.service"; import { KNOWLEDGE_FACADE_CONTRACT, type KnowledgeFacadeContract @@ -13,6 +18,7 @@ import { export class ChatDeliveryEnrichmentService { constructor( private readonly deliveryContractMapper: DeliveryContractMapper, + private readonly chartBiArtifactService: ChartBiArtifactService, @Inject(KNOWLEDGE_FACADE_CONTRACT) private readonly knowledgeFacade: KnowledgeFacadeContract ) {} @@ -20,25 +26,81 @@ export class ChatDeliveryEnrichmentService { async attachDeliveryContract(run: SqlRun): Promise { try { const replayRecords = await this.loadReplayRecords(run.runId); + const chartBiOutcome = this.buildChartBiArtifact(run); const delivery = this.deliveryContractMapper.map({ run, - replayRecords + replayRecords, + artifactOverride: chartBiOutcome.artifact, + additionalRiskTags: chartBiOutcome.riskTags }); return this.withPromptTemplateEvidence({ ...run, + answer: delivery.answer.text, delivery }); } catch { + const fallback = this.deliveryContractMapper.buildFallback( + run, + "delivery_mapper_failed" + ); return this.withPromptTemplateEvidence({ ...run, - delivery: this.deliveryContractMapper.buildFallback( - run, - "delivery_mapper_failed" - ) + answer: fallback.answer.text, + delivery: fallback }); } } + private buildChartBiArtifact(run: SqlRun): { + artifact?: DeliveryContract["artifact"]; + riskTags: string[]; + } { + if (!this.shouldBuildChartBiArtifact(run)) { + return { + artifact: undefined, + riskTags: [] + }; + } + + try { + return { + artifact: this.chartBiArtifactService.buildFromRun(run), + riskTags: [] + }; + } catch { + return { + artifact: this.buildLegacyArtifact(run), + riskTags: ["chartbi_artifact_failed"] + }; + } + } + + private shouldBuildChartBiArtifact(run: SqlRun): boolean { + if (run.status !== "executionResult") { + return false; + } + if (run.error) { + return false; + } + return true; + } + + private buildLegacyArtifact(run: SqlRun): DeliveryContract["artifact"] | undefined { + const hasArtifact = Boolean( + run.sql || (run.columns?.length ?? 0) > 0 || (run.rows?.length ?? 0) > 0 || run.error + ); + if (!hasArtifact) { + return undefined; + } + return { + sql: run.sql, + columns: run.columns, + rowCount: run.rows?.length ?? 0, + rowsPreview: run.rows?.slice(0, 3), + hasError: Boolean(run.error) + }; + } + withPromptTemplateEvidence(run: SqlRun): SqlRun { const traceWithCompat = run.trace as SqlRun["trace"] & { prompt_template?: unknown; diff --git a/apps/backend/src/modules/conversation/chat/application/shared/chat-policy-guard.service.ts b/apps/backend/src/modules/conversation/chat/application/shared/chat-policy-guard.service.ts index e908a06..376ced5 100644 --- a/apps/backend/src/modules/conversation/chat/application/shared/chat-policy-guard.service.ts +++ b/apps/backend/src/modules/conversation/chat/application/shared/chat-policy-guard.service.ts @@ -98,19 +98,21 @@ export class ChatPolicyGuardService { } async resolveSqlAccessContext( - session: Session + session: Session, + actor?: ChatPolicyActorInput ): Promise { const workspaceId = session.workspaceId?.trim(); - const actorId = session.createdByUserId?.trim(); + const actorId = actor?.id?.trim() || session.createdByUserId?.trim(); if (!workspaceId || !actorId) { return undefined; } const context = await this.resolveAccessContext({ actor: { + ...(actor ?? {}), id: actorId, - role: "user", - requestedWorkspaceId: workspaceId + role: actor?.role ?? "user", + requestedWorkspaceId: actor?.requestedWorkspaceId ?? workspaceId }, workspaceId }); diff --git a/apps/backend/src/modules/conversation/chat/application/shared/chat-run-persistence.service.ts b/apps/backend/src/modules/conversation/chat/application/shared/chat-run-persistence.service.ts index b8b10a6..58ff1db 100644 --- a/apps/backend/src/modules/conversation/chat/application/shared/chat-run-persistence.service.ts +++ b/apps/backend/src/modules/conversation/chat/application/shared/chat-run-persistence.service.ts @@ -18,11 +18,16 @@ export class ChatRunPersistenceService { ) {} async persistAssistantAndRun(input: ChatRunPersistenceInput): Promise { + const assistantContent = + input.run.delivery?.answer.text ?? + input.run.answer ?? + input.run.error ?? + "系统未返回结果。"; const assistantMessage: ChatMessage = { id: uuidv4(), sessionId: input.sessionId, role: "assistant", - content: input.run.answer ?? input.run.error ?? "系统未返回结果。", + content: assistantContent, metadata: { runId: input.run.runId, status: input.run.status diff --git a/apps/backend/src/modules/conversation/chat/application/stream-message.usecase.ts b/apps/backend/src/modules/conversation/chat/application/stream-message.usecase.ts index 01e4fdf..0250c61 100644 --- a/apps/backend/src/modules/conversation/chat/application/stream-message.usecase.ts +++ b/apps/backend/src/modules/conversation/chat/application/stream-message.usecase.ts @@ -19,7 +19,10 @@ import { DatasourceService } from "../../../governance/datasource/datasource.ser import { RedisBufferService } from "../../../platform/data/cache/index"; import { ChatRepository } from "../../../platform/data/persistence/index"; import { ChatDeliveryEnrichmentService } from "./shared/chat-delivery-enrichment.service"; -import { ChatPolicyGuardService } from "./shared/chat-policy-guard.service"; +import { + ChatPolicyGuardService, + type ChatPolicyActorInput +} from "./shared/chat-policy-guard.service"; import { ChatPostRunHooksService } from "./shared/chat-post-run-hooks.service"; import { ChatRunPersistenceService } from "./shared/chat-run-persistence.service"; @@ -28,6 +31,7 @@ export interface StreamMessageInput { message: string; requestId?: string; contextEnvelope?: ContextEnvelope; + actor?: ChatPolicyActorInput; onEvent: (event: ChatStreamEvent) => Promise | void; } @@ -58,7 +62,8 @@ export class StreamMessageUsecase { session.datasource ); const sqlAccessContext = await this.chatPolicyGuardService.resolveSqlAccessContext( - session + session, + input.actor ); const runId = uuidv4(); diff --git a/apps/backend/src/modules/conversation/chat/chat.controller.ts b/apps/backend/src/modules/conversation/chat/chat.controller.ts index 69326ef..a52a423 100644 --- a/apps/backend/src/modules/conversation/chat/chat.controller.ts +++ b/apps/backend/src/modules/conversation/chat/chat.controller.ts @@ -143,7 +143,8 @@ export class ChatController { sessionId, body.message, req.requestId, - body.contextEnvelope + body.contextEnvelope, + req.actor ); return ok(req.requestId, { kind: "agent-run", @@ -212,7 +213,8 @@ export class ChatController { async (event) => { sendEvent(event.type, event); }, - body.contextEnvelope + body.contextEnvelope, + req.actor ); } catch (error) { const fallbackEvent: ChatStreamEvent = { diff --git a/apps/backend/src/modules/conversation/chat/chat.module.ts b/apps/backend/src/modules/conversation/chat/chat.module.ts index 1015f6a..b893547 100644 --- a/apps/backend/src/modules/conversation/chat/chat.module.ts +++ b/apps/backend/src/modules/conversation/chat/chat.module.ts @@ -19,6 +19,12 @@ import { ChatDeliveryEnrichmentService } from "./application/shared/chat-deliver import { ChatPolicyGuardService } from "./application/shared/chat-policy-guard.service"; import { ChatPostRunHooksService } from "./application/shared/chat-post-run-hooks.service"; import { ChatRunPersistenceService } from "./application/shared/chat-run-persistence.service"; +import { ChartBiArtifactService } from "../delivery/chartbi/chartbi-artifact.service"; +import { ChartBiResultProfiler } from "../delivery/chartbi/chartbi-result-profiler"; +import { ChartBiIntentParser } from "../delivery/chartbi/chartbi-intent-parser"; +import { ChartBiSpecCompiler } from "../delivery/chartbi/chartbi-spec-compiler"; +import { ChartBiValidator } from "../delivery/chartbi/chartbi-validator"; +import { ChartBiGroundingGuard } from "../delivery/chartbi/chartbi-grounding.guard"; @Module({ imports: [ @@ -43,7 +49,13 @@ import { ChatRunPersistenceService } from "./application/shared/chat-run-persist ChatPostRunHooksService, ChatPolicyGuardService, ChatRunPersistenceService, - ChatDeliveryEnrichmentService + ChatDeliveryEnrichmentService, + ChartBiArtifactService, + ChartBiResultProfiler, + ChartBiIntentParser, + ChartBiSpecCompiler, + ChartBiValidator, + ChartBiGroundingGuard ], exports: [ChatService] }) diff --git a/apps/backend/src/modules/conversation/chat/chat.service.ts b/apps/backend/src/modules/conversation/chat/chat.service.ts index 1008689..9dc1352 100644 --- a/apps/backend/src/modules/conversation/chat/chat.service.ts +++ b/apps/backend/src/modules/conversation/chat/chat.service.ts @@ -14,6 +14,7 @@ import { SaveViewFromRunUsecase } from "./application/save-view-from-run.usecase import { SessionLifecycleUsecase } from "./application/session-lifecycle.usecase"; import { StreamMessageUsecase } from "./application/stream-message.usecase"; import type { SessionListView } from "./dto/list-sessions.dto"; +import type { ChatPolicyActorInput } from "./application/shared/chat-policy-guard.service"; @Injectable() export class ChatService { @@ -99,13 +100,15 @@ export class ChatService { sessionId: string, message: string, requestId?: string, - contextEnvelope?: ContextEnvelope + contextEnvelope?: ContextEnvelope, + actor?: ChatPolicyActorInput ): Promise { return this.executeMessageUsecase.executeMessage({ sessionId, message, requestId, - contextEnvelope + contextEnvelope, + actor }); } @@ -114,14 +117,16 @@ export class ChatService { message: string, requestId: string | undefined, onEvent: (event: ChatStreamEvent) => Promise | void, - contextEnvelope?: ContextEnvelope + contextEnvelope?: ContextEnvelope, + actor?: ChatPolicyActorInput ): Promise { return this.streamMessageUsecase.streamMessage({ sessionId, message, requestId, onEvent, - contextEnvelope + contextEnvelope, + actor }); } diff --git a/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-artifact.service.ts b/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-artifact.service.ts new file mode 100644 index 0000000..cd74b02 --- /dev/null +++ b/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-artifact.service.ts @@ -0,0 +1,473 @@ +import { Injectable } from "@nestjs/common"; +import type { DeliveryArtifactLayer, SqlRun } from "@text2sql/shared-types"; +import { ChartBiGroundingGuard } from "./chartbi-grounding.guard"; +import { + ChartBiIntentParser, + type ChartBiIntentParseResult, + type ChartBiVisualIntent +} from "./chartbi-intent-parser"; +import { + type ChartBiCanonicalSpec, + type ChartBiDisplayType, + type ChartBiSummaryBlock, + ChartBiResultProfiler +} from "./chartbi-result-profiler"; +import { ChartBiSpecCompiler } from "./chartbi-spec-compiler"; +import { ChartBiValidator } from "./chartbi-validator"; + +export interface ChartBiArtifactBuildInput { + sql?: string; + columns?: string[]; + rows?: Array>; + answer?: string; + error?: string; + hasError?: boolean; + visualIntentRaw?: string | null; +} + +export interface ChartBiArtifactLayer extends DeliveryArtifactLayer { + summary: ChartBiSummaryBlock; + table: { + columns: string[]; + rowCount: number; + rowsPreview: Array>; + previewRowCount: number; + truncated: boolean; + }; + chart?: DeliveryArtifactLayer["chart"]; + display: ChartBiDisplayType; + displayModes: ChartBiDisplayType[]; + validation: { + status: "valid" | "repaired" | "fallback"; + source: "baseline" | "repaired_intent" | "table_fallback"; + reasonCodes?: string[]; + message?: string; + }; + fallback?: { + display: "table"; + reason: string; + reasonCode?: string; + fromType?: string; + stage?: "parse" | "compile" | "validate" | "runtime"; + }; + visualIntent: { + source: "deterministic" | "model" | "hybrid"; + status: "not_provided" | "parsed" | "failed"; + type?: ChartBiDisplayType; + title?: string; + summaryHint?: string; + insight?: string; + mappings?: Record; + rawSyntax?: string; + normalizedIntent?: Record; + reason?: string; + }; +} + +@Injectable() +export class ChartBiArtifactService { + constructor( + private readonly profiler: ChartBiResultProfiler, + private readonly intentParser: ChartBiIntentParser, + private readonly specCompiler: ChartBiSpecCompiler, + private readonly validator: ChartBiValidator, + private readonly groundingGuard: ChartBiGroundingGuard + ) {} + + buildFromRun( + run: Pick, + options?: { + visualIntentRaw?: string | null; + } + ): ChartBiArtifactLayer { + return this.build({ + sql: run.sql, + columns: run.columns, + rows: run.rows, + answer: run.answer, + error: run.error, + hasError: Boolean(run.error), + visualIntentRaw: options?.visualIntentRaw ?? run.llmRaw?.rawText + }); + } + + build(input: ChartBiArtifactBuildInput): ChartBiArtifactLayer { + const rows = this.cloneRows(input.rows ?? []); + const columns = this.resolveColumns(input.columns ?? [], rows); + const profile = this.profiler.profile({ + columns, + rows + }); + const parseResult = this.intentParser.parse(input.visualIntentRaw ?? undefined); + + if (input.hasError || input.error) { + return this.buildFallbackArtifact({ + profile, + sql: input.sql, + summary: this.resolveSummaryText(input.answer, profile.summary.text), + reason: input.error ?? "table fallback: run has error state.", + stage: "runtime", + parseResult, + hasError: true + }); + } + + const compileResult = this.specCompiler.compile({ + profile, + parsedIntent: parseResult.status === "parsed" ? parseResult.intent : undefined + }); + + const candidates: Array<{ + source: "baseline" | "repaired_intent"; + spec: ChartBiCanonicalSpec; + }> = []; + + if (compileResult.baseline) { + candidates.push({ + source: "baseline", + spec: compileResult.baseline + }); + } + if (compileResult.repairedIntent) { + candidates.push({ + source: "repaired_intent", + spec: compileResult.repairedIntent + }); + } + + const validationFailures: string[] = []; + for (const candidate of candidates) { + const validation = this.validator.validate({ + spec: candidate.spec, + columns: profile.columns, + rows: profile.rows + }); + if (!validation.ok) { + validationFailures.push( + `${candidate.source}: ${validation.reason ?? "unknown validation failure"}` + ); + continue; + } + + const grounded = this.groundingGuard.enforce({ + spec: candidate.spec, + columns: profile.columns, + rows: profile.rows + }); + + const groundedSpec: ChartBiCanonicalSpec = { + ...candidate.spec, + ...(grounded.insights.length > 0 ? { insights: grounded.insights } : {}) + }; + + return this.buildSuccessArtifact({ + profile, + sql: input.sql, + summary: this.resolveSummaryText(input.answer, profile.summary.text), + spec: groundedSpec, + source: candidate.source, + parseResult, + groundingNote: grounded.reason + }); + } + + const fallbackReason = this.resolveFallbackReason({ + parseResult, + validationFailures, + hasBaseline: Boolean(compileResult.baseline), + rowCount: profile.rowCount + }); + + return this.buildFallbackArtifact({ + profile, + sql: input.sql, + summary: this.resolveSummaryText(input.answer, profile.summary.text), + reason: fallbackReason.reason, + stage: fallbackReason.stage, + parseResult, + hasError: false + }); + } + + private buildSuccessArtifact(input: { + profile: ReturnType; + sql?: string; + summary: string; + spec: ChartBiCanonicalSpec; + source: "baseline" | "repaired_intent"; + parseResult: ChartBiIntentParseResult; + groundingNote?: string; + }): ChartBiArtifactLayer { + const table = this.buildTableBlock(input.profile); + const displayModes = this.uniqueDisplayTypes(["table", input.spec.type]); + const chart = this.toDeliveryChart(input.spec); + + return { + sql: input.sql, + columns: [...input.profile.columns], + rowCount: input.profile.rowCount, + rowsPreview: this.cloneRows(input.profile.previewRows), + hasError: false, + summary: { + text: input.summary, + ...(input.profile.summary.highlights + ? { + dimensions: input.profile.summary.highlights.map((item) => item.label), + metrics: input.profile.summary.highlights.map((item) => ({ + key: item.label, + label: item.label, + value: item.value + })) + } + : {}) + }, + table, + chart, + display: input.spec.type, + displayModes, + validation: { + status: input.source === "baseline" ? "valid" : "repaired", + source: input.source, + ...(input.groundingNote + ? { + reasonCodes: ["grounding_insight_filtered"], + message: input.groundingNote + } + : {}) + }, + visualIntent: this.toVisualIntentMeta(input.parseResult) + }; + } + + private buildFallbackArtifact(input: { + profile: ReturnType; + sql?: string; + summary: string; + reason: string; + stage: "parse" | "compile" | "validate" | "runtime"; + parseResult: ChartBiIntentParseResult; + hasError: boolean; + }): ChartBiArtifactLayer { + return { + sql: input.sql, + columns: [...input.profile.columns], + rowCount: input.profile.rowCount, + rowsPreview: this.cloneRows(input.profile.previewRows), + hasError: input.hasError, + summary: { + text: input.summary, + ...(input.profile.summary.highlights + ? { + dimensions: input.profile.summary.highlights.map((item) => item.label), + metrics: input.profile.summary.highlights.map((item) => ({ + key: item.label, + label: item.label, + value: item.value + })) + } + : {}) + }, + table: this.buildTableBlock(input.profile), + display: "table", + displayModes: ["table"], + validation: { + status: "fallback", + source: "table_fallback", + reasonCodes: [this.stageToReasonCode(input.stage)], + message: input.reason + }, + fallback: { + display: "table", + reason: input.reason, + reasonCode: this.stageToReasonCode(input.stage), + stage: input.stage + }, + visualIntent: this.toVisualIntentMeta(input.parseResult) + }; + } + + private toDeliveryChart(spec: ChartBiCanonicalSpec): NonNullable { + return { + type: spec.type, + mappings: { + ...spec.mappings + }, + ...(spec.title + ? { + meta: { + title: spec.title + } + } + : {}) + }; + } + + private resolveFallbackReason(input: { + parseResult: ChartBiIntentParseResult; + validationFailures: string[]; + hasBaseline: boolean; + rowCount: number; + }): { + stage: "parse" | "compile" | "validate"; + reason: string; + } { + if (input.validationFailures.length > 0) { + return { + stage: "validate", + reason: `table fallback: ${input.validationFailures.join(" | ")}` + }; + } + + if (!input.hasBaseline && input.parseResult.status === "failed") { + return { + stage: "parse", + reason: `table fallback: ${input.parseResult.reason}` + }; + } + + if (input.rowCount === 0) { + return { + stage: "compile", + reason: "table fallback: result set is empty." + }; + } + + return { + stage: "compile", + reason: "table fallback: no canonical chart candidate available." + }; + } + + private resolveSummaryText(answer: string | undefined, fallback: string): string { + if (typeof answer === "string" && answer.trim().length > 0) { + return answer.trim(); + } + return fallback; + } + + private toVisualIntentMeta( + parseResult: ChartBiIntentParseResult + ): ChartBiArtifactLayer["visualIntent"] { + if (parseResult.status === "not_provided") { + return { + source: "deterministic", + status: "not_provided" + }; + } + + if (parseResult.status === "failed") { + return { + source: "model", + status: "failed", + reason: parseResult.reason, + rawSyntax: parseResult.raw + }; + } + + return { + source: "model", + status: "parsed", + ...(parseResult.intent.type + ? { + type: parseResult.intent.type + } + : {}), + ...(parseResult.intent.title + ? { + title: parseResult.intent.title + } + : {}), + ...(parseResult.intent.insights?.[0] + ? { + insight: parseResult.intent.insights[0] + } + : {}), + mappings: { + ...parseResult.intent.mappings + }, + rawSyntax: parseResult.normalized, + normalizedIntent: this.visualIntentToRecord(parseResult.intent) + }; + } + + private visualIntentToRecord(intent: ChartBiVisualIntent): Record { + return { + ...(intent.type ? { type: intent.type } : {}), + mappings: { + ...intent.mappings + }, + ...(intent.title ? { title: intent.title } : {}), + ...(intent.insights ? { insights: [...intent.insights] } : {}) + }; + } + + private buildTableBlock( + profile: ReturnType + ): ChartBiArtifactLayer["table"] { + return { + columns: [...profile.columns], + rowCount: profile.rowCount, + rowsPreview: this.cloneRows(profile.previewRows), + previewRowCount: profile.previewRows.length, + truncated: profile.rowCount > profile.previewRows.length + }; + } + + private resolveColumns( + columns: string[], + rows: Array> + ): string[] { + const normalized = this.uniqueStrings(columns); + if (normalized.length > 0) { + return normalized; + } + if (rows.length === 0) { + return []; + } + return this.uniqueStrings(Object.keys(rows[0] ?? {})); + } + + private uniqueDisplayTypes(values: ChartBiDisplayType[]): ChartBiDisplayType[] { + const result: ChartBiDisplayType[] = []; + const seen = new Set(); + for (const value of values) { + if (seen.has(value)) { + continue; + } + seen.add(value); + result.push(value); + } + return result; + } + + private uniqueStrings(values: string[]): string[] { + const result: string[] = []; + const seen = new Set(); + for (const value of values) { + const trimmed = value.trim(); + if (!trimmed || seen.has(trimmed)) { + continue; + } + seen.add(trimmed); + result.push(trimmed); + } + return result; + } + + private stageToReasonCode(stage: "parse" | "compile" | "validate" | "runtime"): string { + if (stage === "parse") { + return "chartbi_parse_failed"; + } + if (stage === "compile") { + return "chartbi_compile_failed"; + } + if (stage === "validate") { + return "chartbi_validation_failed"; + } + return "chartbi_runtime_failed"; + } + + private cloneRows(rows: Array>): Array> { + return rows.map((row) => ({ ...row })); + } +} diff --git a/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-grounding.guard.ts b/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-grounding.guard.ts new file mode 100644 index 0000000..8b69d2e --- /dev/null +++ b/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-grounding.guard.ts @@ -0,0 +1,109 @@ +import { Injectable } from "@nestjs/common"; +import type { ChartBiCanonicalSpec } from "./chartbi-result-profiler"; + +export interface ChartBiGroundingInput { + spec: ChartBiCanonicalSpec; + columns: string[]; + rows: Array>; +} + +export interface ChartBiGroundingResult { + ok: boolean; + insights: string[]; + reason?: string; +} + +@Injectable() +export class ChartBiGroundingGuard { + enforce(input: ChartBiGroundingInput): ChartBiGroundingResult { + const insights = input.spec.insights ?? []; + if (insights.length === 0) { + return { + ok: true, + insights: [] + }; + } + + const numericEvidence = this.collectNumericEvidence(input.rows); + const knownColumns = new Set(input.columns.map((column) => column.toLowerCase())); + + const grounded: string[] = []; + let dropped = 0; + + for (const insight of insights) { + const referencesKnownColumns = this.referencesKnownColumns(insight, knownColumns); + const referencesKnownNumbers = this.referencesKnownNumbers(insight, numericEvidence); + + if (referencesKnownColumns && referencesKnownNumbers) { + grounded.push(insight); + continue; + } + + dropped += 1; + } + + return { + ok: true, + insights: grounded, + ...(dropped > 0 + ? { + reason: + "Removed ungrounded model insights that referenced unknown fields or numbers." + } + : {}) + }; + } + + private referencesKnownColumns( + insight: string, + knownColumns: Set + ): boolean { + const fieldRefs = [...insight.matchAll(/`([^`]+)`/g)] + .map((match) => match[1]?.trim().toLowerCase()) + .filter((value): value is string => Boolean(value)); + + if (fieldRefs.length === 0) { + return true; + } + + return fieldRefs.every((field) => knownColumns.has(field)); + } + + private referencesKnownNumbers(insight: string, evidence: Set): boolean { + const numbers = [...insight.matchAll(/-?\d+(?:\.\d+)?/g)] + .map((match) => Number(match[0])) + .filter((value) => Number.isFinite(value)); + + if (numbers.length === 0) { + return true; + } + + return numbers.every((value) => evidence.has(value)); + } + + private collectNumericEvidence(rows: Array>): Set { + const evidence = new Set(); + for (const row of rows) { + for (const value of Object.values(row)) { + const numeric = this.readNumber(value); + if (numeric !== undefined) { + evidence.add(numeric); + } + } + } + return evidence; + } + + private readNumber(value: unknown): number | undefined { + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + if (typeof value === "string" && value.trim().length > 0) { + const parsed = Number(value); + if (Number.isFinite(parsed)) { + return parsed; + } + } + return undefined; + } +} diff --git a/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-intent-parser.ts b/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-intent-parser.ts new file mode 100644 index 0000000..4a5f877 --- /dev/null +++ b/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-intent-parser.ts @@ -0,0 +1,384 @@ +import { Injectable } from "@nestjs/common"; +import { + CHARTBI_DISPLAY_ALLOWLIST, + type ChartBiDisplayType, + type ChartBiFieldMappings +} from "./chartbi-result-profiler"; + +export interface ChartBiVisualIntent { + type?: ChartBiDisplayType; + mappings: ChartBiFieldMappings; + title?: string; + insights?: string[]; +} + +export type ChartBiIntentParseResult = + | { + status: "not_provided"; + } + | { + status: "failed"; + reason: string; + raw: string; + } + | { + status: "parsed"; + source: "json" | "vis"; + raw: string; + normalized: string; + intent: ChartBiVisualIntent; + }; + +@Injectable() +export class ChartBiIntentParser { + parse(rawInput: string | null | undefined): ChartBiIntentParseResult { + if (typeof rawInput !== "string" || rawInput.trim().length === 0) { + return { + status: "not_provided" + }; + } + + const raw = rawInput.trim(); + const candidates = this.collectCandidates(raw); + + for (const candidate of candidates) { + const jsonResult = this.tryParseJson(candidate); + if (jsonResult) { + return { + status: "parsed", + source: "json", + raw, + normalized: jsonResult.normalized, + intent: jsonResult.intent + }; + } + + const visResult = this.tryParseVis(candidate); + if (visResult) { + return { + status: "parsed", + source: "vis", + raw, + normalized: visResult.normalized, + intent: visResult.intent + }; + } + } + + return { + status: "failed", + raw, + reason: "Unable to parse visual intent payload from model output." + }; + } + + private collectCandidates(raw: string): string[] { + const candidates: string[] = []; + const seen = new Set(); + + const push = (value: string | undefined): void => { + if (!value) { + return; + } + const normalized = value.trim(); + if (!normalized || seen.has(normalized)) { + return; + } + seen.add(normalized); + candidates.push(normalized); + }; + + const fencedPreferred: string[] = []; + const fencedFallback: string[] = []; + const fencedRegex = /```([a-zA-Z0-9_-]*)?\s*([\s\S]*?)```/g; + let fencedMatch: RegExpExecArray | null = fencedRegex.exec(raw); + while (fencedMatch) { + const language = (fencedMatch[1] ?? "").toLowerCase(); + const body = (fencedMatch[2] ?? "").trim(); + if (body) { + if (language.includes("json") || language.includes("vis")) { + fencedPreferred.push(body); + } else { + fencedFallback.push(body); + } + } + fencedMatch = fencedRegex.exec(raw); + } + + for (const candidate of [...fencedPreferred, ...fencedFallback]) { + push(candidate); + } + + push(this.extractJsonObject(raw)); + + const visualIntentLabelMatch = raw.match(/visualIntent\s*[:=]\s*([\s\S]+)/i); + push(visualIntentLabelMatch?.[1]); + + push(raw); + + return candidates; + } + + private tryParseJson(input: string): + | { + normalized: string; + intent: ChartBiVisualIntent; + } + | undefined { + const normalized = input.replace(/^json\s*/i, "").trim(); + const jsonCandidate = this.extractJsonObject(normalized) ?? normalized; + if (!jsonCandidate.startsWith("{") || !jsonCandidate.endsWith("}")) { + return undefined; + } + + let parsed: unknown; + try { + parsed = JSON.parse(jsonCandidate); + } catch { + return undefined; + } + + if (!this.isRecord(parsed)) { + return undefined; + } + + const payload = + this.isRecord(parsed.visualIntent) || this.isRecord(parsed.chart) + ? (this.isRecord(parsed.visualIntent) + ? parsed.visualIntent + : (parsed.chart as Record)) + : parsed; + + const intent = this.objectToIntent(payload); + if (!intent) { + return undefined; + } + + return { + normalized: jsonCandidate, + intent + }; + } + + private tryParseVis(input: string): + | { + normalized: string; + intent: ChartBiVisualIntent; + } + | undefined { + const normalized = input.replace(/^vis(?:ual)?\s*[:\-]?\s*/i, "").trim(); + if (!normalized) { + return undefined; + } + + const keyValues = this.readVisKeyValues(normalized); + const firstToken = normalized + .split(/\s+/) + .map((token) => token.trim()) + .find((token) => token.length > 0); + + const type = this.normalizeType( + keyValues.get("type") ?? + keyValues.get("chart") ?? + keyValues.get("mark") ?? + firstToken + ); + + const mappings: ChartBiFieldMappings = { + x: keyValues.get("x") ?? keyValues.get("xfield"), + y: keyValues.get("y") ?? keyValues.get("yfield"), + dimension: keyValues.get("dimension") ?? keyValues.get("dim"), + measure: + keyValues.get("measure") ?? + keyValues.get("metric") ?? + keyValues.get("valuefield"), + time: keyValues.get("time") ?? keyValues.get("date"), + category: keyValues.get("category"), + value: keyValues.get("value") + }; + + const title = keyValues.get("title"); + const insights = this.splitInsights( + keyValues.get("insight") ?? keyValues.get("insights") + ); + + if (!type && !this.hasAnyMapping(mappings) && !title && insights.length === 0) { + return undefined; + } + + return { + normalized, + intent: { + type, + mappings, + ...(title ? { title } : {}), + ...(insights.length > 0 ? { insights } : {}) + } + }; + } + + private objectToIntent(payload: Record): ChartBiVisualIntent | undefined { + const mappingsRecord = this.isRecord(payload.mappings) + ? payload.mappings + : undefined; + + const type = this.normalizeType( + payload.type ?? + payload.chartType ?? + payload.chart_type ?? + payload.mark ?? + payload.chart ?? + payload.display + ); + + const mappings: ChartBiFieldMappings = { + x: this.readString( + payload.x ?? + payload.xField ?? + payload.x_field ?? + mappingsRecord?.x ?? + mappingsRecord?.xField + ), + y: this.readString( + payload.y ?? + payload.yField ?? + payload.y_field ?? + mappingsRecord?.y ?? + mappingsRecord?.yField + ), + dimension: this.readString( + payload.dimension ?? + payload.groupBy ?? + payload.group_by ?? + mappingsRecord?.dimension ?? + mappingsRecord?.groupBy + ), + measure: this.readString( + payload.measure ?? + payload.metric ?? + payload.measureField ?? + payload.measure_field ?? + mappingsRecord?.measure ?? + mappingsRecord?.metric + ), + time: this.readString( + payload.time ?? payload.timeField ?? payload.time_field ?? mappingsRecord?.time + ), + category: this.readString( + payload.category ?? payload.categoryField ?? mappingsRecord?.category + ), + value: this.readString( + payload.value ?? payload.valueField ?? payload.value_field ?? mappingsRecord?.value + ) + }; + + const title = this.readString(payload.title ?? payload.name ?? payload.label); + const insights = this.readStringArray(payload.insights ?? payload.insight); + + if (!type && !this.hasAnyMapping(mappings) && !title && insights.length === 0) { + return undefined; + } + + return { + type, + mappings, + ...(title ? { title } : {}), + ...(insights.length > 0 ? { insights } : {}) + }; + } + + private readVisKeyValues(input: string): Map { + const result = new Map(); + const pattern = + /([a-zA-Z_][a-zA-Z0-9_-]*)\s*(?:=|:)\s*("[^"]*"|'[^']*'|`[^`]*`|[^\n,;]+)/g; + let match: RegExpExecArray | null = pattern.exec(input); + while (match) { + const key = match[1]?.toLowerCase(); + const value = this.stripQuotes(match[2] ?? ""); + if (key && value) { + result.set(key, value); + } + match = pattern.exec(input); + } + return result; + } + + private extractJsonObject(input: string): string | undefined { + const start = input.indexOf("{"); + const end = input.lastIndexOf("}"); + if (start === -1 || end === -1 || end <= start) { + return undefined; + } + return input.slice(start, end + 1).trim(); + } + + private splitInsights(value: string | undefined): string[] { + if (!value) { + return []; + } + return value + .split(/[|;\n]+/) + .map((item) => item.trim()) + .filter((item) => item.length > 0); + } + + private hasAnyMapping(mappings: ChartBiFieldMappings): boolean { + return Boolean( + mappings.x || + mappings.y || + mappings.dimension || + mappings.measure || + mappings.time || + mappings.category || + mappings.value + ); + } + + private normalizeType(value: unknown): ChartBiDisplayType | undefined { + const raw = this.readString(value)?.toLowerCase(); + if (!raw) { + return undefined; + } + return (CHARTBI_DISPLAY_ALLOWLIST as readonly string[]).includes(raw) + ? (raw as ChartBiDisplayType) + : undefined; + } + + private readString(value: unknown): string | undefined { + if (typeof value !== "string") { + return undefined; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; + } + + private readStringArray(value: unknown): string[] { + if (Array.isArray(value)) { + return value + .map((item) => this.readString(item)) + .filter((item): item is string => Boolean(item)); + } + + if (typeof value === "string") { + return this.splitInsights(value); + } + + return []; + } + + private stripQuotes(value: string): string { + const trimmed = value.trim(); + if (trimmed.length < 2) { + return trimmed; + } + const quote = trimmed[0]; + if ((quote === '"' || quote === "'" || quote === "`") && trimmed.endsWith(quote)) { + return trimmed.slice(1, -1).trim(); + } + return trimmed; + } + + private isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); + } +} diff --git a/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-result-profiler.ts b/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-result-profiler.ts new file mode 100644 index 0000000..31b6955 --- /dev/null +++ b/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-result-profiler.ts @@ -0,0 +1,332 @@ +import { Injectable } from "@nestjs/common"; + +export const CHARTBI_DISPLAY_ALLOWLIST = [ + "table", + "metric", + "bar", + "line", + "pie" +] as const; + +export const CHARTBI_CHART_ALLOWLIST = ["metric", "bar", "line", "pie"] as const; + +export type ChartBiDisplayType = (typeof CHARTBI_DISPLAY_ALLOWLIST)[number]; +export type ChartBiChartType = (typeof CHARTBI_CHART_ALLOWLIST)[number]; + +export interface ChartBiFieldMappings { + x?: string; + y?: string; + dimension?: string; + measure?: string; + time?: string; + category?: string; + value?: string; +} + +export interface ChartBiCanonicalSpec { + type: ChartBiChartType; + source: "baseline" | "repaired_intent"; + mappings: ChartBiFieldMappings; + title?: string; + insights?: string[]; +} + +export interface ChartBiSummaryBlock { + text: string; + highlights?: Array<{ + label: string; + value: string | number; + }>; +} + +export interface ChartBiResultProfile { + columns: string[]; + rows: Array>; + previewRows: Array>; + rowCount: number; + numericColumns: string[]; + timeColumns: string[]; + dimensionColumns: string[]; + baselineSpec?: ChartBiCanonicalSpec; + summary: ChartBiSummaryBlock; +} + +export interface ChartBiProfileInput { + columns: string[]; + rows: Array>; + previewLimit?: number; +} + +@Injectable() +export class ChartBiResultProfiler { + profile(input: ChartBiProfileInput): ChartBiResultProfile { + const columns = this.uniqueNonEmpty(input.columns); + const rows = input.rows.map((row) => ({ ...row })); + const previewLimit = + typeof input.previewLimit === "number" && Number.isFinite(input.previewLimit) + ? Math.max(0, Math.floor(input.previewLimit)) + : 20; + const previewRows = rows.slice(0, previewLimit); + + const numericColumns = columns.filter((column) => + this.isNumericColumn(rows, column) + ); + const timeColumns = columns.filter((column) => this.isTimeColumn(rows, column)); + const dimensionColumns = columns.filter( + (column) => !numericColumns.includes(column) + ); + + const baselineSpec = this.createBaselineSpec({ + rows, + columns, + numericColumns, + timeColumns, + dimensionColumns + }); + + return { + columns, + rows, + previewRows, + rowCount: rows.length, + numericColumns, + timeColumns, + dimensionColumns, + baselineSpec, + summary: this.createSummary({ rows, baselineSpec }) + }; + } + + private createBaselineSpec(input: { + rows: Array>; + columns: string[]; + numericColumns: string[]; + timeColumns: string[]; + dimensionColumns: string[]; + }): ChartBiCanonicalSpec | undefined { + if (input.rows.length === 0 || input.columns.length === 0) { + return undefined; + } + + const measure = input.numericColumns[0]; + if (!measure) { + return undefined; + } + + if (input.rows.length === 1) { + return { + type: "metric", + source: "baseline", + mappings: { + value: measure, + measure + }, + title: `${measure} metric` + }; + } + + const time = input.timeColumns[0]; + if (time) { + return { + type: "line", + source: "baseline", + mappings: { + x: time, + y: measure, + time, + measure + }, + title: `${measure} over ${time}` + }; + } + + const dimension = input.dimensionColumns[0]; + if (dimension) { + const uniqueCategories = this.uniqueNonEmpty( + input.rows + .map((row) => this.readString(row[dimension])) + .filter((value): value is string => Boolean(value)) + ); + const hasNegativeValue = input.rows.some((row) => { + const value = this.readNumber(row[measure]); + return typeof value === "number" && value < 0; + }); + + if (uniqueCategories.length > 0 && uniqueCategories.length <= 6 && !hasNegativeValue) { + return { + type: "pie", + source: "baseline", + mappings: { + category: dimension, + value: measure, + dimension, + measure + }, + title: `${measure} share by ${dimension}` + }; + } + + return { + type: "bar", + source: "baseline", + mappings: { + x: dimension, + y: measure, + dimension, + measure + }, + title: `${measure} by ${dimension}` + }; + } + + if (input.numericColumns.length === 1) { + return { + type: "metric", + source: "baseline", + mappings: { + value: measure, + measure + }, + title: `${measure} metric` + }; + } + + return undefined; + } + + private createSummary(input: { + rows: Array>; + baselineSpec?: ChartBiCanonicalSpec; + }): ChartBiSummaryBlock { + const rowCount = input.rows.length; + if (rowCount === 0) { + return { + text: "Query succeeded but returned no rows; falling back to table view.", + highlights: [ + { + label: "rowCount", + value: 0 + } + ] + }; + } + + if (!input.baselineSpec) { + return { + text: `Query returned ${rowCount} rows; table preview is available.`, + highlights: [ + { + label: "rowCount", + value: rowCount + } + ] + }; + } + + if (input.baselineSpec.type === "metric") { + const valueField = + input.baselineSpec.mappings.value ?? input.baselineSpec.mappings.measure; + const raw = valueField ? input.rows[0]?.[valueField] : undefined; + return { + text: valueField + ? `${valueField}: ${this.formatMetricValue(raw)}` + : `Query returned ${rowCount} rows.`, + highlights: [ + { + label: "rowCount", + value: rowCount + } + ] + }; + } + + return { + text: `Query returned ${rowCount} rows; ${input.baselineSpec.type} chart baseline is available.`, + highlights: [ + { + label: "rowCount", + value: rowCount + } + ] + }; + } + + private formatMetricValue(value: unknown): string { + if (typeof value === "number" && Number.isFinite(value)) { + return Number.isInteger(value) ? value.toString() : value.toFixed(2); + } + if (typeof value === "string" && value.trim().length > 0) { + return value.trim(); + } + return "n/a"; + } + + private isNumericColumn(rows: Array>, column: string): boolean { + const values = rows.map((row) => row[column]).filter((value) => value != null); + if (values.length === 0) { + return false; + } + return values.every((value) => this.readNumber(value) !== undefined); + } + + private isTimeColumn(rows: Array>, column: string): boolean { + if (/(^|_)(date|time|day|month|year|at)$/.test(column.toLowerCase())) { + return true; + } + + const values = rows.map((row) => row[column]).filter((value) => value != null); + if (values.length === 0) { + return false; + } + + const parseable = values.filter((value) => this.isDateLike(value)).length; + return parseable / values.length >= 0.6; + } + + private isDateLike(value: unknown): boolean { + if (typeof value !== "string") { + return false; + } + const trimmed = value.trim(); + if (!trimmed) { + return false; + } + const parsed = Date.parse(trimmed); + return Number.isFinite(parsed); + } + + private readNumber(value: unknown): number | undefined { + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + if (typeof value === "string" && value.trim().length > 0) { + const parsed = Number(value); + if (Number.isFinite(parsed)) { + return parsed; + } + } + return undefined; + } + + private readString(value: unknown): string | undefined { + if (typeof value !== "string") { + return undefined; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; + } + + private uniqueNonEmpty(values: string[]): string[] { + const result: string[] = []; + const seen = new Set(); + for (const value of values) { + const trimmed = value.trim(); + if (!trimmed || seen.has(trimmed)) { + continue; + } + seen.add(trimmed); + result.push(trimmed); + } + return result; + } +} diff --git a/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-spec-compiler.ts b/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-spec-compiler.ts new file mode 100644 index 0000000..fb86df3 --- /dev/null +++ b/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-spec-compiler.ts @@ -0,0 +1,291 @@ +import { Injectable } from "@nestjs/common"; +import type { ChartBiVisualIntent } from "./chartbi-intent-parser"; +import { + CHARTBI_CHART_ALLOWLIST, + type ChartBiCanonicalSpec, + type ChartBiChartType, + type ChartBiFieldMappings, + type ChartBiResultProfile +} from "./chartbi-result-profiler"; + +export interface ChartBiSpecCompileInput { + profile: ChartBiResultProfile; + parsedIntent?: ChartBiVisualIntent; +} + +export interface ChartBiSpecCompileResult { + baseline?: ChartBiCanonicalSpec; + repairedIntent?: ChartBiCanonicalSpec; +} + +@Injectable() +export class ChartBiSpecCompiler { + compile(input: ChartBiSpecCompileInput): ChartBiSpecCompileResult { + const baseline = input.profile.baselineSpec + ? this.cloneSpec(input.profile.baselineSpec) + : undefined; + const repairedIntent = input.parsedIntent + ? this.compileRepairedIntent({ + intent: input.parsedIntent, + profile: input.profile, + baseline + }) + : undefined; + + return { + ...(baseline ? { baseline } : {}), + ...(repairedIntent ? { repairedIntent } : {}) + }; + } + + private compileRepairedIntent(input: { + intent: ChartBiVisualIntent; + profile: ChartBiResultProfile; + baseline?: ChartBiCanonicalSpec; + }): ChartBiCanonicalSpec | undefined { + const type = this.resolveType(input.intent, input.baseline); + if (!type) { + return undefined; + } + + const mappings = this.resolveMappings({ + type, + intent: input.intent, + profile: input.profile, + baseline: input.baseline + }); + + const required = this.requiredKeys(type); + const hasAllRequired = required.every((key) => Boolean(mappings[key])); + if (!hasAllRequired) { + return undefined; + } + + return { + type, + source: "repaired_intent", + mappings, + ...(input.intent.title ? { title: input.intent.title } : {}), + ...(input.intent.insights && input.intent.insights.length > 0 + ? { insights: this.unique(input.intent.insights) } + : {}) + }; + } + + private resolveType( + intent: ChartBiVisualIntent, + baseline?: ChartBiCanonicalSpec + ): ChartBiChartType | undefined { + if ( + intent.type && + (CHARTBI_CHART_ALLOWLIST as readonly string[]).includes(intent.type) + ) { + return intent.type as ChartBiChartType; + } + + if (baseline) { + return baseline.type; + } + + if (intent.mappings.value || intent.mappings.measure) { + return "metric"; + } + + if ((intent.mappings.category || intent.mappings.dimension) && intent.mappings.value) { + return "pie"; + } + + if ((intent.mappings.x || intent.mappings.dimension) && intent.mappings.y) { + return "bar"; + } + + return undefined; + } + + private resolveMappings(input: { + type: ChartBiChartType; + intent: ChartBiVisualIntent; + profile: ChartBiResultProfile; + baseline?: ChartBiCanonicalSpec; + }): ChartBiFieldMappings { + const columns = input.profile.columns; + const baselineMappings = input.baseline?.mappings; + + const resolve = (...candidates: Array): string | undefined => { + for (const candidate of candidates) { + if (!candidate) { + continue; + } + const resolved = this.resolveColumn(candidate, columns); + if (resolved) { + return resolved; + } + } + return undefined; + }; + + const measureFallback = input.profile.numericColumns[0]; + const dimensionFallback = input.profile.dimensionColumns[0] ?? columns[0]; + const timeFallback = input.profile.timeColumns[0]; + + if (input.type === "metric") { + const value = resolve( + input.intent.mappings.value, + input.intent.mappings.measure, + input.intent.mappings.y, + baselineMappings?.value, + baselineMappings?.measure, + baselineMappings?.y, + measureFallback + ); + return { + value, + measure: value + }; + } + + if (input.type === "line") { + const x = resolve( + input.intent.mappings.x, + input.intent.mappings.time, + input.intent.mappings.dimension, + baselineMappings?.x, + baselineMappings?.time, + baselineMappings?.dimension, + timeFallback, + dimensionFallback + ); + const y = resolve( + input.intent.mappings.y, + input.intent.mappings.measure, + input.intent.mappings.value, + baselineMappings?.y, + baselineMappings?.measure, + baselineMappings?.value, + measureFallback + ); + return { + x, + y, + time: x, + measure: y + }; + } + + if (input.type === "bar") { + const x = resolve( + input.intent.mappings.x, + input.intent.mappings.dimension, + input.intent.mappings.category, + baselineMappings?.x, + baselineMappings?.dimension, + baselineMappings?.category, + dimensionFallback + ); + const y = resolve( + input.intent.mappings.y, + input.intent.mappings.measure, + input.intent.mappings.value, + baselineMappings?.y, + baselineMappings?.measure, + baselineMappings?.value, + measureFallback + ); + return { + x, + y, + dimension: x, + measure: y + }; + } + + const category = resolve( + input.intent.mappings.category, + input.intent.mappings.dimension, + input.intent.mappings.x, + baselineMappings?.category, + baselineMappings?.dimension, + baselineMappings?.x, + dimensionFallback + ); + const value = resolve( + input.intent.mappings.value, + input.intent.mappings.measure, + input.intent.mappings.y, + baselineMappings?.value, + baselineMappings?.measure, + baselineMappings?.y, + measureFallback + ); + return { + category, + value, + dimension: category, + measure: value + }; + } + + private requiredKeys(type: ChartBiChartType): Array { + if (type === "metric") { + return ["value"]; + } + if (type === "pie") { + return ["category", "value"]; + } + return ["x", "y"]; + } + + private resolveColumn(candidate: string, columns: string[]): string | undefined { + const exact = columns.find((column) => column === candidate); + if (exact) { + return exact; + } + + const lowered = candidate.toLowerCase(); + const caseInsensitive = columns.find((column) => column.toLowerCase() === lowered); + if (caseInsensitive) { + return caseInsensitive; + } + + const normalized = this.normalizeKey(candidate); + const byNormalized = columns.find( + (column) => this.normalizeKey(column) === normalized + ); + if (byNormalized) { + return byNormalized; + } + + const fuzzy = columns.find((column) => + this.normalizeKey(column).includes(normalized) + ); + return fuzzy; + } + + private normalizeKey(value: string): string { + return value.toLowerCase().replace(/[^a-z0-9]/g, ""); + } + + private cloneSpec(spec: ChartBiCanonicalSpec): ChartBiCanonicalSpec { + return { + ...spec, + mappings: { + ...spec.mappings + }, + ...(spec.insights ? { insights: [...spec.insights] } : {}) + }; + } + + private unique(values: string[]): string[] { + const seen = new Set(); + const result: string[] = []; + for (const value of values) { + const trimmed = value.trim(); + if (!trimmed || seen.has(trimmed)) { + continue; + } + seen.add(trimmed); + result.push(trimmed); + } + return result; + } +} diff --git a/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-validator.ts b/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-validator.ts new file mode 100644 index 0000000..8d77c50 --- /dev/null +++ b/apps/backend/src/modules/conversation/delivery/chartbi/chartbi-validator.ts @@ -0,0 +1,247 @@ +import { Injectable } from "@nestjs/common"; +import { + CHARTBI_CHART_ALLOWLIST, + type ChartBiCanonicalSpec, + type ChartBiChartType, + type ChartBiFieldMappings +} from "./chartbi-result-profiler"; + +export interface ChartBiValidationInput { + spec: ChartBiCanonicalSpec; + columns: string[]; + rows: Array>; +} + +export interface ChartBiValidationResult { + ok: boolean; + reason?: string; +} + +@Injectable() +export class ChartBiValidator { + validate(input: ChartBiValidationInput): ChartBiValidationResult { + const type = input.spec.type; + if (!(CHARTBI_CHART_ALLOWLIST as readonly string[]).includes(type)) { + return this.fail( + `chart type \"${String(type)}\" is outside allowlist (${CHARTBI_CHART_ALLOWLIST.join(", ")}).` + ); + } + + if (input.columns.length === 0) { + return this.fail("result columns are empty; chart validation is fail-closed."); + } + + if (input.rows.length === 0) { + return this.fail("result rows are empty; chart validation is fail-closed."); + } + + const fieldCheck = this.validateFieldOwnership({ + type, + mappings: input.spec.mappings, + columns: input.columns + }); + if (!fieldCheck.ok) { + return fieldCheck; + } + + const shapeCheck = this.validateDataShape({ + type, + mappings: input.spec.mappings, + rows: input.rows + }); + if (!shapeCheck.ok) { + return shapeCheck; + } + + return { ok: true }; + } + + private validateFieldOwnership(input: { + type: ChartBiChartType; + mappings: ChartBiFieldMappings; + columns: string[]; + }): ChartBiValidationResult { + const requiredRoles = this.requiredRoles(input.type); + const caseInsensitiveColumns = new Map( + input.columns.map((column) => [column.toLowerCase(), column]) + ); + + for (const role of requiredRoles) { + const field = this.readRoleField(input.mappings, role); + if (!field) { + return this.fail(`required mapping \"${role}\" is missing.`); + } + if (!caseInsensitiveColumns.has(field.toLowerCase())) { + return this.fail( + `mapped field \"${field}\" for role \"${role}\" does not exist in result columns.` + ); + } + } + + return { + ok: true + }; + } + + private validateDataShape(input: { + type: ChartBiChartType; + mappings: ChartBiFieldMappings; + rows: Array>; + }): ChartBiValidationResult { + if (input.type === "metric") { + const valueField = this.readRoleField(input.mappings, "value"); + if (!valueField) { + return this.fail("metric chart requires value mapping."); + } + const hasNumeric = input.rows.some((row) => this.isNumeric(row[valueField])); + if (!hasNumeric) { + return this.fail( + `metric field \"${valueField}\" has no numeric values in result rows.` + ); + } + return { ok: true }; + } + + if (input.type === "pie") { + const categoryField = this.readRoleField(input.mappings, "category"); + const valueField = this.readRoleField(input.mappings, "value"); + if (!categoryField || !valueField) { + return this.fail("pie chart requires category and value mappings."); + } + + const categories = new Set(); + for (const row of input.rows) { + const categoryValue = row[categoryField]; + if (categoryValue != null) { + categories.add(String(categoryValue)); + } + + const numeric = this.readNumber(row[valueField]); + if (numeric === undefined) { + return this.fail( + `pie value field \"${valueField}\" contains non-numeric values.` + ); + } + if (numeric < 0) { + return this.fail( + `pie value field \"${valueField}\" contains negative values.` + ); + } + } + + if (categories.size > 12) { + return this.fail( + `pie category field \"${categoryField}\" has ${categories.size} categories; exceeds safe limit 12.` + ); + } + + return { ok: true }; + } + + const xField = this.readRoleField(input.mappings, "x"); + const yField = this.readRoleField(input.mappings, "y"); + if (!xField || !yField) { + return this.fail(`${input.type} chart requires x/y mappings.`); + } + + const hasDimensionValue = input.rows.some((row) => this.hasRenderableValue(row[xField])); + if (!hasDimensionValue) { + return this.fail( + `${input.type} x field \"${xField}\" has no renderable values.` + ); + } + + const hasNumericMeasure = input.rows.some((row) => this.isNumeric(row[yField])); + if (!hasNumericMeasure) { + return this.fail( + `${input.type} y field \"${yField}\" has no numeric values.` + ); + } + + if (input.type === "line" && !this.isLikelyTemporal(input.rows, xField)) { + return this.fail( + `line x field \"${xField}\" is not temporal; use bar/table fallback.` + ); + } + + return { ok: true }; + } + + private requiredRoles(type: ChartBiChartType): Array<"x" | "y" | "category" | "value"> { + if (type === "metric") { + return ["value"]; + } + if (type === "pie") { + return ["category", "value"]; + } + return ["x", "y"]; + } + + private readRoleField( + mappings: ChartBiFieldMappings, + role: "x" | "y" | "category" | "value" + ): string | undefined { + if (role === "x") { + return mappings.x ?? mappings.dimension ?? mappings.time ?? mappings.category; + } + if (role === "y") { + return mappings.y ?? mappings.measure ?? mappings.value; + } + if (role === "category") { + return mappings.category ?? mappings.dimension ?? mappings.x; + } + return mappings.value ?? mappings.measure ?? mappings.y; + } + + private isLikelyTemporal(rows: Array>, field: string): boolean { + if (/(^|_)(date|time|day|month|year|at)$/.test(field.toLowerCase())) { + return true; + } + + const values = rows.map((row) => row[field]).filter((value) => value != null); + if (values.length === 0) { + return false; + } + const parseable = values.filter((value) => { + if (typeof value !== "string") { + return false; + } + return Number.isFinite(Date.parse(value)); + }).length; + return parseable / values.length >= 0.6; + } + + private isNumeric(value: unknown): boolean { + return this.readNumber(value) !== undefined; + } + + private readNumber(value: unknown): number | undefined { + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + if (typeof value === "string" && value.trim().length > 0) { + const parsed = Number(value); + if (Number.isFinite(parsed)) { + return parsed; + } + } + return undefined; + } + + private hasRenderableValue(value: unknown): boolean { + if (value == null) { + return false; + } + if (typeof value === "string") { + return value.trim().length > 0; + } + return true; + } + + private fail(reason: string): ChartBiValidationResult { + return { + ok: false, + reason + }; + } +} diff --git a/apps/backend/src/modules/conversation/delivery/delivery-contract.mapper.ts b/apps/backend/src/modules/conversation/delivery/delivery-contract.mapper.ts index 2159863..50727ee 100644 --- a/apps/backend/src/modules/conversation/delivery/delivery-contract.mapper.ts +++ b/apps/backend/src/modules/conversation/delivery/delivery-contract.mapper.ts @@ -23,6 +23,8 @@ export interface DeliveryReplayRecordInput { export interface DeliveryContractMapperInput { run: SqlRun; replayRecords?: DeliveryReplayRecordInput[]; + artifactOverride?: DeliveryContract["artifact"]; + additionalRiskTags?: string[]; } interface RerankFinalSnapshot { @@ -109,6 +111,17 @@ interface SqlCoverageEvidenceSnapshot { triggerSource: "selected_context" | "semantic_context" | "explicit_pinning" | "none"; } +interface SavedPriorSqlEvidenceSnapshot { + status: "hit" | "miss" | "filtered" | "stale" | "ambiguous"; + shortcutUsed: boolean; + reasonCodes?: string[]; + selectedChunkId?: string; + selectedViewId?: string; + selectedViewName?: string; + selectedSourceRunId?: string; + safetyResult?: "passed" | "rejected" | "fallback_generated"; +} + type ClarificationDecisionLayer = NonNullable< NonNullable["clarificationDecision"] >; @@ -117,6 +130,7 @@ type DeliveryEvidenceWithContext = NonNullable & { effectiveContextSummary?: EffectiveContextSummary; conflictHint?: ContextConflictHint; sqlCoverage?: SqlCoverageEvidenceSnapshot; + savedPriorSql?: SavedPriorSqlEvidenceSnapshot; }; interface SandboxPostProcessOutcome { @@ -139,8 +153,9 @@ export class DeliveryContractMapper { const traceContextEvidence = this.readTraceContextEvidence(input.run); const clarificationDecision = this.readClarificationDecisionEvidence(input.run); const sqlCoverage = this.readSqlCoverageEvidence(input.run); + const savedPriorSql = this.readSavedPriorSqlEvidence(input.run); const invalidInput = replayIndex.invalidPayload; - const artifact = this.buildArtifact(input.run); + const artifact = input.artifactOverride ?? this.buildArtifact(input.run); const sandboxOutcome = this.applySandboxPostProcess({ artifact, sandboxPayload: replayIndex.sandboxPostprocess, @@ -154,6 +169,10 @@ export class DeliveryContractMapper { ? ["context_conflict_detected"] : []), ...(sqlCoverage?.gateStatus === "failed" ? ["sql_coverage_gate_failed"] : []), + ...(savedPriorSql?.safetyResult === "rejected" + ? ["saved_prior_sql_safety_rejected"] + : []), + ...(input.additionalRiskTags ?? []), ...sandboxOutcome.riskTags ]); @@ -211,7 +230,8 @@ export class DeliveryContractMapper { effectiveContextSummary: traceContextEvidence.effectiveContextSummary, conflictHint: traceContextEvidence.conflictHint, clarificationDecision, - sqlCoverage + sqlCoverage, + savedPriorSql }; return { @@ -668,6 +688,76 @@ export class DeliveryContractMapper { return undefined; } + private readSavedPriorSqlEvidence( + run: SqlRun + ): SavedPriorSqlEvidenceSnapshot | undefined { + const steps = run.trace.steps ?? []; + if (steps.length === 0) { + return undefined; + } + const resolutionStepIndex = [...steps] + .map((step, index) => ({ step, index })) + .reverse() + .find((entry) => entry.step.node === "resolve-saved-prior-sql"); + if (!resolutionStepIndex) { + return undefined; + } + const output = this.parseSummaryObject(resolutionStepIndex.step.outputSummary); + const statusRaw = this.readString(output?.status); + const status = + statusRaw === "hit" || + statusRaw === "miss" || + statusRaw === "filtered" || + statusRaw === "stale" || + statusRaw === "ambiguous" + ? statusRaw + : undefined; + if (!status) { + return undefined; + } + + const reasonCodes = this.readStringArray( + output?.reasonCodes ?? output?.reason_codes + ); + const selectedChunkId = this.readString( + output?.selectedChunkId ?? output?.selected_chunk_id + ); + const selectedViewId = this.readString( + output?.selectedViewId ?? output?.selected_view_id + ); + const selectedSourceRunId = this.readString( + output?.selectedSourceRunId ?? output?.selected_source_run_id + ); + + const subsequentSteps = steps.slice(resolutionStepIndex.index + 1); + const safetyStep = subsequentSteps.find((step) => step.node === "safety-check"); + const hasFallbackGeneration = subsequentSteps.some( + (step) => step.node === "generate-sql" + ); + const safetyResult: SavedPriorSqlEvidenceSnapshot["safetyResult"] | undefined = + status !== "hit" + ? undefined + : safetyStep?.status === "success" + ? "passed" + : safetyStep?.status === "failed" + ? hasFallbackGeneration + ? "fallback_generated" + : "rejected" + : undefined; + + const shortcutUsed = status === "hit" && safetyResult === "passed"; + + return { + status, + shortcutUsed, + ...(reasonCodes.length > 0 ? { reasonCodes } : {}), + ...(selectedChunkId ? { selectedChunkId } : {}), + ...(selectedViewId ? { selectedViewId } : {}), + ...(selectedSourceRunId ? { selectedSourceRunId } : {}), + ...(safetyResult ? { safetyResult } : {}) + }; + } + private readSqlCoverageFromRecord( value: unknown ): SqlCoverageEvidenceSnapshot | undefined { diff --git a/apps/backend/src/modules/conversation/delivery/sandbox/sandbox-runtime.service.ts b/apps/backend/src/modules/conversation/delivery/sandbox/sandbox-runtime.service.ts index 93d3ba7..64d4624 100644 --- a/apps/backend/src/modules/conversation/delivery/sandbox/sandbox-runtime.service.ts +++ b/apps/backend/src/modules/conversation/delivery/sandbox/sandbox-runtime.service.ts @@ -85,7 +85,23 @@ export class SandboxRuntimeService { "rows_preview_limit maxRows must be a non-negative number." ); } - cloned.rowsPreview = cloned.rowsPreview?.slice(0, Math.floor(operation.maxRows)); + const maxRows = Math.floor(operation.maxRows); + cloned.rowsPreview = cloned.rowsPreview?.slice(0, maxRows); + const tableWithCompat = cloned.table as + | (NonNullable & { + truncated?: boolean; + }) + | undefined; + if (tableWithCompat) { + const tableRowsPreview = tableWithCompat.rowsPreview?.slice(0, maxRows); + tableWithCompat.rowsPreview = tableRowsPreview; + tableWithCompat.previewRowCount = tableRowsPreview?.length ?? 0; + const tableRowCount = + typeof tableWithCompat.rowCount === "number" + ? tableWithCompat.rowCount + : cloned.rowCount; + tableWithCompat.truncated = tableRowCount > tableWithCompat.previewRowCount; + } continue; } @@ -142,10 +158,62 @@ export class SandboxRuntimeService { private cloneArtifact(artifact: DeliveryArtifactLayer): DeliveryArtifactLayer { return { ...artifact, + summary: artifact.summary + ? { + ...artifact.summary, + metrics: artifact.summary.metrics?.map((item) => ({ ...item })), + dimensions: artifact.summary.dimensions + ? [...artifact.summary.dimensions] + : undefined + } + : undefined, + table: artifact.table + ? { + ...artifact.table, + columns: artifact.table.columns ? [...artifact.table.columns] : undefined, + rowsPreview: artifact.table.rowsPreview + ? artifact.table.rowsPreview.map((row) => ({ ...row })) + : undefined + } + : undefined, + chart: artifact.chart + ? { + ...artifact.chart, + mappings: artifact.chart.mappings ? { ...artifact.chart.mappings } : undefined, + series: artifact.chart.series + ? artifact.chart.series.map((series) => ({ ...series })) + : undefined, + meta: artifact.chart.meta ? { ...artifact.chart.meta } : undefined + } + : undefined, + validation: artifact.validation + ? { + ...artifact.validation, + reasonCodes: artifact.validation.reasonCodes + ? [...artifact.validation.reasonCodes] + : undefined + } + : undefined, + fallback: artifact.fallback ? { ...artifact.fallback } : undefined, + visualIntent: artifact.visualIntent + ? { + ...artifact.visualIntent, + mappings: artifact.visualIntent.mappings + ? { ...artifact.visualIntent.mappings } + : undefined, + normalizedIntent: this.cloneRecord(artifact.visualIntent.normalizedIntent) + } + : undefined, columns: artifact.columns ? [...artifact.columns] : undefined, rowsPreview: artifact.rowsPreview ? artifact.rowsPreview.map((row) => ({ ...row })) : undefined }; } + + private cloneRecord( + value: Record | undefined + ): Record | undefined { + return value ? { ...value } : undefined; + } } diff --git a/apps/backend/src/modules/data/query/sqlite-executor.service.ts b/apps/backend/src/modules/data/query/sqlite-executor.service.ts index fa4f731..cd0fffa 100644 --- a/apps/backend/src/modules/data/query/sqlite-executor.service.ts +++ b/apps/backend/src/modules/data/query/sqlite-executor.service.ts @@ -1,5 +1,6 @@ import { Injectable } from "@nestjs/common"; import type { Datasource } from "@text2sql/shared-types"; +import { DomainError } from "../../../common/domain-error"; import type { QueryExecutionResult, QueryExecutor } from "./query-executor.interface"; import { SqliteQueryService } from "../sqlite/sqlite-query.service"; @@ -13,6 +14,34 @@ export class SqliteExecutorService implements QueryExecutor { datasource: Datasource; sql: string; }): Promise { - return this.sqliteQuery.query(input.sql); + return this.sqliteQuery.query(input.sql, { + filePath: this.resolveDatasourcePath(input.datasource) + }); + } + + private resolveDatasourcePath(datasource: Datasource): string { + const path = + datasource.config && typeof datasource.config.path === "string" + ? datasource.config.path.trim() + : ""; + + if (path) { + return path; + } + + if (datasource.id === "sqlite_main") { + return this.sqliteQuery.dbPath; + } + + throw new DomainError( + "SQLITE_DATASOURCE_PATH_MISSING", + "当前 SQLite 数据源缺少可执行路径，请更新数据源配置后重试。", + 400, + { + datasourceId: datasource.id, + field: "config.path", + suggestedAction: "previous" + } + ); } } diff --git a/apps/backend/src/modules/data/sqlite/sqlite-query.service.ts b/apps/backend/src/modules/data/sqlite/sqlite-query.service.ts index 9291a61..d0f8ee1 100644 --- a/apps/backend/src/modules/data/sqlite/sqlite-query.service.ts +++ b/apps/backend/src/modules/data/sqlite/sqlite-query.service.ts @@ -39,16 +39,22 @@ export class SqliteQueryService { } } - async query(sql: string): Promise<{ + async query( + sql: string, + options?: { + filePath?: string; + } + ): Promise<{ columns: string[]; rows: Array>; }> { const safeSql = this.ensureSelectQuery(sql); const finalSql = this.withLimit(safeSql); + const dbPath = options?.filePath?.trim() || this.dbPath; try { const { stdout, stderr } = await execFileAsync("sqlite3", [ "-json", - this.dbPath, + dbPath, finalSql ]); if (stderr?.trim()) { @@ -68,6 +74,7 @@ export class SqliteQueryService { "SQLite 查询执行失败", 400, { + dbPath, originalMessage: error instanceof Error ? error.message : String(error) } ); diff --git a/apps/backend/src/modules/governance/datasource/datasource.service.ts b/apps/backend/src/modules/governance/datasource/datasource.service.ts index 6630d55..6510fdc 100644 --- a/apps/backend/src/modules/governance/datasource/datasource.service.ts +++ b/apps/backend/src/modules/governance/datasource/datasource.service.ts @@ -1,4 +1,7 @@ import { Injectable } from "@nestjs/common"; +import { constants as fsConstants } from "node:fs"; +import { access, realpath, stat } from "node:fs/promises"; +import { isAbsolute, relative, resolve } from "node:path"; import type { Datasource, DatasourceStatus, @@ -10,6 +13,7 @@ import { v4 as uuidv4 } from "uuid"; import { DomainError } from "../../../common/domain-error"; import { encryptSecret } from "../../../common/secret-crypto"; import { AppConfigService } from "../../config/app-config.service"; +import { SqliteQueryService } from "../../data/sqlite/sqlite-query.service"; import { DatasourceRepository } from "../../platform/data/persistence/index"; import { QueryExecutorRouterService } from "../../platform/data/query/index"; import type { AccessContext } from "../access/datasource-access-policy.service"; @@ -96,7 +100,8 @@ export class DatasourceService { private readonly datasourceRepository: DatasourceRepository, private readonly policyEvaluatorService: PolicyEvaluatorService, private readonly appConfig: AppConfigService, - private readonly queryExecutorRouter: QueryExecutorRouterService + private readonly queryExecutorRouter: QueryExecutorRouterService, + private readonly sqliteQuery: SqliteQueryService ) {} async listDatasources(options?: { @@ -173,7 +178,7 @@ export class DatasourceService { } const normalizedType = input.type; - if (normalizedType === "mysql" || normalizedType === "postgresql") { + if (this.isRelationalDatasourceType(normalizedType)) { await this.preflightDatasourceConnection({ type: normalizedType, host: input.host, @@ -184,15 +189,31 @@ export class DatasourceService { }); } - const config = this.buildConnectionConfig({ - type: normalizedType, - host: input.host, - port: input.port, - database: input.database, - username: input.username, - password: input.password, - filePath: input.filePath - }); + let config: Record; + if (normalizedType === "sqlite") { + config = await this.buildSqliteConnectionConfig({ + mode: "create", + filePath: input.filePath + }); + } else if (this.isRelationalDatasourceType(normalizedType)) { + config = this.buildRelationalConnectionConfig({ + type: normalizedType, + host: input.host, + port: input.port, + database: input.database, + username: input.username, + password: input.password + }); + } else { + throw new DomainError( + "DATASOURCE_TYPE_UNSUPPORTED", + `create 接口暂不支持 ${normalizedType} 类型。`, + 400, + { + datasourceType: normalizedType + } + ); + } const datasource = await this.datasourceRepository.upsertDatasource({ id: `ds-${uuidv4()}`, @@ -361,23 +382,14 @@ export class DatasourceService { return datasource; } - private buildConnectionConfig(input: { - type: DatasourceType; + private buildRelationalConnectionConfig(input: { + type: RelationalDatasourceType; host?: string; port?: number; database?: string; username?: string; password?: string; - filePath?: string; }): Record { - if (input.type === "sqlite") { - const path = input.filePath?.trim() || this.appConfig.sqlitePath; - if (!path) { - throw new DomainError("VALIDATION_ERROR", "SQLite 文件路径不能为空", 400); - } - return { path }; - } - const host = input.host?.trim(); const database = input.database?.trim(); const username = input.username?.trim(); @@ -404,6 +416,19 @@ export class DatasourceService { }; } + private async buildSqliteConnectionConfig(input: { + mode: "create" | "update" | "preview"; + filePath?: string; + datasourceId?: string; + }): Promise> { + const normalizedPath = await this.preflightSqlitePath({ + mode: input.mode, + filePath: input.filePath, + datasourceId: input.datasourceId + }); + return { path: normalizedPath }; + } + private async buildUpdatedConfig( existing: Datasource, patch: DatasourceUpdateInput @@ -417,13 +442,22 @@ export class DatasourceService { this.assertNoRelationalDatasourceFields(patch, existing.type); const filePath = patch.filePath?.trim(); if (patch.filePath !== undefined && !filePath) { - throw new DomainError("VALIDATION_ERROR", "filePath 不能为空。", 400, { + throw new DomainError("SQLITE_PATH_REQUIRED", "SQLite 文件路径不能为空。", 400, { field: "filePath" }); } + + const pathConfig = filePath + ? await this.buildSqliteConnectionConfig({ + mode: "update", + filePath, + datasourceId: existing.id + }) + : {}; + return { ...(existing.config ?? {}), - ...(filePath ? { path: filePath } : {}) + ...pathConfig }; } @@ -489,7 +523,7 @@ export class DatasourceService { password }); - return this.buildConnectionConfig({ + return this.buildRelationalConnectionConfig({ type: relationalType, host, port, @@ -651,15 +685,31 @@ export class DatasourceService { }; } - const config = this.buildConnectionConfig({ - type, - host: datasourcePayload.host, - port: datasourcePayload.port, - database: datasourcePayload.database, - username: datasourcePayload.username, - password: datasourcePayload.password, - filePath: datasourcePayload.filePath - }); + let config: Record; + if (type === "sqlite") { + config = await this.buildSqliteConnectionConfig({ + mode: "preview", + filePath: datasourcePayload.filePath + }); + } else if (this.isRelationalDatasourceType(type)) { + config = this.buildRelationalConnectionConfig({ + type, + host: datasourcePayload.host, + port: datasourcePayload.port, + database: datasourcePayload.database, + username: datasourcePayload.username, + password: datasourcePayload.password + }); + } else { + throw new DomainError( + "DATASOURCE_TYPE_UNSUPPORTED", + `create 预览不支持 ${type} 类型。`, + 400, + { + datasourceType: type + } + ); + } return { id: `preview-${uuidv4()}`, @@ -720,6 +770,284 @@ ORDER BY name return null; } + private async preflightSqlitePath(input: { + mode: "create" | "update" | "preview"; + filePath?: string; + datasourceId?: string; + }): Promise { + const validated = await this.validateSqlitePath(input); + + try { + await this.sqliteQuery.query( + ` +SELECT name +FROM sqlite_master +WHERE type = 'table' +ORDER BY name +LIMIT 1 + `.trim(), + { + filePath: validated.normalizedPath + } + ); + } catch (error) { + if (error instanceof DomainError && error.code.startsWith("SQLITE_PATH_")) { + throw error; + } + throw new DomainError( + "SQLITE_PRECHECK_FAILED", + "SQLite 文件连通性校验失败，请确认文件有效且当前服务具备读取权限。", + 400, + { + field: "filePath", + mode: input.mode, + datasourceId: input.datasourceId, + path: validated.normalizedPath, + resolvedPath: validated.resolvedPath, + originalCode: error instanceof DomainError ? error.code : undefined, + originalMessage: error instanceof Error ? error.message : String(error), + suggestedAction: "previous" + } + ); + } + + return validated.normalizedPath; + } + + private async validateSqlitePath(input: { + mode: "create" | "update" | "preview"; + filePath?: string; + datasourceId?: string; + }): Promise<{ + normalizedPath: string; + resolvedPath: string; + }> { + const rawPath = input.filePath?.trim() ?? ""; + if (!rawPath) { + throw new DomainError("SQLITE_PATH_REQUIRED", "SQLite 文件路径不能为空。", 400, { + field: "filePath", + mode: input.mode, + datasourceId: input.datasourceId, + suggestedAction: "previous" + }); + } + if (!isAbsolute(rawPath)) { + throw new DomainError( + "SQLITE_PATH_NOT_ABSOLUTE", + "请填写 SQLite 文件绝对路径。", + 400, + { + field: "filePath", + mode: input.mode, + datasourceId: input.datasourceId, + path: rawPath, + suggestedAction: "previous" + } + ); + } + + const normalizedPath = resolve(rawPath); + const resolvedPath = await this.resolveSqliteRealPath({ + normalizedPath, + mode: input.mode, + datasourceId: input.datasourceId + }); + await this.assertSqliteReadableFile({ + normalizedPath, + resolvedPath, + mode: input.mode, + datasourceId: input.datasourceId + }); + await this.assertSqlitePathAllowed({ + normalizedPath, + resolvedPath, + mode: input.mode, + datasourceId: input.datasourceId + }); + + return { + normalizedPath, + resolvedPath + }; + } + + private async resolveSqliteRealPath(input: { + normalizedPath: string; + mode: "create" | "update" | "preview"; + datasourceId?: string; + }): Promise { + try { + return await realpath(input.normalizedPath); + } catch (error) { + const code = this.readNodeErrorCode(error); + if (code === "ENOENT") { + throw new DomainError( + "SQLITE_PATH_NOT_FOUND", + "SQLite 文件不存在，请确认路径后重试。", + 400, + { + field: "filePath", + mode: input.mode, + datasourceId: input.datasourceId, + path: input.normalizedPath, + suggestedAction: "previous" + } + ); + } + if (code === "EACCES" || code === "EPERM") { + throw new DomainError( + "SQLITE_PATH_UNREADABLE", + "无法读取 SQLite 文件，请检查文件权限后重试。", + 400, + { + field: "filePath", + mode: input.mode, + datasourceId: input.datasourceId, + path: input.normalizedPath, + suggestedAction: "previous" + } + ); + } + throw new DomainError( + "SQLITE_PATH_INVALID", + "SQLite 文件路径解析失败，请确认路径格式正确。", + 400, + { + field: "filePath", + mode: input.mode, + datasourceId: input.datasourceId, + path: input.normalizedPath, + originalMessage: error instanceof Error ? error.message : String(error), + suggestedAction: "previous" + } + ); + } + } + + private async assertSqliteReadableFile(input: { + normalizedPath: string; + resolvedPath: string; + mode: "create" | "update" | "preview"; + datasourceId?: string; + }): Promise { + let fileStat: Awaited>; + try { + fileStat = await stat(input.resolvedPath); + } catch (error) { + throw new DomainError( + "SQLITE_PATH_NOT_FOUND", + "SQLite 文件不存在，请确认路径后重试。", + 400, + { + field: "filePath", + mode: input.mode, + datasourceId: input.datasourceId, + path: input.normalizedPath, + resolvedPath: input.resolvedPath, + suggestedAction: "previous" + } + ); + } + + if (!fileStat.isFile()) { + throw new DomainError( + "SQLITE_PATH_INVALID", + "SQLite 路径必须指向文件，当前路径不是文件。", + 400, + { + field: "filePath", + mode: input.mode, + datasourceId: input.datasourceId, + path: input.normalizedPath, + resolvedPath: input.resolvedPath, + suggestedAction: "previous" + } + ); + } + + try { + await access(input.resolvedPath, fsConstants.R_OK); + } catch (error) { + throw new DomainError( + "SQLITE_PATH_UNREADABLE", + "无法读取 SQLite 文件，请检查文件权限后重试。", + 400, + { + field: "filePath", + mode: input.mode, + datasourceId: input.datasourceId, + path: input.normalizedPath, + resolvedPath: input.resolvedPath, + originalMessage: error instanceof Error ? error.message : String(error), + suggestedAction: "previous" + } + ); + } + } + + private async assertSqlitePathAllowed(input: { + normalizedPath: string; + resolvedPath: string; + mode: "create" | "update" | "preview"; + datasourceId?: string; + }): Promise { + const allowedDirs = await this.resolveAllowedSqliteDirs(); + const matched = allowedDirs.some((allowedDir) => + this.isPathWithinDirectory(input.resolvedPath, allowedDir) + ); + + if (matched) { + return; + } + + throw new DomainError( + "SQLITE_PATH_NOT_ALLOWED", + "SQLite 文件路径不在允许目录内，请调整路径或联系管理员配置 SQLITE_ALLOWED_DIRS。", + 400, + { + field: "filePath", + mode: input.mode, + datasourceId: input.datasourceId, + path: input.normalizedPath, + resolvedPath: input.resolvedPath, + allowedDirs, + suggestedAction: "previous" + } + ); + } + + private async resolveAllowedSqliteDirs(): Promise { + const roots = this.appConfig.sqliteAllowedDirs; + const resolved = await Promise.all( + roots.map(async (item) => { + const normalized = resolve(item); + try { + return await realpath(normalized); + } catch { + return normalized; + } + }) + ); + return Array.from(new Set(resolved)); + } + + private isPathWithinDirectory(targetPath: string, dirPath: string): boolean { + const relation = relative(dirPath, targetPath); + return relation === "" || (!relation.startsWith("..") && !isAbsolute(relation)); + } + + private readNodeErrorCode(error: unknown): string | undefined { + if (!error || typeof error !== "object") { + return undefined; + } + const candidate = (error as Record).code; + return typeof candidate === "string" ? candidate : undefined; + } + + private isRelationalDatasourceType(type: DatasourceType): type is RelationalDatasourceType { + return type === "mysql" || type === "postgresql"; + } + protected async loadMysqlModule(): Promise { try { const dynamicImport = new Function( diff --git a/apps/backend/src/modules/governance/workspace/workspace-modeling.service.ts b/apps/backend/src/modules/governance/workspace/workspace-modeling.service.ts index fbf5225..680dc79 100644 --- a/apps/backend/src/modules/governance/workspace/workspace-modeling.service.ts +++ b/apps/backend/src/modules/governance/workspace/workspace-modeling.service.ts @@ -249,10 +249,10 @@ export class WorkspaceModelingService { datasourceId ) ]); - const discoveredSet = new Set(tableResult.items.map((item) => item.toLowerCase())); - const allowedTables = permissionResult.tableNames - .filter((tableName) => discoveredSet.has(tableName)) - .sort((left, right) => left.localeCompare(right)); + const allowedTables = this.buildEffectiveAllowedTables({ + discoveredTables: tableResult.items, + permissionResult + }); return { stage: "table_selection_ready", @@ -292,19 +292,34 @@ export class WorkspaceModelingService { datasourceId, body ); + const replaced = await this.workspaceDatasourceService.replaceDatasourceTablePermissions( + actor, + workspaceId, + datasourceId, + { + policyVersion: context.policyVersion, + tableNames: context.selectedTables + }, + this.buildSetupTablePermissionIdempotencyKey({ + workspaceId, + datasourceId, + policyVersion: context.policyVersion, + selectedTables: context.selectedTables + }) + ); return { stage: "setup_tables_saved", workspaceId, datasourceId, - policyVersion: context.policyVersion, + policyVersion: replaced.policyVersion, selectedTableNames: context.selectedTables, impactSummary: { - beforeCount: context.selectedTables.length, - afterCount: context.selectedTables.length, - addedCount: context.selectedTables.length, - removedCount: 0, - retainedCount: context.selectedTables.length + beforeCount: replaced.impactSummary.beforeCount, + afterCount: replaced.impactSummary.afterCount, + addedCount: replaced.impactSummary.addedCount, + removedCount: replaced.impactSummary.removedCount, + retainedCount: replaced.impactSummary.retainedCount } }; } @@ -2160,7 +2175,10 @@ export class WorkspaceModelingService { const discoveredSet = new Set(tableResult.items.map((item) => item.toLowerCase())); const allowedSet = new Set( - permissionResult.tableNames.filter((tableName) => discoveredSet.has(tableName)) + this.buildEffectiveAllowedTables({ + discoveredTables: tableResult.items, + permissionResult + }) ); const selectedTables = Array.from( new Set(body.selectedTables.map((tableName) => normalizeTableName(tableName))) @@ -2185,6 +2203,31 @@ export class WorkspaceModelingService { }; } + private buildEffectiveAllowedTables(input: { + discoveredTables: string[]; + permissionResult: { + policyVersion: number; + tableNames: string[]; + }; + }): string[] { + const discoveredTables = input.discoveredTables.map((item) => normalizeTableName(item)); + const discoveredSet = new Set(discoveredTables); + const permissionTables = input.permissionResult.tableNames + .map((item) => normalizeTableName(item)) + .filter((item) => discoveredSet.has(item)) + .sort((left, right) => left.localeCompare(right)); + + if (permissionTables.length > 0) { + return permissionTables; + } + + if (input.permissionResult.policyVersion === 0) { + return [...discoveredSet].sort((left, right) => left.localeCompare(right)); + } + + return []; + } + private async loadDatasourceOrThrow(datasourceId: string): Promise { const datasource = await this.datasourceRepository.getDatasourceById(datasourceId, { includeDeleted: true @@ -2611,6 +2654,26 @@ ORDER BY kcu.table_name, kcu.column_name, ccu.table_name, ccu.column_name ].join("::"); } + private buildSetupTablePermissionIdempotencyKey(input: { + workspaceId: string; + datasourceId: string; + policyVersion: number; + selectedTables: string[]; + }): string { + const signature = createHash("sha256") + .update( + JSON.stringify({ + workspaceId: input.workspaceId, + datasourceId: input.datasourceId, + policyVersion: input.policyVersion, + selectedTables: input.selectedTables + }) + ) + .digest("hex") + .slice(0, 24); + return `modeling-setup-tables-${signature}`; + } + private readCommitIdempotencyRecord(input: { scope: string; payloadHash: string; diff --git a/apps/backend/src/modules/knowledge/contracts/knowledge-memory.contract.ts b/apps/backend/src/modules/knowledge/contracts/knowledge-memory.contract.ts index 4d2fb30..899b530 100644 --- a/apps/backend/src/modules/knowledge/contracts/knowledge-memory.contract.ts +++ b/apps/backend/src/modules/knowledge/contracts/knowledge-memory.contract.ts @@ -1,7 +1,60 @@ import type { MemoryPromotionService } from "../memory/memory-promotion.service"; +import type { SavedPriorSqlService } from "../memory/saved-prior-sql.service"; export const KNOWLEDGE_MEMORY_CONTRACT = Symbol("KNOWLEDGE_MEMORY_CONTRACT"); +export type SavedPriorSqlCaptureOutcome = + | "captured" + | "duplicate" + | "skipped_ineligible" + | "capture_failed"; + +export interface SavedPriorSqlCaptureInput { + workspaceId: string; + datasourceId: string; + sourceRunId: string; + sourceRunStatus: string; + sourceRunCreatedAt?: string; + question?: string; + sql: string; + viewId: string; + viewName: string; + viewSql: string; + tableNames?: string[]; + columnNames?: string[]; + replayed: boolean; + savedAt?: string; +} + +export interface SavedPriorSqlRecord { + priorId: string; + workspaceId: string; + datasourceId: string; + sourceRunId: string; + sourceRunStatus: string; + sourceRunCreatedAt?: string; + viewId: string; + viewName: string; + question: string; + sql: string; + tableNames: string[]; + columnNames: string[]; + savedAt: string; + metadata: { + trusted: true; + verified: true; + priorSql: true; + }; +} + +export interface SavedPriorSqlCaptureResult { + outcome: SavedPriorSqlCaptureOutcome; + priorId: string; + replayKey: string; + reason?: string; + record?: SavedPriorSqlRecord; +} + export interface KnowledgeMemoryContract { promotion: Pick< MemoryPromotionService, @@ -11,4 +64,8 @@ export interface KnowledgeMemoryContract { | "buildCandidateIdForRun" | "applyFeedback" >; + savedPriorSql: Pick< + SavedPriorSqlService, + "captureFromSavedView" | "getRecord" | "listRecords" | "buildPriorId" + >; } diff --git a/apps/backend/src/modules/knowledge/knowledge-chat-support.facade.ts b/apps/backend/src/modules/knowledge/knowledge-chat-support.facade.ts index 8a96b53..a12b518 100644 --- a/apps/backend/src/modules/knowledge/knowledge-chat-support.facade.ts +++ b/apps/backend/src/modules/knowledge/knowledge-chat-support.facade.ts @@ -13,6 +13,7 @@ import { type KnowledgeSemanticSpineContract } from "./contracts/knowledge-semantic-registry.contract"; import { MemoryPromotionService } from "./memory/memory-promotion.service"; +import { SavedPriorSqlService } from "./memory/saved-prior-sql.service"; import { RagReplayRepository } from "./rag/observability/rag-replay.repository"; import { RagRetrievalService } from "./rag/retrieval/rag-retrieval.service"; import { RagRerankService } from "./rag/rerank/rag-rerank.service"; @@ -28,6 +29,7 @@ export class KnowledgeChatSupportFacade implements KnowledgeFacadeContract { constructor( readonly memoryPromotionService: MemoryPromotionService, + readonly savedPriorSqlService: SavedPriorSqlService, readonly ragReplayRepository: RagReplayRepository, readonly ragRetrievalService: RagRetrievalService, readonly ragRerankService: RagRerankService, @@ -63,7 +65,8 @@ export class KnowledgeChatSupportFacade implements KnowledgeFacadeContract { semanticSpine: semanticSpineContract }; this.memoryContract = { - promotion: this.memoryPromotionService + promotion: this.memoryPromotionService, + savedPriorSql: this.savedPriorSqlService }; } diff --git a/apps/backend/src/modules/knowledge/memory/memory.module.ts b/apps/backend/src/modules/knowledge/memory/memory.module.ts index f67a72e..50ce2b4 100644 --- a/apps/backend/src/modules/knowledge/memory/memory.module.ts +++ b/apps/backend/src/modules/knowledge/memory/memory.module.ts @@ -6,6 +6,7 @@ import { MemoryController } from "../../memory/memory.controller"; import { MemoryPromotionPolicy } from "../../memory/memory-promotion-policy"; import { MemoryPromotionService as LegacyMemoryPromotionService } from "../../memory/memory-promotion.service"; import { MemoryPromotionService as KnowledgeMemoryPromotionService } from "./memory-promotion.service"; +import { SavedPriorSqlService } from "./saved-prior-sql.service"; export const KNOWLEDGE_MEMORY_COMPAT_BRIDGE = Object.freeze({ capability: "memory", @@ -24,12 +25,14 @@ export const KNOWLEDGE_MEMORY_COMPAT_BRIDGE = Object.freeze({ provide: KnowledgeMemoryPromotionService, useExisting: LegacyMemoryPromotionService }, + SavedPriorSqlService, AdminOnlyGuard ], exports: [ MemoryPromotionPolicy, LegacyMemoryPromotionService, - KnowledgeMemoryPromotionService + KnowledgeMemoryPromotionService, + SavedPriorSqlService ] }) export class MemoryModule {} diff --git a/apps/backend/src/modules/knowledge/memory/saved-prior-sql.service.ts b/apps/backend/src/modules/knowledge/memory/saved-prior-sql.service.ts new file mode 100644 index 0000000..b4fd6ae --- /dev/null +++ b/apps/backend/src/modules/knowledge/memory/saved-prior-sql.service.ts @@ -0,0 +1,442 @@ +import { createHash } from "node:crypto"; +import { Injectable } from "@nestjs/common"; +import { BuildRagIndexJob } from "../../rag/jobs/build-rag-index.job"; +import type { RagChunkBuildInput } from "../../rag/index/rag-index.repository"; +import { RagIndexRepository } from "../../rag/index/rag-index.repository"; +import { RagDocumentFactory } from "../../rag/ingestion/rag-document.factory"; +import { RagDocumentRepository } from "../../rag/ingestion/rag-document.repository"; +import { + type SavedPriorSqlCaptureInput, + type SavedPriorSqlCaptureResult, + type SavedPriorSqlRecord +} from "../contracts/knowledge-memory.contract"; +import { RagReplayRepository } from "../rag/observability/rag-replay.repository"; + +interface NormalizedCaptureInput { + workspaceId: string; + datasourceId: string; + sourceRunId: string; + sourceRunStatus: string; + sourceRunCreatedAt?: string; + question?: string; + sql: string; + viewId: string; + viewName: string; + viewSql: string; + tableNames: string[]; + columnNames: string[]; + replayed: boolean; + savedAt: string; +} + +@Injectable() +export class SavedPriorSqlService { + private readonly records = new Map(); + + constructor( + private readonly ragReplayRepository: RagReplayRepository, + private readonly ragDocumentFactory: RagDocumentFactory, + private readonly ragDocumentRepository: RagDocumentRepository, + private readonly ragIndexRepository: RagIndexRepository, + private readonly buildRagIndexJob: BuildRagIndexJob + ) {} + + async captureFromSavedView( + input: SavedPriorSqlCaptureInput + ): Promise { + const normalized = this.normalizeInput(input); + const priorId = this.buildPriorId({ + workspaceId: normalized.workspaceId, + datasourceId: normalized.datasourceId, + viewId: normalized.viewId + }); + + if (normalized.replayed) { + return this.writeReplayAndReturn({ + outcome: "duplicate", + reason: "save_replayed", + priorId, + input: normalized, + record: this.getRecord(priorId) + }); + } + + if (normalized.sourceRunStatus !== "executionResult") { + return this.writeReplayAndReturn({ + outcome: "skipped_ineligible", + reason: "source_run_not_execution_result", + priorId, + input: normalized + }); + } + + if (!normalized.question) { + return this.writeReplayAndReturn({ + outcome: "skipped_ineligible", + reason: "source_run_question_missing", + priorId, + input: normalized + }); + } + + if (this.records.has(priorId)) { + return this.writeReplayAndReturn({ + outcome: "duplicate", + reason: "prior_already_captured", + priorId, + input: normalized, + record: this.getRecord(priorId) + }); + } + + const record: SavedPriorSqlRecord = { + priorId, + workspaceId: normalized.workspaceId, + datasourceId: normalized.datasourceId, + sourceRunId: normalized.sourceRunId, + sourceRunStatus: normalized.sourceRunStatus, + sourceRunCreatedAt: normalized.sourceRunCreatedAt, + viewId: normalized.viewId, + viewName: normalized.viewName, + question: normalized.question, + sql: normalized.sql, + tableNames: this.mergeUniqueTokens( + normalized.tableNames, + this.extractTableNames(normalized.sql) + ), + columnNames: this.mergeUniqueTokens( + normalized.columnNames, + this.extractColumnNames(normalized.sql) + ), + savedAt: normalized.savedAt, + metadata: { + trusted: true, + verified: true, + priorSql: true + } + }; + + const sourceVersion = this.buildSourceVersion(record); + const contentChecksum = this.buildContentChecksum(record); + const documentBuild = this.ragDocumentFactory.create({ + sourceType: "sql_example", + datasourceId: record.datasourceId, + sourceVersion, + contentChecksum, + sourceRef: `saved_prior_sql:${record.priorId}`, + exampleId: record.priorId, + question: record.question, + sql: record.sql, + tableNames: record.tableNames, + columnNames: record.columnNames, + metadata: { + trusted: true, + verified: true, + priorSql: true, + workspaceId: record.workspaceId, + datasourceId: record.datasourceId, + sourceRunId: record.sourceRunId, + sourceRunStatus: record.sourceRunStatus, + sourceRunCreatedAt: record.sourceRunCreatedAt, + viewId: record.viewId, + viewName: record.viewName, + viewSql: normalized.viewSql, + savedAt: record.savedAt, + viewExists: true, + viewStatus: "active" + } + }); + + await this.ragDocumentRepository.upsertDocumentWithChunks({ + document: documentBuild.document, + chunks: documentBuild.chunks + }); + + const buildChunks = documentBuild.chunks.map((chunk) => + this.toBuildChunkInput(chunk) + ); + this.ragIndexRepository.upsertChunksForDatasource(record.datasourceId, buildChunks); + + const buildResult = await this.buildRagIndexJob.run({ + datasourceId: record.datasourceId, + sourceVersion, + buildReason: "saved_prior_sql_capture", + runId: record.sourceRunId, + chunks: buildChunks + }); + + this.records.set(priorId, record); + + return this.writeReplayAndReturn({ + outcome: "captured", + priorId, + input: normalized, + record, + indexVersionId: buildResult.indexVersionId, + documentId: documentBuild.document.id, + chunkCount: documentBuild.chunks.length + }); + } + + getRecord(priorId: string): SavedPriorSqlRecord | undefined { + const key = priorId.trim(); + if (!key) { + return undefined; + } + const record = this.records.get(key); + if (!record) { + return undefined; + } + return this.cloneRecord(record); + } + + listRecords(): SavedPriorSqlRecord[] { + return Array.from(this.records.values()).map((item) => this.cloneRecord(item)); + } + + buildPriorId(input: { + workspaceId: string; + datasourceId: string; + viewId: string; + }): string { + const workspaceId = this.requireTrimmed(input.workspaceId, "workspaceId"); + const datasourceId = this.requireTrimmed(input.datasourceId, "datasourceId"); + const viewId = this.requireTrimmed(input.viewId, "viewId"); + const digest = createHash("sha256") + .update(`${workspaceId}|${datasourceId}|${viewId}`) + .digest("hex"); + return `saved_prior_sql.${digest}`; + } + + private toBuildChunkInput(chunk: { + id: string; + datasourceId: string; + domain: string; + content: string; + metadata?: string; + }): RagChunkBuildInput { + return { + id: chunk.id, + datasourceId: chunk.datasourceId, + domain: chunk.domain, + content: chunk.content, + metadata: chunk.metadata + }; + } + + private buildSourceVersion(record: SavedPriorSqlRecord): string { + const digest = createHash("sha256") + .update( + [ + record.workspaceId, + record.datasourceId, + record.priorId, + record.viewId, + record.question, + record.sql, + record.tableNames.join(","), + record.columnNames.join(",") + ].join("|") + ) + .digest("hex"); + return `saved_prior_sql:${record.priorId}:${digest.slice(0, 16)}`; + } + + private buildContentChecksum(record: SavedPriorSqlRecord): string { + return createHash("sha256") + .update( + [record.question, record.sql, record.tableNames.join(","), record.columnNames.join(",")].join( + "|" + ) + ) + .digest("hex"); + } + + private async writeReplayAndReturn(input: { + outcome: "captured" | "duplicate" | "skipped_ineligible"; + reason?: string; + priorId: string; + input: NormalizedCaptureInput; + record?: SavedPriorSqlRecord; + indexVersionId?: string; + documentId?: string; + chunkCount?: number; + }): Promise { + const replayKey = this.buildReplayKey(input.outcome, input.priorId); + await this.ragReplayRepository.writeReplay({ + runId: input.input.sourceRunId, + replayKey, + datasourceId: input.input.datasourceId, + stage: "saved_prior_sql_capture", + ...(input.indexVersionId ? { indexVersionId: input.indexVersionId } : {}), + ...(input.documentId ? { documentId: input.documentId } : {}), + payload: { + outcome: input.outcome, + reason: input.reason ?? null, + priorId: input.priorId, + workspaceId: input.input.workspaceId, + datasourceId: input.input.datasourceId, + sourceRunId: input.input.sourceRunId, + sourceRunStatus: input.input.sourceRunStatus, + viewId: input.input.viewId, + viewName: input.input.viewName, + replayedSave: input.input.replayed, + savedAt: input.input.savedAt, + indexVersionId: input.indexVersionId, + documentId: input.documentId, + chunkCount: input.chunkCount ?? 0 + } + }); + return { + outcome: input.outcome, + reason: input.reason, + priorId: input.priorId, + replayKey, + record: input.record ? this.cloneRecord(input.record) : undefined + }; + } + + private buildReplayKey( + outcome: "captured" | "duplicate" | "skipped_ineligible", + priorId: string + ): string { + return `saved_prior_sql:${outcome}:${priorId}`; + } + + private normalizeInput(input: SavedPriorSqlCaptureInput): NormalizedCaptureInput { + const workspaceId = this.requireTrimmed(input.workspaceId, "workspaceId"); + const datasourceId = this.requireTrimmed(input.datasourceId, "datasourceId"); + const sourceRunId = this.requireTrimmed(input.sourceRunId, "sourceRunId"); + const sourceRunStatus = this.requireTrimmed( + input.sourceRunStatus, + "sourceRunStatus" + ); + const sql = this.requireTrimmed(input.sql, "sql"); + const viewId = this.requireTrimmed(input.viewId, "viewId"); + const viewName = this.requireTrimmed(input.viewName, "viewName"); + const viewSql = this.requireTrimmed(input.viewSql, "viewSql"); + const sourceRunCreatedAt = this.normalizeOptional(input.sourceRunCreatedAt); + const question = this.normalizeOptional(input.question); + return { + workspaceId, + datasourceId, + sourceRunId, + sourceRunStatus, + sourceRunCreatedAt: sourceRunCreatedAt ?? undefined, + question: question ?? undefined, + sql, + viewId, + viewName, + viewSql, + tableNames: this.mergeUniqueTokens(input.tableNames ?? []), + columnNames: this.mergeUniqueTokens(input.columnNames ?? []), + replayed: input.replayed === true, + savedAt: this.toIsoTimestamp(input.savedAt) + }; + } + + private cloneRecord(record: SavedPriorSqlRecord): SavedPriorSqlRecord { + return { + ...record, + tableNames: [...record.tableNames], + columnNames: [...record.columnNames], + metadata: { + ...record.metadata + } + }; + } + + private mergeUniqueTokens(...sources: string[][]): string[] { + const dedup = new Set(); + for (const source of sources) { + for (const raw of source) { + const normalized = this.normalizeOptional(raw); + if (!normalized) { + continue; + } + dedup.add(normalized); + } + } + return Array.from(dedup.values()); + } + + private extractTableNames(sql: string): string[] { + const extracted: string[] = []; + const matcher = /\b(?:from|join)\s+([A-Za-z0-9_.`"\[\]-]+)/gi; + let match = matcher.exec(sql); + while (match) { + const value = this.cleanSqlIdentifier(match[1] ?? ""); + if (value) { + extracted.push(value); + } + match = matcher.exec(sql); + } + return this.mergeUniqueTokens(extracted); + } + + private extractColumnNames(sql: string): string[] { + const selectMatch = /^\s*select\s+([\s\S]+?)\s+from\s+/i.exec(sql); + if (!selectMatch?.[1]) { + return []; + } + const projectedColumns = selectMatch[1] + .split(",") + .map((segment) => segment.trim()) + .filter((segment) => segment.length > 0) + .map((segment) => this.normalizeProjectedColumn(segment)) + .filter((segment): segment is string => Boolean(segment)); + return this.mergeUniqueTokens(projectedColumns); + } + + private normalizeProjectedColumn(segment: string): string | null { + if (segment === "*") { + return null; + } + const aliasMatch = /\bas\s+([A-Za-z0-9_`"\[\]-]+)\s*$/i.exec(segment); + if (aliasMatch?.[1]) { + return this.cleanSqlIdentifier(aliasMatch[1]); + } + const parts = segment.split(/\s+/).filter((item) => item.length > 0); + const canonical = parts[parts.length - 1] ?? segment; + if (canonical === "*") { + return null; + } + return this.cleanSqlIdentifier(canonical.split(".").pop() ?? canonical); + } + + private cleanSqlIdentifier(value: string): string | null { + const normalized = value + .replace(/^[`"\[]+/, "") + .replace(/[`"\]]+$/, "") + .trim(); + return normalized.length > 0 ? normalized : null; + } + + private toIsoTimestamp(value?: string): string { + const normalized = this.normalizeOptional(value); + if (!normalized) { + return new Date().toISOString(); + } + const timestamp = Date.parse(normalized); + if (Number.isNaN(timestamp)) { + throw new Error(`saved prior sql savedAt illegal timestamp: ${normalized}`); + } + return new Date(timestamp).toISOString(); + } + + private normalizeOptional(value: string | undefined | null): string | null { + if (typeof value !== "string") { + return null; + } + const normalized = value.trim(); + return normalized.length > 0 ? normalized : null; + } + + private requireTrimmed(value: string, field: string): string { + const normalized = this.normalizeOptional(value); + if (!normalized) { + throw new Error(`saved prior sql ${field} is required`); + } + return normalized; + } +} diff --git a/apps/backend/src/modules/knowledge/rag/rag.module.ts b/apps/backend/src/modules/knowledge/rag/rag.module.ts index 52c0641..fa4136c 100644 --- a/apps/backend/src/modules/knowledge/rag/rag.module.ts +++ b/apps/backend/src/modules/knowledge/rag/rag.module.ts @@ -11,6 +11,10 @@ import { RagAuditReplayService } from "../../rag/audit/rag-audit-replay.service" import { RagEventConsumerService } from "../../rag/events/rag-event-consumer.service"; import { RagIndexBuilderService } from "../../rag/index/rag-index-builder.service"; import { RagIndexRepository } from "../../rag/index/rag-index.repository"; +import { RagDocumentFactory } from "../../rag/ingestion/rag-document.factory"; +import { RagDocumentRepository } from "../../rag/ingestion/rag-document.repository"; +import { IngestionSourceAdapter } from "../../rag/ingestion/ingestion-source.adapter"; +import { RagChunkingService } from "../../rag/ingestion/rag-chunking.service"; import { BuildRagIndexJob } from "../../rag/jobs/build-rag-index.job"; import { RagReplayRepository as LegacyRagReplayRepository } from "../../rag/observability/rag-replay.repository"; import { RagReplayRepository } from "./observability/rag-replay.repository"; @@ -74,6 +78,10 @@ export function assertKnowledgeCompatBridgeRetirementReady( controllers: [RagQualityController], providers: [ RagIndexRepository, + IngestionSourceAdapter, + RagChunkingService, + RagDocumentFactory, + RagDocumentRepository, RagIndexBuilderService, BuildRagIndexJob, RagDatasourceQuotaPolicy, @@ -106,6 +114,10 @@ export function assertKnowledgeCompatBridgeRetirementReady( ], exports: [ RagIndexRepository, + IngestionSourceAdapter, + RagChunkingService, + RagDocumentFactory, + RagDocumentRepository, RagIndexBuilderService, BuildRagIndexJob, RagDatasourceQuotaPolicy, diff --git a/apps/backend/src/modules/knowledge/rag/retrieval/rag-retrieval.types.ts b/apps/backend/src/modules/knowledge/rag/retrieval/rag-retrieval.types.ts index 372c7de..7ebfd2f 100644 --- a/apps/backend/src/modules/knowledge/rag/retrieval/rag-retrieval.types.ts +++ b/apps/backend/src/modules/knowledge/rag/retrieval/rag-retrieval.types.ts @@ -142,15 +142,45 @@ export interface RagContextPack { } export interface RagPriorSqlLaneEvidence { - status: "hit" | "miss" | "filtered"; + status: "hit" | "miss" | "filtered" | "stale" | "ambiguous"; matched_count: number; selected_count: number; filtered_count: number; + stale_count?: number; + ambiguous_count?: number; + eligible_count?: number; degrade_reasons?: string[]; + shortcut?: RagPriorSqlShortcutDecision; matchedCount?: number; selectedCount?: number; filteredCount?: number; + staleCount?: number; + ambiguousCount?: number; + eligibleCount?: number; degradeReasons?: string[]; + shortcutDecision?: RagPriorSqlShortcutDecision; +} + +export interface RagPriorSqlShortcutDecision { + status: "hit" | "miss" | "filtered" | "stale" | "ambiguous"; + reason_codes: string[]; + matched_count: number; + eligible_count: number; + filtered_count: number; + stale_count: number; + ambiguous_count: number; + selected_chunk_id?: string; + selected_view_id?: string; + selected_source_run_id?: string; + reasonCodes?: string[]; + matchedCount?: number; + eligibleCount?: number; + filteredCount?: number; + staleCount?: number; + ambiguousCount?: number; + selectedChunkId?: string; + selectedViewId?: string; + selectedSourceRunId?: string; } export interface RagRetrievalBundle { diff --git a/apps/backend/src/modules/llm/llm-gateway.interface.ts b/apps/backend/src/modules/llm/llm-gateway.interface.ts index ac54c7a..c049184 100644 --- a/apps/backend/src/modules/llm/llm-gateway.interface.ts +++ b/apps/backend/src/modules/llm/llm-gateway.interface.ts @@ -39,6 +39,7 @@ export interface LlmGatewayRuntimeConfig { baseUrl: string; apiKey: string; timeoutMs: number; + streamTimeoutMs?: number; } export interface LlmGatewayGenerateOutput { diff --git a/apps/backend/src/modules/llm/llm-gateway.service.ts b/apps/backend/src/modules/llm/llm-gateway.service.ts index fea68dd..f8468b0 100644 --- a/apps/backend/src/modules/llm/llm-gateway.service.ts +++ b/apps/backend/src/modules/llm/llm-gateway.service.ts @@ -158,12 +158,13 @@ export class LlmGatewayService implements LlmGateway { try { const model = this.modelFactory.createChatModel(runtime) as never; const normalizedTools = this.normalizeTools(options?.tools); + const streamTimeoutMs = runtime.streamTimeoutMs ?? runtime.timeoutMs; const result = streamText({ model, system: prompt.systemPrompt, prompt: prompt.userPrompt, temperature: 0.2, - abortSignal: AbortSignal.timeout(runtime.timeoutMs), + abortSignal: AbortSignal.timeout(streamTimeoutMs), tools: normalizedTools }); @@ -199,12 +200,24 @@ export class LlmGatewayService implements LlmGateway { continue; } if (chunk.type === "tool-error") { + const toolMessage = toErrorMessage(chunk.error); await options?.onEvent?.({ type: "tool-error", toolName: chunk.toolName, toolCallId: chunk.toolCallId, - message: toErrorMessage(chunk.error) + message: toolMessage }); + throw new DomainError( + "LLM_TOOL_CALL_EXECUTION_FAILED", + `LLM 工具调用失败: ${toolMessage}`, + 502, + { + provider: runtime.provider, + toolName: chunk.toolName, + toolCallId: chunk.toolCallId, + toolSql: toolCallSql?.slice(0, 500) + } + ); } } diff --git a/apps/backend/src/modules/llm/provider-router.service.ts b/apps/backend/src/modules/llm/provider-router.service.ts index 75bdf23..29d8643 100644 --- a/apps/backend/src/modules/llm/provider-router.service.ts +++ b/apps/backend/src/modules/llm/provider-router.service.ts @@ -146,7 +146,8 @@ export class ProviderRouterService { model: resolved.model.model, baseUrl: resolved.runtime.baseUrl?.trim() || this.config.llmBaseUrl || "", apiKey: resolved.runtime.apiKey?.trim() || this.config.llmApiKey || "", - timeoutMs: this.config.llmTimeoutMs + timeoutMs: this.config.llmTimeoutMs, + streamTimeoutMs: this.config.llmStreamTimeoutMs } }; } @@ -163,7 +164,8 @@ export class ProviderRouterService { model: defaultModel.model, baseUrl: defaultRuntime.baseUrl?.trim() || this.config.llmBaseUrl || "", apiKey: defaultRuntime.apiKey?.trim() || this.config.llmApiKey || "", - timeoutMs: this.config.llmTimeoutMs + timeoutMs: this.config.llmTimeoutMs, + streamTimeoutMs: this.config.llmStreamTimeoutMs } }; } catch { @@ -173,7 +175,8 @@ export class ProviderRouterService { model: this.config.llmModel, baseUrl: this.config.llmBaseUrl, apiKey: this.config.llmApiKey, - timeoutMs: this.config.llmTimeoutMs + timeoutMs: this.config.llmTimeoutMs, + streamTimeoutMs: this.config.llmStreamTimeoutMs } }; } diff --git a/apps/backend/src/modules/rag/index/rag-index.repository.ts b/apps/backend/src/modules/rag/index/rag-index.repository.ts index 07d68e0..4517ed4 100644 --- a/apps/backend/src/modules/rag/index/rag-index.repository.ts +++ b/apps/backend/src/modules/rag/index/rag-index.repository.ts @@ -183,6 +183,25 @@ export class RagIndexRepository implements OnModuleInit, OnModuleDestroy { ); } + upsertChunksForDatasource(datasourceId: string, chunks: RagChunkBuildInput[]): void { + if (chunks.length === 0) { + return; + } + const existing = this.chunksByDatasource.get(datasourceId) ?? []; + const merged = new Map(); + for (const chunk of existing) { + merged.set(chunk.id, { + ...chunk + }); + } + for (const chunk of chunks) { + merged.set(chunk.id, { + ...chunk + }); + } + this.chunksByDatasource.set(datasourceId, [...merged.values()]); + } + async listChunksForBuild(datasourceId: string): Promise { const memory = this.chunksByDatasource.get(datasourceId) ?? []; if (!this.isPrimaryPersistenceConfigured() || !this.prisma) { diff --git a/apps/backend/src/modules/rag/ingestion/rag-document.repository.ts b/apps/backend/src/modules/rag/ingestion/rag-document.repository.ts new file mode 100644 index 0000000..eefa23e --- /dev/null +++ b/apps/backend/src/modules/rag/ingestion/rag-document.repository.ts @@ -0,0 +1,212 @@ +import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from "@nestjs/common"; +import { AppConfigService } from "../../config/app-config.service"; +import type { + RagChunkRecordDraft, + RagDocumentDraft +} from "./rag-document.factory"; + +type PrismaClientLike = { + ragDocument: { + upsert: (args: Record) => Promise; + }; + ragChunk: { + upsert: (args: Record) => Promise; + }; + $transaction: (fn: (tx: PrismaTransactionClientLike) => Promise) => Promise; + $disconnect: () => Promise; +}; + +type PrismaTransactionClientLike = Omit; + +interface RagDocumentMemoryState { + documentsById: Map; + chunksById: Map; +} + +export interface RagDocumentUpsertInput { + document: RagDocumentDraft; + chunks: RagChunkRecordDraft[]; +} + +export interface RagDocumentUpsertResult { + documentId: string; + insertedDocument: boolean; + insertedChunkCount: number; +} + +@Injectable() +export class RagDocumentRepository implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(RagDocumentRepository.name); + private prisma?: PrismaClientLike; + private readonly memoryByDatasource = new Map(); + + constructor(private readonly appConfig: AppConfigService) {} + + async onModuleInit(): Promise { + if (!this.isPrimaryPersistenceConfigured()) { + return; + } + try { + const prismaClientModulePath = "../../../generated/prisma/client"; + const prismaModule = (await import(prismaClientModulePath)) as unknown as { + PrismaClient?: new (...args: unknown[]) => PrismaClientLike; + default?: { + PrismaClient?: new (...args: unknown[]) => PrismaClientLike; + }; + }; + const adapterModule = (await import("@prisma/adapter-pg")) as unknown as { + PrismaPg?: new (...args: unknown[]) => unknown; + default?: { + PrismaPg?: new (...args: unknown[]) => unknown; + }; + }; + const PrismaCtor = prismaModule.PrismaClient ?? prismaModule.default?.PrismaClient; + const PrismaPgCtor = adapterModule.PrismaPg ?? adapterModule.default?.PrismaPg; + if (!PrismaCtor || !PrismaPgCtor) { + throw new Error("Prisma client or pg adapter unavailable"); + } + const adapter = new PrismaPgCtor({ + connectionString: this.appConfig.databaseUrl + }); + this.prisma = new PrismaCtor({ adapter }) as PrismaClientLike; + this.logger.log("RAG Document 仓储已启用 PostgreSQL 持久化。"); + } catch (error) { + this.logger.warn( + `RAG Document 仓储初始化失败，降级为内存模式: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + async onModuleDestroy(): Promise { + if (this.prisma) { + await this.prisma.$disconnect(); + } + } + + async upsertDocumentWithChunks( + input: RagDocumentUpsertInput + ): Promise { + const datasourceState = this.ensureMemoryState(input.document.datasourceId); + const insertedDocument = !datasourceState.documentsById.has(input.document.id); + datasourceState.documentsById.set(input.document.id, { ...input.document }); + let insertedChunkCount = 0; + for (const chunk of input.chunks) { + if (!datasourceState.chunksById.has(chunk.id)) { + insertedChunkCount += 1; + } + datasourceState.chunksById.set(chunk.id, { ...chunk }); + } + + if (!this.isPrimaryPersistenceConfigured() || !this.prisma) { + return { + documentId: input.document.id, + insertedDocument, + insertedChunkCount + }; + } + + await this.prisma.$transaction(async (tx) => { + await tx.ragDocument.upsert({ + where: { id: input.document.id }, + create: this.toPrismaDocument(input.document), + update: this.toPrismaDocumentUpdate(input.document) + }); + for (const chunk of input.chunks) { + await tx.ragChunk.upsert({ + where: { id: chunk.id }, + create: this.toPrismaChunk(chunk), + update: this.toPrismaChunkUpdate(chunk) + }); + } + }); + + return { + documentId: input.document.id, + insertedDocument, + insertedChunkCount + }; + } + + private ensureMemoryState(datasourceId: string): RagDocumentMemoryState { + const normalized = datasourceId.trim(); + const existing = this.memoryByDatasource.get(normalized); + if (existing) { + return existing; + } + const created: RagDocumentMemoryState = { + documentsById: new Map(), + chunksById: new Map() + }; + this.memoryByDatasource.set(normalized, created); + return created; + } + + private toPrismaDocument(input: RagDocumentDraft): Record { + return { + id: input.id, + datasourceId: input.datasourceId, + domain: input.domain, + sourceType: input.sourceType, + sourceRef: input.sourceRef ?? null, + sourceVersion: input.sourceVersion, + contentChecksum: input.contentChecksum, + title: input.title ?? null, + content: input.content, + tableNames: input.tableNames, + columnNames: input.columnNames, + metadata: input.metadata ?? null + }; + } + + private toPrismaDocumentUpdate(input: RagDocumentDraft): Record { + return { + domain: input.domain, + sourceType: input.sourceType, + sourceRef: input.sourceRef ?? null, + sourceVersion: input.sourceVersion, + contentChecksum: input.contentChecksum, + title: input.title ?? null, + content: input.content, + tableNames: input.tableNames, + columnNames: input.columnNames, + metadata: input.metadata ?? null + }; + } + + private toPrismaChunk(input: RagChunkRecordDraft): Record { + return { + id: input.id, + documentId: input.documentId, + datasourceId: input.datasourceId, + domain: input.domain, + chunkProfile: input.chunkProfile, + chunkOrder: input.chunkOrder, + content: input.content, + contentChecksum: input.contentChecksum, + tableNames: input.tableNames, + columnNames: input.columnNames, + metadata: input.metadata ?? null + }; + } + + private toPrismaChunkUpdate(input: RagChunkRecordDraft): Record { + return { + documentId: input.documentId, + datasourceId: input.datasourceId, + domain: input.domain, + chunkProfile: input.chunkProfile, + chunkOrder: input.chunkOrder, + content: input.content, + contentChecksum: input.contentChecksum, + tableNames: input.tableNames, + columnNames: input.columnNames, + metadata: input.metadata ?? null + }; + } + + private isPrimaryPersistenceConfigured(): boolean { + return this.appConfig.databaseUrl.trim().length > 0; + } +} diff --git a/apps/backend/src/modules/rag/quality/rag-quality.service.ts b/apps/backend/src/modules/rag/quality/rag-quality.service.ts index 989dbd0..bf3b05a 100644 --- a/apps/backend/src/modules/rag/quality/rag-quality.service.ts +++ b/apps/backend/src/modules/rag/quality/rag-quality.service.ts @@ -26,8 +26,13 @@ export interface RagQualityEvaluationInput { export interface RagPriorSqlLaneMetricsInput { totalCount: number; hitCount: number; + missCount?: number; filteredCount?: number; staleCount?: number; + ambiguousCount?: number; + duplicateCount?: number; + safetyRejectedCount?: number; + fallbackToGenerationCount?: number; } interface RagQualityEvaluationRecord extends RagQualityEvaluationInput { @@ -38,8 +43,13 @@ interface RagQualityEvaluationRecord extends RagQualityEvaluationInput { interface RagPriorSqlLaneMetricsRecord { totalCount: number; hitCount: number; + missCount: number; filteredCount: number; staleCount: number; + ambiguousCount: number; + duplicateCount: number; + safetyRejectedCount: number; + fallbackToGenerationCount: number; } export interface RagDatasourceOrchestrationSample { @@ -145,8 +155,13 @@ export interface RagQualityGateReport { export interface RagPriorSqlLaneGateReport { sampleSize: number; priorSqlHitRate: number; + priorSqlMissCount: number; priorSqlFilteredCount: number; priorSqlStaleRate: number; + priorSqlAmbiguousCount: number; + priorSqlDuplicateCount: number; + priorSqlSafetyRejectedCount: number; + priorSqlFallbackToGenerationCount: number; } interface GlossarySelectedContextSampleRow { @@ -639,11 +654,27 @@ export class RagQualityService { const hitCount = Math.min(totalCount, this.normalizeCount(input.hitCount)); const filteredCount = this.normalizeCount(input.filteredCount); const staleCount = Math.min(totalCount, this.normalizeCount(input.staleCount)); + const ambiguousCount = this.normalizeCount(input.ambiguousCount); + const duplicateCount = this.normalizeCount(input.duplicateCount); + const safetyRejectedCount = this.normalizeCount(input.safetyRejectedCount); + const fallbackToGenerationCount = this.normalizeCount( + input.fallbackToGenerationCount + ); + const missCount = Math.max( + 0, + this.normalizeCount(input.missCount) || + totalCount - hitCount - filteredCount - staleCount - ambiguousCount + ); return { totalCount, hitCount, + missCount, filteredCount, - staleCount + staleCount, + ambiguousCount, + duplicateCount, + safetyRejectedCount, + fallbackToGenerationCount }; } @@ -690,15 +721,31 @@ export class RagQualityService { return { totalCount: accumulator.totalCount + record.priorSqlLane.totalCount, hitCount: accumulator.hitCount + record.priorSqlLane.hitCount, + missCount: accumulator.missCount + record.priorSqlLane.missCount, filteredCount: accumulator.filteredCount + record.priorSqlLane.filteredCount, - staleCount: accumulator.staleCount + record.priorSqlLane.staleCount + staleCount: accumulator.staleCount + record.priorSqlLane.staleCount, + ambiguousCount: + accumulator.ambiguousCount + record.priorSqlLane.ambiguousCount, + duplicateCount: + accumulator.duplicateCount + record.priorSqlLane.duplicateCount, + safetyRejectedCount: + accumulator.safetyRejectedCount + + record.priorSqlLane.safetyRejectedCount, + fallbackToGenerationCount: + accumulator.fallbackToGenerationCount + + record.priorSqlLane.fallbackToGenerationCount }; }, { totalCount: 0, hitCount: 0, + missCount: 0, filteredCount: 0, - staleCount: 0 + staleCount: 0, + ambiguousCount: 0, + duplicateCount: 0, + safetyRejectedCount: 0, + fallbackToGenerationCount: 0 } ); return { @@ -707,11 +754,16 @@ export class RagQualityService { summary.totalCount === 0 ? 0 : Number((summary.hitCount / summary.totalCount).toFixed(6)), + priorSqlMissCount: summary.missCount, priorSqlFilteredCount: summary.filteredCount, priorSqlStaleRate: summary.totalCount === 0 ? 0 - : Number((summary.staleCount / summary.totalCount).toFixed(6)) + : Number((summary.staleCount / summary.totalCount).toFixed(6)), + priorSqlAmbiguousCount: summary.ambiguousCount, + priorSqlDuplicateCount: summary.duplicateCount, + priorSqlSafetyRejectedCount: summary.safetyRejectedCount, + priorSqlFallbackToGenerationCount: summary.fallbackToGenerationCount }; } diff --git a/apps/backend/src/modules/rag/retrieval/rag-retrieval.service.ts b/apps/backend/src/modules/rag/retrieval/rag-retrieval.service.ts index 4aa5119..4a2ba82 100644 --- a/apps/backend/src/modules/rag/retrieval/rag-retrieval.service.ts +++ b/apps/backend/src/modules/rag/retrieval/rag-retrieval.service.ts @@ -16,6 +16,7 @@ import { fuseWithRrf } from "./fusion/rrf-fusion"; import { RAG_RETRIEVAL_LANES, type RagPriorSqlLaneEvidence, + type RagPriorSqlShortcutDecision, type RagRetrievalCandidate, type RagRetrievalChunkMetadata, type RagRetrievalChunkPayload, @@ -248,11 +249,16 @@ export class RagRetrievalService { graph: laneResults.graph.hits }; const fused = fuseWithRrf({ laneHits }); + const activeModelingRevision = await this.resolveActiveModelingRevision( + workspaceId, + datasourceId + ); const priorSqlSelection = this.selectTrustedPriorSqlCandidates({ candidates: fused, datasourceId, workspaceId, - allowedTables + allowedTables, + activeModelingRevision }); const priorSqlFiltered = this.filterBlockedPriorSqlCandidates( fused, @@ -713,15 +719,18 @@ export class RagRetrievalService { datasourceId: string; workspaceId?: string; allowedTables: string[]; + activeModelingRevision?: number; }): PriorSqlSelectionResult { const trustedSqlExampleCandidates = input.candidates.filter( (candidate) => candidate.chunk.metadata.domain === "sql_example" && this.isTrustedPriorSqlCandidate(candidate) ); - const selectedCandidates: RagRetrievalCandidate[] = []; + const eligibleCandidates: RagRetrievalCandidate[] = []; const blockedChunkIds = new Set(); - const degradeReasons: string[] = []; + const filteredReasons: string[] = []; + const staleReasons: string[] = []; + let staleCandidateCount = 0; const allowedTables = new Set(input.allowedTables); const workspaceId = input.workspaceId?.trim(); @@ -734,47 +743,102 @@ export class RagRetrievalService { }); if (filterReasons.length > 0) { blockedChunkIds.add(candidate.chunk_id); - degradeReasons.push(...filterReasons); + filteredReasons.push(...filterReasons); + continue; + } + + const candidateStaleReasons = this.collectPriorSqlStaleReasons({ + candidate, + activeModelingRevision: input.activeModelingRevision + }); + if (candidateStaleReasons.length > 0) { + staleCandidateCount += 1; + staleReasons.push(...candidateStaleReasons); continue; } - selectedCandidates.push({ + eligibleCandidates.push({ ...candidate, evidence: this.unique([...candidate.evidence, "prior_sql:trusted"]) }); } - if (selectedCandidates.length > 0) { - const lane: RagPriorSqlLaneEvidence = { - status: "hit", + const selectedCandidates = + eligibleCandidates.length === 1 ? [eligibleCandidates[0]] : []; + const shortcutStatus: RagPriorSqlShortcutDecision["status"] = + selectedCandidates.length === 1 + ? "hit" + : eligibleCandidates.length > 1 + ? "ambiguous" + : blockedChunkIds.size > 0 + ? "filtered" + : staleCandidateCount > 0 + ? "stale" + : "miss"; + const selectedCandidate = selectedCandidates[0]; + const shortcutReasons = this.unique([ + ...(shortcutStatus === "hit" + ? ["prior_sql_shortcut_hit"] + : []), + ...(shortcutStatus === "miss" + ? ["prior_sql_no_trusted_match"] + : []), + ...(shortcutStatus === "filtered" + ? ["prior_sql_shortcut_filtered", ...filteredReasons] + : []), + ...(shortcutStatus === "stale" + ? ["prior_sql_shortcut_stale", ...staleReasons] + : []), + ...(shortcutStatus === "ambiguous" + ? ["prior_sql_shortcut_ambiguous"] + : []) + ]); + const laneDegradeReasons = this.unique([ + ...filteredReasons, + ...staleReasons, + ...(shortcutStatus === "miss" ? ["prior_sql_no_trusted_match"] : []), + ...(shortcutStatus === "ambiguous" ? ["prior_sql_shortcut_ambiguous"] : []), + ...(shortcutStatus === "filtered" ? ["prior_sql_shortcut_filtered"] : []), + ...(shortcutStatus === "stale" ? ["prior_sql_shortcut_stale"] : []) + ]); + const shortcutDecision: RagPriorSqlShortcutDecision = + this.withPriorSqlShortcutCompatFields({ + status: shortcutStatus, + reason_codes: shortcutReasons, matched_count: trustedSqlExampleCandidates.length, - selected_count: selectedCandidates.length, + eligible_count: eligibleCandidates.length, filtered_count: blockedChunkIds.size, - ...(degradeReasons.length > 0 + stale_count: staleCandidateCount, + ambiguous_count: shortcutStatus === "ambiguous" ? eligibleCandidates.length : 0, + ...(selectedCandidate ? { - degrade_reasons: this.unique(degradeReasons) + selected_chunk_id: selectedCandidate.chunk_id, + selected_view_id: this.readPriorSqlViewId( + selectedCandidate.chunk.metadata.sourceMetadata + ), + selected_source_run_id: this.readPriorSqlSourceRunId( + selectedCandidate.chunk.metadata.sourceMetadata + ) } : {}) - }; - return { - lane: this.withPriorSqlLaneCompatFields(lane), - selectedCandidates, - blockedChunkIds - }; - } - + }); const lane: RagPriorSqlLaneEvidence = { - status: trustedSqlExampleCandidates.length > 0 ? "filtered" : "miss", + status: shortcutStatus, matched_count: trustedSqlExampleCandidates.length, - selected_count: 0, + selected_count: selectedCandidates.length, filtered_count: blockedChunkIds.size, - degrade_reasons: - trustedSqlExampleCandidates.length > 0 - ? this.unique(degradeReasons) - : ["prior_sql_no_trusted_match"] + stale_count: staleCandidateCount, + ambiguous_count: shortcutStatus === "ambiguous" ? eligibleCandidates.length : 0, + eligible_count: eligibleCandidates.length, + ...(laneDegradeReasons.length > 0 + ? { + degrade_reasons: laneDegradeReasons + } + : {}), + shortcut: shortcutDecision }; return { lane: this.withPriorSqlLaneCompatFields(lane), - selectedCandidates: [], + selectedCandidates, blockedChunkIds }; } @@ -836,6 +900,85 @@ export class RagRetrievalService { return reasons; } + private collectPriorSqlStaleReasons(input: { + candidate: RagRetrievalCandidate; + activeModelingRevision?: number; + }): string[] { + const sourceMetadata = input.candidate.chunk.metadata.sourceMetadata; + if (!this.isRecord(sourceMetadata)) { + return []; + } + const reasons: string[] = []; + if ( + this.readBooleanFlag(sourceMetadata.stale) || + this.readBooleanFlag(sourceMetadata.isStale) || + this.readBooleanFlag(sourceMetadata.priorSqlStale) || + this.readBooleanFlag(sourceMetadata.prior_sql_stale) + ) { + reasons.push("prior_sql_stale_marked"); + } + const viewDeleted = + this.readBooleanFlag(sourceMetadata.viewDeleted) || + this.readBooleanFlag(sourceMetadata.view_deleted); + const viewExists = this.readOptionalBoolean( + sourceMetadata.viewExists ?? sourceMetadata.view_exists + ); + if (viewDeleted || viewExists === false) { + reasons.push("prior_sql_stale_view_missing"); + } + const viewStatus = this.readString(sourceMetadata.viewStatus) ?? + this.readString(sourceMetadata.view_status); + if (viewStatus) { + const normalized = viewStatus.trim().toLowerCase(); + if ( + normalized === "missing" || + normalized === "deleted" || + normalized === "not_found" || + normalized === "archived" || + normalized === "inactive" || + normalized === "stale" + ) { + reasons.push("prior_sql_stale_view_status"); + } + } + + const sourceModelingRevision = this.readPositiveInteger( + sourceMetadata.modelingRevision ?? sourceMetadata.modeling_revision + ); + const currentModelingRevision = this.readPositiveInteger( + sourceMetadata.currentModelingRevision ?? sourceMetadata.current_modeling_revision + ); + if ( + sourceModelingRevision !== undefined && + currentModelingRevision !== undefined && + sourceModelingRevision !== currentModelingRevision + ) { + reasons.push("prior_sql_stale_modeling_revision_mismatch"); + } + if ( + sourceModelingRevision !== undefined && + input.activeModelingRevision !== undefined && + sourceModelingRevision !== input.activeModelingRevision + ) { + reasons.push("prior_sql_stale_modeling_revision_mismatch"); + } + + const sourceSchemaRevision = this.readPositiveInteger( + sourceMetadata.schemaRevision ?? sourceMetadata.schema_revision + ); + const currentSchemaRevision = this.readPositiveInteger( + sourceMetadata.currentSchemaRevision ?? sourceMetadata.current_schema_revision + ); + if ( + sourceSchemaRevision !== undefined && + currentSchemaRevision !== undefined && + sourceSchemaRevision !== currentSchemaRevision + ) { + reasons.push("prior_sql_stale_schema_revision_mismatch"); + } + return this.unique(reasons); + } + private readWorkspaceIdFromSourceMetadata(sourceMetadata: unknown): string | undefined { if (!this.isRecord(sourceMetadata)) { return undefined; @@ -843,6 +986,21 @@ export class RagRetrievalService { return this.readString(sourceMetadata.workspaceId) ?? this.readString(sourceMetadata.workspace_id); } + private readPriorSqlViewId(sourceMetadata: unknown): string | undefined { + if (!this.isRecord(sourceMetadata)) { + return undefined; + } + return this.readString(sourceMetadata.viewId) ?? this.readString(sourceMetadata.view_id); + } + + private readPriorSqlSourceRunId(sourceMetadata: unknown): string | undefined { + if (!this.isRecord(sourceMetadata)) { + return undefined; + } + return this.readString(sourceMetadata.sourceRunId) ?? + this.readString(sourceMetadata.source_run_id); + } + private filterBlockedPriorSqlCandidates( candidates: RagRetrievalCandidate[], blockedChunkIds: Set @@ -1476,16 +1634,80 @@ export class RagRetrievalService { lane: RagPriorSqlLaneEvidence ): RagPriorSqlLaneEvidence { const degradeReasons = lane.degrade_reasons ?? lane.degradeReasons; + const shortcut = lane.shortcut ?? lane.shortcutDecision; return { ...lane, matched_count: lane.matched_count, selected_count: lane.selected_count, filtered_count: lane.filtered_count, + ...(lane.stale_count !== undefined ? { stale_count: lane.stale_count } : {}), + ...(lane.ambiguous_count !== undefined ? { ambiguous_count: lane.ambiguous_count } : {}), + ...(lane.eligible_count !== undefined ? { eligible_count: lane.eligible_count } : {}), ...(degradeReasons ? { degrade_reasons: degradeReasons } : {}), matchedCount: lane.matched_count, selectedCount: lane.selected_count, filteredCount: lane.filtered_count, - ...(degradeReasons ? { degradeReasons } : {}) + ...(lane.stale_count !== undefined ? { staleCount: lane.stale_count } : {}), + ...(lane.ambiguous_count !== undefined ? { ambiguousCount: lane.ambiguous_count } : {}), + ...(lane.eligible_count !== undefined ? { eligibleCount: lane.eligible_count } : {}), + ...(degradeReasons ? { degradeReasons } : {}), + ...(shortcut + ? { + shortcut: this.withPriorSqlShortcutCompatFields(shortcut), + shortcutDecision: this.withPriorSqlShortcutCompatFields(shortcut) + } + : {}) + }; + } + + private withPriorSqlShortcutCompatFields( + decision: RagPriorSqlShortcutDecision + ): RagPriorSqlShortcutDecision { + const reasonCodes = decision.reason_codes ?? decision.reasonCodes ?? []; + return { + ...decision, + reason_codes: reasonCodes, + matched_count: decision.matched_count, + eligible_count: decision.eligible_count, + filtered_count: decision.filtered_count, + stale_count: decision.stale_count, + ambiguous_count: decision.ambiguous_count, + ...(decision.selected_chunk_id + ? { + selected_chunk_id: decision.selected_chunk_id + } + : {}), + ...(decision.selected_view_id + ? { + selected_view_id: decision.selected_view_id + } + : {}), + ...(decision.selected_source_run_id + ? { + selected_source_run_id: decision.selected_source_run_id + } + : {}), + reasonCodes, + matchedCount: decision.matched_count, + eligibleCount: decision.eligible_count, + filteredCount: decision.filtered_count, + staleCount: decision.stale_count, + ambiguousCount: decision.ambiguous_count, + ...(decision.selected_chunk_id + ? { + selectedChunkId: decision.selected_chunk_id + } + : {}), + ...(decision.selected_view_id + ? { + selectedViewId: decision.selected_view_id + } + : {}), + ...(decision.selected_source_run_id + ? { + selectedSourceRunId: decision.selected_source_run_id + } + : {}) }; } @@ -1597,6 +1819,34 @@ export class RagRetrievalService { return undefined; } + private readPositiveInteger(value: unknown): number | undefined { + const parsed = this.readNumber(value); + if (parsed === undefined || !Number.isInteger(parsed) || parsed <= 0) { + return undefined; + } + return parsed; + } + + private readOptionalBoolean(value: unknown): boolean | undefined { + if (typeof value === "boolean") { + return value; + } + if (typeof value === "number" && Number.isFinite(value)) { + return value > 0; + } + if (typeof value !== "string") { + return undefined; + } + const normalized = value.trim().toLowerCase(); + if (normalized === "true" || normalized === "1" || normalized === "yes") { + return true; + } + if (normalized === "false" || normalized === "0" || normalized === "no") { + return false; + } + return undefined; + } + private readStringArray(value: unknown): string[] { if (!Array.isArray(value)) { return []; diff --git a/apps/backend/src/modules/rag/retrieval/rag-retrieval.types.ts b/apps/backend/src/modules/rag/retrieval/rag-retrieval.types.ts index 6e5f68b..9a40ce0 100644 --- a/apps/backend/src/modules/rag/retrieval/rag-retrieval.types.ts +++ b/apps/backend/src/modules/rag/retrieval/rag-retrieval.types.ts @@ -142,15 +142,45 @@ export interface RagContextPack { } export interface RagPriorSqlLaneEvidence { - status: "hit" | "miss" | "filtered"; + status: "hit" | "miss" | "filtered" | "stale" | "ambiguous"; matched_count: number; selected_count: number; filtered_count: number; + stale_count?: number; + ambiguous_count?: number; + eligible_count?: number; degrade_reasons?: string[]; + shortcut?: RagPriorSqlShortcutDecision; matchedCount?: number; selectedCount?: number; filteredCount?: number; + staleCount?: number; + ambiguousCount?: number; + eligibleCount?: number; degradeReasons?: string[]; + shortcutDecision?: RagPriorSqlShortcutDecision; +} + +export interface RagPriorSqlShortcutDecision { + status: "hit" | "miss" | "filtered" | "stale" | "ambiguous"; + reason_codes: string[]; + matched_count: number; + eligible_count: number; + filtered_count: number; + stale_count: number; + ambiguous_count: number; + selected_chunk_id?: string; + selected_view_id?: string; + selected_source_run_id?: string; + reasonCodes?: string[]; + matchedCount?: number; + eligibleCount?: number; + filteredCount?: number; + staleCount?: number; + ambiguousCount?: number; + selectedChunkId?: string; + selectedViewId?: string; + selectedSourceRunId?: string; } export interface RagRetrievalBundle { diff --git a/apps/backend/test/e2e/chat-api.spec.ts b/apps/backend/test/e2e/chat-api.spec.ts index d180bb9..6f7eda6 100644 --- a/apps/backend/test/e2e/chat-api.spec.ts +++ b/apps/backend/test/e2e/chat-api.spec.ts @@ -54,6 +54,24 @@ describe("chat api (e2e)", () => { expect(runRes.body.data.run.llmRaw).toBeTruthy(); expect(runRes.body.data.run.llmRaw.provider).toBe("volcengine"); expect(runRes.body.data.run.llmRaw.model).toBeTruthy(); + expect(runRes.body.data.delivery).toBeTruthy(); + expect(runRes.body.data.run.delivery).toEqual(runRes.body.data.delivery); + expect(runRes.body.data.run.answer).toBe(runRes.body.data.delivery.answer.text); + expect(runRes.body.data.delivery.evidence.runId).toBe(runRes.body.data.run.runId); + const syncArtifact = runRes.body.data.delivery.artifact as + | { + rowCount?: number; + summary?: { text?: string }; + table?: { rowCount?: number }; + display?: string; + validation?: { status?: string }; + } + | undefined; + expect(syncArtifact).toBeTruthy(); + expect(syncArtifact?.summary?.text).toBeTruthy(); + expect(syncArtifact?.table?.rowCount).toBe(syncArtifact?.rowCount); + expect(syncArtifact?.display).toBeTruthy(); + expect(syncArtifact?.validation?.status).toBeTruthy(); const contextPackStatus = runRes.body.data.delivery?.evidence?.contextPackStatus; if (contextPackStatus !== undefined) { expect(["ready", "degraded"]).toContain(contextPackStatus); @@ -67,6 +85,13 @@ describe("chat api (e2e)", () => { expect(messageViewRes.body.data.session.id).toBe(sessionId); expect(Array.isArray(messageViewRes.body.data.messages)).toBe(true); expect(messageViewRes.body.data.latestRun.runId).toBe(runRes.body.data.run.runId); + expect(messageViewRes.body.data.latestRun.answer).toBe( + messageViewRes.body.data.latestRun.delivery.answer.text + ); + expect(messageViewRes.body.data.latestRun.delivery.evidence.runId).toBe( + messageViewRes.body.data.latestRun.runId + ); + expect(messageViewRes.body.data.latestRun.delivery.artifact.summary.text).toBeTruthy(); }); it("should accept optional contextEnvelope on sync message endpoint", async () => { diff --git a/apps/backend/test/e2e/chat-stream-api.spec.ts b/apps/backend/test/e2e/chat-stream-api.spec.ts index 115bec0..b05ee5c 100644 --- a/apps/backend/test/e2e/chat-stream-api.spec.ts +++ b/apps/backend/test/e2e/chat-stream-api.spec.ts @@ -115,6 +115,14 @@ describe("chat stream api (e2e)", () => { expect(event).toHaveProperty("data"); } + const nonFinishEvents = parsedEvents.filter( + ({ eventType }) => eventType !== "finish" + ); + for (const { event } of nonFinishEvents) { + const dataWithCompat = event.data as Record; + expect(dataWithCompat.delivery).toBeUndefined(); + } + const startEvent = parsedEvents[startIndex]?.event; expect(startEvent).toBeDefined(); expect(startEvent?.type).toBe("start"); @@ -165,7 +173,25 @@ describe("chat stream api (e2e)", () => { status: unknown; rowCount: unknown; delivery?: { + answer?: { + text?: string; + status?: string; + }; + artifact?: { + summary?: { + text?: string; + }; + table?: { + rowCount?: number; + }; + rowCount?: number; + validation?: { + status?: string; + }; + display?: string; + }; evidence?: { + runId?: string; contextPackStatus?: string; }; }; @@ -173,6 +199,14 @@ describe("chat stream api (e2e)", () => { expect(typeof finishData.status).toBe("string"); expect(typeof finishData.rowCount).toBe("number"); expect(finishData.rowCount as number).toBeGreaterThanOrEqual(0); + expect(finishData.delivery?.answer?.status).toBe(finishData.status); + expect(finishData.delivery?.artifact?.summary?.text).toBeTruthy(); + expect(finishData.delivery?.artifact?.table?.rowCount).toBe( + finishData.delivery?.artifact?.rowCount + ); + expect(finishData.delivery?.artifact?.validation?.status).toBeTruthy(); + expect(finishData.delivery?.artifact?.display).toBeTruthy(); + expect(finishData.delivery?.evidence?.runId).toBe(streamRunId); const contextPackStatus = finishData.delivery?.evidence?.contextPackStatus; if (contextPackStatus !== undefined) { expect(["ready", "degraded"]).toContain(contextPackStatus); @@ -193,6 +227,10 @@ describe("chat stream api (e2e)", () => { }; expect(latestRun.runId).toBe(streamRunId); expect(latestRun.sessionId).toBe(sessionId); + expect(messagesRes.body.data.latestRun.answer).toBe( + messagesRes.body.data.latestRun.delivery.answer.text + ); + expect(messagesRes.body.data.latestRun.delivery.evidence.runId).toBe(streamRunId); expect(["completed", "failed"]).toContain(latestRun.trace.streamStatus); if (latestRun.trace.streamStatus === "failed") { expect(typeof latestRun.error).toBe("string"); @@ -201,6 +239,45 @@ describe("chat stream api (e2e)", () => { } }); + it("keeps rejected stream runs in safe delivery semantics without chart artifact", async () => { + const sessionRes = await request(app.getHttpServer()) + .post("/api/v1/sessions") + .send({ datasource: "sqlite_main" }); + const sessionId = sessionRes.body.data.id as string; + + const streamRes = await request(app.getHttpServer()) + .post(`/api/v1/sessions/${sessionId}/messages/stream`) + .send({ message: "DELETE orders where id = 1" }); + + expect(streamRes.status).toBe(200); + const parsedEvents = parseSseEvents(streamRes.text); + const finishEvent = [...parsedEvents] + .reverse() + .find(({ eventType }) => eventType === "finish")?.event; + expect(finishEvent).toBeDefined(); + + const finishData = (finishEvent?.data ?? {}) as { + status?: string; + delivery?: { + answer?: { + status?: string; + }; + artifact?: { + chart?: unknown; + display?: string; + }; + }; + }; + + expect(finishData.status).toBe("rejected"); + expect(finishData.delivery?.answer?.status).toBe("rejected"); + expect(finishData.delivery?.artifact?.chart).toBeUndefined(); + expect(finishData.delivery?.artifact?.display).not.toBe("metric"); + expect(finishData.delivery?.artifact?.display).not.toBe("bar"); + expect(finishData.delivery?.artifact?.display).not.toBe("line"); + expect(finishData.delivery?.artifact?.display).not.toBe("pie"); + }); + it("should accept optional contextEnvelope on stream message endpoint", async () => { const sessionRes = await request(app.getHttpServer()) .post("/api/v1/sessions") diff --git a/apps/backend/test/integration/agent-main-flow.spec.ts b/apps/backend/test/integration/agent-main-flow.spec.ts index 25fec54..f62a130 100644 --- a/apps/backend/test/integration/agent-main-flow.spec.ts +++ b/apps/backend/test/integration/agent-main-flow.spec.ts @@ -34,6 +34,13 @@ describe("agent main flow", () => { expect(run.trace.steps[0]?.lifecycle).toBeTruthy(); expect(run.trace.clarificationDecision).toBeDefined(); expect(["continue", "clarify"]).toContain(run.trace.clarificationDecision?.decision); + if (run.status === "executionResult") { + expect(typeof run.answer).toBe("string"); + expect(run.answer?.trim().length).toBeGreaterThan(0); + expect(run.answer).toContain("已完成分析"); + expect(run.answer).not.toContain("样例结果"); + expect(run.answer).not.toContain("{\""); + } }); it("should reject non-readonly sql intent", async () => { diff --git a/apps/backend/test/integration/agent-saved-prior-sql-shortcut.spec.ts b/apps/backend/test/integration/agent-saved-prior-sql-shortcut.spec.ts new file mode 100644 index 0000000..7b6fef8 --- /dev/null +++ b/apps/backend/test/integration/agent-saved-prior-sql-shortcut.spec.ts @@ -0,0 +1,130 @@ +import { resolve } from "node:path"; +import { Test } from "@nestjs/testing"; +import { AppModule } from "../../src/app.module"; +import { GraphBuilderService } from "../../src/modules/conversation/agent/graph/graph.builder"; +import { SavedPriorSqlService } from "../../src/modules/knowledge/memory/saved-prior-sql.service"; + +describe("agent saved prior sql shortcut integration", () => { + beforeAll(() => { + process.env.SQLITE_PATH = resolve( + __dirname, + "../../../../data/sqlite/text2sql.db" + ); + process.env.DATABASE_URL = ""; + process.env.REDIS_URL = ""; + process.env.LLM_PROVIDER = "volcengine"; + process.env.LLM_MOCK_MODE = "true"; + }); + + it("skips generate-sql when saved prior shortcut hits and safety passes", async () => { + const moduleRef = await Test.createTestingModule({ + imports: [AppModule] + }).compile(); + + const graph = moduleRef.get(GraphBuilderService); + const savedPriorSql = moduleRef.get(SavedPriorSqlService); + + await savedPriorSql.captureFromSavedView({ + workspaceId: "ws-agent-shortcut-hit", + datasourceId: "sqlite_main", + sourceRunId: "run-agent-shortcut-source-hit", + sourceRunStatus: "executionResult", + question: "统计订单总数", + sql: "SELECT COUNT(*) AS total_orders FROM orders", + viewId: "view.chat_run.run-agent-shortcut-source-hit", + viewName: "saved_shortcut_select_one", + viewSql: "SELECT COUNT(*) AS total_orders FROM orders", + replayed: false, + savedAt: "2026-04-25T10:00:00.000Z" + }); + + const run = await graph.run({ + runId: "run-agent-shortcut-hit", + sessionId: "session-agent-shortcut-hit", + question: "统计订单总数", + datasourceId: "sqlite_main", + datasourceType: "sqlite", + contextEnvelope: { + metricDefinition: "订单总数", + timeRange: { + from: "2026-01-01", + to: "2026-01-31" + } + }, + accessContext: { + actorId: "user-agent-shortcut-hit", + workspaceId: "ws-agent-shortcut-hit", + allowedTables: ["orders"] + } + }); + + expect(run.status).not.toBe("clarification"); + expect(run.trace.steps.some((step) => step.node === "resolve-saved-prior-sql")).toBe( + true + ); + expect(run.sql?.toLowerCase()).toContain("select"); + const resolveStep = run.trace.steps.find( + (step) => step.node === "resolve-saved-prior-sql" + ); + expect(resolveStep?.status).toMatch(/success|skipped/); + + await moduleRef.close(); + }); + + it("falls back to generate-sql when saved prior shortcut is safety-rejected", async () => { + const moduleRef = await Test.createTestingModule({ + imports: [AppModule] + }).compile(); + + const graph = moduleRef.get(GraphBuilderService); + const savedPriorSql = moduleRef.get(SavedPriorSqlService); + + await savedPriorSql.captureFromSavedView({ + workspaceId: "ws-agent-shortcut-fallback", + datasourceId: "sqlite_main", + sourceRunId: "run-agent-shortcut-source-fallback", + sourceRunStatus: "executionResult", + question: "统计订单总数（安全回退测试）", + sql: "DELETE FROM orders WHERE id = 1", + viewId: "view.chat_run.run-agent-shortcut-source-fallback", + viewName: "saved_shortcut_delete_orders", + viewSql: "DELETE FROM orders WHERE id = 1", + replayed: false, + savedAt: "2026-04-25T10:01:00.000Z" + }); + + const run = await graph.run({ + runId: "run-agent-shortcut-fallback", + sessionId: "session-agent-shortcut-fallback", + question: "统计订单总数（安全回退测试）", + datasourceId: "sqlite_main", + datasourceType: "sqlite", + contextEnvelope: { + metricDefinition: "订单总数", + timeRange: { + from: "2026-01-01", + to: "2026-01-31" + } + }, + accessContext: { + actorId: "user-agent-shortcut-fallback", + workspaceId: "ws-agent-shortcut-fallback", + allowedTables: ["orders"] + } + }); + + expect(run.status).not.toBe("clarification"); + expect(run.trace.steps.some((step) => step.node === "resolve-saved-prior-sql")).toBe( + true + ); + const resolveStep = run.trace.steps.find( + (step) => step.node === "resolve-saved-prior-sql" + ); + expect(resolveStep?.status).toMatch(/success|skipped/); + expect( + run.trace.steps.some((step) => step.node === "safety-check") + ).toBe(true); + + await moduleRef.close(); + }); +}); diff --git a/apps/backend/test/integration/datasource-service.spec.ts b/apps/backend/test/integration/datasource-service.spec.ts index c4e3e25..8f309fd 100644 --- a/apps/backend/test/integration/datasource-service.spec.ts +++ b/apps/backend/test/integration/datasource-service.spec.ts @@ -1,68 +1,103 @@ -import { resolve } from "node:path"; +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; import { Test } from "@nestjs/testing"; import { AppModule } from "../../src/app.module"; import { DomainError } from "../../src/common/domain-error"; import { DatasourceService } from "../../src/modules/governance/datasource/datasource.service"; +const execFileAsync = promisify(execFile); +const SYSTEM_ADMIN_ACTOR = { + id: "system-admin", + role: "admin" as const, + isSystemAdmin: true +}; + +async function createSqliteFile(path: string, rows = 1): Promise { + const values = Array.from({ length: rows }, (_, index) => `(${index + 1})`).join(","); + await execFileAsync("sqlite3", [ + path, + ` +DROP TABLE IF EXISTS orders; +CREATE TABLE orders (id INTEGER PRIMARY KEY); +INSERT INTO orders (id) VALUES ${values}; + `.trim() + ]); +} + describe("datasource service", () => { beforeAll(() => { process.env.SQLITE_PATH = resolve( __dirname, "../../../../data/sqlite/text2sql.db" ); + process.env.SQLITE_ALLOWED_DIRS = [ + resolve(__dirname, "../../../../data/sqlite"), + resolve(__dirname, "../../../../data/uploads/datasources") + ].join(","); process.env.DATABASE_URL = ""; process.env.REDIS_URL = ""; process.env.LLM_PROVIDER = "volcengine"; process.env.LLM_MOCK_MODE = "true"; }); + afterEach(() => { + jest.restoreAllMocks(); + }); + it("creates mysql/postgresql datasources and lists them", async () => { const moduleRef = await Test.createTestingModule({ imports: [AppModule] }).compile(); - const datasourceService = moduleRef.get(DatasourceService); - jest - .spyOn(datasourceService as any, "loadMysqlModule") - .mockResolvedValue({ - createConnection: jest.fn().mockResolvedValue({ - end: jest.fn().mockResolvedValue(undefined) - }) + try { + const datasourceService = moduleRef.get(DatasourceService); + jest + .spyOn(datasourceService as any, "loadMysqlModule") + .mockResolvedValue({ + createConnection: jest.fn().mockResolvedValue({ + end: jest.fn().mockResolvedValue(undefined) + }) + }); + class FakePgClient { + async connect(): Promise {} + async end(): Promise {} + } + jest + .spyOn(datasourceService as any, "loadPostgresModule") + .mockResolvedValue({ + Client: FakePgClient + }); + + await datasourceService.createDatasource({ + name: "MySQL 业务库", + type: "mysql", + host: "127.0.0.1", + port: 3306, + database: "biz", + username: "root", + password: "secret" }); - class FakePgClient { - async connect(): Promise {} - async end(): Promise {} - } - jest - .spyOn(datasourceService as any, "loadPostgresModule") - .mockResolvedValue({ - Client: FakePgClient + + await datasourceService.createDatasource({ + name: "PostgreSQL 行为库", + type: "postgresql", + host: "127.0.0.1", + port: 5432, + database: "analytics", + username: "postgres", + password: "secret" }); - await datasourceService.createDatasource({ - name: "MySQL 业务库", - type: "mysql", - host: "127.0.0.1", - port: 3306, - database: "biz", - username: "root", - password: "secret" - }); - - await datasourceService.createDatasource({ - name: "PostgreSQL 行为库", - type: "postgresql", - host: "127.0.0.1", - port: 5432, - database: "analytics", - username: "postgres", - password: "secret" - }); - - const list = await datasourceService.listDatasources(); - - expect(list.some((item) => item.type === "mysql")).toBe(true); - expect(list.some((item) => item.type === "postgresql")).toBe(true); + const list = await datasourceService.listDatasources(); + + expect(list.some((item) => item.type === "mysql")).toBe(true); + expect(list.some((item) => item.type === "postgresql")).toBe(true); + } finally { + await moduleRef.close(); + } }); it("rejects invalid connection config before creating datasource", async () => { @@ -70,15 +105,19 @@ describe("datasource service", () => { imports: [AppModule] }).compile(); - const datasourceService = moduleRef.get(DatasourceService); + try { + const datasourceService = moduleRef.get(DatasourceService); - await expect( - datasourceService.createDatasource({ - name: "无效配置", - type: "mysql", - host: "127.0.0.1" - }) - ).rejects.toBeInstanceOf(DomainError); + await expect( + datasourceService.createDatasource({ + name: "无效配置", + type: "mysql", + host: "127.0.0.1" + }) + ).rejects.toBeInstanceOf(DomainError); + } finally { + await moduleRef.close(); + } }); it("creates reusable csv datasource metadata from uploaded file", async () => { @@ -86,19 +125,140 @@ describe("datasource service", () => { imports: [AppModule] }).compile(); - const datasourceService = moduleRef.get(DatasourceService); + try { + const datasourceService = moduleRef.get(DatasourceService); - const datasource = await datasourceService.createDatasourceFromUpload({ - uploadedFile: { - originalname: "orders.csv", - mimetype: "text/csv", - size: 1024, - path: "/tmp/orders.csv" - } - }); + const datasource = await datasourceService.createDatasourceFromUpload({ + uploadedFile: { + originalname: "orders.csv", + mimetype: "text/csv", + size: 1024, + path: "/tmp/orders.csv" + } + }); + + expect(datasource.type).toBe("csv"); + expect(datasource.shared).toBe(true); + expect(datasource.fileMeta?.originalName).toBe("orders.csv"); + } finally { + await moduleRef.close(); + } + }); + + it("rejects sqlite create/preview paths outside SQLITE_ALLOWED_DIRS", async () => { + const tempRoot = await mkdtemp(join(tmpdir(), "text2sql-sqlite-allowed-")); + const allowedDir = join(tempRoot, "allowed"); + const blockedDir = join(tempRoot, "blocked"); + await mkdir(allowedDir, { recursive: true }); + await mkdir(blockedDir, { recursive: true }); + const blockedPath = join(blockedDir, "blocked.db"); + await createSqliteFile(blockedPath, 2); + + process.env.SQLITE_ALLOWED_DIRS = allowedDir; + const moduleRef = await Test.createTestingModule({ + imports: [AppModule] + }).compile(); + + try { + const datasourceService = moduleRef.get(DatasourceService); + + await expect( + datasourceService.createDatasource({ + name: "blocked-sqlite", + type: "sqlite", + filePath: blockedPath + }) + ).rejects.toMatchObject({ + code: "SQLITE_PATH_NOT_ALLOWED" + }); + + await expect( + datasourceService.previewDatasourceTables(SYSTEM_ADMIN_ACTOR, { + mode: "create", + datasource: { + type: "sqlite", + name: "blocked-preview", + filePath: blockedPath + } + }) + ).rejects.toMatchObject({ + code: "SQLITE_PATH_NOT_ALLOWED" + }); + } finally { + await moduleRef.close(); + await rm(tempRoot, { recursive: true, force: true }); + } + }); + + it("blocks symlink escape when realpath points outside SQLITE_ALLOWED_DIRS", async () => { + const tempRoot = await mkdtemp(join(tmpdir(), "text2sql-sqlite-symlink-")); + const allowedDir = join(tempRoot, "allowed"); + const blockedDir = join(tempRoot, "blocked"); + await mkdir(allowedDir, { recursive: true }); + await mkdir(blockedDir, { recursive: true }); + const blockedPath = join(blockedDir, "blocked.db"); + await createSqliteFile(blockedPath, 1); + const symlinkPath = join(allowedDir, "through-link.db"); + await symlink(blockedPath, symlinkPath); + + process.env.SQLITE_ALLOWED_DIRS = allowedDir; + const moduleRef = await Test.createTestingModule({ + imports: [AppModule] + }).compile(); + + try { + const datasourceService = moduleRef.get(DatasourceService); + + await expect( + datasourceService.createDatasource({ + name: "symlink-escape", + type: "sqlite", + filePath: symlinkPath + }) + ).rejects.toMatchObject({ + code: "SQLITE_PATH_NOT_ALLOWED" + }); + } finally { + await moduleRef.close(); + await rm(tempRoot, { recursive: true, force: true }); + } + }); + + it("keeps sqlite config unchanged when update preflight fails", async () => { + const tempRoot = await mkdtemp(join(tmpdir(), "text2sql-sqlite-update-")); + const allowedDir = join(tempRoot, "allowed"); + await mkdir(allowedDir, { recursive: true }); + const validDbPath = join(allowedDir, "valid.db"); + await createSqliteFile(validDbPath, 2); + const invalidDbPath = join(allowedDir, "invalid.db"); + await writeFile(invalidDbPath, "not-a-sqlite-file", "utf8"); + + process.env.SQLITE_ALLOWED_DIRS = allowedDir; + const moduleRef = await Test.createTestingModule({ + imports: [AppModule] + }).compile(); + + try { + const datasourceService = moduleRef.get(DatasourceService); + const datasource = await datasourceService.createDatasource({ + name: "sqlite-update-target", + type: "sqlite", + filePath: validDbPath + }); - expect(datasource.type).toBe("csv"); - expect(datasource.shared).toBe(true); - expect(datasource.fileMeta?.originalName).toBe("orders.csv"); + await expect( + datasourceService.updateDatasource(SYSTEM_ADMIN_ACTOR, datasource.id, { + filePath: invalidDbPath + }) + ).rejects.toMatchObject({ + code: "SQLITE_PRECHECK_FAILED" + }); + + const persisted = await datasourceService.getDatasourceOrThrow(datasource.id); + expect((persisted.config as Record)?.path).toBe(validDbPath); + } finally { + await moduleRef.close(); + await rm(tempRoot, { recursive: true, force: true }); + } }); }); diff --git a/apps/backend/test/integration/delivery-contract.spec.ts b/apps/backend/test/integration/delivery-contract.spec.ts index 1098fa9..096cead 100644 --- a/apps/backend/test/integration/delivery-contract.spec.ts +++ b/apps/backend/test/integration/delivery-contract.spec.ts @@ -82,9 +82,28 @@ describe("delivery contract integration", () => { expect(syncRes.body.data.delivery).toBeTruthy(); expect(syncRes.body.data.run.delivery).toEqual(syncRes.body.data.delivery); expect(syncRes.body.data.delivery.answer).toBeTruthy(); + expect(syncRes.body.data.run.answer).toBe(syncRes.body.data.delivery.answer.text); expect(syncRes.body.data.delivery.evidence.runId).toBe( syncRes.body.data.run.runId as string ); + const syncArtifact = syncRes.body.data.delivery.artifact as + | { + rowCount: number; + summary?: { text?: string }; + table?: { + rowCount?: number; + }; + validation?: { + status?: string; + }; + display?: string; + } + | undefined; + expect(syncArtifact).toBeTruthy(); + expect(syncArtifact?.summary?.text).toBe(syncRes.body.data.run.answer); + expect(syncArtifact?.table?.rowCount).toBe(syncArtifact?.rowCount); + expect(syncArtifact?.validation?.status).toBeTruthy(); + expect(syncArtifact?.display).toBeTruthy(); const streamRes = await request(app.getHttpServer()) .post(`/api/v1/sessions/${sessionId}/messages/stream`) @@ -104,6 +123,13 @@ describe("delivery contract integration", () => { expect(finishData.delivery).toBeTruthy(); expect(finishData.delivery).toHaveProperty("answer"); expect(finishData.delivery).toHaveProperty("evidence"); + expect((finishData.delivery as { answer?: { text?: string } }).answer?.text).toBe( + syncRes.body.data.run.answer + ); + expect((finishData.delivery as { artifact?: { summary?: { text?: string } } }).artifact?.summary?.text).toBeTruthy(); + expect( + (finishData.delivery as { evidence?: { runId?: string } }).evidence?.runId + ).toBe(finish?.event.runId); const messagesRes = await request(app.getHttpServer()) .get(`/api/v1/sessions/${sessionId}/messages`) @@ -117,6 +143,9 @@ describe("delivery contract integration", () => { expect(messagesRes.body.data.latestRun.delivery.artifact).toEqual( (finishData.delivery as { artifact?: unknown })?.artifact ); + expect(messagesRes.body.data.latestRun.answer).toBe( + messagesRes.body.data.latestRun.delivery.answer.text + ); expect(messagesRes.body.data.latestRun.delivery.evidence.runId).toBe( finish?.event.runId ); @@ -141,6 +170,7 @@ describe("delivery contract integration", () => { expect(syncRes.body.data.delivery.evidence.riskTags).toEqual( expect.arrayContaining(["delivery_mapper_failed"]) ); + expect(syncRes.body.data.run.answer).toBe(syncRes.body.data.delivery.answer.text); expect(syncRes.body.data.run.delivery.evidence.riskTags).toEqual( expect.arrayContaining(["delivery_mapper_failed"]) ); diff --git a/apps/backend/test/integration/query-executors.spec.ts b/apps/backend/test/integration/query-executors.spec.ts index f4b5c78..2c0963e 100644 --- a/apps/backend/test/integration/query-executors.spec.ts +++ b/apps/backend/test/integration/query-executors.spec.ts @@ -1,12 +1,28 @@ +import { execFile } from "node:child_process"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { promisify } from "node:util"; import { Test } from "@nestjs/testing"; import { AppModule } from "../../src/app.module"; import { DatasourceRepository } from "../../src/modules/data/persistence/datasource.repository"; import { QueryExecutorRouterService } from "../../src/modules/data/query/query-executor-router.service"; import { createSeededSqliteFixture } from "../support/sqlite-fixture"; +const execFileAsync = promisify(execFile); + +async function createSimpleSqliteDb(path: string, rows: number): Promise { + const values = Array.from({ length: rows }, (_, index) => `(${index + 1})`).join(","); + await execFileAsync("sqlite3", [ + path, + ` +DROP TABLE IF EXISTS metrics; +CREATE TABLE metrics (id INTEGER PRIMARY KEY); +INSERT INTO metrics (id) VALUES ${values}; + `.trim() + ]); +} + describe("query executors", () => { let cleanupFixture: (() => Promise) | undefined; @@ -93,4 +109,105 @@ describe("query executors", () => { await rm(tempDir, { recursive: true, force: true }); } }); + + it("isolates sqlite execution by datasource config.path", async () => { + const tempDir = await mkdtemp(join(tmpdir(), "text2sql-sqlite-isolation-")); + const dbA = join(tempDir, "a.db"); + const dbB = join(tempDir, "b.db"); + await createSimpleSqliteDb(dbA, 2); + await createSimpleSqliteDb(dbB, 5); + + try { + const moduleRef = await Test.createTestingModule({ + imports: [AppModule] + }).compile(); + + try { + const datasourceRepository = moduleRef.get(DatasourceRepository); + const router = moduleRef.get(QueryExecutorRouterService); + + const datasourceA = await datasourceRepository.upsertDatasource({ + id: "ds-sqlite-iso-a", + name: "SQLite A", + type: "sqlite", + status: "available", + readonly: true, + shared: true, + config: { path: dbA } + }); + const datasourceB = await datasourceRepository.upsertDatasource({ + id: "ds-sqlite-iso-b", + name: "SQLite B", + type: "sqlite", + status: "available", + readonly: true, + shared: true, + config: { path: dbB } + }); + + const resultA = await router.execute({ + datasource: datasourceA, + sql: "SELECT COUNT(*) AS total FROM metrics" + }); + const resultB = await router.execute({ + datasource: datasourceB, + sql: "SELECT COUNT(*) AS total FROM metrics" + }); + + expect(Number(resultA.rows[0]?.total)).toBe(2); + expect(Number(resultB.rows[0]?.total)).toBe(5); + } finally { + await moduleRef.close(); + } + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }); + + it("rejects sqlite datasource without config.path except sqlite_main fallback", async () => { + const moduleRef = await Test.createTestingModule({ + imports: [AppModule] + }).compile(); + + try { + const datasourceRepository = moduleRef.get(DatasourceRepository); + const router = moduleRef.get(QueryExecutorRouterService); + + const invalidDatasource = await datasourceRepository.upsertDatasource({ + id: "ds-sqlite-missing-path", + name: "SQLite 缺路径", + type: "sqlite", + status: "available", + readonly: true, + shared: true, + config: {} + }); + + await expect( + router.execute({ + datasource: invalidDatasource, + sql: "SELECT 1 AS value" + }) + ).rejects.toMatchObject({ + code: "SQLITE_DATASOURCE_PATH_MISSING" + }); + + const sqliteMain = await datasourceRepository.upsertDatasource({ + id: "sqlite_main", + name: "SQLite 主数据源", + type: "sqlite", + status: "available", + readonly: true, + shared: true, + config: {} + }); + const mainResult = await router.execute({ + datasource: sqliteMain, + sql: "SELECT 1 AS value" + }); + expect(mainResult.rows[0]?.value).toBe(1); + } finally { + await moduleRef.close(); + } + }); }); diff --git a/apps/backend/test/integration/rag-quality.spec.ts b/apps/backend/test/integration/rag-quality.spec.ts index b8c152c..14495bb 100644 --- a/apps/backend/test/integration/rag-quality.spec.ts +++ b/apps/backend/test/integration/rag-quality.spec.ts @@ -41,8 +41,13 @@ describe("rag quality gate integration", () => { expect(snapshot.priorSqlLane).toEqual({ sampleSize: 0, priorSqlHitRate: 0, + priorSqlMissCount: 0, priorSqlFilteredCount: 0, - priorSqlStaleRate: 0 + priorSqlStaleRate: 0, + priorSqlAmbiguousCount: 0, + priorSqlDuplicateCount: 0, + priorSqlSafetyRejectedCount: 0, + priorSqlFallbackToGenerationCount: 0 }); await moduleRef.close(); @@ -91,8 +96,13 @@ describe("rag quality gate integration", () => { priorSqlLane: { totalCount: 20, hitCount: 15, + missCount: 2, filteredCount: 2, - staleCount: 1 + staleCount: 1, + ambiguousCount: 1, + duplicateCount: 1, + safetyRejectedCount: 1, + fallbackToGenerationCount: 1 } }); quality.recordEvaluation({ @@ -106,8 +116,13 @@ describe("rag quality gate integration", () => { priorSqlLane: { totalCount: 10, hitCount: 5, + missCount: 1, filteredCount: 1, - staleCount: 2 + staleCount: 2, + ambiguousCount: 1, + duplicateCount: 0, + safetyRejectedCount: 1, + fallbackToGenerationCount: 1 } }); @@ -115,8 +130,13 @@ describe("rag quality gate integration", () => { expect(snapshot.priorSqlLane).toEqual({ sampleSize: 30, priorSqlHitRate: 0.666667, + priorSqlMissCount: 3, priorSqlFilteredCount: 3, - priorSqlStaleRate: 0.1 + priorSqlStaleRate: 0.1, + priorSqlAmbiguousCount: 2, + priorSqlDuplicateCount: 1, + priorSqlSafetyRejectedCount: 2, + priorSqlFallbackToGenerationCount: 2 }); expect(snapshot.gatePass).toBe(true); diff --git a/apps/backend/test/integration/rag-retrieval.service.spec.ts b/apps/backend/test/integration/rag-retrieval.service.spec.ts index be1f29b..5b2cb57 100644 --- a/apps/backend/test/integration/rag-retrieval.service.spec.ts +++ b/apps/backend/test/integration/rag-retrieval.service.spec.ts @@ -367,6 +367,132 @@ describe("rag retrieval service integration", () => { await moduleRef.close(); }); + it("marks trusted prior SQL as ambiguous when multiple high-confidence candidates remain", async () => { + const moduleRef = await Test.createTestingModule({ + imports: [AppModule] + }).compile(); + + const indexRepository = moduleRef.get(RagIndexRepository); + const indexBuilder = moduleRef.get(RagIndexBuilderService); + const retrievalService = moduleRef.get(RagRetrievalService); + + const datasourceId = "ds-rag-retrieval-prior-sql-ambiguous"; + const workspaceId = "ws-rag-prior-ambiguous"; + indexRepository.seedChunksForDatasource(datasourceId, [ + { + id: "chunk-prior-ambiguous-schema", + datasourceId, + domain: "schema", + content: "table orders(id, amount, status)" + }, + { + id: "chunk-prior-ambiguous-sql-1", + datasourceId, + domain: "sql_example", + content: "Question: paid orders\\nSQL:\\nSELECT SUM(amount) FROM orders WHERE status = 'paid'", + metadata: JSON.stringify({ + trusted: true, + priorSql: true, + workspaceId, + tableNames: ["orders"], + columnNames: ["amount", "status"] + }) + }, + { + id: "chunk-prior-ambiguous-sql-2", + datasourceId, + domain: "sql_example", + content: "Question: paid order gmv\\nSQL:\\nSELECT AVG(amount) FROM orders WHERE status = 'paid'", + metadata: JSON.stringify({ + trusted: true, + priorSql: true, + workspaceId, + tableNames: ["orders"], + columnNames: ["amount", "status"] + }) + } + ]); + await indexBuilder.buildAndActivate({ + datasourceId, + sourceVersion: "source-rag-retrieval-prior-sql-ambiguous-v1", + createdByRunId: "run-rag-retrieval-prior-sql-ambiguous-build-v1", + activatedByRunId: "run-rag-retrieval-prior-sql-ambiguous-build-v1" + }); + + const response = await retrievalService.retrieve({ + query: "orders paid gmv", + datasourceId, + workspaceId, + allowedTables: ["orders"], + runId: "run-rag-retrieval-prior-sql-ambiguous-v1" + }); + + const priorSqlLane = response.retrieval_bundle.prior_sql_lane; + expect(priorSqlLane?.status).toBe("ambiguous"); + expect(priorSqlLane?.selected_count).toBe(0); + expect(priorSqlLane?.eligible_count).toBe(2); + expect(priorSqlLane?.shortcut?.status).toBe("ambiguous"); + + await moduleRef.close(); + }); + + it("marks trusted prior SQL as stale when stale flags exist in source metadata", async () => { + const moduleRef = await Test.createTestingModule({ + imports: [AppModule] + }).compile(); + + const indexRepository = moduleRef.get(RagIndexRepository); + const indexBuilder = moduleRef.get(RagIndexBuilderService); + const retrievalService = moduleRef.get(RagRetrievalService); + + const datasourceId = "ds-rag-retrieval-prior-sql-stale"; + const workspaceId = "ws-rag-prior-stale"; + indexRepository.seedChunksForDatasource(datasourceId, [ + { + id: "chunk-prior-stale-schema", + datasourceId, + domain: "schema", + content: "table orders(id, amount, status)" + }, + { + id: "chunk-prior-stale-sql-1", + datasourceId, + domain: "sql_example", + content: "Question: paid orders\\nSQL:\\nSELECT SUM(amount) FROM orders WHERE status = 'paid'", + metadata: JSON.stringify({ + trusted: true, + priorSql: true, + stale: true, + workspaceId, + tableNames: ["orders"], + columnNames: ["amount", "status"] + }) + } + ]); + await indexBuilder.buildAndActivate({ + datasourceId, + sourceVersion: "source-rag-retrieval-prior-sql-stale-v1", + createdByRunId: "run-rag-retrieval-prior-sql-stale-build-v1", + activatedByRunId: "run-rag-retrieval-prior-sql-stale-build-v1" + }); + + const response = await retrievalService.retrieve({ + query: "orders paid gmv", + datasourceId, + workspaceId, + allowedTables: ["orders"], + runId: "run-rag-retrieval-prior-sql-stale-v1" + }); + + const priorSqlLane = response.retrieval_bundle.prior_sql_lane; + expect(priorSqlLane?.status).toBe("stale"); + expect(priorSqlLane?.selected_count).toBe(0); + expect(priorSqlLane?.stale_count).toBe(1); + expect(priorSqlLane?.shortcut?.status).toBe("stale"); + + await moduleRef.close(); + }); + it("uses conservative table-first/field-second pruning on wide tables and safely degrades when evidence is weak", async () => { const moduleRef = await Test.createTestingModule({ imports: [AppModule] diff --git a/apps/backend/test/integration/save-view-from-run.spec.ts b/apps/backend/test/integration/save-view-from-run.spec.ts index 2429488..edc6d90 100644 --- a/apps/backend/test/integration/save-view-from-run.spec.ts +++ b/apps/backend/test/integration/save-view-from-run.spec.ts @@ -1,9 +1,10 @@ import { ChatRepository } from "../../src/modules/data/persistence/chat.repository"; import { SaveViewFromRunUsecase } from "../../src/modules/conversation/chat/application/save-view-from-run.usecase"; +import type { KnowledgeMemoryContract } from "../../src/modules/knowledge/contracts/knowledge-memory.contract"; import { ModelingGraphRepository } from "../../src/modules/platform/data/persistence/modeling-graph.repository"; import { ModelingGraphValidator } from "../../src/modules/platform/data/persistence/modeling-graph.validator"; -const buildUsecase = () => { +const buildUsecase = (knowledgeMemoryContract?: KnowledgeMemoryContract) => { const chatRepository = new ChatRepository({ databaseUrl: "" } as never); @@ -14,7 +15,8 @@ const buildUsecase = () => { const usecase = new SaveViewFromRunUsecase( chatRepository, modelingGraphRepository, - modelingGraphValidator + modelingGraphValidator, + knowledgeMemoryContract ); return { usecase, @@ -243,4 +245,77 @@ describe("save view from run integration", () => { code: "RUN_NOT_FOUND" }); }); + + it("keeps save-as-view successful when saved prior sql capture throws", async () => { + const captureFromSavedView = jest + .fn() + .mockRejectedValue(new Error("replay write failed")); + const { usecase, chatRepository, modelingGraphRepository } = buildUsecase({ + promotion: { + promoteFromRun: jest.fn(), + getRecord: jest.fn(), + listCompensations: jest.fn(), + buildCandidateIdForRun: jest.fn(), + applyFeedback: jest.fn() + } as never, + savedPriorSql: { + captureFromSavedView, + getRecord: jest.fn(), + listRecords: jest.fn(), + buildPriorId: jest + .fn() + .mockReturnValue("saved_prior_sql.test-prior-id") + } + } as KnowledgeMemoryContract); + + await chatRepository.createSession({ + id: "session-1", + datasource: "ds-1", + workspaceId: "ws-1", + title: "test", + createdAt: "2026-04-23T00:00:00.000Z" + }); + await chatRepository.persistRun({ + runId: "run-1", + sessionId: "session-1", + question: "recent orders", + status: "executionResult", + provider: "openai", + sql: "SELECT * FROM orders LIMIT 10", + trace: { + runId: "run-1", + provider: "openai", + retryCount: 0, + steps: [] + }, + createdAt: "2026-04-23T01:00:00.000Z" + }); + + const result = await usecase.execute({ + runId: "run-1", + name: "orders_recent_10", + actorId: "user-admin" + }); + + expect(result.replayed).toBe(false); + expect(result.savedPriorSql).toEqual( + expect.objectContaining({ + outcome: "capture_failed", + priorId: "saved_prior_sql.test-prior-id", + reason: "saved_prior_sql_capture_exception" + }) + ); + expect(captureFromSavedView).toHaveBeenCalledTimes(1); + + const snapshot = await modelingGraphRepository.getLatestScopeState({ + workspaceId: "ws-1", + datasourceId: "ds-1" + }); + expect(snapshot.draft?.graphPayload.views).toEqual([ + expect.objectContaining({ + id: "view.chat_run.run-1", + name: "orders_recent_10" + }) + ]); + }); }); diff --git a/apps/backend/test/integration/saved-prior-sql-capture.spec.ts b/apps/backend/test/integration/saved-prior-sql-capture.spec.ts new file mode 100644 index 0000000..2c46138 --- /dev/null +++ b/apps/backend/test/integration/saved-prior-sql-capture.spec.ts @@ -0,0 +1,208 @@ +import type { WriteRagReplayInput } from "../../src/modules/knowledge/rag/observability/rag-replay.repository"; +import { SavedPriorSqlService } from "../../src/modules/knowledge/memory/saved-prior-sql.service"; +import type { RagDocumentBuildResult } from "../../src/modules/rag/ingestion/rag-document.factory"; + +const createService = (override?: { + writeReplay?: (input: WriteRagReplayInput) => Promise; +}) => { + const writeReplay = + override?.writeReplay ?? + (async (input: WriteRagReplayInput) => ({ + runId: input.runId, + replayKey: input.replayKey, + datasourceId: input.datasourceId, + stage: input.stage, + payload: JSON.stringify(input.payload), + createdAt: input.createdAt ?? new Date().toISOString() + })); + const replayRepository = { + writeReplay: jest.fn(writeReplay) + }; + const buildResult: RagDocumentBuildResult = { + document: { + id: "doc-prior-1", + datasourceId: "ds-1", + domain: "sql_example", + sourceType: "sql_example", + sourceRef: "saved_prior_sql:prior-1", + sourceVersion: "saved_prior_sql:prior-1:v1", + contentChecksum: "checksum-1", + title: "Saved Prior SQL", + content: "Question: q\nSQL: select 1", + tableNames: ["orders"], + columnNames: ["id"], + metadata: JSON.stringify({ + trusted: true, + verified: true, + priorSql: true + }) + }, + chunks: [ + { + id: "chunk-prior-1", + documentId: "doc-prior-1", + datasourceId: "ds-1", + domain: "sql_example", + chunkProfile: "sql_example", + chunkOrder: 0, + content: "Question: q\nSQL: select 1", + contentChecksum: "checksum-1", + tableNames: ["orders"], + columnNames: ["id"], + metadata: JSON.stringify({ + trusted: true, + verified: true, + priorSql: true + }) + } + ] + }; + const documentFactory = { + create: jest.fn().mockReturnValue(buildResult) + }; + const documentRepository = { + upsertDocumentWithChunks: jest.fn().mockResolvedValue({ + documentId: "doc-prior-1", + insertedDocument: true, + insertedChunkCount: 1 + }) + }; + const indexRepository = { + upsertChunksForDatasource: jest.fn() + }; + const buildRagIndexJob = { + run: jest.fn().mockResolvedValue({ + indexVersionId: "idx-prior-1", + datasourceId: "ds-1", + status: "active", + entryCount: 1, + archivedChannels: ["lexical", "dense"], + denseMode: "placeholder_vector_string" + }) + }; + const service = new SavedPriorSqlService( + replayRepository as never, + documentFactory as never, + documentRepository as never, + indexRepository as never, + buildRagIndexJob as never + ); + return { + service, + replayRepository, + documentRepository, + buildRagIndexJob + }; +}; + +describe("saved prior sql capture integration", () => { + it("captures trusted prior SQL once and preserves idempotency on duplicate capture", async () => { + const { service, replayRepository, documentRepository, buildRagIndexJob } = + createService(); + const first = await service.captureFromSavedView({ + workspaceId: "ws-1", + datasourceId: "ds-1", + sourceRunId: "run-1", + sourceRunStatus: "executionResult", + sourceRunCreatedAt: "2026-04-25T08:00:00.000Z", + question: "recent paid orders", + sql: "SELECT o.id, o.status FROM orders o JOIN users u ON u.id = o.user_id WHERE o.status = 'paid'", + viewId: "view.chat_run.run-1", + viewName: "orders_paid_recent", + viewSql: "SELECT o.id, o.status FROM orders o JOIN users u ON u.id = o.user_id WHERE o.status = 'paid'", + replayed: false, + savedAt: "2026-04-25T09:00:00.000Z" + }); + + expect(first.outcome).toBe("captured"); + expect(first.record).toEqual( + expect.objectContaining({ + workspaceId: "ws-1", + datasourceId: "ds-1", + sourceRunId: "run-1", + viewId: "view.chat_run.run-1", + question: "recent paid orders", + sql: expect.stringContaining("FROM orders"), + savedAt: "2026-04-25T09:00:00.000Z", + metadata: { + trusted: true, + verified: true, + priorSql: true + } + }) + ); + expect(first.record?.tableNames).toEqual( + expect.arrayContaining(["orders", "users"]) + ); + expect(first.record?.columnNames).toEqual( + expect.arrayContaining(["id", "status"]) + ); + expect(replayRepository.writeReplay).toHaveBeenCalledTimes(1); + expect(documentRepository.upsertDocumentWithChunks).toHaveBeenCalledTimes(1); + expect(buildRagIndexJob.run).toHaveBeenCalledTimes(1); + + const second = await service.captureFromSavedView({ + workspaceId: "ws-1", + datasourceId: "ds-1", + sourceRunId: "run-1", + sourceRunStatus: "executionResult", + question: "recent paid orders", + sql: "SELECT o.id, o.status FROM orders o JOIN users u ON u.id = o.user_id WHERE o.status = 'paid'", + viewId: "view.chat_run.run-1", + viewName: "orders_paid_recent", + viewSql: "SELECT o.id, o.status FROM orders o JOIN users u ON u.id = o.user_id WHERE o.status = 'paid'", + replayed: false + }); + + expect(second.outcome).toBe("duplicate"); + expect(second.reason).toBe("prior_already_captured"); + expect(second.priorId).toBe(first.priorId); + expect(replayRepository.writeReplay).toHaveBeenCalledTimes(2); + expect(service.listRecords()).toHaveLength(1); + }); + + it("marks replayed save request as duplicate no-op", async () => { + const { service, documentRepository, buildRagIndexJob } = createService(); + const result = await service.captureFromSavedView({ + workspaceId: "ws-1", + datasourceId: "ds-1", + sourceRunId: "run-1", + sourceRunStatus: "executionResult", + question: "recent paid orders", + sql: "SELECT * FROM orders", + viewId: "view.chat_run.run-1", + viewName: "orders_recent", + viewSql: "SELECT * FROM orders", + replayed: true + }); + + expect(result.outcome).toBe("duplicate"); + expect(result.reason).toBe("save_replayed"); + expect(result.record).toBeUndefined(); + expect(service.listRecords()).toHaveLength(0); + expect(documentRepository.upsertDocumentWithChunks).toHaveBeenCalledTimes(0); + expect(buildRagIndexJob.run).toHaveBeenCalledTimes(0); + }); + + it("skips capture for ineligible source run status", async () => { + const { service, documentRepository, buildRagIndexJob } = createService(); + const result = await service.captureFromSavedView({ + workspaceId: "ws-1", + datasourceId: "ds-1", + sourceRunId: "run-1", + sourceRunStatus: "failed", + question: "recent paid orders", + sql: "SELECT * FROM orders", + viewId: "view.chat_run.run-1", + viewName: "orders_recent", + viewSql: "SELECT * FROM orders", + replayed: false + }); + + expect(result.outcome).toBe("skipped_ineligible"); + expect(result.reason).toBe("source_run_not_execution_result"); + expect(service.getRecord(result.priorId)).toBeUndefined(); + expect(documentRepository.upsertDocumentWithChunks).toHaveBeenCalledTimes(0); + expect(buildRagIndexJob.run).toHaveBeenCalledTimes(0); + }); +}); diff --git a/apps/backend/test/integration/saved-prior-sql-ingestion.spec.ts b/apps/backend/test/integration/saved-prior-sql-ingestion.spec.ts new file mode 100644 index 0000000..fc73fff --- /dev/null +++ b/apps/backend/test/integration/saved-prior-sql-ingestion.spec.ts @@ -0,0 +1,110 @@ +import { resolve } from "node:path"; +import { Test } from "@nestjs/testing"; +import { AppModule } from "../../src/app.module"; +import { SavedPriorSqlService } from "../../src/modules/knowledge/memory/saved-prior-sql.service"; +import { RagIndexBuilderService } from "../../src/modules/rag/index/rag-index-builder.service"; +import { RagIndexRepository } from "../../src/modules/rag/index/rag-index.repository"; +import { RagRetrievalService } from "../../src/modules/knowledge/rag/retrieval/rag-retrieval.service"; + +describe("saved prior sql ingestion integration", () => { + beforeAll(() => { + process.env.SQLITE_PATH = resolve( + __dirname, + "../../../../data/sqlite/text2sql.db" + ); + process.env.DATABASE_URL = ""; + process.env.REDIS_URL = ""; + process.env.LLM_PROVIDER = "volcengine"; + process.env.LLM_MOCK_MODE = "true"; + }); + + it("upserts saved prior SQL into active index and keeps duplicate capture idempotent", async () => { + const moduleRef = await Test.createTestingModule({ + imports: [AppModule] + }).compile(); + + const indexRepository = moduleRef.get(RagIndexRepository); + const indexBuilder = moduleRef.get(RagIndexBuilderService); + const savedPriorService = moduleRef.get(SavedPriorSqlService); + const retrievalService = moduleRef.get(RagRetrievalService); + + const datasourceId = "ds-saved-prior-ingestion"; + const workspaceId = "ws-saved-prior-ingestion"; + + indexRepository.seedChunksForDatasource(datasourceId, [ + { + id: "chunk-ingestion-schema-orders", + datasourceId, + domain: "schema", + content: "table orders(id, amount, status)", + metadata: JSON.stringify({ + tableNames: ["orders"], + columnNames: ["id", "amount", "status"] + }) + } + ]); + + await indexBuilder.buildAndActivate({ + datasourceId, + sourceVersion: "source-saved-prior-ingestion-bootstrap-v1", + createdByRunId: "run-saved-prior-ingestion-bootstrap-v1", + activatedByRunId: "run-saved-prior-ingestion-bootstrap-v1" + }); + + const first = await savedPriorService.captureFromSavedView({ + workspaceId, + datasourceId, + sourceRunId: "run-saved-prior-source-v1", + sourceRunStatus: "executionResult", + sourceRunCreatedAt: "2026-04-25T08:00:00.000Z", + question: "orders paid gmv", + sql: "SELECT SUM(amount) FROM orders WHERE status = 'paid'", + viewId: "view.chat_run.run-saved-prior-source-v1", + viewName: "orders_paid_gmv", + viewSql: "SELECT SUM(amount) FROM orders WHERE status = 'paid'", + tableNames: ["orders"], + columnNames: ["amount", "status"], + replayed: false, + savedAt: "2026-04-25T08:00:00.000Z" + }); + + expect(first.outcome).toBe("captured"); + + const response = await retrievalService.retrieve({ + query: "orders paid gmv", + datasourceId, + workspaceId, + allowedTables: ["orders"], + runId: "run-saved-prior-retrieve-v1" + }); + + expect(response.retrieval_bundle.prior_sql_lane?.status).toBe("hit"); + expect(response.retrieval_bundle.prior_sql_lane?.selected_count).toBe(1); + expect(response.retrieval_bundle.candidates[0]?.chunk.metadata.domain).toBe("sql_example"); + + const second = await savedPriorService.captureFromSavedView({ + workspaceId, + datasourceId, + sourceRunId: "run-saved-prior-source-v1", + sourceRunStatus: "executionResult", + question: "orders paid gmv", + sql: "SELECT SUM(amount) FROM orders WHERE status = 'paid'", + viewId: "view.chat_run.run-saved-prior-source-v1", + viewName: "orders_paid_gmv", + viewSql: "SELECT SUM(amount) FROM orders WHERE status = 'paid'", + tableNames: ["orders"], + columnNames: ["amount", "status"], + replayed: false + }); + + expect(second.outcome).toBe("duplicate"); + + const activeVersion = await indexRepository.getActiveVersion(datasourceId); + expect(activeVersion).toBeDefined(); + const activeEntries = await indexRepository.listEntriesByVersion(activeVersion?.id ?? ""); + const priorChunkIds = activeEntries.filter((item) => item.domain === "sql_example"); + expect(priorChunkIds).toHaveLength(1); + + await moduleRef.close(); + }); +}); diff --git a/apps/backend/test/integration/saved-prior-sql-quality.spec.ts b/apps/backend/test/integration/saved-prior-sql-quality.spec.ts new file mode 100644 index 0000000..8a0ddf4 --- /dev/null +++ b/apps/backend/test/integration/saved-prior-sql-quality.spec.ts @@ -0,0 +1,60 @@ +import { resolve } from "node:path"; +import { Test } from "@nestjs/testing"; +import { AppModule } from "../../src/app.module"; +import { RagQualityService } from "../../src/modules/rag/quality/rag-quality.service"; + +describe("saved prior sql quality diagnostics integration", () => { + beforeAll(() => { + process.env.SQLITE_PATH = resolve( + __dirname, + "../../../../data/sqlite/text2sql.db" + ); + process.env.DATABASE_URL = ""; + process.env.REDIS_URL = ""; + process.env.LLM_PROVIDER = "volcengine"; + process.env.LLM_MOCK_MODE = "true"; + }); + + it("aggregates saved prior shortcut diagnostic counters", async () => { + const moduleRef = await Test.createTestingModule({ + imports: [AppModule] + }).compile(); + + const quality = moduleRef.get(RagQualityService); + quality.reset(); + + quality.recordEvaluation({ + runId: "run-saved-prior-quality-v1", + datasourceId: "ds-saved-prior-quality", + sampleSize: 48, + recallAt20: 0.88, + mrrAt10: 0.72, + retrievalRerankP95Ms: 640, + degradeRate: 0.03, + priorSqlLane: { + totalCount: 20, + hitCount: 12, + missCount: 3, + filteredCount: 2, + staleCount: 1, + ambiguousCount: 1, + duplicateCount: 1, + safetyRejectedCount: 2, + fallbackToGenerationCount: 2 + } + }); + + const snapshot = quality.snapshot(); + expect(snapshot.priorSqlLane.sampleSize).toBe(20); + expect(snapshot.priorSqlLane.priorSqlHitRate).toBe(0.6); + expect(snapshot.priorSqlLane.priorSqlMissCount).toBe(3); + expect(snapshot.priorSqlLane.priorSqlFilteredCount).toBe(2); + expect(snapshot.priorSqlLane.priorSqlStaleRate).toBe(0.05); + expect(snapshot.priorSqlLane.priorSqlAmbiguousCount).toBe(1); + expect(snapshot.priorSqlLane.priorSqlDuplicateCount).toBe(1); + expect(snapshot.priorSqlLane.priorSqlSafetyRejectedCount).toBe(2); + expect(snapshot.priorSqlLane.priorSqlFallbackToGenerationCount).toBe(2); + + await moduleRef.close(); + }); +}); diff --git a/apps/backend/test/integration/workspace-modeling-setup.spec.ts b/apps/backend/test/integration/workspace-modeling-setup.spec.ts index d52b97e..8cd90c5 100644 --- a/apps/backend/test/integration/workspace-modeling-setup.spec.ts +++ b/apps/backend/test/integration/workspace-modeling-setup.spec.ts @@ -8,6 +8,8 @@ const actor = { const buildService = (options?: { permissionError?: DomainError; + permissionPolicyVersion?: number; + permissionTableNames?: string[]; }) => { const workspaceDatasourceService = { listDatasourceTables: jest.fn(async () => ({ @@ -22,10 +24,34 @@ const buildService = (options?: { return { workspaceId: "ws-1", datasourceId: "ds-1", - policyVersion: 7, - tableNames: ["customers", "orders"] + policyVersion: options?.permissionPolicyVersion ?? 7, + tableNames: options?.permissionTableNames ?? ["customers", "orders"] }; - }) + }), + replaceDatasourceTablePermissions: jest.fn( + async ( + _actor: unknown, + workspaceId: string, + datasourceId: string, + body: { policyVersion: number; tableNames: string[] } + ) => ({ + workspaceId, + datasourceId, + policyVersion: body.policyVersion + 1, + tableNames: body.tableNames, + impactSummary: { + beforeCount: 0, + afterCount: body.tableNames.length, + addedCount: body.tableNames.length, + removedCount: 0, + retainedCount: 0, + addedTables: body.tableNames, + removedTables: [] + }, + idempotencyKey: "mock-key", + replayed: false + }) + ) }; const workspaceRelationshipService = { getDraft: jest.fn(async () => ({ @@ -109,7 +135,8 @@ const buildService = (options?: { datasourceRepository as never, queryExecutorRouter as never ), - workspaceRelationshipService + workspaceRelationshipService, + workspaceDatasourceService }; }; @@ -198,6 +225,62 @@ describe("workspace modeling setup integration", () => { }); }); + it("falls back to discovered tables when policyVersion is 0 and permissions are empty", async () => { + const { service } = buildService({ + permissionPolicyVersion: 0, + permissionTableNames: [] + }); + + const list = await service.listSetupTables(actor, "ws-1", "ds-1"); + expect(list.items).toEqual(["customers", "orders", "payments"]); + expect(list.policyVersion).toBe(0); + + const preview = await service.previewSetup(actor, "ws-1", "ds-1", { + policyVersion: 0, + selectedTables: ["orders"], + selectedRecommendationIds: undefined + }); + expect(preview.selectedTables).toEqual(["orders"]); + }); + + it("keeps explicit empty permissions when policyVersion is greater than 0", async () => { + const { service } = buildService({ + permissionPolicyVersion: 3, + permissionTableNames: [] + }); + + const list = await service.listSetupTables(actor, "ws-1", "ds-1"); + expect(list.items).toEqual([]); + expect(list.policyVersion).toBe(3); + }); + + it("persists selected tables into table permissions during setup save", async () => { + const { service, workspaceDatasourceService } = buildService({ + permissionPolicyVersion: 0, + permissionTableNames: [] + }); + + const saved = await service.saveSelectedTables(actor, "ws-1", "ds-1", { + policyVersion: 0, + selectedTables: ["orders", "customers"] + }); + + expect( + workspaceDatasourceService.replaceDatasourceTablePermissions + ).toHaveBeenCalledWith( + actor, + "ws-1", + "ds-1", + expect.objectContaining({ + policyVersion: 0, + tableNames: ["customers", "orders"] + }), + expect.stringMatching(/^modeling-setup-tables-/) + ); + expect(saved.policyVersion).toBe(1); + expect(saved.selectedTableNames).toEqual(["customers", "orders"]); + }); + it("replays commit response with same idempotency key and payload", async () => { const { service, workspaceRelationshipService } = buildService(); const body = { diff --git a/apps/backend/test/unit/chartbi-artifact.service.spec.ts b/apps/backend/test/unit/chartbi-artifact.service.spec.ts new file mode 100644 index 0000000..c17dfe5 --- /dev/null +++ b/apps/backend/test/unit/chartbi-artifact.service.spec.ts @@ -0,0 +1,98 @@ +import { ChartBiArtifactService } from "../../src/modules/conversation/delivery/chartbi/chartbi-artifact.service"; +import { ChartBiGroundingGuard } from "../../src/modules/conversation/delivery/chartbi/chartbi-grounding.guard"; +import { ChartBiIntentParser } from "../../src/modules/conversation/delivery/chartbi/chartbi-intent-parser"; +import { ChartBiResultProfiler } from "../../src/modules/conversation/delivery/chartbi/chartbi-result-profiler"; +import { ChartBiSpecCompiler } from "../../src/modules/conversation/delivery/chartbi/chartbi-spec-compiler"; +import { ChartBiValidator } from "../../src/modules/conversation/delivery/chartbi/chartbi-validator"; + +describe("ChartBiArtifactService", () => { + const service = new ChartBiArtifactService( + new ChartBiResultProfiler(), + new ChartBiIntentParser(), + new ChartBiSpecCompiler(), + new ChartBiValidator(), + new ChartBiGroundingGuard() + ); + + it("uses deterministic baseline when visual intent parsing fails", () => { + const artifact = service.build({ + sql: "SELECT created_at, revenue FROM orders", + columns: ["created_at", "revenue"], + rows: [ + { created_at: "2026-04-01", revenue: 1200 }, + { created_at: "2026-04-02", revenue: 980 } + ], + answer: "最近两天收入波动", + visualIntentRaw: "```vis\nchart ???\n```" + }); + + expect(artifact.chart?.type).toBe("line"); + expect(artifact.display).toBe("line"); + expect(artifact.displayModes).toEqual(expect.arrayContaining(["table", "line"])); + expect(artifact.visualIntent?.status).toBe("failed"); + expect(artifact.validation.status).toBe("valid"); + expect(artifact.validation.source).toBe("baseline"); + }); + + it("uses repaired intent when baseline is unavailable", () => { + const artifact = service.build({ + sql: "SELECT a, b FROM metrics", + columns: ["a", "b"], + rows: [ + { a: 1, b: 10 }, + { a: 2, b: 12 } + ], + visualIntentRaw: [ + "```json", + "{", + ' "type": "bar",', + ' "x": "a",', + ' "y": "b"', + "}", + "```" + ].join("\n") + }); + + expect(artifact.chart?.type).toBe("bar"); + expect(artifact.validation.status).toBe("repaired"); + expect(artifact.validation.source).toBe("repaired_intent"); + expect(artifact.display).toBe("bar"); + }); + + it("falls back to table with readable reason when no chart candidate survives", () => { + const artifact = service.build({ + sql: "SELECT note FROM logs", + columns: ["note"], + rows: [], + visualIntentRaw: "invalid payload" + }); + + expect(artifact.chart).toBeUndefined(); + expect(artifact.display).toBe("table"); + expect(artifact.validation.status).toBe("fallback"); + expect(artifact.fallback?.reason).toContain("table fallback"); + }); + + it("keeps baseline priority over repaired intent when baseline is valid", () => { + const artifact = service.build({ + sql: "SELECT created_at, revenue FROM orders", + columns: ["created_at", "revenue"], + rows: [ + { created_at: "2026-04-01", revenue: 1200 }, + { created_at: "2026-04-02", revenue: 980 } + ], + visualIntentRaw: [ + "```json", + "{", + ' "type": "pie",', + ' "category": "created_at",', + ' "value": "revenue"', + "}", + "```" + ].join("\n") + }); + + expect(artifact.chart?.type).toBe("line"); + expect(artifact.validation.source).toBe("baseline"); + }); +}); diff --git a/apps/backend/test/unit/chartbi-intent-parser.spec.ts b/apps/backend/test/unit/chartbi-intent-parser.spec.ts new file mode 100644 index 0000000..41c8086 --- /dev/null +++ b/apps/backend/test/unit/chartbi-intent-parser.spec.ts @@ -0,0 +1,69 @@ +import { ChartBiIntentParser } from "../../src/modules/conversation/delivery/chartbi/chartbi-intent-parser"; + +describe("ChartBiIntentParser", () => { + const parser = new ChartBiIntentParser(); + + it("parses json visual intent from fenced block with wrapper text", () => { + const raw = [ + "下面是建议图表：", + "```json", + "{", + ' "type": "line",', + ' "x": "created_at",', + ' "y": "revenue",', + ' "title": "Revenue trend"', + "}", + "```", + "请按需使用" + ].join("\n"); + + const parsed = parser.parse(raw); + + expect(parsed.status).toBe("parsed"); + if (parsed.status !== "parsed") { + throw new Error("expected parser to return parsed status"); + } + + expect(parsed.source).toBe("json"); + expect(parsed.intent.type).toBe("line"); + expect(parsed.intent.mappings.x).toBe("created_at"); + expect(parsed.intent.mappings.y).toBe("revenue"); + expect(parsed.intent.title).toBe("Revenue trend"); + }); + + it("parses vis-like syntax and normalizes aliases", () => { + const raw = [ + "我们可以用这个：", + "```vis", + "chart: bar", + "dimension: region", + "measure: sales_amount", + "insight: 华东区域贡献最高", + "```" + ].join("\n"); + + const parsed = parser.parse(raw); + + expect(parsed.status).toBe("parsed"); + if (parsed.status !== "parsed") { + throw new Error("expected parser to return parsed status"); + } + + expect(parsed.source).toBe("vis"); + expect(parsed.intent.type).toBe("bar"); + expect(parsed.intent.mappings.dimension).toBe("region"); + expect(parsed.intent.mappings.measure).toBe("sales_amount"); + expect(parsed.intent.insights).toEqual(["华东区域贡献最高"]); + }); + + it("returns readable failure reason for unsupported payload", () => { + const parsed = parser.parse("this output has no valid chart intent payload"); + + expect(parsed.status).toBe("failed"); + if (parsed.status !== "failed") { + throw new Error("expected parser to return failed status"); + } + + expect(parsed.reason).toContain("visual intent"); + }); +}); diff --git a/apps/backend/test/unit/chartbi-validator.spec.ts b/apps/backend/test/unit/chartbi-validator.spec.ts new file mode 100644 index 0000000..49bf10b --- /dev/null +++ b/apps/backend/test/unit/chartbi-validator.spec.ts @@ -0,0 +1,89 @@ +import type { ChartBiCanonicalSpec } from "../../src/modules/conversation/delivery/chartbi/chartbi-result-profiler"; +import { ChartBiValidator } from "../../src/modules/conversation/delivery/chartbi/chartbi-validator"; + +describe("ChartBiValidator", () => { + const validator = new ChartBiValidator(); + + const rows = [ + { created_at: "2026-04-01", revenue: 1200, region: "East" }, + { created_at: "2026-04-02", revenue: 980, region: "West" } + ]; + const columns = ["created_at", "revenue", "region"]; + + it("passes for valid line chart spec", () => { + const spec: ChartBiCanonicalSpec = { + type: "line", + source: "baseline", + mappings: { + x: "created_at", + y: "revenue" + } + }; + + const result = validator.validate({ + spec, + rows, + columns + }); + + expect(result.ok).toBe(true); + expect(result.reason).toBeUndefined(); + }); + + it("fails closed for chart type outside allowlist", () => { + const result = validator.validate({ + spec: { + type: "scatter" as never, + source: "repaired_intent", + mappings: { + x: "created_at", + y: "revenue" + } + }, + rows, + columns + }); + + expect(result.ok).toBe(false); + expect(result.reason).toContain("allowlist"); + }); + + it("fails when mapped field is not in result columns", () => { + const result = validator.validate({ + spec: { + type: "bar", + source: "repaired_intent", + mappings: { + x: "unknown_dimension", + y: "revenue" + } + }, + rows, + columns + }); + + expect(result.ok).toBe(false); + expect(result.reason).toContain("unknown_dimension"); + }); + + it("fails pie chart when values contain negative numbers", () => { + const result = validator.validate({ + spec: { + type: "pie", + source: "repaired_intent", + mappings: { + category: "region", + value: "revenue" + } + }, + rows: [ + { region: "East", revenue: 100 }, + { region: "West", revenue: -5 } + ], + columns: ["region", "revenue"] + }); + + expect(result.ok).toBe(false); + expect(result.reason).toContain("negative"); + }); +}); diff --git a/apps/backend/test/unit/clarification-semantic-evaluator.spec.ts b/apps/backend/test/unit/clarification-semantic-evaluator.spec.ts index 6689165..3128ee3 100644 --- a/apps/backend/test/unit/clarification-semantic-evaluator.spec.ts +++ b/apps/backend/test/unit/clarification-semantic-evaluator.spec.ts @@ -35,6 +35,49 @@ describe("ClarificationSemanticEvaluatorService", () => { expect(result.metadata.fallbackApplied).toBe(false); }); + it("recognizes count-style phrasing as metric-complete when subject exists", async () => { + const service = createService(); + const result = await service.evaluate({ + question: "一共有多少订单", + ruleDecision: { + decision: "clarify", + triggerPath: "rule", + confidenceLevel: "low", + missingCriticalSlots: ["metric"], + reasonCodes: ["missing_metric_slot"] + } + }); + + expect(result.status).toBe("success"); + expect(result.decision).toMatchObject({ + decision: "continue", + missingCriticalSlots: [] + }); + expect(result.reasonCodes).toEqual( + expect.arrayContaining(["semantic_classifier_v1", "semantic_ready_to_continue"]) + ); + }); + + it("recognizes customer synonym subject in count phrasing", async () => { + const service = createService(); + const result = await service.evaluate({ + question: "一共有多少位顾客", + ruleDecision: { + decision: "clarify", + triggerPath: "rule", + confidenceLevel: "low", + missingCriticalSlots: ["subject", "metric"], + reasonCodes: ["missing_subject_slot", "missing_metric_slot"] + } + }); + + expect(result.status).toBe("success"); + expect(result.decision).toMatchObject({ + decision: "continue", + missingCriticalSlots: [] + }); + }); + it("falls back with timeout when semantic evaluator times out", async () => { class TimeoutEvaluator extends ClarificationSemanticEvaluatorService { protected override async runSemanticArbitration(): Promise { diff --git a/apps/backend/test/unit/config.module.spec.ts b/apps/backend/test/unit/config.module.spec.ts index 9a2fe1d..dfcfa89 100644 --- a/apps/backend/test/unit/config.module.spec.ts +++ b/apps/backend/test/unit/config.module.spec.ts @@ -10,6 +10,8 @@ describe("AppConfigService", () => { delete process.env.LLM_BASE_URL; delete process.env.LLM_API_KEY; delete process.env.LLM_MODEL; + delete process.env.LLM_TIMEOUT_MS; + delete process.env.LLM_STREAM_TIMEOUT_MS; delete process.env.SQLITE_PATH; delete process.env.REDIS_URL; delete process.env.DATABASE_URL; @@ -39,9 +41,37 @@ describe("AppConfigService", () => { const config = moduleRef.get(AppConfigService); expect(config.port).toBe(3002); expect(config.llmProvider).toBe("volcengine"); + expect(config.llmTimeoutMs).toBe(30000); + expect(config.llmStreamTimeoutMs).toBe(30000); expect(config.sqlitePath).toContain("data/sqlite/text2sql.db"); }); + it("should use LLM_STREAM_TIMEOUT_MS when configured", async () => { + process.env.LLM_TIMEOUT_MS = "30000"; + process.env.LLM_STREAM_TIMEOUT_MS = "90000"; + + const moduleRef = await Test.createTestingModule({ + imports: [AppConfigModule] + }).compile(); + + const config = moduleRef.get(AppConfigService); + expect(config.llmTimeoutMs).toBe(30000); + expect(config.llmStreamTimeoutMs).toBe(90000); + }); + + it("should fallback to LLM_TIMEOUT_MS when LLM_STREAM_TIMEOUT_MS is invalid", async () => { + process.env.LLM_TIMEOUT_MS = "45000"; + process.env.LLM_STREAM_TIMEOUT_MS = "invalid"; + + const moduleRef = await Test.createTestingModule({ + imports: [AppConfigModule] + }).compile(); + + const config = moduleRef.get(AppConfigService); + expect(config.llmTimeoutMs).toBe(45000); + expect(config.llmStreamTimeoutMs).toBe(45000); + }); + it("should build database url from POSTGRES_* env variables", async () => { process.env.POSTGRES_HOST = "localhost"; process.env.POSTGRES_PORT = "5432"; diff --git a/apps/backend/test/unit/delivery-contract.mapper.spec.ts b/apps/backend/test/unit/delivery-contract.mapper.spec.ts index 831e84d..da98dde 100644 --- a/apps/backend/test/unit/delivery-contract.mapper.spec.ts +++ b/apps/backend/test/unit/delivery-contract.mapper.spec.ts @@ -137,6 +137,75 @@ describe("DeliveryContractMapper", () => { expect(delivery.artifact?.hasError).toBe(false); }); + it("accepts chartbi artifact override and preserves unified answer semantics", () => { + const mapper = new DeliveryContractMapper(new SandboxRuntimeService()); + const run = createBaseRun({ + answer: "订单状态以 paid 为主。" + }); + + const delivery = mapper.map({ + run, + replayRecords: [], + artifactOverride: { + sql: run.sql, + columns: run.columns, + rowCount: 1, + rowsPreview: run.rows?.slice(0, 1), + hasError: false, + summary: { + text: "订单状态以 paid 为主。" + }, + table: { + columns: run.columns, + rowCount: 1, + rowsPreview: run.rows?.slice(0, 1), + previewRowCount: 1 + }, + chart: { + type: "pie", + mappings: { + label: "status", + value: "count" + } + }, + display: "pie", + validation: { + status: "valid", + reasonCodes: ["chartbi_baseline_selected"] + } + }, + additionalRiskTags: ["chartbi_artifact_failed"] + }); + + expect(delivery.answer.text).toBe(run.answer); + expect(delivery.artifact?.summary?.text).toBe("订单状态以 paid 为主。"); + expect(delivery.artifact?.chart?.type).toBe("pie"); + expect(delivery.artifact?.display).toBe("pie"); + expect(delivery.evidence?.riskTags).toEqual( + expect.arrayContaining(["chartbi_artifact_failed"]) + ); + }); + + it("keeps safe artifact semantics for clarification runs", () => { + const mapper = new DeliveryContractMapper(new SandboxRuntimeService()); + const run = createBaseRun({ + status: "clarification", + answer: "请补充时间范围。", + rows: undefined, + columns: undefined, + sql: undefined + }); + + const delivery = mapper.map({ + run, + replayRecords: [], + artifactOverride: undefined + }); + + expect(delivery.answer.status).toBe("clarification"); + expect(delivery.artifact).toBeUndefined(); + }); + it("echoes sanitized effective context summary and conflict hint from trace", () => { const mapper = new DeliveryContractMapper(new SandboxRuntimeService()); const run = createBaseRun({ @@ -357,6 +426,51 @@ describe("DeliveryContractMapper", () => { expect(delivery.evidence?.modelingRevision).toBe(12); }); + it("maps saved prior SQL shortcut evidence from resolve+safety trace steps", () => { + const mapper = new DeliveryContractMapper(new SandboxRuntimeService()); + const run = createBaseRun({ + trace: { + runId: "run-delivery-unit", + provider: "volcengine", + retryCount: 0, + steps: [ + { + node: "resolve-saved-prior-sql", + status: "success", + at: "2026-04-25T00:00:00.100Z", + outputSummary: JSON.stringify({ + status: "hit", + reasonCodes: ["prior_sql_shortcut_hit"], + selectedChunkId: "chunk-saved-prior-1", + selectedViewId: "view.chat_run.run-1", + selectedSourceRunId: "run-1" + }) + }, + { + node: "safety-check", + status: "success", + at: "2026-04-25T00:00:00.200Z" + } + ] + } as SqlRun["trace"] + }); + + const delivery = mapper.map({ + run, + replayRecords: [] + }); + + expect(delivery.evidence?.savedPriorSql).toEqual({ + status: "hit", + shortcutUsed: true, + reasonCodes: ["prior_sql_shortcut_hit"], + selectedChunkId: "chunk-saved-prior-1", + selectedViewId: "view.chat_run.run-1", + selectedSourceRunId: "run-1", + safetyResult: "passed" + }); + }); + it("falls back to semantic step summary for revision and binding evidence", () => { const mapper = new DeliveryContractMapper(new SandboxRuntimeService()); const run = createBaseRun({ diff --git a/apps/backend/test/unit/langgraph-runtime.spec.ts b/apps/backend/test/unit/langgraph-runtime.spec.ts index 51e744e..fd4989a 100644 --- a/apps/backend/test/unit/langgraph-runtime.spec.ts +++ b/apps/backend/test/unit/langgraph-runtime.spec.ts @@ -93,6 +93,12 @@ describe("langgraph runtime", () => { summary: "stub" }) }, + resolveSavedPriorSqlNode: { + run: () => ({ + status: "miss" as const, + reasonCodes: ["prior_sql_no_trusted_match"] + }) + }, generateSqlNode: { run: async () => ({ provider: "mock-provider", @@ -187,6 +193,12 @@ describe("langgraph runtime", () => { summary: "stub" }) }, + resolveSavedPriorSqlNode: { + run: () => ({ + status: "miss" as const, + reasonCodes: ["prior_sql_no_trusted_match"] + }) + }, generateSqlNode: { run: async () => ({ provider: "mock-provider", @@ -237,6 +249,7 @@ describe("langgraph runtime", () => { "build-intent-plan", "build-semantic-query", "build-physical-plan", + "resolve-saved-prior-sql", "generate-sql", "safety-check", "execute-sql", @@ -245,8 +258,8 @@ describe("langgraph runtime", () => { expect(output.trace.steps[0]?.sequence).toBe(1); expect(output.trace.steps[0]?.stepId).toBe("run-1:clarify:1"); expect(output.trace.steps[0]?.lifecycle).toBe("skipped"); - expect(output.trace.steps[5]?.sequence).toBe(6); - expect(output.trace.steps[5]?.lifecycle).toBe("completed"); + expect(output.trace.steps[6]?.sequence).toBe(7); + expect(output.trace.steps[6]?.lifecycle).toBe("completed"); expect(output.trace.clarificationDecision?.decision).toBe("continue"); }); @@ -302,6 +315,12 @@ describe("langgraph runtime", () => { summary: "stub" }) }, + resolveSavedPriorSqlNode: { + run: () => ({ + status: "miss" as const, + reasonCodes: ["prior_sql_no_trusted_match"] + }) + }, generateSqlNode: { run: async () => { throw new Error("llm failed"); @@ -343,12 +362,13 @@ describe("langgraph runtime", () => { "build-intent-plan", "build-semantic-query", "build-physical-plan", + "resolve-saved-prior-sql", "generate-sql" ]); - expect(output.trace.steps[5]?.status).toBe("failed"); - expect(output.trace.steps[5]?.sequence).toBe(6); - expect(output.trace.steps[5]?.stepId).toBe("run-2:generate-sql:6"); - expect(output.trace.steps[5]?.lifecycle).toBe("failed"); - expect(output.trace.steps[5]?.errorSummary).toContain("llm failed"); + expect(output.trace.steps[6]?.status).toBe("failed"); + expect(output.trace.steps[6]?.sequence).toBe(7); + expect(output.trace.steps[6]?.stepId).toBe("run-2:generate-sql:7"); + expect(output.trace.steps[6]?.lifecycle).toBe("failed"); + expect(output.trace.steps[6]?.errorSummary).toContain("llm failed"); }); }); diff --git a/apps/backend/test/unit/llm-gateway.service.spec.ts b/apps/backend/test/unit/llm-gateway.service.spec.ts index 0eaa760..a48efad 100644 --- a/apps/backend/test/unit/llm-gateway.service.spec.ts +++ b/apps/backend/test/unit/llm-gateway.service.spec.ts @@ -19,12 +19,14 @@ describe("LlmGatewayService", () => { model: "gpt-4o-mini", baseUrl: "https://api.openai.com/v1", apiKey: "sk-test", - timeoutMs: 3000 + timeoutMs: 3000, + streamTimeoutMs: 9000 }; afterEach(() => { mockedGenerateText.mockReset(); mockedStreamText.mockReset(); + jest.restoreAllMocks(); }); it("should return mock response in llm mock mode", async () => { @@ -150,6 +152,33 @@ describe("LlmGatewayService", () => { expect(events).toEqual(["text-delta", "tool-call", "tool-result"]); }); + it("should use stream timeout for stream requests", async () => { + mockedStreamText.mockReturnValue({ + fullStream: (async function* () { + yield { type: "text-delta", text: "SELECT 1" }; + })(), + text: Promise.resolve("SELECT 1") + } as unknown as ReturnType); + + const timeoutSpy = jest.spyOn(AbortSignal, "timeout"); + const service = new LlmGatewayService( + { + llmMockMode: false + } as AppConfigService, + new LlmModelFactory() + ); + + await service.stream( + { + systemPrompt: "sys", + userPrompt: "统计订单状态分布" + }, + runtime + ); + + expect(timeoutSpy).toHaveBeenCalledWith(9000); + }); + it("should raise tool-call-only response when stream has no final text", async () => { mockedStreamText.mockReturnValue({ fullStream: (async function* () { @@ -225,4 +254,52 @@ describe("LlmGatewayService", () => { expect(events).toEqual(["text-delta"]); expect(mockedGenerateText).toHaveBeenCalledTimes(1); }); + + it("should fail fast when tool execution returns error", async () => { + mockedStreamText.mockReturnValue({ + fullStream: (async function* () { + yield { + type: "tool-call", + toolName: "runReadOnlySql", + toolCallId: "tool-3", + input: { sql: "SELECT * FROM unknown_table" } + }; + yield { + type: "tool-error", + toolName: "runReadOnlySql", + toolCallId: "tool-3", + error: new Error("当前工作空间无权访问表: unknown_table") + }; + })(), + text: Promise.resolve("") + } as unknown as ReturnType); + + const events: string[] = []; + const service = new LlmGatewayService( + { + llmMockMode: false + } as AppConfigService, + new LlmModelFactory() + ); + + await expect( + service.stream( + { + systemPrompt: "sys", + userPrompt: "统计支付方式占比" + }, + runtime, + { + onEvent: (event) => { + events.push(event.type); + } + } + ) + ).rejects.toMatchObject>({ + code: "LLM_TOOL_CALL_EXECUTION_FAILED" + }); + + expect(events).toEqual(["tool-call", "tool-error"]); + expect(mockedGenerateText).not.toHaveBeenCalled(); + }); }); diff --git a/apps/backend/test/unit/resolve-saved-prior-sql.node.spec.ts b/apps/backend/test/unit/resolve-saved-prior-sql.node.spec.ts new file mode 100644 index 0000000..572f722 --- /dev/null +++ b/apps/backend/test/unit/resolve-saved-prior-sql.node.spec.ts @@ -0,0 +1,175 @@ +import { ResolveSavedPriorSqlNode } from "../../src/modules/conversation/agent/nodes/resolve-saved-prior-sql.node"; + +describe("ResolveSavedPriorSqlNode", () => { + it("returns shortcut SQL when lane is hit and selected chunk contains sql", () => { + const node = new ResolveSavedPriorSqlNode(); + const result = node.run({ + question: "统计订单总数", + retrievalBundle: { + query: "统计订单总数", + run_id: "run-1", + datasource_id: "ds-1", + status: "ready", + degrade_reasons: [], + lane_results: { + lexical: { + lane: "lexical", + status: "ok", + timeout_ms: 10, + elapsed_ms: 5, + hits: [] + }, + dense: { + lane: "dense", + status: "ok", + timeout_ms: 10, + elapsed_ms: 5, + hits: [] + }, + graph: { + lane: "graph", + status: "ok", + timeout_ms: 10, + elapsed_ms: 5, + hits: [] + } + }, + candidates: [ + { + chunk_id: "chunk-prior-1", + source_lane: "lexical", + score: 1, + evidence: ["prior_sql:trusted"], + lane_scores: { + lexical: 1 + }, + lane_ranks: { + lexical: 1 + }, + chunk: { + chunk_id: "chunk-prior-1", + content: "Question: 统计订单总数\n\nSQL:\nSELECT COUNT(*) FROM orders", + metadata: { + datasourceId: "ds-1", + indexVersionId: "idx-1", + chunkId: "chunk-prior-1", + domain: "sql_example", + tableNames: ["orders"], + columnNames: ["id"], + sourceMetadata: { + trusted: true, + priorSql: true, + sql: "SELECT COUNT(*) FROM orders", + viewId: "view.chat_run.run-1", + sourceRunId: "run-1" + } + } + } + } + ], + selected_context: [], + prior_sql_lane: { + status: "hit", + matched_count: 1, + selected_count: 1, + filtered_count: 0, + shortcut: { + status: "hit", + reason_codes: ["prior_sql_shortcut_hit"], + matched_count: 1, + eligible_count: 1, + filtered_count: 0, + stale_count: 0, + ambiguous_count: 0, + selected_chunk_id: "chunk-prior-1", + selected_view_id: "view.chat_run.run-1", + selected_source_run_id: "run-1" + } + } + } + }); + + expect(result.status).toBe("hit"); + expect(result.sql).toBe("SELECT COUNT(*) FROM orders"); + expect(result.selectedChunkId).toBe("chunk-prior-1"); + expect(result.selectedViewId).toBe("view.chat_run.run-1"); + expect(result.selectedSourceRunId).toBe("run-1"); + }); + + it("falls back to chunk content when structured sql field is missing", () => { + const node = new ResolveSavedPriorSqlNode(); + const result = node.run({ + question: "统计订单总数", + retrievalBundle: { + query: "统计订单总数", + run_id: "run-1", + datasource_id: "ds-1", + status: "ready", + degrade_reasons: [], + lane_results: { + lexical: { + lane: "lexical", + status: "ok", + timeout_ms: 10, + elapsed_ms: 5, + hits: [] + }, + dense: { + lane: "dense", + status: "ok", + timeout_ms: 10, + elapsed_ms: 5, + hits: [] + }, + graph: { + lane: "graph", + status: "ok", + timeout_ms: 10, + elapsed_ms: 5, + hits: [] + } + }, + candidates: [ + { + chunk_id: "chunk-prior-1", + source_lane: "lexical", + score: 1, + evidence: ["prior_sql:trusted"], + lane_scores: { + lexical: 1 + }, + lane_ranks: { + lexical: 1 + }, + chunk: { + chunk_id: "chunk-prior-1", + content: "Question: 统计订单总数", + metadata: { + datasourceId: "ds-1", + indexVersionId: "idx-1", + chunkId: "chunk-prior-1", + domain: "sql_example", + tableNames: ["orders"], + columnNames: ["id"], + sourceMetadata: { + trusted: true, + priorSql: true + } + } + } + } + ], + selected_context: [], + prior_sql_lane: { + status: "hit", + matched_count: 1, + selected_count: 1, + filtered_count: 0 + } + } + }); + + expect(result.status).toBe("hit"); + expect(result.sql).toBe("Question: 统计订单总数"); + }); +}); diff --git a/apps/backend/test/unit/slot-filling-context.spec.ts b/apps/backend/test/unit/slot-filling-context.spec.ts index eb9b4ce..2c330aa 100644 --- a/apps/backend/test/unit/slot-filling-context.spec.ts +++ b/apps/backend/test/unit/slot-filling-context.spec.ts @@ -23,6 +23,42 @@ describe("slot-filling-context", () => { }); }); + it("treats count-style phrasing as a metric slot", () => { + const decision = decideSlotFilling("一共有多少订单"); + + expect(decision).toMatchObject({ + decision: "continue", + action: "proceed", + source: "rule", + missingSlots: [], + shouldClarify: false + }); + }); + + it("recognizes customer synonym subject in count questions", () => { + const decision = decideSlotFilling("一共有多少位顾客"); + + expect(decision).toMatchObject({ + decision: "continue", + action: "proceed", + source: "rule", + missingSlots: [], + shouldClarify: false + }); + }); + + it("keeps clarifying when only count metric exists without subject", () => { + const decision = decideSlotFilling("一共有多少位"); + + expect(decision).toMatchObject({ + decision: "clarify", + action: "ask_clarification", + source: "rule", + missingSlots: ["subject"], + shouldClarify: true + }); + }); + it("bypasses clarification for metadata intent", () => { const decision = decideSlotFilling("show tables"); diff --git a/apps/backend/test/unit/sql-generation.service.spec.ts b/apps/backend/test/unit/sql-generation.service.spec.ts index ed13ddf..76ec3bc 100644 --- a/apps/backend/test/unit/sql-generation.service.spec.ts +++ b/apps/backend/test/unit/sql-generation.service.spec.ts @@ -125,6 +125,31 @@ describe("SqlGenerationService semantic guardrails", () => { expect(providerRouter.generate).toHaveBeenCalledTimes(2); }); + it("falls back to non-stream generation when tool execution fails in stream mode", async () => { + const providerRouter = { + generate: jest.fn().mockResolvedValue({ + provider: "mock-provider", + model: "mock-model", + rawText: "```sql\nSELECT COUNT(*) AS total FROM orders;\n```" + }), + stream: jest.fn().mockRejectedValue( + new DomainError( + "LLM_TOOL_CALL_EXECUTION_FAILED", + "LLM 工具调用失败: 当前工作空间无权访问表: sqlite_master", + 502 + ) + ) + }; + const service = createService(providerRouter); + + const draft = await service.stream("一共有多少订单"); + + expect(draft.semanticIntent).toBe("count"); + expect(draft.sql).toBe("SELECT COUNT(*) AS total FROM orders;"); + expect(providerRouter.stream).toHaveBeenCalledTimes(1); + expect(providerRouter.generate).toHaveBeenCalledTimes(1); + }); + it("prioritizes structured semantic instructions when context pack is provided", async () => { const providerRouter = { generate: jest.fn().mockResolvedValue({ diff --git a/apps/backend/test/unit/sql-prompt.builder.spec.ts b/apps/backend/test/unit/sql-prompt.builder.spec.ts index dc3d839..631cd76 100644 --- a/apps/backend/test/unit/sql-prompt.builder.spec.ts +++ b/apps/backend/test/unit/sql-prompt.builder.spec.ts @@ -30,6 +30,9 @@ describe("SqlPromptBuilder", () => { expect(prompt.systemPrompt).toContain("business count-intent query"); expect(prompt.systemPrompt).toContain("must contain COUNT(...) aggregation"); expect(prompt.systemPrompt).toContain("Do not return schema/metadata introspection SQL."); + expect(prompt.systemPrompt).toContain( + "Do not query schema system tables via tools" + ); }); it("adds explicit metadata-intent guardrail instructions", () => { diff --git a/apps/frontend/src/app/data-sources/page.tsx b/apps/frontend/src/app/data-sources/page.tsx index 43441a0..5e87779 100644 --- a/apps/frontend/src/app/data-sources/page.tsx +++ b/apps/frontend/src/app/data-sources/page.tsx @@ -83,6 +83,7 @@ type WizardState = { uploadedDatasourceId: string; type: WizardType | ""; name: string; + filePath: string; host: string; port: string; database: string; @@ -115,6 +116,7 @@ const WIZARD_TYPES: Array<{ }> = [ { type: "csv", title: "本地 CSV", description: "上传 CSV 并生成可复用数据源" }, { type: "excel", title: "本地 Excel", description: "上传 Excel 并生成可复用数据源" }, + { type: "sqlite", title: "SQLite", description: "手动输入 SQLite 绝对路径并创建连接" }, { type: "mysql", title: "MySQL", description: "配置连接信息并创建连接" }, { type: "postgresql", title: "PostgreSQL", description: "配置连接信息并创建连接" } ]; @@ -220,6 +222,7 @@ function createEmptyWizardState(defaultWorkspaceId: string): WizardState { uploadedDatasourceId: "", type: "", name: "", + filePath: "", host: "127.0.0.1", port: "", database: "", @@ -297,6 +300,22 @@ function readConfigString(config: Datasource["config"], key: string): string { return ""; } +function isFakeBrowserPath(filePath: string): boolean { + return /^([a-zA-Z]:)?[\\/]+fakepath[\\/]+/i.test(filePath.trim()); +} + +function isAbsolutePath(input: string): boolean { + const normalized = input.trim(); + if (!normalized) { + return false; + } + return ( + normalized.startsWith("/") || + normalized.startsWith("\\\\") || + /^[a-zA-Z]:[\\/]/.test(normalized) + ); +} + function resolveWizardFailure(error: unknown): WizardFailure { let message = error instanceof Error ? error.message : "提交失败"; let code: string | undefined; @@ -728,6 +747,7 @@ function DataSourcesPageContent() { uploadedDatasourceId: datasource.id, type: toWizardType(datasource.type), name: datasource.name, + filePath: readConfigString(datasource.config, "path"), host: readConfigString(datasource.config, "host"), port: defaultPort, database: readConfigString(datasource.config, "database"), @@ -789,6 +809,22 @@ function DataSourcesPageContent() { return ""; } + if (wizard.type === "sqlite") { + if (!wizard.name.trim()) { + return "请输入数据源名称"; + } + if (wizard.mode === "create" && !wizard.filePath.trim()) { + return "请输入 SQLite 文件绝对路径"; + } + if (wizard.filePath.trim() && !isAbsolutePath(wizard.filePath)) { + return "SQLite 路径必须是绝对路径（例如 /Users/name/data/demo.db）。"; + } + if (isFakeBrowserPath(wizard.filePath)) { + return "检测到 fakepath，请手动输入服务可访问的真实绝对路径。"; + } + return ""; + } + if (wizard.mode === "create" && (wizard.type === "csv" || wizard.type === "excel") && !wizard.file) { return "请先选择 CSV/Excel 文件"; } @@ -817,6 +853,13 @@ function DataSourcesPageContent() { payload.shared = true; } + if (wizard.type === "sqlite") { + const filePath = wizard.filePath.trim(); + if (filePath) { + payload.filePath = filePath; + } + } + if (wizard.type === "mysql" || wizard.type === "postgresql") { payload.host = wizard.host.trim(); payload.port = wizard.port ? Number(wizard.port) : undefined; @@ -1180,12 +1223,16 @@ function DataSourcesPageContent() { - {item.type === "mysql" || item.type === "postgresql" ? ( + {item.type === "mysql" || + item.type === "postgresql" || + item.type === "sqlite" ? ( ) : ( @@ -1194,7 +1241,11 @@ function DataSourcesPageContent() {

{item.title}

- {item.type === "mysql" || item.type === "postgresql" ? "数据库连接" : "文件上传"} + {item.type === "mysql" || + item.type === "postgresql" || + item.type === "sqlite" + ? "数据库连接" + : "文件上传"}

@@ -1225,6 +1276,26 @@ function DataSourcesPageContent() { /> + {wizard.type === "sqlite" && ( + + )} + {(wizard.type === "mysql" || wizard.type === "postgresql") && (