diff --git a/cli/src/modelable/llm/conversation_engine.py b/cli/src/modelable/llm/conversation_engine.py index be7f01a..a80f192 100644 --- a/cli/src/modelable/llm/conversation_engine.py +++ b/cli/src/modelable/llm/conversation_engine.py @@ -1,5 +1,9 @@ from __future__ import annotations +from collections.abc import Callable +from dataclasses import dataclass +from uuid import uuid4 + from modelable.llm.conversation_backend import ConversationBackend, ConversationReply from modelable.llm.conversation_plan import ( ChangeSetPlan, @@ -16,10 +20,25 @@ ResumableConversationPlanner, parse_compile_command, ) +from modelable.llm.provider_types import LLMRequest type ConversationOutcome = ConversationReply | PendingPlanRequest +_ANSWER_SYSTEM_PROMPT = """You are Modelable's conversational assistant. +Answer the user's question using only the workspace facts provided below. +Be concise and factual. Use markdown formatting where it helps readability: +bullet lists for multiple points, inline code for field and type names, and bold for emphasis. +If the facts do not contain enough information to answer, say what is missing.""" + + +@dataclass(frozen=True) +class _PendingSynthesis: + request_id: str + message: str + fallback_text: str + + class ConversationEngine: def __init__( self, @@ -28,22 +47,28 @@ def __init__( planner: ResumableConversationPlanner, focused_ref: str | None = None, completion_enabled: bool = True, + id_factory: Callable[[], str] | None = None, ) -> None: self.backend = backend self.planner = planner self.focused_ref = focused_ref self.completion_enabled = completion_enabled + self.id_factory = id_factory or (lambda: str(uuid4())) self.history: list[tuple[str, str]] = [] self.pending_action_id: str | None = None self.pending_operation_kind: str | None = None self._pending_change_plan: ChangeSetPlan | None = None self._pending_request_id: str | None = None self._pending_message: str | None = None + self._pending_synthesis: _PendingSynthesis | None = None def begin_turn(self, message: str) -> ConversationOutcome: - if self._pending_request_id is not None: + if self._pending_request_id is not None or self._pending_synthesis is not None: + pending_id = self._pending_request_id or ( + self._pending_synthesis.request_id if self._pending_synthesis is not None else None + ) raise PlanningRequestError( - f"Planning request {self._pending_request_id} must be resumed or reset before starting another turn" + f"Planning request {pending_id} must be resumed or reset before starting another turn" ) normalized = message.strip() lowered = normalized.lower() @@ -100,6 +125,8 @@ def begin_turn(self, message: str) -> ConversationOutcome: return self._complete_plan(normalized, outcome) def resume_turn(self, request_id: str, content: str) -> ConversationOutcome: + if self._pending_synthesis is not None and request_id == self._pending_synthesis.request_id: + return self._complete_synthesis(content) if request_id != self._pending_request_id or self._pending_message is None: raise PlanningRequestError(f"Unknown or completed planning request: {request_id}") message = self._pending_message @@ -112,13 +139,17 @@ def resume_turn(self, request_id: str, content: str) -> ConversationOutcome: return self._complete_plan(message, outcome) def fail_turn(self, request_id: str, error: Exception) -> ConversationReply: + if self._pending_synthesis is not None and request_id == self._pending_synthesis.request_id: + return self._complete_synthesis_error() if request_id != self._pending_request_id or self._pending_message is None: raise PlanningRequestError(f"Unknown or completed planning request: {request_id}") message = self._pending_message plan = self.planner.fail(request_id, error) self._pending_request_id = None self._pending_message = None - return self._complete_plan(message, plan) + reply = self._complete_plan(message, plan) + assert isinstance(reply, ConversationReply) + return reply def apply(self, action_id: str) -> ConversationReply: self._assert_pending_action(action_id) @@ -139,6 +170,7 @@ def reset(self) -> None: self._pending_change_plan = None self._pending_request_id = None self._pending_message = None + self._pending_synthesis = None def synchronize_pending_action( self, @@ -153,7 +185,10 @@ def synchronize_pending_action( def record_completed_reply(self, message: str, reply: ConversationReply) -> ConversationReply: return self._complete_turn(message, reply) - def _complete_plan(self, message: str, plan: ConversationPlan) -> ConversationReply: + def _complete_plan(self, message: str, plan: ConversationPlan) -> ConversationOutcome: + if isinstance(plan, QueryPlan) and self.completion_enabled and _is_conversational(message): + reply = self.backend.execute_query(plan) + return self._begin_synthesis(message, reply.text) reply = self._execute_plan(plan) return self._complete_turn(message, reply) @@ -225,3 +260,66 @@ def _complete_turn(self, message: str, reply: ConversationReply) -> Conversation self.history.append(("user", message)) self.history.append(("assistant", reply.text)) return reply + + def _begin_synthesis(self, message: str, facts: str) -> PendingPlanRequest: + request_id = self.id_factory() + self._pending_synthesis = _PendingSynthesis( + request_id=request_id, + message=message, + fallback_text=facts, + ) + return PendingPlanRequest( + request_id=request_id, + request=_synthesis_request( + message=message, + facts=facts, + focused_ref=self.focused_ref, + history=self.history, + ), + attempt=0, + ) + + def _complete_synthesis(self, content: str) -> ConversationReply: + synthesis = self._pending_synthesis + if synthesis is None: + raise PlanningRequestError("No pending synthesis request") + self._pending_synthesis = None + text = content.strip() or synthesis.fallback_text + return self._complete_turn(synthesis.message, ConversationReply(kind="answer", text=text)) + + def _complete_synthesis_error(self) -> ConversationReply: + synthesis = self._pending_synthesis + if synthesis is None: + raise PlanningRequestError("No pending synthesis request") + self._pending_synthesis = None + return self._complete_turn( + synthesis.message, + ConversationReply(kind="answer", text=synthesis.fallback_text), + ) + + +def _is_conversational(message: str) -> bool: + normalized = message.strip().lower() + return not (normalized.startswith("/context") or normalized.startswith("/describe")) + + +def _synthesis_request( + *, + message: str, + facts: str, + focused_ref: str | None, + history: list[tuple[str, str]], +) -> LLMRequest: + lines = ["Workspace facts:", facts] + if focused_ref is not None: + lines.append(f"\nFocused reference: {focused_ref}") + lines.append(f"\nUser question:\n{message}") + if history: + lines.append("\nConversation history:") + lines.extend(f"{role}: {text}" for role, text in history[-6:]) + return LLMRequest( + system=_ANSWER_SYSTEM_PROMPT, + user="\n".join(lines), + temperature=0.2, + response_format="text", + ) diff --git a/cli/tests/support/conversation_simulator.py b/cli/tests/support/conversation_simulator.py index 5ca3613..0222eb3 100644 --- a/cli/tests/support/conversation_simulator.py +++ b/cli/tests/support/conversation_simulator.py @@ -21,6 +21,12 @@ def complete(self, request: LLMRequest) -> LLMResponse: self.requests.append(request) if self.failure is not None: raise RuntimeError(self.failure) + if request.response_format == "text": + return LLMResponse( + content=_answer_for(request.user), + provider="simulator", + model="semantic-v1", + ) if ( self.malformed_first and len(self.requests) == 1 @@ -45,6 +51,19 @@ def _user_request(prompt: str) -> str: return match.group("message").strip() if match is not None else prompt.strip() +def _answer_for(user_prompt: str) -> str: + facts_match = re.search( + r"Workspace facts:\n(?P.*?)\n\nUser question:\n(?P.*?)$", + user_prompt, + flags=re.DOTALL, + ) + if facts_match is None: + return "Here is what I found in the workspace." + facts = facts_match.group("facts").strip() + question = facts_match.group("question").strip() + return f"Based on the workspace facts, here is the answer to '{question}':\n\n{facts}" + + def _plan_for(message: str) -> dict[str, object]: normalized = message.lower() if "describe the workspace" in normalized: diff --git a/cli/tests/test_conversation.py b/cli/tests/test_conversation.py index 5dbd132..c8f793d 100644 --- a/cli/tests/test_conversation.py +++ b/cli/tests/test_conversation.py @@ -1,4 +1,5 @@ import dataclasses +import re from pathlib import Path import pytest @@ -47,6 +48,18 @@ def __init__(self, *plans: ConversationPlan) -> None: def complete(self, request: LLMRequest) -> LLMResponse: self.requests.append(request) + if request.response_format == "text": + facts_match = re.search( + r"Workspace facts:\n(?P.*?)\n\nUser question:\n", + request.user, + flags=re.DOTALL, + ) + facts = facts_match.group("facts").strip() if facts_match else "workspace facts" + return LLMResponse( + content=f"Here is the answer:\n\n{facts}", + provider="fake", + model="test-model", + ) return LLMResponse( content=self.plans.pop(0).model_dump_json(), provider="fake", diff --git a/cli/tests/test_conversation_engine.py b/cli/tests/test_conversation_engine.py index d46164c..3a9dd9a 100644 --- a/cli/tests/test_conversation_engine.py +++ b/cli/tests/test_conversation_engine.py @@ -47,11 +47,13 @@ class RecordingBackend: discarded_ids: list[str] = field(default_factory=list) reset_calls: int = 0 next_action: int = 1 + execute_query_called: bool = False def workspace_summary(self) -> str: return "domain customer\n owner: customer-team" def execute_query(self, plan: QueryPlan) -> ConversationReply: + self.execute_query_called = True return ConversationReply(kind="answer", text=f"query:{plan.query_kind}") def preview_source_change( @@ -241,3 +243,54 @@ def test_semantic_simulator_drives_typed_conversation_planner() -> None: assert isinstance(plan, ChangeSetPlan) assert plan.operations[0].kind == "create_model" + + +def test_engine_synthesizes_conversational_query_with_provider() -> None: + engine, backend = engine_with_request_ids("synthesis-1") + + pending = engine.begin_turn("describe the workspace") + + assert isinstance(pending, PendingPlanRequest) + assert pending.request.response_format == "text" + assert "Workspace facts:" in pending.request.user + reply = engine.resume_turn(pending.request_id, "A concise explanation of the workspace.") + + assert reply.kind == "answer" + assert reply.text == "A concise explanation of the workspace." + assert engine.history[-1] == ("assistant", reply.text) + assert backend.execute_query_called + + +def test_engine_keeps_slash_describe_deterministic() -> None: + engine, backend = engine_with_request_ids("unused") + + reply = engine.begin_turn("/describe customer.Customer@1") + + assert isinstance(reply, ConversationReply) + assert reply.kind == "answer" + assert reply.text == "query:summary" + assert backend.execute_query_called + + +def test_engine_synthesis_failure_falls_back_to_facts() -> None: + engine, backend = engine_with_request_ids("synthesis-1") + + pending = engine.begin_turn("describe the workspace") + + assert isinstance(pending, PendingPlanRequest) + reply = engine.fail_turn(pending.request_id, RuntimeError("provider unavailable")) + + assert reply.kind == "answer" + assert reply.text == "query:summary" + assert backend.execute_query_called + + +def test_engine_synthesis_empty_content_falls_back_to_facts() -> None: + engine, backend = engine_with_request_ids("synthesis-1") + + pending = engine.begin_turn("describe the workspace") + reply = engine.resume_turn(pending.request_id, " ") + + assert reply.kind == "answer" + assert reply.text == "query:summary" + assert backend.execute_query_called diff --git a/web/src/ai/simulator-provider.test.ts b/web/src/ai/simulator-provider.test.ts index efbca1c..1f26077 100644 --- a/web/src/ai/simulator-provider.test.ts +++ b/web/src/ai/simulator-provider.test.ts @@ -70,4 +70,25 @@ describe('SimulatorProvider', () => { expect(projection.operations[0].kind).toBe('create_projection'); expect(clarification.kind).toBe('clarification'); }); + + test('returns a grounded prose answer for text synthesis requests', async () => { + const provider = new SimulatorProvider(); + + const response = await provider.complete({ + system: 'Answer using the workspace facts.', + user: [ + 'Workspace facts:', + 'domain customer\n entity Customer @ 1', + '', + 'User question:', + 'Describe the focused definition or workspace', + ].join('\n'), + temperature: 0.2, + responseFormat: 'text', + }); + + expect(response.content).toContain('Based on the workspace facts'); + expect(response.content).toContain('entity Customer @ 1'); + expect(response.provider).toBe('simulator'); + }); }); diff --git a/web/src/ai/simulator-provider.ts b/web/src/ai/simulator-provider.ts index 5872f20..1a16154 100644 --- a/web/src/ai/simulator-provider.ts +++ b/web/src/ai/simulator-provider.ts @@ -19,6 +19,13 @@ export class SimulatorProvider implements LlmProvider { if (this.options.failure !== undefined) { throw new Error(this.options.failure); } + if (request.responseFormat === 'text') { + return { + content: answerFor(request.user), + provider: this.id, + model: this.model, + }; + } const isRepair = request.user.includes( 'Previous response validation error', ); @@ -43,6 +50,17 @@ function userRequest(prompt: string): string { return (match?.[1] ?? prompt).trim(); } +function answerFor(userPrompt: string): string { + const match = /Workspace facts:\n([\s\S]*?)\n\nUser question:\n([\s\S]*?)$/u + .exec(userPrompt); + if (match === null) { + return 'Here is what I found in the workspace.'; + } + const facts = (match[1] ?? '').trim(); + const question = (match[2] ?? '').trim(); + return `Based on the workspace facts, here is the answer to '${question}':\n\n${facts}`; +} + function planFor(message: string): Record { const normalized = message.toLowerCase(); if (normalized.includes('describe the workspace')) {