From f46e88f278690024d2807c48e759d1f5067e074b Mon Sep 17 00:00:00 2001 From: Francesco Scalzo Date: Sat, 21 Mar 2026 17:09:45 +0100 Subject: [PATCH 01/15] feat(learn): optional enrollment for LMS integrations (#42) - Optional enrollment mode for LMS integrations (FEAT-003) - Widget UX: collapsible sources, markdown-ready citations, a11y improvements - Sparse embedding generation during ingest (BUG-011), conversation auto-ID (BUG-010) - Registry None guard in health check, sparse embedding count validation - Combo D RAG tuning defaults, OpenAI-compatible API base, parameterized ports Reviews: 16/16 addressed (CodeRabbit 12, Gemini 1, manual 3) Tests: 578 passed Refs: FEAT-003, BUG-010, BUG-011, DEBT-002, DEBT-003, DEBT-008 --- .env.example | 34 +- .gitignore | 1 + .s2s/BACKLOG.md | 516 +++++++++++++++++- docker-compose.yml | 6 +- vektra-admin/src/vektra_admin/health.py | 46 +- vektra-admin/tests/test_health.py | 26 + vektra-app/src/vektra_app/main.py | 10 +- .../vektra_core/providers/litellm_provider.py | 10 +- vektra-ingest/src/vektra_ingest/pipeline.py | 23 +- vektra-ingest/tests/test_pipeline.py | 1 + vektra-ingest/tests/test_versioning.py | 1 + vektra-learn/src/vektra_learn/api.py | 51 +- vektra-learn/src/vektra_learn/service.py | 9 +- vektra-learn/tests/test_api.py | 263 +++++++++ vektra-learn/tests/test_service.py | 25 + vektra-learn/widget/src/api-client.js | 8 +- vektra-learn/widget/src/chat-ui.js | 63 ++- vektra-learn/widget/src/index.js | 3 + vektra-learn/widget/src/styles.js | 44 +- vektra-shared/src/vektra_shared/config.py | 38 +- vektra-shared/tests/test_config.py | 4 +- 21 files changed, 1119 insertions(+), 63 deletions(-) diff --git a/.env.example b/.env.example index 15493c6d..3147cf15 100644 --- a/.env.example +++ b/.env.example @@ -20,6 +20,11 @@ # Generic LLM API key (passed to litellm) # VEKTRA_LLM_API_KEY= +# Custom API base URL for OpenAI-compatible providers (vLLM, LM Studio, etc.) +# When set, litellm routes requests to this URL instead of the provider default. +# Example: VEKTRA_LLM_API_BASE=http://localhost:8000/v1 +# VEKTRA_LLM_API_BASE= + # Or set provider-specific keys directly: # OPENAI_API_KEY=sk-... # ANTHROPIC_API_KEY=sk-ant-... @@ -53,7 +58,7 @@ # Embedding provider and model. # VEKTRA_EMBEDDING_PROVIDER=sentence-transformers -# VEKTRA_EMBEDDING_MODEL=all-MiniLM-L6-v2 +# VEKTRA_EMBEDDING_MODEL=paraphrase-multilingual-MiniLM-L12-v2 # -------------------------------------------------------------------------- # Vector store @@ -66,10 +71,10 @@ # Query pipeline # -------------------------------------------------------------------------- -# VEKTRA_QUERY_PIPELINE=simple +# VEKTRA_QUERY_PIPELINE=advanced # VEKTRA_MIN_RELEVANCE_SCORE=0.3 # VEKTRA_CHUNK_DEDUP_ENABLED=true -# VEKTRA_RESPONSE_TOKEN_RESERVE=1024 +# VEKTRA_RESPONSE_TOKEN_RESERVE=2048 # VEKTRA_CONTEXT_CHUNK_RATIO=0.6 # VEKTRA_PROMPT_TEMPLATES_DIR= @@ -78,8 +83,8 @@ # -------------------------------------------------------------------------- # VEKTRA_CHUNKING_STRATEGY=fixed -# VEKTRA_CHUNK_SIZE=1000 -# VEKTRA_CHUNK_OVERLAP=200 +# VEKTRA_CHUNK_SIZE=500 +# VEKTRA_CHUNK_OVERLAP=100 # VEKTRA_MAX_FILE_SIZE_MB=50 # VEKTRA_DOCUMENT_EXTRACTOR=pdfplumber @@ -89,7 +94,7 @@ # Fallback model when primary times out. # VEKTRA_LLM_FALLBACK_MODEL= -# VEKTRA_LLM_FALLBACK_TIMEOUT_MS=30000 +# VEKTRA_LLM_FALLBACK_TIMEOUT_MS=60000 # VEKTRA_LLM_CONTEXT_ONLY_ENABLED=true # -------------------------------------------------------------------------- @@ -99,6 +104,11 @@ # VEKTRA_PORT=8000 # VEKTRA_STARTUP_LLM_CHECK=true +# Optional service ports (host-side). Override to avoid port conflicts. +# OLLAMA_PORT=11434 +# QDRANT_PORT=6333 +# TEI_PORT=8080 + # -------------------------------------------------------------------------- # Observability # -------------------------------------------------------------------------- @@ -118,3 +128,15 @@ # VEKTRA_ANALYTICS_RETENTION_DAYS= # VEKTRA_SPARSE_EMBEDDING_PROVIDER= # VEKTRA_SPARSE_EMBEDDING_MODEL= + +# -------------------------------------------------------------------------- +# Learn (e-learning vertical) +# -------------------------------------------------------------------------- + +# JWT signing secret for dashboard tokens. Required when vektra-learn is active. +# VEKTRA_LEARN_JWT_SECRET= + +# Require Vektra enrollment record for learn queries. +# Set to false when an external LMS (e.g. Moodle) manages enrollment. +# When false, namespace is derived from JWT (namespace claim or course_id fallback). +# VEKTRA_LEARN_REQUIRE_ENROLLMENT=true diff --git a/.gitignore b/.gitignore index aa952f14..6fdd6674 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ logs/ .env.*.local .claude/settings.local.json CLAUDE.local.md +docker-compose.ports.yml # Build outputs dist/ diff --git a/.s2s/BACKLOG.md b/.s2s/BACKLOG.md index 4b74e8fe..e38b0ab2 100644 --- a/.s2s/BACKLOG.md +++ b/.s2s/BACKLOG.md @@ -462,14 +462,528 @@ Plan generation follows a three-phase approach (lesson learned from Phase 1): --- +### FEAT-005: LLM fallback for no-context queries (greetings, courtesies, off-topic) + +**Status**: draft | **Priority**: medium | **Created**: 2026-03-17 +**Origin**: Moodle integration testing — greetings like "ciao", "buongiorno" trigger `no_relevant_context` and produce empty/unhelpful responses + +**Context**: Both `SimpleQueryPipeline` and `AdvancedQueryPipeline` short-circuit when no chunks pass the relevance threshold (`min_relevance_score`): they return `answer: null` + `no_relevant_context: true` without ever calling the LLM. This is correct for retrieval quality (ARCH-056, REQ-066) — the system should not hallucinate answers from non-relevant chunks. + +However, for the learn chatbot widget (and any conversational interface), this creates a poor UX for: +- **Greetings and courtesies**: "ciao", "buongiorno", "come stai?" — the assistant should acknowledge and redirect to course materials +- **Meta questions**: "chi sei?", "cosa puoi fare?" — the assistant should explain its role +- **Off-topic but harmless**: "che ore sono?" — the assistant should politely decline + +The existing **query rewriting** (ADR-0023, `AdvancedQueryPipeline`) only activates when conversation history exists and resolves pronoun references — it does not help with greetings. + +The existing **SafeguardHook** (`pre_query`, `post_retrieval`, `pre_response`) could catch prompt injection and abusive content at the `pre_query` stage, but with `VEKTRA_SAFEGUARD_MODE=passthrough` they are no-ops. Even with safeguards active, they filter/block — they don't generate friendly responses. + +**Proposed approach**: When `no_relevant_context` is detected, instead of short-circuiting, invoke the LLM with a modified system prompt that instructs it to respond to greetings, explain its role, and redirect to course-related questions — without fabricating information from missing context. This keeps the retrieval quality gate intact while allowing the LLM to handle conversational basics. + +**Alternatives considered**: +- **Intent classification pre-retrieval**: separate LLM call to classify intent before retrieval. More precise but adds latency and cost for every query. +- **Client-side pattern matching**: widget detects greetings and responds locally. Fragile, language-dependent, doesn't help other clients. + +**Traceability**: ARCH-056, REQ-066, ADR-0021, ADR-0025 + +**Acceptance Criteria** (tentative): +- [ ] Greetings/courtesies receive a friendly response acknowledging the user and explaining the assistant's role +- [ ] Off-topic queries receive a polite redirect to course-related questions +- [ ] The LLM is NOT given fabricated context — it knows no relevant chunks were found +- [ ] Retrieval quality gate unchanged — `no_relevant_context` flag still set in QueryTrace +- [ ] Safeguard hooks still apply (pre_query can block before LLM call) +- [ ] Works with both SimpleQueryPipeline and AdvancedQueryPipeline +- [ ] System prompt for no-context fallback is configurable via Jinja2 template + +--- + +### FEAT-007: Markdown rendering in widget chat messages + +**Status**: draft | **Priority**: medium | **Created**: 2026-03-20 +**Origin**: Moodle integration testing (2026-03-20) + +**Context**: The learn chatbot widget (`vektra-chat.js`) renders all messages as plain text via `textContent`. LLM responses typically contain Markdown formatting (bold, italic, lists, code blocks, headings) which is displayed as raw syntax. This makes responses harder to read, especially for structured answers with bullet points or code examples. + +The widget is deliberately vanilla JS with zero dependencies (ADR-0025). Adding Markdown rendering requires either a lightweight library (e.g., `marked`, ~7KB minified) or a minimal custom parser for the most common patterns. + +**Scope**: only assistant messages need rendering. User messages stay as plain text. Sources section is already structured HTML. + +**Security**: rendered HTML must be sanitized to prevent XSS. The LLM output is not user-controlled but defense in depth applies. Use a sanitizer or restrict to a safe subset of HTML tags. + +**Traceability**: ADR-0025, ARCH-063 + +**Acceptance Criteria** (tentative): +- [ ] Assistant messages render bold, italic, lists (ordered/unordered), code inline/blocks, and headings +- [ ] User messages remain plain text +- [ ] Streaming tokens render progressively (Markdown applied incrementally or on completion) +- [ ] Output is sanitized against XSS +- [ ] Widget bundle size increase is documented and reasonable (<10KB) + +--- + +### FEAT-008: Per-namespace prompt template customization + +**Status**: draft | **Priority**: medium | **Created**: 2026-03-20 +**Origin**: Moodle integration testing - generic system prompt not suitable for diverse course contexts + +**Context**: The current prompt template system (ARCH-054, ADR-0020) supports only global customization via `VEKTRA_PROMPT_TEMPLATES_DIR`. All namespaces share the same system.j2, context.j2, and conversation.j2. This is limiting for the e-learning vertical where each course (namespace) may have different needs: + +- A professor wants a specific greeting or tone ("you are the teaching assistant for Advanced Calculus, taught by Prof. Rossi") +- A course requires answers in a specific language regardless of the student's UI language +- Some courses want the assistant to refuse certain question types (e.g., "do not solve exercises directly, guide the student step by step") +- A course may need domain-specific instructions ("when discussing legal cases, always cite the article number") +- Info about the course, the professor, office hours, exam dates, etc. + +**Proposed approach**: Two complementary sources of template variables, plus per-namespace template override. + +### Template variable sources + +**A. Dynamic metadata via JWT claims (preferred for LMS integrations)**: +The upstream system (Moodle, or any LMS/application) passes metadata in the token generation request. Vektra includes them as JWT claims. At query time, the pipeline extracts the claims and injects them as Jinja2 variables. This requires no storage in Vektra - data comes from the source system at each page load and is always up to date. + +Flow: `LMS page load -> read course/instructor info -> POST /learn/tokens { ..., metadata: { course_name, instructor, ... } } -> JWT claims -> query -> pipeline extracts claims -> template variables` + +This is LMS-agnostic: any system that calls the token endpoint can pass arbitrary key-value metadata. Moodle, Canvas, custom apps - all use the same mechanism. + +**B. Static metadata in Vektra database (fallback for non-LMS use cases)**: +For namespaces not backed by an LMS (e.g., standalone knowledge bases, internal tools), metadata is stored in the namespace entity (ARCH-047) and managed via admin API. The pipeline reads namespace metadata at query time. + +**C. Merge strategy**: dynamic JWT claims take precedence over static DB metadata. Both are merged and passed to the Jinja2 context. A template can use variables from either source transparently. + +### Per-namespace template override + +1. **Resolution order**: namespace-specific template > global override (`VEKTRA_PROMPT_TEMPLATES_DIR`) > built-in default. If a namespace defines only `system.j2`, the global `context.j2` and `conversation.j2` still apply. +2. **Storage**: namespace templates stored in the database (via admin API) or as files in a convention-based directory structure (`{PROMPT_TEMPLATES_DIR}/{namespace}/system.j2`). +3. **Management**: API endpoints for CRUD on namespace prompt templates (admin scope). In the learn vertical, the Moodle plugin or admin UI could expose this to course coordinators. + +### Example + +A Moodle plugin sends metadata at token generation: +```json +{ "student_id": "jdoe", "course_id": "calc-201", "metadata": { + "course_name": "Advanced Calculus", + "instructor": "Prof. Rossi", + "custom_instructions": "Guide students step by step, do not solve exercises directly." +}} +``` + +The namespace `calc-201` has a custom `system.j2`: +```jinja2 +You are the teaching assistant for {{ course_name }}, taught by {{ instructor }}. +{{ custom_instructions }} +Answer based on the provided context. Do not invent information. +``` + +If no custom template exists, the global system.j2 still has access to the same variables (they just won't be referenced unless the template uses them). + +**Not in scope**: per-student templates (per-namespace only). Runtime template editing by students (admin/instructor only). + +**Traceability**: ARCH-054, ADR-0020, ARCH-047 (namespace as first-class entity) + +**Acceptance Criteria** (tentative): +- [ ] Token generation endpoint accepts optional `metadata` dict (arbitrary key-value pairs) +- [ ] Metadata included as JWT claims, extracted at query time +- [ ] Namespace can store static metadata in database (admin API) +- [ ] JWT claims override DB metadata on key collision +- [ ] All metadata available as Jinja2 template variables +- [ ] Namespace can define custom system.j2 that overrides the global one +- [ ] Missing namespace templates fall back to global, then built-in +- [ ] API endpoints for managing namespace prompt templates (admin scope) +- [ ] Existing global `VEKTRA_PROMPT_TEMPLATES_DIR` continues to work unchanged + +--- + +### FEAT-006: Widget error feedback when Vektra API is unreachable + +**Status**: draft | **Priority**: medium | **Created**: 2026-03-20 +**Origin**: Moodle integration testing on remote machine (2026-03-20) + +**Context**: When the chatbot widget JS (`vektra-chat.js`) cannot reach the Vektra API (missing SSH tunnel, CORS misconfiguration, Vektra container down), the floating chat button silently fails to appear. No error is shown to the user or the admin. The Moodle block still displays "AI Assistant is active" because the server-side token generation succeeded (PHP runs inside Docker, reaches Vektra on the internal network), but the browser-side widget cannot load or connect. + +This makes troubleshooting difficult: the admin sees "active" but students see nothing. The root cause (network/CORS/port) is invisible without opening browser dev tools. + +**Proposed approach**: The widget JS should detect load/connection failures and surface them: +- If the script loads but cannot reach the API (fetch error, CORS block): show a subtle error state in the chat button or a dismissible banner +- If the script itself fails to load (network error): the Moodle plugin could add a `