From 12b38f18a3ea2902ed337c49a23d26a249daafec Mon Sep 17 00:00:00 2001 From: CIKR-Repos Date: Tue, 17 Feb 2026 23:00:01 -0500 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20pipeline=20builder=20UI=20=E2=80=94?= =?UTF-8?q?=20visual=20pipeline=20config=20with=20step=20wizard,=20model?= =?UTF-8?q?=20selection,=20status=20monitoring?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/package-lock.json | 21 ++- client/package.json | 1 + .../pipeline-builder/pipeline.service.ts | 165 ++++++++++++++++++ client/src/app/pages/pipeline/pipeline.ts | 97 ++++++++-- client/src/styles.scss | 1 + docs/features/pr-008-pipeline-builder-ui.md | 34 ++++ .../Services/QueryEngineService.cs | 4 +- 7 files changed, 305 insertions(+), 18 deletions(-) create mode 100644 client/src/app/features/pipeline-builder/pipeline.service.ts create mode 100644 docs/features/pr-008-pipeline-builder-ui.md diff --git a/client/package-lock.json b/client/package-lock.json index da014a3..ff97909 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -8,6 +8,7 @@ "name": "client", "version": "0.0.0", "dependencies": { + "@angular/cdk": "^21.1.4", "@angular/common": "^21.1.0", "@angular/compiler": "^21.1.0", "@angular/core": "^21.1.0", @@ -438,6 +439,22 @@ } } }, + "node_modules/@angular/cdk": { + "version": "21.1.4", + "resolved": "https://registry.npmjs.org/@angular/cdk/-/cdk-21.1.4.tgz", + "integrity": "sha512-PElA4Ww4TIa3+B/ND+fm8ZPDKONTIqc9a/s0qNxhcAD9IpDqjaBVi/fyg+ZWBtS+x0DQgJtKeCsSZ6sr2aFQaQ==", + "license": "MIT", + "dependencies": { + "parse5": "^8.0.0", + "tslib": "^2.3.0" + }, + "peerDependencies": { + "@angular/common": "^21.0.0 || ^22.0.0", + "@angular/core": "^21.0.0 || ^22.0.0", + "@angular/platform-browser": "^21.0.0 || ^22.0.0", + "rxjs": "^6.5.3 || ^7.4.0" + } + }, "node_modules/@angular/cli": { "version": "21.1.4", "resolved": "https://registry.npmjs.org/@angular/cli/-/cli-21.1.4.tgz", @@ -6185,7 +6202,6 @@ "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", "dev": true, "license": "MIT", - "peer": true, "bin": { "jiti": "lib/jiti-cli.mjs" } @@ -6321,7 +6337,6 @@ "integrity": "sha512-utfs7Pr5uJyyvDETitgsaqSyjCb2qNRAtuqUeWIAKztsOYdcACf2KtARYXg2pSvhkt+9NfoaNY7fxjl6nuMjIQ==", "dev": true, "license": "MPL-2.0", - "peer": true, "dependencies": { "detect-libc": "^2.0.3" }, @@ -7462,7 +7477,6 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.0.tgz", "integrity": "sha512-9m4m5GSgXjL4AjumKzq1Fgfp3Z8rsvjRNbnkVwfu2ImRqE5D0LnY2QfDen18FSY9C573YU5XxSapdHZTZ2WolA==", - "dev": true, "license": "MIT", "dependencies": { "entities": "^6.0.0" @@ -7516,7 +7530,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", - "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=0.12" diff --git a/client/package.json b/client/package.json index 861ec9b..99ce0ba 100644 --- a/client/package.json +++ b/client/package.json @@ -23,6 +23,7 @@ "private": true, "packageManager": "npm@11.6.2", "dependencies": { + "@angular/cdk": "^21.1.4", "@angular/common": "^21.1.0", "@angular/compiler": "^21.1.0", "@angular/core": "^21.1.0", diff --git a/client/src/app/features/pipeline-builder/pipeline.service.ts b/client/src/app/features/pipeline-builder/pipeline.service.ts new file mode 100644 index 0000000..fd90268 --- /dev/null +++ b/client/src/app/features/pipeline-builder/pipeline.service.ts @@ -0,0 +1,165 @@ +import { Injectable, inject, signal, computed } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { firstValueFrom } from 'rxjs'; + +export interface PipelineConfig { + chunkSize: number; + chunkOverlap: number; + chunkingStrategy: string; + embeddingModel: string; + embeddingDimensions: number; + retrievalStrategy: string; + topK: number; + scoreThreshold: number; + generationModel: string; + temperature: number; + maxTokens: number; + systemPrompt: string; +} + +export interface PipelineResponse { + id: string; + projectId: string; + name: string; + description: string | null; + config: PipelineConfig; + status: string; + createdAt: string; + updatedAt: string | null; +} + +export interface DocumentInfo { + id: string; + fileName: string; + fileType: string; + fileSize: number; + status: string; +} + +export interface ChunkPreviewItem { + content: string; + tokenCount: number; + index: number; +} + +export interface ChunkPreviewResponse { + chunks: ChunkPreviewItem[]; + totalCount: number; + page: number; + pageSize: number; +} + +export interface ModelSelection { + embeddingModel: string; + embeddingDimensions: number; + chatModel: string; + maxTokensPerRequest: number; + maxDocumentsPerProject: number; +} + +const DEFAULT_CONFIG: PipelineConfig = { + chunkSize: 512, + chunkOverlap: 50, + chunkingStrategy: 'recursive', + embeddingModel: 'text-embedding-3-small', + embeddingDimensions: 1536, + retrievalStrategy: 'semantic', + topK: 5, + scoreThreshold: 0.7, + generationModel: 'gpt-4o-mini', + temperature: 0.7, + maxTokens: 2048, + systemPrompt: 'You are a helpful assistant. Answer questions based on the provided context.', +}; + +@Injectable({ providedIn: 'root' }) +export class PipelineService { + private http = inject(HttpClient); + + readonly pipeline = signal(null); + readonly config = signal({ ...DEFAULT_CONFIG }); + readonly savedConfig = signal({ ...DEFAULT_CONFIG }); + readonly documents = signal([]); + readonly models = signal(null); + readonly chunkPreview = signal(null); + readonly isSaving = signal(false); + readonly isLoading = signal(false); + readonly saveError = signal(null); + + readonly hasUnsavedChanges = computed(() => + JSON.stringify(this.config()) !== JSON.stringify(this.savedConfig()) + ); + + async loadPipeline(projectId: string): Promise { + this.isLoading.set(true); + try { + const pipelines = await firstValueFrom( + this.http.get('/api/projects/' + projectId + '/pipelines') + ); + if (pipelines.length > 0) { + const p = pipelines[0]; + this.pipeline.set(p); + const merged = { ...DEFAULT_CONFIG, ...p.config }; + this.config.set(merged); + this.savedConfig.set(merged); + } + try { + const docs = await firstValueFrom( + this.http.get('/api/projects/' + projectId + '/documents') + ); + this.documents.set(docs); + } catch { this.documents.set([]); } + try { + const m = await firstValueFrom(this.http.get('/api/models')); + this.models.set(m); + } catch { /* ignore */ } + } finally { + this.isLoading.set(false); + } + } + + async savePipeline(projectId: string): Promise { + const p = this.pipeline(); + if (!p) return; + this.isSaving.set(true); + this.saveError.set(null); + try { + const updated = await firstValueFrom( + this.http.put( + '/api/projects/' + projectId + '/pipelines/' + p.id, + { name: p.name, description: p.description, config: this.config() } + ) + ); + this.pipeline.set(updated); + const merged = { ...DEFAULT_CONFIG, ...updated.config }; + this.savedConfig.set(merged); + this.config.set(merged); + } catch (err: any) { + this.saveError.set(err?.error?.error ?? 'Failed to save pipeline'); + throw err; + } finally { + this.isSaving.set(false); + } + } + + resetDefaults(): void { + this.config.set({ ...DEFAULT_CONFIG }); + } + + updateConfig(partial: Partial): void { + this.config.update(c => ({ ...c, ...partial })); + } + + async loadChunkPreview(projectId: string, documentId: string): Promise { + try { + const cfg = this.config(); + const preview = await firstValueFrom( + this.http.get( + '/api/projects/' + projectId + '/documents/' + documentId + '/chunks', + { params: { chunkSize: cfg.chunkSize.toString(), chunkOverlap: cfg.chunkOverlap.toString(), page: '1', pageSize: '3' } } + ) + ); + this.chunkPreview.set(preview); + } catch { this.chunkPreview.set(null); } + } +} diff --git a/client/src/app/pages/pipeline/pipeline.ts b/client/src/app/pages/pipeline/pipeline.ts index c31bbb4..354e497 100644 --- a/client/src/app/pages/pipeline/pipeline.ts +++ b/client/src/app/pages/pipeline/pipeline.ts @@ -1,17 +1,90 @@ -import { Component } from '@angular/core'; -import { NavbarComponent } from '../../shared/components/navbar/navbar'; +import { Component, inject, OnInit } from '@angular/core'; +import { ActivatedRoute } from '@angular/router'; +import { CdkDropList, CdkDrag, CdkDragDrop, CdkDragHandle, CdkDragPlaceholder } from '@angular/cdk/drag-drop'; +import { PipelineService, PipelineBlock, BlockType } from './pipeline.service'; + +const BLOCK_COLORS: Record = { + source: { border: 'border-blue-300', bg: 'bg-blue-50', iconBg: 'bg-blue-100', badge: 'text-blue-700' }, + chunking: { border: 'border-amber-300', bg: 'bg-amber-50', iconBg: 'bg-amber-100', badge: 'text-amber-700' }, + embedding: { border: 'border-purple-300', bg: 'bg-purple-50', iconBg: 'bg-purple-100', badge: 'text-purple-700' }, + retrieval: { border: 'border-emerald-300', bg: 'bg-emerald-50', iconBg: 'bg-emerald-100', badge: 'text-emerald-700' }, + generation: { border: 'border-rose-300', bg: 'bg-rose-50', iconBg: 'bg-rose-100', badge: 'text-rose-700' }, +}; @Component({ selector: 'app-pipeline', standalone: true, - imports: [NavbarComponent], - template: ` - -
-
🔧
-

Pipeline Builder

-

Visual pipeline configuration coming soon.

-
- `, + imports: [CdkDropList, CdkDrag, CdkDragHandle, CdkDragPlaceholder], + providers: [PipelineService], + templateUrl: './pipeline.html', }) -export class PipelineComponent {} +export class PipelineComponent implements OnInit { + private route = inject(ActivatedRoute); + readonly svc: PipelineService = inject(PipelineService); + private projectId = ''; + + ngOnInit() { + this.projectId = this.route.snapshot.paramMap.get('projectId') ?? ''; + if (this.projectId) { + this.svc.loadPipeline(this.projectId); + } + } + + onDrop(event: CdkDragDrop) { + if (event.previousIndex !== event.currentIndex) { + this.svc.reorderBlocks(event.previousIndex, event.currentIndex); + } + } + + save() { + if (this.projectId) { + this.svc.savePipeline(this.projectId); + } + } + + blockClass(block: PipelineBlock): string { + const c = BLOCK_COLORS[block.type]; + const selected = this.svc.selectedBlockId() === block.id; + return `${c.border} ${c.bg} ${selected ? 'ring-2 ring-indigo-500 shadow-md' : ''}`; + } + + blockIconBg(block: PipelineBlock): string { + return BLOCK_COLORS[block.type].iconBg; + } + + blockTypeBadge(block: PipelineBlock): string { + return BLOCK_COLORS[block.type].badge; + } + + blockSummary(block: PipelineBlock): string { + const c = block.config; + switch (block.type) { + case 'source': return `${c['sourceType']} · ${c['fileTypes']}`; + case 'chunking': return `${c['strategy']} · ${c['chunkSize']} chars · ${c['chunkOverlap']} overlap`; + case 'embedding': return `${c['model']} · ${c['dimensions']}d`; + case 'retrieval': return `${c['strategy']} · top ${c['topK']} · threshold ${c['scoreThreshold']}`; + case 'generation': return `${c['model']} · temp ${c['temperature']} · ${c['maxTokens']} tokens`; + default: return ''; + } + } + + onConfigChange(blockId: string, key: string, event: Event) { + const el = event.target as HTMLInputElement | HTMLSelectElement | HTMLTextAreaElement; + this.svc.updateBlockConfig(blockId, key, el.value); + } + + onConfigChangeNum(blockId: string, key: string, event: Event) { + const el = event.target as HTMLInputElement; + this.svc.updateBlockConfig(blockId, key, parseInt(el.value, 10)); + } + + onConfigChangeFloat(blockId: string, key: string, event: Event) { + const el = event.target as HTMLInputElement; + this.svc.updateBlockConfig(blockId, key, parseFloat(el.value)); + } + + onConfigChangeBool(blockId: string, key: string, event: Event) { + const el = event.target as HTMLInputElement; + this.svc.updateBlockConfig(blockId, key, el.checked); + } +} diff --git a/client/src/styles.scss b/client/src/styles.scss index cea3024..cc656a3 100644 --- a/client/src/styles.scss +++ b/client/src/styles.scss @@ -1,3 +1,4 @@ +@import "tailwindcss"; @import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&display=swap'); *, *::before, *::after { diff --git a/docs/features/pr-008-pipeline-builder-ui.md b/docs/features/pr-008-pipeline-builder-ui.md new file mode 100644 index 0000000..26d95b8 --- /dev/null +++ b/docs/features/pr-008-pipeline-builder-ui.md @@ -0,0 +1,34 @@ +# PR #8 — Pipeline Builder UI + +## Overview +Visual pipeline builder with card-based interface for configuring RAG pipeline stages: +Source → Chunking → Embedding → Retrieval → Generation. + +## Components +- **PipelineBuilderComponent** — `client/src/app/features/pipeline-builder/pipeline-builder.component.ts` +- **PipelineService** — `client/src/app/features/pipeline-builder/pipeline.service.ts` + +## Pipeline Blocks +| Block | Settings | +|-------|----------| +| Source | Document list, file types, total size | +| Chunking | Chunk size (128-2048), overlap (0-200), strategy (recursive/sentence) | +| Embedding | Model selector (tier-gated), dimension display | +| Retrieval | Strategy (semantic/hybrid), top-k (1-20), score threshold (0-1) | +| Generation | Model (tier-gated), temperature, max tokens, system prompt | + +## Chunk Preview Panel +Side panel showing real-time chunking preview with chunk count, avg tokens, and first 3 chunks. + +## API Endpoints Used +- `GET /api/projects/{id}/pipelines` — Load config +- `PUT /api/projects/{id}/pipelines/{pipelineId}` — Save config +- `GET /api/projects/{id}/documents` — Document list +- `GET /api/projects/{id}/documents/{docId}/chunks` — Chunk preview +- `GET /api/models` — Tier-gated models + +## Backend Changes +Extended `PipelineConfigDto` with: ChunkingStrategy, EmbeddingDimensions, GenerationModel, Temperature, MaxTokens, SystemPrompt. + +## Route +`/projects/:projectId/pipeline` diff --git a/src/PipeRAG.Infrastructure/Services/QueryEngineService.cs b/src/PipeRAG.Infrastructure/Services/QueryEngineService.cs index 4a1bee5..c537c2f 100644 --- a/src/PipeRAG.Infrastructure/Services/QueryEngineService.cs +++ b/src/PipeRAG.Infrastructure/Services/QueryEngineService.cs @@ -63,7 +63,7 @@ public async Task QueryAsync( var kernel = BuildChatKernel(models.ChatModel); var chatService = kernel.GetRequiredService(); - var chatHistory = BuildChatHistory(conversationHistory, sources); + var chatHistory = BuildChatHistory(conversationHistory, sources, userMessage); var result = await chatService.GetChatMessageContentAsync(chatHistory, cancellationToken: ct); var responseText = result.Content ?? string.Empty; var tokensUsed = responseText.Length / 4; @@ -92,7 +92,7 @@ public async IAsyncEnumerable QueryStreamAsync( var conversationHistory = await _memory.GetConversationWindowAsync(sessionId, ct: ct); var kernel = BuildChatKernel(models.ChatModel); var chatService = kernel.GetRequiredService(); - var chatHistory = BuildChatHistory(conversationHistory, sources); + var chatHistory = BuildChatHistory(conversationHistory, sources, userMessage); var fullResponse = new StringBuilder(); From cca29d91ff9f4edafeca5eca2c4d7c1139428eb2 Mon Sep 17 00:00:00 2001 From: CIKR-Repos Date: Tue, 17 Feb 2026 23:03:09 -0500 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20address=20Sourcery=20review=20?= =?UTF-8?q?=E2=80=94=20import=20path,=20provider,=20NaN=20guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pipeline-block.model.spec.ts | 93 ++++ .../pipeline-builder/pipeline.service.ts | 274 +++++----- client/src/app/pages/pipeline/pipeline.html | 466 ++++++++++++++++++ client/src/app/pages/pipeline/pipeline.ts | 76 ++- client/src/styles.scss | 2 +- docs/features/pr-008-pipeline-builder-ui.md | 27 +- 6 files changed, 766 insertions(+), 172 deletions(-) create mode 100644 client/src/app/features/pipeline-builder/pipeline-block.model.spec.ts create mode 100644 client/src/app/pages/pipeline/pipeline.html diff --git a/client/src/app/features/pipeline-builder/pipeline-block.model.spec.ts b/client/src/app/features/pipeline-builder/pipeline-block.model.spec.ts new file mode 100644 index 0000000..6349cd8 --- /dev/null +++ b/client/src/app/features/pipeline-builder/pipeline-block.model.spec.ts @@ -0,0 +1,93 @@ +import { defaultBlocks, getBlockFields, BLOCK_ICONS, BLOCK_LABELS, BlockType } from './pipeline-block.model'; + +describe('PipelineBlock Model', () => { + describe('defaultBlocks', () => { + it('should return 5 blocks', () => { + expect(defaultBlocks().length).toBe(5); + }); + + it('should have correct order: source → chunking → embedding → retrieval → generation', () => { + const types = defaultBlocks().map(b => b.type); + expect(types).toEqual(['source', 'chunking', 'embedding', 'retrieval', 'generation']); + }); + + it('should have unique IDs', () => { + const ids = defaultBlocks().map(b => b.id); + expect(new Set(ids).size).toBe(5); + }); + + it('should have default config values for chunking', () => { + const chunking = defaultBlocks().find(b => b.type === 'chunking')!; + expect(chunking.config['chunkSize']).toBe(512); + expect(chunking.config['chunkOverlap']).toBe(50); + expect(chunking.config['strategy']).toBe('recursive'); + }); + + it('should have default config values for embedding', () => { + const embedding = defaultBlocks().find(b => b.type === 'embedding')!; + expect(embedding.config['model']).toBe('text-embedding-3-small'); + expect(embedding.config['dimensions']).toBe(1536); + }); + + it('should have default config values for generation', () => { + const gen = defaultBlocks().find(b => b.type === 'generation')!; + expect(gen.config['model']).toBe('gpt-4o-mini'); + expect(gen.config['temperature']).toBe(0.7); + expect(gen.config['maxTokens']).toBe(2048); + }); + + it('should return independent copies each call', () => { + const a = defaultBlocks(); + const b = defaultBlocks(); + a[0].config['test'] = 'modified'; + expect(b[0].config['test']).toBeUndefined(); + }); + }); + + describe('BLOCK_ICONS', () => { + it('should have icons for all 5 block types', () => { + const types: BlockType[] = ['source', 'chunking', 'embedding', 'retrieval', 'generation']; + for (const t of types) { + expect(BLOCK_ICONS[t]).toBeTruthy(); + } + }); + }); + + describe('BLOCK_LABELS', () => { + it('should have labels for all block types', () => { + expect(BLOCK_LABELS['source']).toBe('Data Source'); + expect(BLOCK_LABELS['generation']).toBe('Generation'); + }); + }); + + describe('getBlockFields', () => { + it('should return fields for chunking block', () => { + const fields = getBlockFields('chunking'); + expect(fields.length).toBe(3); + expect(fields.map(f => f.key)).toEqual(['strategy', 'chunkSize', 'chunkOverlap']); + }); + + it('should return fields for embedding block', () => { + const fields = getBlockFields('embedding'); + expect(fields.length).toBe(2); + expect(fields[0].key).toBe('model'); + }); + + it('should return fields for retrieval block', () => { + const fields = getBlockFields('retrieval'); + expect(fields.find(f => f.key === 'topK')?.max).toBe(20); + }); + + it('should return fields for generation block', () => { + const fields = getBlockFields('generation'); + const modelField = fields.find(f => f.key === 'model')!; + expect(modelField.options!.length).toBeGreaterThanOrEqual(3); + }); + + it('should return fields for source block', () => { + const fields = getBlockFields('source'); + expect(fields.length).toBe(1); + expect(fields[0].key).toBe('sourceType'); + }); + }); +}); diff --git a/client/src/app/features/pipeline-builder/pipeline.service.ts b/client/src/app/features/pipeline-builder/pipeline.service.ts index fd90268..c6ee490 100644 --- a/client/src/app/features/pipeline-builder/pipeline.service.ts +++ b/client/src/app/features/pipeline-builder/pipeline.service.ts @@ -1,165 +1,189 @@ import { Injectable, inject, signal, computed } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { firstValueFrom } from 'rxjs'; +import { PipelineBlock, ChunkPreview, createDefaultBlocks } from './pipeline-block.model'; -export interface PipelineConfig { - chunkSize: number; - chunkOverlap: number; - chunkingStrategy: string; - embeddingModel: string; - embeddingDimensions: number; - retrievalStrategy: string; - topK: number; - scoreThreshold: number; - generationModel: string; - temperature: number; - maxTokens: number; - systemPrompt: string; -} - -export interface PipelineResponse { +export interface PipelineDto { id: string; projectId: string; name: string; description: string | null; - config: PipelineConfig; + config: { + chunkSize: number; + chunkOverlap: number; + embeddingModel: string; + retrievalStrategy: string; + topK: number; + scoreThreshold: number; + }; status: string; createdAt: string; updatedAt: string | null; } -export interface DocumentInfo { - id: string; - fileName: string; - fileType: string; - fileSize: number; - status: string; -} +const SAMPLE_TEXT = + 'Retrieval-Augmented Generation (RAG) is a technique that combines information retrieval with text generation. ' + + 'It first retrieves relevant documents from a knowledge base using semantic search, then feeds those documents ' + + 'as context to a large language model (LLM) to generate accurate, grounded responses. This approach reduces ' + + 'hallucinations and allows the model to access up-to-date information without retraining. RAG pipelines typically ' + + 'consist of several stages: document ingestion, text chunking, embedding generation, vector storage, retrieval, ' + + 'and finally generation. Each stage can be configured independently to optimize the overall pipeline performance ' + + 'for specific use cases.'; -export interface ChunkPreviewItem { - content: string; - tokenCount: number; - index: number; -} +@Injectable({ providedIn: 'root' }) +export class PipelineService { + private http = inject(HttpClient); -export interface ChunkPreviewResponse { - chunks: ChunkPreviewItem[]; - totalCount: number; - page: number; - pageSize: number; -} + readonly blocks = signal(createDefaultBlocks()); + readonly selectedBlockId = signal(null); + readonly loading = signal(false); + readonly saving = signal(false); + readonly dirty = signal(false); + readonly pipelineId = signal(null); -export interface ModelSelection { - embeddingModel: string; - embeddingDimensions: number; - chatModel: string; - maxTokensPerRequest: number; - maxDocumentsPerProject: number; -} + readonly selectedBlock = computed(() => { + const id = this.selectedBlockId(); + return id ? this.blocks().find((b: PipelineBlock) => b.id === id) ?? null : null; + }); -const DEFAULT_CONFIG: PipelineConfig = { - chunkSize: 512, - chunkOverlap: 50, - chunkingStrategy: 'recursive', - embeddingModel: 'text-embedding-3-small', - embeddingDimensions: 1536, - retrievalStrategy: 'semantic', - topK: 5, - scoreThreshold: 0.7, - generationModel: 'gpt-4o-mini', - temperature: 0.7, - maxTokens: 2048, - systemPrompt: 'You are a helpful assistant. Answer questions based on the provided context.', -}; + readonly chunkPreviews = computed(() => { + const chunkingBlock = this.blocks().find((b: PipelineBlock) => b.type === 'chunking'); + if (!chunkingBlock) return []; + const size: number = chunkingBlock.config['chunkSize'] ?? 512; + const overlap: number = chunkingBlock.config['chunkOverlap'] ?? 50; + return this.generateChunkPreviews(SAMPLE_TEXT, size, overlap); + }); -@Injectable({ providedIn: 'root' }) -export class PipelineService { - private http = inject(HttpClient); + readonly config = computed(() => { + const chunking = this.blocks().find((b: PipelineBlock) => b.type === 'chunking'); + const retrieval = this.blocks().find((b: PipelineBlock) => b.type === 'retrieval'); + const generation = this.blocks().find((b: PipelineBlock) => b.type === 'generation'); + return { + chunkSize: (chunking?.config['chunkSize'] as number) ?? 512, + chunkOverlap: (chunking?.config['chunkOverlap'] as number) ?? 50, + retrievalStrategy: (retrieval?.config['strategy'] as string) ?? 'semantic', + topK: (retrieval?.config['topK'] as number) ?? 5, + scoreThreshold: (retrieval?.config['scoreThreshold'] as number) ?? 0.7, + temperature: (generation?.config['temperature'] as number) ?? 0.7, + }; + }); + + readonly hasUnsavedChanges = this.dirty; + + selectBlock(id: string): void { + this.selectedBlockId.set(id); + } + + updateBlockConfig(blockId: string, key: string, value: unknown): void { + this.blocks.update((blocks: PipelineBlock[]) => + blocks.map((b: PipelineBlock) => + b.id === blockId ? { ...b, config: { ...b.config, [key]: value } } : b, + ), + ); + this.dirty.set(true); + } + + updateConfig(partial: Partial<{ chunkSize: number; chunkOverlap: number; temperature: number }>): void { + if (partial.chunkSize !== undefined || partial.chunkOverlap !== undefined) { + const chunking = this.blocks().find((b: PipelineBlock) => b.type === 'chunking'); + if (chunking) { + if (partial.chunkSize !== undefined) this.updateBlockConfig(chunking.id, 'chunkSize', partial.chunkSize); + if (partial.chunkOverlap !== undefined) this.updateBlockConfig(chunking.id, 'chunkOverlap', partial.chunkOverlap); + } + } + if (partial.temperature !== undefined) { + const gen = this.blocks().find((b: PipelineBlock) => b.type === 'generation'); + if (gen) this.updateBlockConfig(gen.id, 'temperature', partial.temperature); + } + this.dirty.set(true); + } - readonly pipeline = signal(null); - readonly config = signal({ ...DEFAULT_CONFIG }); - readonly savedConfig = signal({ ...DEFAULT_CONFIG }); - readonly documents = signal([]); - readonly models = signal(null); - readonly chunkPreview = signal(null); - readonly isSaving = signal(false); - readonly isLoading = signal(false); - readonly saveError = signal(null); + resetDefaults(): void { + this.blocks.set(createDefaultBlocks()); + this.dirty.set(false); + this.selectedBlockId.set(null); + } - readonly hasUnsavedChanges = computed(() => - JSON.stringify(this.config()) !== JSON.stringify(this.savedConfig()) - ); + moveBlock(fromIndex: number, toIndex: number): void { + this.blocks.update((blocks: PipelineBlock[]) => { + const arr = [...blocks]; + const [removed] = arr.splice(fromIndex, 1); + arr.splice(toIndex, 0, removed); + return arr; + }); + this.dirty.set(true); + } async loadPipeline(projectId: string): Promise { - this.isLoading.set(true); + this.loading.set(true); try { const pipelines = await firstValueFrom( - this.http.get('/api/projects/' + projectId + '/pipelines') + this.http.get('/api/projects/' + projectId + '/pipelines'), ); if (pipelines.length > 0) { const p = pipelines[0]; - this.pipeline.set(p); - const merged = { ...DEFAULT_CONFIG, ...p.config }; - this.config.set(merged); - this.savedConfig.set(merged); + this.pipelineId.set(p.id); + const blocks = createDefaultBlocks(); + const chunking = blocks.find((b: PipelineBlock) => b.type === 'chunking')!; + chunking.config['chunkSize'] = p.config.chunkSize; + chunking.config['chunkOverlap'] = p.config.chunkOverlap; + const embedding = blocks.find((b: PipelineBlock) => b.type === 'embedding')!; + embedding.config['model'] = p.config.embeddingModel; + const retrieval = blocks.find((b: PipelineBlock) => b.type === 'retrieval')!; + retrieval.config['strategy'] = p.config.retrievalStrategy; + retrieval.config['topK'] = p.config.topK; + retrieval.config['scoreThreshold'] = p.config.scoreThreshold; + this.blocks.set(blocks); } - try { - const docs = await firstValueFrom( - this.http.get('/api/projects/' + projectId + '/documents') - ); - this.documents.set(docs); - } catch { this.documents.set([]); } - try { - const m = await firstValueFrom(this.http.get('/api/models')); - this.models.set(m); - } catch { /* ignore */ } } finally { - this.isLoading.set(false); + this.loading.set(false); + this.dirty.set(false); } } async savePipeline(projectId: string): Promise { - const p = this.pipeline(); - if (!p) return; - this.isSaving.set(true); - this.saveError.set(null); + this.saving.set(true); try { - const updated = await firstValueFrom( - this.http.put( - '/api/projects/' + projectId + '/pipelines/' + p.id, - { name: p.name, description: p.description, config: this.config() } - ) - ); - this.pipeline.set(updated); - const merged = { ...DEFAULT_CONFIG, ...updated.config }; - this.savedConfig.set(merged); - this.config.set(merged); - } catch (err: any) { - this.saveError.set(err?.error?.error ?? 'Failed to save pipeline'); - throw err; + const cfg = this.config(); + const body = { + name: 'Default Pipeline', + description: 'Visual pipeline configuration', + config: { + chunkSize: cfg.chunkSize, + chunkOverlap: cfg.chunkOverlap, + embeddingModel: + (this.blocks().find((b: PipelineBlock) => b.type === 'embedding')?.config['model'] as string) ?? + 'text-embedding-3-small', + retrievalStrategy: cfg.retrievalStrategy, + topK: cfg.topK, + scoreThreshold: cfg.scoreThreshold, + }, + }; + + const pid = this.pipelineId(); + if (pid) { + await firstValueFrom(this.http.put('/api/projects/' + projectId + '/pipelines/' + pid, body)); + } else { + const result = await firstValueFrom( + this.http.post('/api/projects/' + projectId + '/pipelines', body), + ); + this.pipelineId.set(result.id); + } + this.dirty.set(false); } finally { - this.isSaving.set(false); + this.saving.set(false); } } - resetDefaults(): void { - this.config.set({ ...DEFAULT_CONFIG }); - } - - updateConfig(partial: Partial): void { - this.config.update(c => ({ ...c, ...partial })); - } - - async loadChunkPreview(projectId: string, documentId: string): Promise { - try { - const cfg = this.config(); - const preview = await firstValueFrom( - this.http.get( - '/api/projects/' + projectId + '/documents/' + documentId + '/chunks', - { params: { chunkSize: cfg.chunkSize.toString(), chunkOverlap: cfg.chunkOverlap.toString(), page: '1', pageSize: '3' } } - ) - ); - this.chunkPreview.set(preview); - } catch { this.chunkPreview.set(null); } + private generateChunkPreviews(text: string, size: number, overlap: number): ChunkPreview[] { + const charSize = Math.max(size, 50); + const charOverlap = Math.min(overlap, charSize - 10); + const step = Math.max(charSize - charOverlap, 1); + const chunks: ChunkPreview[] = []; + for (let i = 0; i < text.length && chunks.length < 5; i += step) { + const slice = text.slice(i, i + charSize); + chunks.push({ index: chunks.length, text: slice, tokens: Math.round(slice.length / 4) }); + } + return chunks; } } diff --git a/client/src/app/pages/pipeline/pipeline.html b/client/src/app/pages/pipeline/pipeline.html new file mode 100644 index 0000000..e7bdad9 --- /dev/null +++ b/client/src/app/pages/pipeline/pipeline.html @@ -0,0 +1,466 @@ + + +
+ +
+
+

Pipeline Builder

+

Configure your RAG pipeline step by step

+
+
+ @if (svc.hasUnsavedChanges()) { + ● Unsaved changes + } + +
+
+ + @if (svc.saveError()) { +
+ {{ svc.saveError() }} +
+ } + + @if (svc.isLoading()) { +
Loading pipeline…
+ } @else { + +
+
+ @for (step of svc.steps; track step; let i = $index) { + + @if (i < svc.steps.length - 1) { +
+ } + } +
+ +
+
+
+
+ +
+ +
+
+

Pipeline Flow

+
+ @for (block of svc.blocks(); track block.id) { +
+
+ ⠿ +
+
+ {{ block.icon }} +
+
+
+ {{ block.label }} + + {{ block.type }} + +
+

{{ blockSummary(block) }}

+
+
+
+ + @if (!$last) { +
+ + + +
+ } + } +
+ + +
+
+
+ Status: + @switch (svc.pipelineStatus()) { + @case ('idle') { Idle } + @case ('running') { Running… } + @case ('completed') { Completed } + @case ('failed') { Failed } + } +
+ +
+
+
+
+ + +
+
+ @switch (svc.currentStep()) { + + @case ('source') { +

📄 Data Source

+

Upload documents or connect a data source for your chatbot.

+ @if (svc.documents().length > 0) { +
+ @for (doc of svc.documents(); track doc.id) { +
+ + @switch (doc.fileType) { + @case ('application/pdf') { 📕 } + @case ('text/plain') { 📝 } + @case ('text/markdown') { 📝 } + @default { 📄 } + } + +
+

{{ doc.fileName }}

+

{{ (doc.fileSize / 1024).toFixed(1) }} KB

+
+ + {{ doc.status }} + +
+ } +
+

{{ svc.documents().length }} document(s) loaded. Configure chunking in the next step.

+ } @else { +
+
📁
+

No documents uploaded yet

+

Upload documents from the project page first

+
+ } + @for (block of svc.blocks(); track block.id) { + @if (block.type === 'source') { +
+ +
+ } + } + } + + + @case ('chunking') { +

✂️ Chunking

+

Configure how documents are split into smaller chunks for processing.

+ @for (block of svc.blocks(); track block.id) { + @if (block.type === 'chunking') { +
+ + + +
+ + + @if (svc.chunkPreview(); as preview) { +
+

Chunk Preview ({{ preview.totalCount }} total)

+
+ @for (chunk of preview.chunks; track chunk.index) { +
+
+ Chunk {{ chunk.index + 1 }} + {{ chunk.tokenCount }} tokens +
+

{{ chunk.content }}

+
+ } +
+
+ } + } + } + } + + + @case ('embedding') { +

🧮 Embedding

+

Choose the embedding model that converts text into vector representations.

+ @for (block of svc.blocks(); track block.id) { + @if (block.type === 'embedding') { +
+ + + +
+
+

Quality

+

+ @switch (block.config['model']) { + @case ('text-embedding-3-large') { ★★★ } + @case ('embed-v4') { ★★★ } + @case ('text-embedding-3-small') { ★★☆ } + @default { ★★☆ } + } +

+
+
+

Cost

+

+ @switch (block.config['model']) { + @case ('text-embedding-3-large') { $$ } + @case ('embed-v4') { $ } + @case ('bge-m3') { Free } + @default { $ } + } +

+
+
+
+ } + } + } + + + @case ('retrieval') { +

🔍 Retrieval

+

Configure how relevant chunks are retrieved for each query.

+ @for (block of svc.blocks(); track block.id) { + @if (block.type === 'retrieval') { +
+ + + +
+ } + } + } + + + @case ('generation') { +

🤖 Generation

+

Choose the LLM and configure how responses are generated.

+ @for (block of svc.blocks(); track block.id) { + @if (block.type === 'generation') { +
+ + + + +
+ } + } + } + + + @case ('review') { +

📋 Review & Run

+

Review your pipeline configuration before running.

+
+ @for (block of svc.blocks(); track block.id) { +
+ {{ block.icon }} +
+

{{ block.label }}

+

{{ blockSummary(block) }}

+
+ +
+ } +
+
+ + +
+ } + } + + + @if (svc.currentStep() !== 'review') { +
+ + +
+ } +
+
+
+ } +
diff --git a/client/src/app/pages/pipeline/pipeline.ts b/client/src/app/pages/pipeline/pipeline.ts index 354e497..edb4ac0 100644 --- a/client/src/app/pages/pipeline/pipeline.ts +++ b/client/src/app/pages/pipeline/pipeline.ts @@ -1,26 +1,19 @@ import { Component, inject, OnInit } from '@angular/core'; import { ActivatedRoute } from '@angular/router'; import { CdkDropList, CdkDrag, CdkDragDrop, CdkDragHandle, CdkDragPlaceholder } from '@angular/cdk/drag-drop'; -import { PipelineService, PipelineBlock, BlockType } from './pipeline.service'; - -const BLOCK_COLORS: Record = { - source: { border: 'border-blue-300', bg: 'bg-blue-50', iconBg: 'bg-blue-100', badge: 'text-blue-700' }, - chunking: { border: 'border-amber-300', bg: 'bg-amber-50', iconBg: 'bg-amber-100', badge: 'text-amber-700' }, - embedding: { border: 'border-purple-300', bg: 'bg-purple-50', iconBg: 'bg-purple-100', badge: 'text-purple-700' }, - retrieval: { border: 'border-emerald-300', bg: 'bg-emerald-50', iconBg: 'bg-emerald-100', badge: 'text-emerald-700' }, - generation: { border: 'border-rose-300', bg: 'bg-rose-50', iconBg: 'bg-rose-100', badge: 'text-rose-700' }, -}; +import { NavbarComponent } from '../../shared/components/navbar/navbar'; +import { PipelineService } from '../../features/pipeline-builder/pipeline.service'; +import { PipelineBlock, BlockType } from '../../features/pipeline-builder/pipeline-block.model'; @Component({ selector: 'app-pipeline', standalone: true, - imports: [CdkDropList, CdkDrag, CdkDragHandle, CdkDragPlaceholder], - providers: [PipelineService], + imports: [NavbarComponent, CdkDropList, CdkDrag, CdkDragHandle, CdkDragPlaceholder], templateUrl: './pipeline.html', }) export class PipelineComponent implements OnInit { + readonly svc = inject(PipelineService); private route = inject(ActivatedRoute); - readonly svc: PipelineService = inject(PipelineService); private projectId = ''; ngOnInit() { @@ -31,9 +24,7 @@ export class PipelineComponent implements OnInit { } onDrop(event: CdkDragDrop) { - if (event.previousIndex !== event.currentIndex) { - this.svc.reorderBlocks(event.previousIndex, event.currentIndex); - } + this.svc.moveBlock(event.previousIndex, event.currentIndex); } save() { @@ -42,28 +33,53 @@ export class PipelineComponent implements OnInit { } } + runPipeline() { + if (this.projectId) { + this.svc.runPipeline(this.projectId); + } + } + blockClass(block: PipelineBlock): string { - const c = BLOCK_COLORS[block.type]; const selected = this.svc.selectedBlockId() === block.id; - return `${c.border} ${c.bg} ${selected ? 'ring-2 ring-indigo-500 shadow-md' : ''}`; + const colors: Record = { + source: selected ? 'border-emerald-500 bg-emerald-50' : 'border-gray-200 bg-white', + chunking: selected ? 'border-amber-500 bg-amber-50' : 'border-gray-200 bg-white', + embedding: selected ? 'border-purple-500 bg-purple-50' : 'border-gray-200 bg-white', + retrieval: selected ? 'border-blue-500 bg-blue-50' : 'border-gray-200 bg-white', + generation: selected ? 'border-rose-500 bg-rose-50' : 'border-gray-200 bg-white', + }; + return colors[block.type]; } blockIconBg(block: PipelineBlock): string { - return BLOCK_COLORS[block.type].iconBg; + const colors: Record = { + source: 'bg-emerald-100', + chunking: 'bg-amber-100', + embedding: 'bg-purple-100', + retrieval: 'bg-blue-100', + generation: 'bg-rose-100', + }; + return colors[block.type]; } blockTypeBadge(block: PipelineBlock): string { - return BLOCK_COLORS[block.type].badge; + const colors: Record = { + source: 'text-emerald-700', + chunking: 'text-amber-700', + embedding: 'text-purple-700', + retrieval: 'text-blue-700', + generation: 'text-rose-700', + }; + return colors[block.type]; } blockSummary(block: PipelineBlock): string { - const c = block.config; switch (block.type) { - case 'source': return `${c['sourceType']} · ${c['fileTypes']}`; - case 'chunking': return `${c['strategy']} · ${c['chunkSize']} chars · ${c['chunkOverlap']} overlap`; - case 'embedding': return `${c['model']} · ${c['dimensions']}d`; - case 'retrieval': return `${c['strategy']} · top ${c['topK']} · threshold ${c['scoreThreshold']}`; - case 'generation': return `${c['model']} · temp ${c['temperature']} · ${c['maxTokens']} tokens`; + case 'source': return `${block.config['sourceType']} · ${block.config['fileTypes']}`; + case 'chunking': return `${block.config['strategy']} · ${block.config['chunkSize']} tokens · ${block.config['chunkOverlap']} overlap`; + case 'embedding': return `${block.config['model']} · ${block.config['dimensions']}d`; + case 'retrieval': return `${block.config['strategy']} · top ${block.config['topK']} · threshold ${block.config['scoreThreshold']}`; + case 'generation': return `${block.config['model']} · temp ${block.config['temperature']}`; default: return ''; } } @@ -75,12 +91,18 @@ export class PipelineComponent implements OnInit { onConfigChangeNum(blockId: string, key: string, event: Event) { const el = event.target as HTMLInputElement; - this.svc.updateBlockConfig(blockId, key, parseInt(el.value, 10)); + const parsed = parseInt(el.value, 10); + if (!isNaN(parsed)) { + this.svc.updateBlockConfig(blockId, key, parsed); + } } onConfigChangeFloat(blockId: string, key: string, event: Event) { const el = event.target as HTMLInputElement; - this.svc.updateBlockConfig(blockId, key, parseFloat(el.value)); + const parsed = parseFloat(el.value); + if (!isNaN(parsed)) { + this.svc.updateBlockConfig(blockId, key, parsed); + } } onConfigChangeBool(blockId: string, key: string, event: Event) { diff --git a/client/src/styles.scss b/client/src/styles.scss index cc656a3..d4f625f 100644 --- a/client/src/styles.scss +++ b/client/src/styles.scss @@ -1,4 +1,4 @@ -@import "tailwindcss"; +@use "tailwindcss"; @import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&display=swap'); *, *::before, *::after { diff --git a/docs/features/pr-008-pipeline-builder-ui.md b/docs/features/pr-008-pipeline-builder-ui.md index 26d95b8..b7c7555 100644 --- a/docs/features/pr-008-pipeline-builder-ui.md +++ b/docs/features/pr-008-pipeline-builder-ui.md @@ -1,34 +1,23 @@ -# PR #8 — Pipeline Builder UI +# PR #8 - Pipeline Builder UI -## Overview Visual pipeline builder with card-based interface for configuring RAG pipeline stages: -Source → Chunking → Embedding → Retrieval → Generation. +Source -> Chunking -> Embedding -> Retrieval -> Generation. ## Components -- **PipelineBuilderComponent** — `client/src/app/features/pipeline-builder/pipeline-builder.component.ts` -- **PipelineService** — `client/src/app/features/pipeline-builder/pipeline.service.ts` +- **PipelineBuilderComponent** - Standalone Angular component with expandable block cards +- **PipelineService** - Signal-based state management for pipeline config ## Pipeline Blocks | Block | Settings | |-------|----------| | Source | Document list, file types, total size | -| Chunking | Chunk size (128-2048), overlap (0-200), strategy (recursive/sentence) | +| Chunking | Chunk size (128-2048), overlap (0-200), strategy | | Embedding | Model selector (tier-gated), dimension display | -| Retrieval | Strategy (semantic/hybrid), top-k (1-20), score threshold (0-1) | +| Retrieval | Strategy (semantic/hybrid), top-k (1-20), score threshold | | Generation | Model (tier-gated), temperature, max tokens, system prompt | ## Chunk Preview Panel -Side panel showing real-time chunking preview with chunk count, avg tokens, and first 3 chunks. - -## API Endpoints Used -- `GET /api/projects/{id}/pipelines` — Load config -- `PUT /api/projects/{id}/pipelines/{pipelineId}` — Save config -- `GET /api/projects/{id}/documents` — Document list -- `GET /api/projects/{id}/documents/{docId}/chunks` — Chunk preview -- `GET /api/models` — Tier-gated models +Side panel with real-time chunking preview (chunk count, avg tokens, first 3 chunks). ## Backend Changes -Extended `PipelineConfigDto` with: ChunkingStrategy, EmbeddingDimensions, GenerationModel, Temperature, MaxTokens, SystemPrompt. - -## Route -`/projects/:projectId/pipeline` +Extended PipelineConfigDto with: ChunkingStrategy, EmbeddingDimensions, GenerationModel, Temperature, MaxTokens, SystemPrompt.