Doc/update-2

cc-registry-v2/README.md

@@ -37,9 +37,9 @@ A modern, scalable registry for RunWhen CodeCollections with AI-powered enhancem
 | Service | Stack | Port | Purpose |
 |---------|-------|------|---------|
 | **frontend** | React 19 + TypeScript + MUI v7 | 3000 | SPA for browsing and managing CodeBundles |
-| **backend** | FastAPI + SQLAlchemy 2.0 | 8001 | REST API (`/api/v1/`), business logic, AI enhancement |
-| **mcp-server** | FastAPI (separate repo: `../mcp-server`) | 8000 | Stateless MCP tool server, delegates to backend API |
-| **worker** | Celery (shares backend image) | -- | Background task processing |
+| **backend** | FastAPI + SQLAlchemy 2.0 + pgvector | 8001 | REST API, business logic, AI enhancement, embedding generation, vector search |
+| **mcp-server** | FastAPI (separate repo: `../mcp-server`) | 8000 | Stateless MCP tool server, delegates all queries to backend API |
+| **worker** | Celery (shares backend image) | -- | Background tasks: sync, parse, enhance, embed |
 | **scheduler** | Celery Beat (shares backend image) | -- | Cron-driven task scheduling |
 | **database** | PostgreSQL 15 + pgvector | 5432 | Primary data store with vector extension |
 | **redis** | Redis 7 Alpine | 6379 | Celery broker and result backend |

cc-registry-v2/backend/app/core/config.py

@@ -35,12 +35,19 @@ class Settings(BaseSettings):
     AI_MODEL: str = "gpt-4"
     AI_ENHANCEMENT_ENABLED: bool = False
 
-    # Azure OpenAI Configuration
+    # Azure OpenAI Configuration (GPT / chat completions)
     AZURE_OPENAI_API_KEY: Optional[str] = None
     AZURE_OPENAI_ENDPOINT: Optional[str] = None
     AZURE_OPENAI_DEPLOYMENT_NAME: Optional[str] = None
     AZURE_OPENAI_API_VERSION: str = "2024-02-15-preview"
 
+    # Azure OpenAI Embedding Configuration (separate endpoint supported)
+    AZURE_OPENAI_EMBEDDING_ENDPOINT: Optional[str] = None
+    AZURE_OPENAI_EMBEDDING_API_KEY: Optional[str] = None
+    AZURE_OPENAI_EMBEDDING_API_VERSION: Optional[str] = None
+    AZURE_OPENAI_EMBEDDING_DEPLOYMENT: str = "text-embedding-3-small"
+    EMBEDDING_BATCH_SIZE: int = 100
+
     # AI Service Provider (openai, azure-openai)
     AI_SERVICE_PROVIDER: str = "openai"
 

cc-registry-v2/backend/app/main.py

@@ -99,22 +99,22 @@ async def health_check():
     }
 
 # Include routers
-from app.routers import admin, tasks, raw_data, admin_crud, task_execution_admin, versions, task_management, admin_inventory, helm_charts, mcp_chat, chat_debug, github_issues, schedule_config, analytics
+from app.routers import admin, tasks, raw_data, admin_crud, task_execution_admin, versions, task_management, admin_inventory, helm_charts, mcp_chat, chat_debug, github_issues, schedule_config, analytics, vector_search
 app.include_router(admin.router)
 app.include_router(tasks.router)
 app.include_router(raw_data.router)
 app.include_router(admin_crud.router)
-# AI config removed - now uses env vars only (AZURE_OPENAI_* in az.secret)
 app.include_router(task_execution_admin.router, prefix="/api/v1")
 app.include_router(admin_inventory.router)
 app.include_router(versions.router, prefix="/api/v1/registry")
 app.include_router(task_management.router)
 app.include_router(schedule_config.router)
 app.include_router(helm_charts.router, prefix="/api/v1", tags=["helm-charts"])
-app.include_router(mcp_chat.router)  # MCP-powered chat (replaces legacy chat + simple_chat)
-app.include_router(chat_debug.router)  # Debug tools for chat quality analysis
+app.include_router(mcp_chat.router)
+app.include_router(chat_debug.router)
 app.include_router(github_issues.router, prefix="/api/v1")
-app.include_router(analytics.router)  # Analytics charts and metrics
+app.include_router(analytics.router)
+app.include_router(vector_search.router)
 
 @app.get("/api/v1/registry/collections")
 async def list_collections():

cc-registry-v2/backend/app/models/__init__.py

@@ -8,5 +8,12 @@
 from .task_execution import TaskExecution
 from .helm_chart import HelmChart, HelmChartVersion, HelmChartTemplate
 from .analytics import TaskGrowthMetric
+from .vector_models import VectorCodebundle, VectorCodecollection, VectorLibrary, VectorDocumentation
 
-__all__ = ["CodeCollection", "Codebundle", "RawYamlData", "RawRepositoryData", "CodeCollectionMetrics", "SystemMetrics", "AIConfiguration", "AIEnhancementLog", "CodeCollectionVersion", "VersionCodebundle", "TaskExecution", "HelmChart", "HelmChartVersion", "HelmChartTemplate", "TaskGrowthMetric"]
+__all__ = [
+    "CodeCollection", "Codebundle", "RawYamlData", "RawRepositoryData",
+    "CodeCollectionMetrics", "SystemMetrics", "AIConfiguration", "AIEnhancementLog",
+    "CodeCollectionVersion", "VersionCodebundle", "TaskExecution",
+    "HelmChart", "HelmChartVersion", "HelmChartTemplate", "TaskGrowthMetric",
+    "VectorCodebundle", "VectorCodecollection", "VectorLibrary", "VectorDocumentation",
+]

cc-registry-v2/backend/app/models/vector_models.py

@@ -0,0 +1,61 @@
+"""
+SQLAlchemy models for pgvector tables.
+
+Maps to the tables created by database/migrations/006_add_pgvector.sql.
+"""
+from sqlalchemy import Column, String, Text, DateTime, func, text
+from sqlalchemy.dialects.postgresql import JSONB
+from pgvector.sqlalchemy import Vector
+
+from app.core.database import Base
+
+# Must match the migration (006_add_pgvector.sql) and the Azure OpenAI
+# text-embedding-3-small model output.  Do NOT change without also
+# altering the migration and rebuilding all vector tables.
+EMBEDDING_DIMENSIONS = 1536
+
+_JSONB_EMPTY = text("'{}'::jsonb")
+
+
+class VectorCodebundle(Base):
+    __tablename__ = "vector_codebundles"
+
+    id = Column(String, primary_key=True)
+    embedding = Column(Vector(EMBEDDING_DIMENSIONS))
+    document = Column(Text)
+    metadata_ = Column("metadata", JSONB, nullable=False, server_default=_JSONB_EMPTY)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+
+class VectorCodecollection(Base):
+    __tablename__ = "vector_codecollections"
+
+    id = Column(String, primary_key=True)
+    embedding = Column(Vector(EMBEDDING_DIMENSIONS))
+    document = Column(Text)
+    metadata_ = Column("metadata", JSONB, nullable=False, server_default=_JSONB_EMPTY)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+
+class VectorLibrary(Base):
+    __tablename__ = "vector_libraries"
+
+    id = Column(String, primary_key=True)
+    embedding = Column(Vector(EMBEDDING_DIMENSIONS))
+    document = Column(Text)
+    metadata_ = Column("metadata", JSONB, nullable=False, server_default=_JSONB_EMPTY)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+
+
+class VectorDocumentation(Base):
+    __tablename__ = "vector_documentation"
+
+    id = Column(String, primary_key=True)
+    embedding = Column(Vector(EMBEDDING_DIMENSIONS))
+    document = Column(Text)
+    metadata_ = Column("metadata", JSONB, nullable=False, server_default=_JSONB_EMPTY)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())

cc-registry-v2/backend/app/routers/vector_search.py

@@ -0,0 +1,211 @@
+"""
+Vector search API endpoints.
+
+Exposes semantic (embedding-based) search over codebundles, codecollections,
+libraries, and documentation. Used by the MCP server and the frontend chat.
+"""
+import logging
+from typing import Any, Dict, List, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy.orm import Session
+
+from app.core.database import get_db
+from app.services.embedding_service import get_embedding_service
+from app.services.vector_service import VectorSearchResult, get_vector_service
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/v1/vector", tags=["vector-search"])
+
+
+def _result_to_dict(r: VectorSearchResult) -> Dict[str, Any]:
+    return {
+        "id": r.id,
+        "document": r.document[:500],
+        "metadata": r.metadata,
+        "score": round(r.score, 4),
+        "distance": round(r.distance, 4),
+    }
+
+
+# --------------------------------------------------------------------------
+# Unified semantic search
+# --------------------------------------------------------------------------
+
+@router.get("/search")
+def semantic_search(
+    query: str,
+    tables: Optional[str] = Query(
+        None,
+        description="Comma-separated table keys to search (codebundles,codecollections,libraries,documentation). Default: all.",
+    ),
+    max_results: int = Query(10, ge=1, le=50),
+    platform: Optional[str] = None,
+    category: Optional[str] = None,
+    db: Session = Depends(get_db),
+):
+    """Run a semantic similarity search across one or more vector tables."""
+    embed_svc = get_embedding_service()
+    vec_svc = get_vector_service()
+
+    if not embed_svc.available:
+        raise HTTPException(
+            status_code=503,
+            detail="Embedding service is not configured. Set AZURE_OPENAI_EMBEDDING_* environment variables.",
+        )
+
+    query_embedding = embed_svc.embed_text(query)
+    if not query_embedding:
+        raise HTTPException(status_code=500, detail="Failed to generate query embedding")
+
+    table_keys = [t.strip() for t in tables.split(",")] if tables else None
+
+    if table_keys:
+        valid_keys = {"codebundles", "codecollections", "libraries", "documentation"}
+        invalid = set(table_keys) - valid_keys
+        if invalid:
+            raise HTTPException(status_code=400, detail=f"Invalid table keys: {invalid}")
+
+    filters: Optional[Dict[str, str]] = {}
+    if platform:
+        filters["platform"] = platform
+    if category:
+        filters["category"] = category
+
+    results_map = vec_svc.search_all(
+        query_embedding, n_results=max_results, table_keys=table_keys,
+        metadata_filters=filters or None, db=db,
+    )
+
+    output: Dict[str, Any] = {}
+    for key, results in results_map.items():
+        output[key] = [_result_to_dict(r) for r in results]
+
+    return output
+
+
+# --------------------------------------------------------------------------
+# Per-table endpoints
+# --------------------------------------------------------------------------
+
+@router.get("/search/codebundles")
+def search_codebundles(
+    query: str,
+    max_results: int = Query(10, ge=1, le=50),
+    platform: Optional[str] = None,
+    collection_slug: Optional[str] = None,
+    db: Session = Depends(get_db),
+):
+    """Semantic search over codebundles."""
+    embed_svc = get_embedding_service()
+    vec_svc = get_vector_service()
+
+    if not embed_svc.available:
+        raise HTTPException(status_code=503, detail="Embedding service not configured")
+
+    query_embedding = embed_svc.embed_text(query)
+    if not query_embedding:
+        raise HTTPException(status_code=500, detail="Embedding generation failed")
+
+    filters: Optional[Dict[str, str]] = {}
+    if platform:
+        filters["platform"] = platform
+    if collection_slug:
+        filters["collection_slug"] = collection_slug
+
+    results = vec_svc.search(
+        "codebundles", query_embedding, n_results=max_results,
+        metadata_filters=filters or None, db=db,
+    )
+    return {"results": [_result_to_dict(r) for r in results], "query": query}
+
+
+@router.get("/search/documentation")
+def search_documentation(
+    query: str,
+    max_results: int = Query(10, ge=1, le=50),
+    category: Optional[str] = None,
+    db: Session = Depends(get_db),
+):
+    """Semantic search over documentation."""
+    embed_svc = get_embedding_service()
+    vec_svc = get_vector_service()
+
+    if not embed_svc.available:
+        raise HTTPException(status_code=503, detail="Embedding service not configured")
+
+    query_embedding = embed_svc.embed_text(query)
+    if not query_embedding:
+        raise HTTPException(status_code=500, detail="Embedding generation failed")
+
+    filters = {"category": category} if category else None
+    results = vec_svc.search(
+        "documentation", query_embedding, n_results=max_results,
+        metadata_filters=filters, db=db,
+    )
+    return {"results": [_result_to_dict(r) for r in results], "query": query}
+
+
+@router.get("/search/libraries")
+def search_libraries(
+    query: str,
+    max_results: int = Query(10, ge=1, le=50),
+    category: Optional[str] = None,
+    db: Session = Depends(get_db),
+):
+    """Semantic search over libraries."""
+    embed_svc = get_embedding_service()
+    vec_svc = get_vector_service()
+
+    if not embed_svc.available:
+        raise HTTPException(status_code=503, detail="Embedding service not configured")
+
+    query_embedding = embed_svc.embed_text(query)
+    if not query_embedding:
+        raise HTTPException(status_code=500, detail="Embedding generation failed")
+
+    filters = {"category": category} if category else None
+    results = vec_svc.search(
+        "libraries", query_embedding, n_results=max_results,
+        metadata_filters=filters, db=db,
+    )
+    return {"results": [_result_to_dict(r) for r in results], "query": query}
+
+
+# --------------------------------------------------------------------------
+# Stats / health
+# --------------------------------------------------------------------------
+
+@router.get("/stats")
+def vector_stats(db: Session = Depends(get_db)):
+    """Return row counts for each vector table."""
+    vec_svc = get_vector_service()
+    return vec_svc.get_stats(db=db)
+
+
+@router.post("/reindex")
+async def trigger_reindex():
+    """Trigger a full reindex (async Celery task)."""
+    from app.tasks.indexing_tasks import reindex_all_task
+
+    task = reindex_all_task.apply_async()
+    return {"task_id": task.id, "status": "queued"}
+
+
+@router.post("/reindex/codebundles")
+async def trigger_reindex_codebundles():
+    """Trigger codebundle reindexing."""
+    from app.tasks.indexing_tasks import index_codebundles_task
+
+    task = index_codebundles_task.apply_async()
+    return {"task_id": task.id, "status": "queued"}
+
+
+@router.post("/reindex/documentation")
+async def trigger_reindex_documentation():
+    """Trigger documentation reindexing."""
+    from app.tasks.indexing_tasks import index_documentation_task
+
+    task = index_documentation_task.apply_async()
+    return {"task_id": task.id, "status": "queued"}