diff --git a/api/main.py b/api/main.py index 5499a62..ddcb96c 100644 --- a/api/main.py +++ b/api/main.py @@ -7,6 +7,7 @@ python -m uvicorn api.main:app --host 0.0.0.0 --port 8000 """ +from collections.abc import AsyncGenerator from contextlib import asynccontextmanager from fastapi import FastAPI @@ -19,7 +20,7 @@ @asynccontextmanager -async def lifespan(app: FastAPI): # type: ignore +async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]: """Startup and shutdown events.""" # Startup logger.info("Starting Streaming Video-RAG API...") @@ -69,7 +70,11 @@ def health() -> dict[str, str]: return { "status": "ok", "llm_provider": settings.llm_provider.value, - "llm_model": settings.llm_model if settings.llm_provider.value != "ollama" else settings.ollama_model, + "llm_model": ( + settings.llm_model + if settings.llm_provider.value != "ollama" + else settings.ollama_model + ), "whisper_mode": settings.whisper_mode.value, "whisper_model": settings.whisper_model_size, "embedding_mode": settings.embedding_mode.value, diff --git a/api/models.py b/api/models.py index 7f259cc..6dd9384 100644 --- a/api/models.py +++ b/api/models.py @@ -8,14 +8,23 @@ class IngestRequest(BaseModel): - source: str = Field(..., description="YouTube URL, local file path, stream URL, or video API URL") + source: str = Field( + ..., description="YouTube URL, local file path, stream URL, or video API URL" + ) source_type: str | None = Field( None, description="Hint for source type: 'youtube' | 'local_file' | 'live_stream' | 'video_api'. Auto-detected if omitted.", ) - language: str | None = Field(None, description="ISO 639-1 language code (e.g. 'en'). Auto-detected if omitted.") - platform: str | None = Field(None, description="For video_api source: 'vimeo' | 'twitch'") - api_credentials: dict[str, str] | None = Field(None, description="API credentials for video_api source") + language: str | None = Field( + None, + description="ISO 639-1 language code (e.g. 'en'). Auto-detected if omitted.", + ) + platform: str | None = Field( + None, description="For video_api source: 'vimeo' | 'twitch'" + ) + api_credentials: dict[str, str] | None = Field( + None, description="API credentials for video_api source" + ) class IngestResponse(BaseModel): @@ -59,9 +68,15 @@ class VideoListResponse(BaseModel): class QueryRequest(BaseModel): - question: str = Field(..., description="Natural language question to answer from video content") - video_id: str | None = Field(None, description="If set, restrict search to this video") - top_k: int | None = Field(None, description="Number of chunks to retrieve (default: from settings)") + question: str = Field( + ..., description="Natural language question to answer from video content" + ) + video_id: str | None = Field( + None, description="If set, restrict search to this video" + ) + top_k: int | None = Field( + None, description="Number of chunks to retrieve (default: from settings)" + ) class SourceCitation(BaseModel): @@ -86,7 +101,9 @@ class QueryResponse(BaseModel): class SearchRequest(BaseModel): query: str = Field(..., description="Semantic search query") - video_id: str | None = Field(None, description="If set, search only within this video") + video_id: str | None = Field( + None, description="If set, search only within this video" + ) top_k: int = Field(10, description="Maximum number of results") min_score: float = Field(0.0, description="Minimum similarity score (0-1)") @@ -117,7 +134,9 @@ class SearchResponse(BaseModel): class SummarizeRequest(BaseModel): video_id: str - include_chapters: bool = Field(True, description="Also generate per-chapter summaries if available") + include_chapters: bool = Field( + True, description="Also generate per-chapter summaries if available" + ) class ChapterSummary(BaseModel): diff --git a/api/routes/ingest.py b/api/routes/ingest.py index 622e91a..6b68bdd 100644 --- a/api/routes/ingest.py +++ b/api/routes/ingest.py @@ -144,7 +144,9 @@ def _run_ingest_pipeline( db = SessionLocal() - def update_job(status: str, message: str, video_id: str | None = None, error: str | None = None) -> None: + def update_job( + status: str, message: str, video_id: str | None = None, error: str | None = None + ) -> None: try: job = db.query(IngestJob).filter(IngestJob.id == job_id).first() if job: @@ -159,7 +161,9 @@ def update_job(status: str, message: str, video_id: str | None = None, error: st db.commit() except Exception: db.rollback() - logger.warning(f"[Ingest] Job {job_id} — failed to update status to {status!r}") + logger.warning( + f"[Ingest] Job {job_id} — failed to update status to {status!r}" + ) try: settings.ensure_dirs() @@ -169,7 +173,11 @@ def update_job(status: str, message: str, video_id: str | None = None, error: st ingester = _get_ingester(source_type, platform, api_credentials) asset = ingester.ingest(source) - update_job("transcribing", "Transcribing audio with Whisper...", video_id=asset.video_id) + update_job( + "transcribing", + "Transcribing audio with Whisper...", + video_id=asset.video_id, + ) # ── Step 2: Save video record ───────────────────────────────────────── # Use merge so a concurrent job for the same video_id doesn't raise IntegrityError. @@ -191,11 +199,15 @@ def update_job(status: str, message: str, video_id: str | None = None, error: st db.rollback() video = db.query(Video).filter(Video.id == asset.video_id).first() if video is None: - raise RuntimeError(f"Video {asset.video_id} not found after merge") from None + raise RuntimeError( + f"Video {asset.video_id} not found after merge" + ) from None # ── Step 3: Transcribe ──────────────────────────────────────────────── transcriber = _get_transcriber() - transcript = transcriber.transcribe(asset.local_audio_path, asset.video_id, language) + transcript = transcriber.transcribe( + asset.local_audio_path, asset.video_id, language + ) # Save transcript to disk transcript_path = Path(settings.transcript_dir) / f"{asset.video_id}.json" @@ -210,7 +222,9 @@ def update_job(status: str, message: str, video_id: str | None = None, error: st chunks = chunker.chunk(transcript, asset) if not chunks: - raise ValueError("No chunks produced from transcript — audio may be silent or too short") + raise ValueError( + "No chunks produced from transcript — audio may be silent or too short" + ) # ── Step 5: Embed ───────────────────────────────────────────────────── embedder = _get_embedder() @@ -229,7 +243,11 @@ def update_job(status: str, message: str, video_id: str | None = None, error: st video.indexed_at = datetime.utcnow() db.commit() - update_job("done", f"Indexed {len(chunks)} chunks successfully", video_id=asset.video_id) + update_job( + "done", + f"Indexed {len(chunks)} chunks successfully", + video_id=asset.video_id, + ) logger.success(f"[Ingest] Job {job_id} complete — {len(chunks)} chunks indexed") except Exception as e: @@ -251,7 +269,9 @@ def update_job(status: str, message: str, video_id: str | None = None, error: st def _detect_source_type(source: str) -> str: - if source.startswith("http") and ("youtu" in source or "vimeo" in source or "twitch" in source): + if source.startswith("http") and ( + "youtu" in source or "vimeo" in source or "twitch" in source + ): return "youtube" if source.startswith("rtmp") or ".m3u8" in source: return "live_stream" @@ -260,7 +280,11 @@ def _detect_source_type(source: str) -> str: return "local_file" -def _get_ingester(source_type: str, platform: str | None = None, credentials: dict[str, str] | None = None) -> Any: +def _get_ingester( + source_type: str, + platform: str | None = None, + credentials: dict[str, str] | None = None, +) -> Any: from ingestion import LiveStreamIngester, LocalFileIngester, YouTubeIngester from ingestion.video_api import get_api_ingester @@ -271,6 +295,8 @@ def _get_ingester(source_type: str, platform: str | None = None, credentials: di elif source_type == "live_stream": return LiveStreamIngester(audio_dir=settings.audio_dir) elif source_type == "video_api": - return get_api_ingester(platform or "vimeo", credentials or {}, audio_dir=settings.audio_dir) + return get_api_ingester( + platform or "vimeo", credentials or {}, audio_dir=settings.audio_dir + ) else: return YouTubeIngester(audio_dir=settings.audio_dir) diff --git a/api/routes/query.py b/api/routes/query.py index 44570de..ae366a1 100644 --- a/api/routes/query.py +++ b/api/routes/query.py @@ -46,8 +46,14 @@ def query_videos(request: QueryRequest) -> QueryResponse: ) -@router.post("/summarize", response_model=SummarizeResponse, responses={404: {"description": "Video not found"}}) -def summarize_video(request: SummarizeRequest, db: Annotated[Session, Depends(get_db)]) -> SummarizeResponse: +@router.post( + "/summarize", + response_model=SummarizeResponse, + responses={404: {"description": "Video not found"}}, +) +def summarize_video( + request: SummarizeRequest, db: Annotated[Session, Depends(get_db)] +) -> SummarizeResponse: """ Generate a summary of an indexed video. Uses map-reduce to handle long videos gracefully. @@ -56,7 +62,9 @@ def summarize_video(request: SummarizeRequest, db: Annotated[Session, Depends(ge video = db.query(Video).filter(Video.id == request.video_id).first() if not video: - raise HTTPException(status_code=404, detail=f"Video {request.video_id!r} not found") + raise HTTPException( + status_code=404, detail=f"Video {request.video_id!r} not found" + ) summarizer = Summarizer() result = summarizer.summarize( @@ -68,7 +76,8 @@ def summarize_video(request: SummarizeRequest, db: Annotated[Session, Depends(ge chapter_summaries = None if result.chapter_summaries: chapter_summaries = [ - ChapterSummary(chapter=c["chapter"], summary=c["summary"]) for c in result.chapter_summaries + ChapterSummary(chapter=c["chapter"], summary=c["summary"]) + for c in result.chapter_summaries ] return SummarizeResponse( diff --git a/config.py b/config.py index 525dc59..94b839c 100644 --- a/config.py +++ b/config.py @@ -89,7 +89,12 @@ class Config: def ensure_dirs(self) -> None: """Create all required data directories if they don't exist.""" - for d in [self.data_dir, self.audio_dir, self.transcript_dir, self.chroma_persist_dir]: + for d in [ + self.data_dir, + self.audio_dir, + self.transcript_dir, + self.chroma_persist_dir, + ]: Path(d).mkdir(parents=True, exist_ok=True) diff --git a/ingestion/base.py b/ingestion/base.py index 026fc71..530bbf5 100644 --- a/ingestion/base.py +++ b/ingestion/base.py @@ -31,7 +31,9 @@ class VideoAsset: local_audio_path: Path # Path to extracted 16kHz mono WAV duration_seconds: float | None = None description: str | None = None - chapters: list[dict[str, str | float]] = field(default_factory=list) # [{title, start, end}] + chapters: list[dict[str, str | float]] = field( + default_factory=list + ) # [{title, start, end}] thumbnail_url: str | None = None uploader: str | None = None upload_date: str | None = None @@ -50,7 +52,7 @@ class BaseIngester(ABC): 3. Returns a VideoAsset ready for the transcription stage """ - def __init__(self, audio_dir: str = "./data/audio"): + def __init__(self, audio_dir: str = "./data/audio") -> None: self.audio_dir = Path(audio_dir) self.audio_dir.mkdir(parents=True, exist_ok=True) diff --git a/ingestion/live_stream.py b/ingestion/live_stream.py index 901acc4..2d698fc 100644 --- a/ingestion/live_stream.py +++ b/ingestion/live_stream.py @@ -59,7 +59,10 @@ def ingest(self, source: str, video_id: str | None = None) -> VideoAsset: source_type=SourceType.LIVE_STREAM, local_audio_path=audio_path, duration_seconds=float(segment_seconds), - extra_metadata={"segment_type": "live", "capture_duration": segment_seconds}, + extra_metadata={ + "segment_type": "live", + "capture_duration": segment_seconds, + }, ) def ingest_stream( @@ -93,7 +96,9 @@ def ingest_stream( yield asset - logger.debug(f"[LiveStream] Segment {segment_count} done, capturing next...") + logger.debug( + f"[LiveStream] Segment {segment_count} done, capturing next..." + ) except KeyboardInterrupt: logger.info("[LiveStream] Capture stopped by user.") @@ -103,7 +108,9 @@ def ingest_stream( # ── Private helpers ────────────────────────────────────────────────────── - def _capture_segment(self, source: str, output: Path, duration_seconds: int) -> None: + def _capture_segment( + self, source: str, output: Path, duration_seconds: int + ) -> None: """ Use ffmpeg to read from the stream for `duration_seconds` and save as WAV. """ diff --git a/ingestion/local_file.py b/ingestion/local_file.py index efc82e8..4e89823 100644 --- a/ingestion/local_file.py +++ b/ingestion/local_file.py @@ -77,7 +77,9 @@ def ingest(self, source: str, video_id: str | None = None) -> VideoAsset: self._extract_audio(source_path, audio_path) duration = float(metadata.get("format", {}).get("duration", 0)) or None - title = metadata.get("format", {}).get("tags", {}).get("title") or source_path.stem + title = ( + metadata.get("format", {}).get("tags", {}).get("title") or source_path.stem + ) logger.success(f"[LocalFile] Done → {audio_path}") @@ -120,4 +122,6 @@ def _extract_audio(self, source: Path, output: Path) -> None: .run(quiet=True) ) except ffmpeg.Error as e: - raise RuntimeError(f"ffmpeg audio extraction failed: {e.stderr.decode()}") from e + raise RuntimeError( + f"ffmpeg audio extraction failed: {e.stderr.decode()}" + ) from e diff --git a/ingestion/video_api.py b/ingestion/video_api.py index aaf7661..cc880f2 100644 --- a/ingestion/video_api.py +++ b/ingestion/video_api.py @@ -24,7 +24,9 @@ class VideoAPIIngester(BaseIngester): Subclasses implement _fetch_metadata() and _get_download_url(). """ - def __init__(self, credentials: dict[str, str], audio_dir: str = "./data/audio"): + def __init__( + self, credentials: dict[str, str], audio_dir: str = "./data/audio" + ) -> None: super().__init__(audio_dir) self.credentials = credentials self.session = requests.Session() @@ -199,7 +201,9 @@ def _extract_twitch_vod_id(self, url: str) -> str: def _fetch_vod_metadata(self, vod_id: str) -> dict[str, Any]: try: - resp = self.session.get(f"{self.API_BASE}/videos", params={"id": vod_id}, timeout=30) + resp = self.session.get( + f"{self.API_BASE}/videos", params={"id": vod_id}, timeout=30 + ) resp.raise_for_status() data = resp.json().get("data", []) return data[0] if data else {} @@ -247,7 +251,9 @@ def _download_via_ytdlp(self, url: str, output: Path) -> None: # ── Factory ────────────────────────────────────────────────────────────────── -def get_api_ingester(platform: str, credentials: dict[str, str], audio_dir: str = "./data/audio") -> VideoAPIIngester: +def get_api_ingester( + platform: str, credentials: dict[str, str], audio_dir: str = "./data/audio" +) -> VideoAPIIngester: """ Factory: return the right VideoAPIIngester for the given platform name. @@ -265,5 +271,7 @@ def get_api_ingester(platform: str, credentials: dict[str, str], audio_dir: str } cls = registry.get(platform.lower()) if cls is None: - raise ValueError(f"Unknown platform '{platform}'. Supported: {list(registry.keys())}") + raise ValueError( + f"Unknown platform '{platform}'. Supported: {list(registry.keys())}" + ) return cls(credentials=credentials, audio_dir=audio_dir) diff --git a/ingestion/youtube.py b/ingestion/youtube.py index de13b8a..81a6563 100644 --- a/ingestion/youtube.py +++ b/ingestion/youtube.py @@ -86,16 +86,29 @@ def _fetch_metadata(self, url: str) -> dict[str, Any]: """Run yt-dlp in dump-only mode to get video metadata as JSON.""" try: result = subprocess.run( - ["yt-dlp", "--dump-json", "--no-playlist", "--js-runtimes", "node,deno", url], + [ + "yt-dlp", + "--dump-json", + "--no-playlist", + "--js-runtimes", + "node,deno", + url, + ], capture_output=True, text=True, timeout=60, ) if result.returncode != 0: - logger.warning(f"[YouTube] Metadata fetch warning: {result.stderr[:200]}") + logger.warning( + f"[YouTube] Metadata fetch warning: {result.stderr[:200]}" + ) return {} return cast(dict[str, Any], json.loads(result.stdout)) - except (subprocess.TimeoutExpired, json.JSONDecodeError, FileNotFoundError) as e: + except ( + subprocess.TimeoutExpired, + json.JSONDecodeError, + FileNotFoundError, + ) as e: logger.warning(f"[YouTube] Metadata fetch failed: {e}") return {} @@ -131,4 +144,6 @@ def _download_audio(self, url: str, output_path: Path) -> None: if wav_candidate.exists(): wav_candidate.rename(output_path) else: - raise FileNotFoundError(f"Expected audio at {output_path} but not found") + raise FileNotFoundError( + f"Expected audio at {output_path} but not found" + ) diff --git a/processing/chunker.py b/processing/chunker.py index c975cb8..9643289 100644 --- a/processing/chunker.py +++ b/processing/chunker.py @@ -92,7 +92,7 @@ def __init__( self, chunk_duration: int | None = None, chunk_overlap: int | None = None, - ): + ) -> None: self.chunk_duration = chunk_duration or settings.chunk_duration_seconds self.chunk_overlap = chunk_overlap or settings.chunk_overlap_seconds @@ -171,7 +171,9 @@ def chunk(self, transcript: Transcript, asset: VideoAsset) -> list[VideoChunk]: # ── Private helpers ────────────────────────────────────────────────────── - def _get_chapter(self, time: float, chapters: list[dict[str, str | float]]) -> str | None: + def _get_chapter( + self, time: float, chapters: list[dict[str, str | float]] + ) -> str | None: """Return the chapter title for the given timestamp, if chapters exist.""" for ch in chapters: start = float(ch.get("start", 0)) diff --git a/processing/embedder.py b/processing/embedder.py index 0bce697..0a645c1 100644 --- a/processing/embedder.py +++ b/processing/embedder.py @@ -35,7 +35,7 @@ class Embedder: Lazy-loads the model on first use. """ - def __init__(self): + def __init__(self) -> None: self.mode = settings.embedding_mode self._local_model: SentenceTransformerProtocol | None = None self._openai_client: OpenAIClientProtocol | None = None @@ -46,7 +46,6 @@ def dimension(self) -> int: if self.mode == EmbeddingMode.LOCAL: return 384 # all-MiniLM-L6-v2 else: - if "large" in settings.openai_embedding_model: return 3072 return 1536 @@ -67,9 +66,15 @@ def embed_chunks(self, chunks: list[VideoChunk]) -> list[list[float]]: texts = [c.text for c in chunks] logger.info(f"[Embedder/{self.mode.value}] Embedding {len(texts)} chunks...") - vectors = self._embed_local(texts) if self.mode == EmbeddingMode.LOCAL else self._embed_openai(texts) + vectors = ( + self._embed_local(texts) + if self.mode == EmbeddingMode.LOCAL + else self._embed_openai(texts) + ) - logger.success(f"[Embedder] Done — {len(vectors)} vectors, dim={self.dimension}") + logger.success( + f"[Embedder] Done — {len(vectors)} vectors, dim={self.dimension}" + ) return vectors def embed_query(self, query: str) -> list[float]: @@ -103,7 +108,9 @@ def _get_local_model(self) -> SentenceTransformerProtocol: def _get_openai_client(self) -> OpenAIClientProtocol: if self._openai_client is None: if not settings.openai_api_key: - raise RuntimeError("OPENAI_API_KEY is not set. Required when EMBEDDING_MODE=openai.") + raise RuntimeError( + "OPENAI_API_KEY is not set. Required when EMBEDDING_MODE=openai." + ) from typing import cast from openai import OpenAI @@ -134,7 +141,9 @@ def _embed_openai(self, texts: list[str]) -> list[list[float]]: # Truncate to avoid token limit (8191 tokens for text-embedding-3-*) batch = [t[:8000] for t in batch] response = client.embeddings.create(model=model, input=batch) - batch_embeddings = [item.embedding for item in sorted(response.data, key=lambda x: x.index)] + batch_embeddings = [ + item.embedding for item in sorted(response.data, key=lambda x: x.index) + ] all_embeddings.extend(batch_embeddings) return all_embeddings diff --git a/pyproject.toml b/pyproject.toml index 3dd286c..de23b34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,9 +6,11 @@ authors = [ { name = "Streaming Video-RAG Team" } ] readme = "README.md" -requires-python = "==3.14.*" +requires-python = ">=3.12" classifiers = [ "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", @@ -17,58 +19,58 @@ classifiers = [ ] dependencies = [ # Core - "pydantic>=2.0", - "pydantic-settings>=2.0", - "python-dotenv>=1.0", + "pydantic>=2.7.0", + "pydantic-settings>=2.2.0", + "python-dotenv>=1.0.1", # Ingestion "yt-dlp>=2024.1.1", "ffmpeg-python>=0.2.0", - "requests>=2.31", + "requests>=2.31.0", # Transcription "openai-whisper>=20231117", # LLM Providers - "openai>=1.30", - "anthropic>=0.25", - "langchain>=0.2", - "langchain-openai>=0.1", - "langchain-anthropic>=0.1", - "langchain-ollama>=0.1", - "langchain-community>=0.2", + "openai>=1.30.0", + "anthropic>=0.25.0", + "langchain>=0.2.0", + "langchain-openai>=0.1.0", + "langchain-anthropic>=0.1.0", + "langchain-ollama>=0.1.0", + "langchain-community>=0.2.0", # Embeddings - "sentence-transformers>=2.7", - "torch>=2.0", + "sentence-transformers>=2.7.0", + "torch>=2.2.0", # Vector Stores - "chromadb>=0.5", - "qdrant-client>=1.9", + "chromadb>=0.5.0", + "qdrant-client>=1.9.0", # RAG - "tiktoken>=0.7", + "tiktoken>=0.7.0", "nltk>=3.8.1", # Storage - "sqlalchemy>=2.0", - "alembic>=1.13", - "aiosqlite>=0.20", + "sqlalchemy>=2.0.30", + "alembic>=1.13.1", + "aiosqlite>=0.20.0", # API - "fastapi>=0.111", - "uvicorn[standard]>=0.29", + "fastapi>=0.111.0", + "uvicorn[standard]>=0.29.0", "python-multipart>=0.0.9", - "httpx>=0.27", + "httpx>=0.27.0", # UI - "streamlit>=1.35", - "streamlit-extras>=0.4", + "streamlit>=1.35.0", + "streamlit-extras>=0.4.0", # Utilities - "tqdm>=4.66", - "loguru>=0.7", - "tenacity>=8.3", + "tqdm>=4.66.0", + "loguru>=0.7.2", + "tenacity>=8.3.0", ] [project.urls] @@ -82,14 +84,16 @@ video-rag-ui = "ui.app:main" [project.optional-dependencies] dev = [ - "black>=24.0.0", "ruff>=0.7.0", "mypy>=1.13.0", "pre-commit>=3.7.0", "pydantic[mypy]>=2.0", "sqlalchemy[mypy]>=2.0", "types-requests>=2.31.0", - "types-tqdm>=4.66.0" + "types-tqdm>=4.66.0", + "pytest>=8.0.0", + "pytest-asyncio>=0.23.0", + "pytest-cov>=4.1.0", ] [build-system] @@ -99,34 +103,9 @@ build-backend = "setuptools.build_meta" [tool.setuptools] packages = ["api", "ingestion", "llm", "processing", "rag", "storage", "transcription", "ui", "vector_store"] -[tool.black] -line-length = 120 -target-version = ["py314"] -exclude = ''' -/( - build - | dist - | __pycache__ - | .git - | .venv - | venv - | node_modules - | data - | certs - | .github - | migrations -)/ -| \.env.* -| docker-compose\.yml -| Dockerfile -| railway\.toml -| \.gitignore -| \.pre-commit-config\.yaml -''' - [tool.ruff] line-length = 120 -target-version = "py314" +target-version = "py312" exclude = [ "build/", "dist/", @@ -139,19 +118,31 @@ exclude = [ [tool.ruff.lint] select = [ - "E", # pycodestyle errors - "F", # pyflakes - "I", # isort - "B", # bugbear - "C4", # comprehensions - "SIM",# simplify - "UP", # pyupgrade - "RUF",# ruff specific - "TID",# tidy imports - "TRY",# try/except - "PERF"# performance + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort + "B", # bugbear + "C4", # comprehensions + "SIM", # simplify + "UP", # pyupgrade + "RUF", # ruff specific + "TID", # tidy imports + "TRY", # try/except + "PERF",# performance + "ANN", # annotations + "A", # built-ins + "ARG", # unused arguments + "PTH", # pathlib + "ERA", # eradicate (commented out code) +] +ignore = [ + "E501", # line length (handled by formatter) + "TRY003", # long message in except + "TRY301", # abstract raise to inner function + "ANN101", # Missing type annotation for self + "ANN102", # Missing type annotation for cls + "ANN401", # Dynamically typed expressions (Any) allowed if explicit ] -ignore = ["E501", "TRY003", "TRY301"] fixable = ["ALL"] unfixable = [] @@ -164,27 +155,26 @@ known-first-party = ["api", "ingestion", "llm", "processing", "rag", "storage", section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"] [tool.mypy] -python_version = "3.14" -strict = false -ignore_missing_imports = true -disallow_untyped_defs = false -disallow_untyped_calls = false -disallow_incomplete_defs = false -disallow_subclassing_any = false -disallow_untyped_decorators = false +python_version = "3.12" +strict = true +ignore_missing_imports = false +disallow_untyped_defs = true +disallow_untyped_calls = true +disallow_incomplete_defs = true +disallow_subclassing_any = true +disallow_untyped_decorators = true no_implicit_optional = true -warn_redundant_casts = false -warn_unused_ignores = false -warn_no_return = false -warn_return_any = false -strict_optional = false +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_return_any = true +strict_optional = true plugins = [ "pydantic.mypy", "sqlalchemy.ext.mypy.plugin" ] -show_error_codes = false -pretty = false -color_output = false +show_error_codes = true +pretty = true exclude = [ "build/", "dist/", @@ -193,23 +183,17 @@ exclude = [ "venv/", "node_modules/", ".git/", - ".*/torch/", - ".*/transformers/", - ".*/huggingface_hub/", - ".*/sentence_transformers/", - ".*/langchain/", - ".*/chromadb/", - ".*/qdrant_client/", ] [[tool.mypy.overrides]] module = [ "chromadb.*", "openai.*", - "whisper", + "whisper.*", "sentence_transformers.*", - "ffmpeg", + "ffmpeg.*", "streamlit.*", + "streamlit_extras.*", "langchain.*", "langchain_community.*", "langchain_openai.*", @@ -220,16 +204,20 @@ module = [ "uvicorn.*", "fastapi.*", "alembic.*", - "loguru", + "loguru.*", "tqdm.*", - "tenacity.*" + "tenacity.*", + "requests.*", + "sqlalchemy.*", + "aiosqlite.*" ] -ignore_errors = true +ignore_missing_imports = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_functions = ["test_*"] +python_classes = ["Test*"] +addopts = "-v --cov=api --cov=ingestion --cov=llm --cov=processing --cov=rag --cov=storage --cov=transcription --cov=ui --cov=vector_store --cov-report=term-missing" +asyncio_mode = "auto" -[[tool.mypy.overrides]] -module = [ - "storage.database", - "ui.app", - "transcription.whisper_transcriber" -] -ignore_errors = true diff --git a/rag/qa_chain.py b/rag/qa_chain.py index e0eaca1..5808a55 100644 --- a/rag/qa_chain.py +++ b/rag/qa_chain.py @@ -71,7 +71,7 @@ class QAChain: Retrieves relevant chunks, builds a prompt, and calls the LLM. """ - def __init__(self): + def __init__(self) -> None: self.retriever = get_retriever() def ask( @@ -110,7 +110,10 @@ def ask( # ── Call LLM ────────────────────────────────────────────────────────── llm = get_llm(temperature=0.0) - messages = [SystemMessage(content=SYSTEM_PROMPT), HumanMessage(content=user_message)] + messages = [ + SystemMessage(content=SYSTEM_PROMPT), + HumanMessage(content=user_message), + ] response = llm.invoke(messages) answer = response.content if hasattr(response, "content") else str(response) answer = cast(str, answer) diff --git a/rag/retriever.py b/rag/retriever.py index 9a19b1a..3afb09a 100644 --- a/rag/retriever.py +++ b/rag/retriever.py @@ -12,7 +12,7 @@ _retriever_instance = None -def get_retriever(): +def get_retriever() -> "Retriever": """Return a cached Retriever singleton.""" global _retriever_instance if _retriever_instance is None: @@ -26,7 +26,7 @@ class Retriever: Handles embedding the query and calling the vector store. """ - def __init__(self): + def __init__(self) -> None: self.embedder = Embedder() self.store = get_vector_store() @@ -48,11 +48,15 @@ def retrieve( List of SearchResult sorted by relevance (highest first) """ top_k = top_k or settings.retrieval_top_k - logger.debug(f"[Retriever] Query='{query[:60]}', top_k={top_k}, video_id={video_id}") + logger.debug( + f"[Retriever] Query='{query[:60]}', top_k={top_k}, video_id={video_id}" + ) try: query_vector = self.embedder.embed_query(query) - results = self.store.search(query_vector, top_k=top_k, filter_video_id=video_id) + results = self.store.search( + query_vector, top_k=top_k, filter_video_id=video_id + ) except Exception as e: import traceback @@ -71,7 +75,9 @@ def retrieve( # Validate results before returning for idx, result in enumerate(results): if not hasattr(result, "chunk") or not hasattr(result, "score"): - logger.warning(f"[Retriever] Invalid result object at index {idx}: {result}") + logger.warning( + f"[Retriever] Invalid result object at index {idx}: {result}" + ) return results @@ -82,5 +88,7 @@ def format_context(self, results: list[SearchResult]) -> str: """ parts = [] for i, r in enumerate(results, 1): - parts.append(f'[{i}] "{r.chunk.title}" @ {r.chunk.start_ts}-{r.chunk.end_ts}\n{r.chunk.text}') + parts.append( + f'[{i}] "{r.chunk.title}" @ {r.chunk.start_ts}-{r.chunk.end_ts}\n{r.chunk.text}' + ) return "\n\n---\n\n".join(parts) diff --git a/rag/search.py b/rag/search.py index bd0daea..9f8ba88 100644 --- a/rag/search.py +++ b/rag/search.py @@ -27,7 +27,7 @@ class SearchEngine: Supports optional filtering by video_id. """ - def __init__(self): + def __init__(self) -> None: self.retriever = get_retriever() def search( diff --git a/rag/summarizer.py b/rag/summarizer.py index c92335e..517d186 100644 --- a/rag/summarizer.py +++ b/rag/summarizer.py @@ -62,7 +62,7 @@ class Summarizer: Generates summaries of indexed videos using map-reduce over chunks. """ - def __init__(self): + def __init__(self) -> None: self.store = get_vector_store() def summarize( @@ -90,7 +90,9 @@ def summarize( dim = 1536 if settings.embedding_mode == EmbeddingMode.OPENAI else 384 dummy_vector = [0.0] * dim - all_results = self.store.search(dummy_vector, top_k=9999, filter_video_id=video_id) + all_results = self.store.search( + dummy_vector, top_k=9999, filter_video_id=video_id + ) # Sort by start time all_results.sort(key=lambda r: r.chunk.start_time) @@ -140,7 +142,9 @@ def _map_chunks(self, results: list[SearchResult]) -> list[str]: end_ts=r.chunk.end_ts, text=r.chunk.text, ) - response = llm.invoke([SystemMessage(content=MAP_SYSTEM), HumanMessage(content=prompt)]) + response = llm.invoke( + [SystemMessage(content=MAP_SYSTEM), HumanMessage(content=prompt)] + ) content = cast(str, response.content) summaries.append(content.strip()) @@ -149,13 +153,19 @@ def _map_chunks(self, results: list[SearchResult]) -> list[str]: def _reduce(self, summaries: list[str], title: str) -> str: """Combine chunk summaries into a final overall summary (reduce phase).""" llm = get_llm(temperature=0.3) - combined = "\n\n".join(f"[Section {i + 1}] {s}" for i, s in enumerate(summaries)) + combined = "\n\n".join( + f"[Section {i + 1}] {s}" for i, s in enumerate(summaries) + ) prompt = REDUCE_TEMPLATE.format(title=title, summaries=combined) - response = llm.invoke([SystemMessage(content=REDUCE_SYSTEM), HumanMessage(content=prompt)]) + response = llm.invoke( + [SystemMessage(content=REDUCE_SYSTEM), HumanMessage(content=prompt)] + ) content = cast(str, response.content) return content.strip() - def _group_by_chapter(self, results: list[SearchResult]) -> dict[str, list[SearchResult]]: + def _group_by_chapter( + self, results: list[SearchResult] + ) -> dict[str, list[SearchResult]]: """Group chunks by their chapter label.""" grouped: dict[str, list[SearchResult]] = {} for r in results: @@ -163,14 +173,19 @@ def _group_by_chapter(self, results: list[SearchResult]) -> dict[str, list[Searc grouped.setdefault(key, []).append(r) return grouped - def _summarize_chapters(self, chapters: dict[str, list[SearchResult]]) -> list[dict[str, str]]: + def _summarize_chapters( + self, chapters: dict[str, list[SearchResult]] + ) -> list[dict[str, str]]: """Generate a brief summary for each chapter.""" llm = get_llm(temperature=0.2) chapter_content = "\n\n".join( - f"Chapter: {ch}\nContent: {' '.join(r.chunk.text for r in chunks[:3])}" for ch, chunks in chapters.items() + f"Chapter: {ch}\nContent: {' '.join(r.chunk.text for r in chunks[:3])}" + for ch, chunks in chapters.items() ) prompt = CHAPTER_TEMPLATE.format(chapter_content=chapter_content) - response = llm.invoke([SystemMessage(content=MAP_SYSTEM), HumanMessage(content=prompt)]) + response = llm.invoke( + [SystemMessage(content=MAP_SYSTEM), HumanMessage(content=prompt)] + ) result = [{"chapter": ch, "summary": ""} for ch in chapters] @@ -184,7 +199,9 @@ def _summarize_chapters(self, chapters: dict[str, list[SearchResult]]) -> list[d for i, ch in enumerate(chapters.keys()): if ch.lower() in line.lower(): if buffer and current_chapter_idx < len(result): - result[current_chapter_idx]["summary"] = " ".join(buffer).strip() + result[current_chapter_idx]["summary"] = " ".join( + buffer + ).strip() current_chapter_idx = i buffer = [] matched = True diff --git a/storage/database.py b/storage/database.py index 216e8cc..2766e4c 100644 --- a/storage/database.py +++ b/storage/database.py @@ -17,7 +17,9 @@ engine = create_engine( settings.database_url, - connect_args={"check_same_thread": False} if "sqlite" in settings.database_url else {}, + connect_args=( + {"check_same_thread": False} if "sqlite" in settings.database_url else {} + ), echo=False, ) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) @@ -35,10 +37,14 @@ class Video(Base): __tablename__ = "videos" - id: Mapped[str] = mapped_column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + id: Mapped[str] = mapped_column( + String, primary_key=True, default=lambda: str(uuid.uuid4()) + ) title: Mapped[str] = mapped_column(String, nullable=False) source_url: Mapped[str] = mapped_column(String, nullable=False) - source_type: Mapped[str] = mapped_column(String, nullable=False) # youtube | local_file | live_stream | video_api + source_type: Mapped[str] = mapped_column( + String, nullable=False + ) # youtube | local_file | live_stream | video_api duration_seconds: Mapped[float | None] = mapped_column(Float, nullable=True) description: Mapped[str | None] = mapped_column(Text, nullable=True) uploader: Mapped[str | None] = mapped_column(String, nullable=True) @@ -77,15 +83,31 @@ class IngestJob(Base): __tablename__ = "ingest_jobs" - id: Mapped[str] = mapped_column(String, primary_key=True, default=lambda: str(uuid.uuid4())) - video_id: Mapped[str | None] = mapped_column(String, nullable=True) # populated once ingestion starts - source: Mapped[str] = mapped_column(String, nullable=False) # original source URL or path + id: Mapped[str] = mapped_column( + String, primary_key=True, default=lambda: str(uuid.uuid4()) + ) + video_id: Mapped[str | None] = mapped_column( + String, nullable=True + ) # populated once ingestion starts + source: Mapped[str] = mapped_column( + String, nullable=False + ) # original source URL or path source_type: Mapped[str] = mapped_column(String, nullable=False) status: Mapped[str] = mapped_column( - SAEnum("queued", "ingesting", "transcribing", "indexing", "done", "error", name="job_status"), + SAEnum( + "queued", + "ingesting", + "transcribing", + "indexing", + "done", + "error", + name="job_status", + ), default="queued", ) - progress_message: Mapped[str | None] = mapped_column(String, nullable=True) # e.g. "Transcribing audio..." + progress_message: Mapped[str | None] = mapped_column( + String, nullable=True + ) # e.g. "Transcribing audio..." error_message: Mapped[str | None] = mapped_column(Text, nullable=True) created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) @@ -100,7 +122,9 @@ def to_dict(self) -> dict[str, object]: "progress_message": self.progress_message, "error_message": self.error_message, "created_at": self.created_at.isoformat() if self.created_at else None, - "completed_at": self.completed_at.isoformat() if self.completed_at else None, + "completed_at": ( + self.completed_at.isoformat() if self.completed_at else None + ), } diff --git a/transcription/whisper_transcriber.py b/transcription/whisper_transcriber.py index 1219d9f..deee9cb 100644 --- a/transcription/whisper_transcriber.py +++ b/transcription/whisper_transcriber.py @@ -7,6 +7,8 @@ Local model size controlled by WHISPER_MODEL_SIZE (default: base). """ +from __future__ import annotations + import json from dataclasses import dataclass from pathlib import Path @@ -55,7 +57,7 @@ class Transcript: segments: list[TranscriptSegment] full_text: str = "" - def __post_init__(self): + def __post_init__(self) -> None: if not self.full_text: self.full_text = " ".join(s.text.strip() for s in self.segments) @@ -64,7 +66,9 @@ def to_dict(self) -> dict[str, object]: "video_id": self.video_id, "language": self.language, "full_text": self.full_text, - "segments": [{"start": s.start, "end": s.end, "text": s.text} for s in self.segments], + "segments": [ + {"start": s.start, "end": s.end, "text": s.text} for s in self.segments + ], } def save(self, path: Path) -> None: @@ -75,7 +79,9 @@ def save(self, path: Path) -> None: @classmethod def load(cls, path: Path) -> Transcript: data = json.loads(path.read_text()) - segments = [TranscriptSegment(s["start"], s["end"], s["text"]) for s in data["segments"]] + segments = [ + TranscriptSegment(s["start"], s["end"], s["text"]) for s in data["segments"] + ] return cls( video_id=data["video_id"], language=data["language"], @@ -93,11 +99,13 @@ class WhisperTranscriber: Mode is determined by settings.whisper_mode. """ - def __init__(self): + def __init__(self) -> None: self.mode = settings.whisper_mode self._local_model: Any | None = None # lazy-loaded - def transcribe(self, audio_path: Path, video_id: str, language: str | None = None) -> Transcript: + def transcribe( + self, audio_path: Path, video_id: str, language: str | None = None + ) -> Transcript: """ Transcribe the audio file at `audio_path`. @@ -121,7 +129,9 @@ def transcribe(self, audio_path: Path, video_id: str, language: str | None = Non # ── Local Whisper ──────────────────────────────────────────────────────── - def _transcribe_local(self, audio_path: Path, video_id: str, language: str | None) -> Transcript: + def _transcribe_local( + self, audio_path: Path, video_id: str, language: str | None + ) -> Transcript: """Run openai-whisper locally.""" model = self._get_local_model() @@ -141,9 +151,13 @@ def _transcribe_local(self, audio_path: Path, video_id: str, language: str | Non ] detected_language = result.get("language", language or "en") - logger.success(f"[Whisper/local] Done — {len(segments)} segments, lang={detected_language}") + logger.success( + f"[Whisper/local] Done — {len(segments)} segments, lang={detected_language}" + ) - return Transcript(video_id=video_id, language=detected_language, segments=segments) + return Transcript( + video_id=video_id, language=detected_language, segments=segments + ) def _get_local_model(self) -> Any: """Lazy-load the Whisper model (avoids loading it until needed).""" @@ -158,7 +172,9 @@ def _get_local_model(self) -> Any: # ── OpenAI API Whisper ─────────────────────────────────────────────────── @retry(stop=stop_after_attempt(3), wait=wait_exponential(min=2, max=10)) - def _transcribe_api(self, audio_path: Path, video_id: str, language: str | None) -> Transcript: + def _transcribe_api( + self, audio_path: Path, video_id: str, language: str | None + ) -> Transcript: """Call the OpenAI Whisper API.""" from openai import OpenAI @@ -168,7 +184,7 @@ def _transcribe_api(self, audio_path: Path, video_id: str, language: str | None) max_retries=2, ) - with open(audio_path, "rb") as f: + with audio_path.open("rb") as f: base_kwargs = { "model": "whisper-1", "file": f, @@ -177,7 +193,9 @@ def _transcribe_api(self, audio_path: Path, video_id: str, language: str | None) } if language: - response = client.audio.transcriptions.create(**base_kwargs, language=language) + response = client.audio.transcriptions.create( + **base_kwargs, language=language + ) else: response = client.audio.transcriptions.create(**base_kwargs) @@ -193,6 +211,10 @@ def _transcribe_api(self, audio_path: Path, video_id: str, language: str | None) ] detected_language = getattr(response, "language", language or "en") - logger.success(f"[Whisper/api] Done — {len(segments)} segments, lang={detected_language}") + logger.success( + f"[Whisper/api] Done — {len(segments)} segments, lang={detected_language}" + ) - return Transcript(video_id=video_id, language=detected_language, segments=segments) + return Transcript( + video_id=video_id, language=detected_language, segments=segments + ) diff --git a/ui/tabs/ingest_tab.py b/ui/tabs/ingest_tab.py index 2784fc7..32cf826 100644 --- a/ui/tabs/ingest_tab.py +++ b/ui/tabs/ingest_tab.py @@ -6,7 +6,7 @@ from utils import api_post, poll_job -def render_ingest_tab(): +def render_ingest_tab() -> None: st.header("Ingest a Video") st.caption( "Add a video source to your RAG library. Supports YouTube URLs, local files, live streams, and video APIs." @@ -14,7 +14,12 @@ def render_ingest_tab(): source_type = st.radio( "Source type", - ["YouTube / URL", "Local File Path", "Live Stream (HLS/RTMP)", "Video API (Vimeo/Twitch)"], + [ + "YouTube / URL", + "Local File Path", + "Live Stream (HLS/RTMP)", + "Video API (Vimeo/Twitch)", + ], horizontal=True, ) @@ -30,9 +35,15 @@ def render_ingest_tab(): col1, col2 = st.columns(2) with col1: - language = st.text_input("Language (optional)", placeholder="en, es, fr, de, ...") + language = st.text_input( + "Language (optional)", placeholder="en, es, fr, de, ..." + ) with col2: - platform = st.selectbox("Platform", ["vimeo", "twitch"]) if "Video API" in source_type else None + platform = ( + st.selectbox("Platform", ["vimeo", "twitch"]) + if "Video API" in source_type + else None + ) source_type_map = { "YouTube / URL": "youtube", @@ -55,7 +66,9 @@ def render_ingest_tab(): status_placeholder = st.empty() final = poll_job(job_id, status_placeholder) if final.get("status") == "done": - status_placeholder.success(f"✅ Ingestion complete! Video ID: `{final.get('video_id')}`") + status_placeholder.success( + f"✅ Ingestion complete! Video ID: `{final.get('video_id')}`" + ) st.balloons() elif final.get("status") == "error": status_placeholder.error(f"❌ Error: {final.get('error_message')}") diff --git a/ui/tabs/library_tab.py b/ui/tabs/library_tab.py index 2b2d1fb..c4106ba 100644 --- a/ui/tabs/library_tab.py +++ b/ui/tabs/library_tab.py @@ -7,7 +7,7 @@ from utils import API_BASE, api_get -def render_library_tab(): +def render_library_tab() -> None: st.header("Video Library") st.caption("All videos that have been ingested into your RAG system.") @@ -30,17 +30,24 @@ def render_library_tab(): st.metric("Chunks", v.get("chunk_count", 0)) with col2: dur = v.get("duration_seconds") - st.metric("Duration", f"{int(dur // 60)}m {int(dur % 60)}s" if dur else "—") + st.metric( + "Duration", + f"{int(dur // 60)}m {int(dur % 60)}s" if dur else "—", + ) with col3: st.metric("Language", v.get("language", "—").upper()) - st.caption(f"ID: `{v['id']}` · Type: `{v['source_type']}` · Status: `{v['status']}`") + st.caption( + f"ID: `{v['id']}` · Type: `{v['source_type']}` · Status: `{v['status']}`" + ) if v.get("source_url"): st.caption(f"Source: {v['source_url'][:80]}") if v.get("error_message"): st.error(f"Error: {v['error_message']}") - if v["status"] == "indexed" and st.button("🗑 Delete", key=f"del_{v['id']}"): + if v["status"] == "indexed" and st.button( + "🗑 Delete", key=f"del_{v['id']}" + ): result = httpx.delete(f"{API_BASE}/videos/{v['id']}", timeout=30) if result.status_code == 200: st.success("Deleted") @@ -48,4 +55,6 @@ def render_library_tab(): else: st.error("Delete failed") else: - st.info("No videos in the library yet. Head to the Ingest tab to add your first video.") + st.info( + "No videos in the library yet. Head to the Ingest tab to add your first video." + ) diff --git a/ui/tabs/qa_tab.py b/ui/tabs/qa_tab.py index 1d0ac97..a682fb8 100644 --- a/ui/tabs/qa_tab.py +++ b/ui/tabs/qa_tab.py @@ -6,9 +6,11 @@ from utils import api_post -def render_qa_tab(video_options): +def render_qa_tab(video_options: dict[str, str | None]) -> None: st.header("Ask a Question") - st.caption("Get answers grounded in your indexed video content, with timestamped citations.") + st.caption( + "Get answers grounded in your indexed video content, with timestamped citations." + ) selected_video_label = st.selectbox("Scope (optional)", list(video_options.keys())) selected_video_id = video_options[selected_video_label] diff --git a/ui/tabs/search_tab.py b/ui/tabs/search_tab.py index 1661a3b..80ca321 100644 --- a/ui/tabs/search_tab.py +++ b/ui/tabs/search_tab.py @@ -6,14 +6,18 @@ from utils import api_post -def render_search_tab(video_options): +def render_search_tab(video_options: dict[str, str | None]) -> None: st.header("Semantic Search") st.caption("Find relevant video segments by meaning, not just keywords.") - search_video_label = st.selectbox("Scope (optional)", list(video_options.keys()), key="search_scope") + search_video_label = st.selectbox( + "Scope (optional)", list(video_options.keys()), key="search_scope" + ) search_video_id = video_options[search_video_label] - query = st.text_input("Search query", placeholder="neural network training techniques") + query = st.text_input( + "Search query", placeholder="neural network training techniques" + ) col1, col2 = st.columns(2) with col1: diff --git a/ui/tabs/summarize_tab.py b/ui/tabs/summarize_tab.py index f3ac363..1e3ee3e 100644 --- a/ui/tabs/summarize_tab.py +++ b/ui/tabs/summarize_tab.py @@ -2,21 +2,23 @@ Summarize Tab Component """ +from typing import Any + import streamlit as st from utils import api_get, api_post -def _get_video_choices(videos: list) -> dict: +def _get_video_choices(videos: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: """Build video selection choices from video list.""" return {f"{v['title'][:60]} ({v['id'][:8]})": v for v in videos} -def _render_no_videos_message(): +def _render_no_videos_message() -> None: """Display message when no indexed videos are available.""" st.info("No indexed videos yet. Go to the Ingest tab to add some.") -def _render_summary_results(response: dict): +def _render_summary_results(response: dict[str, Any]) -> None: """Render the summary and chapter results from API response.""" st.subheader(f"Summary: {response['title']}") st.caption(f"Based on {response['chunk_count']} indexed segments") @@ -27,7 +29,7 @@ def _render_summary_results(response: dict): _render_chapter_summaries(chapter_summaries) -def _render_chapter_summaries(chapter_summaries: list): +def _render_chapter_summaries(chapter_summaries: list[dict[str, Any]]) -> None: """Render individual chapter summaries as expanders.""" st.subheader("Chapter Summaries") for chapter in chapter_summaries: @@ -35,27 +37,33 @@ def _render_chapter_summaries(chapter_summaries: list): st.write(chapter["summary"]) -def _fetch_indexed_videos() -> list: +def _fetch_indexed_videos() -> list[dict[str, Any]]: """Fetch list of indexed videos from API.""" videos_data = api_get("/videos", params={"status": "indexed", "limit": 100}) if videos_data is None: return [] - return videos_data.get("videos", []) + videos: list[dict[str, Any]] = videos_data.get("videos", []) + return videos -def _handle_generate_summary(video_id: str, include_chapters: bool) -> dict | None: +def _handle_generate_summary( + video_id: str, include_chapters: bool +) -> dict[str, Any] | None: """Call API to generate summary and return response.""" with st.spinner("Summarizing... this may take a minute for long videos."): - return api_post( + response: dict[str, Any] | None = api_post( "/summarize", {"video_id": video_id, "include_chapters": include_chapters}, ) + return response -def render_summarize_tab(): +def render_summarize_tab() -> None: """Render the summarize tab UI.""" st.header("Summarize a Video") - st.caption("Generate an overall summary and per-chapter breakdown using map-reduce.") + st.caption( + "Generate an overall summary and per-chapter breakdown using map-reduce." + ) videos = _fetch_indexed_videos() diff --git a/ui/utils.py b/ui/utils.py index f526d93..ef24b03 100644 --- a/ui/utils.py +++ b/ui/utils.py @@ -4,6 +4,7 @@ import os import time +from typing import Any import httpx import streamlit as st @@ -12,7 +13,7 @@ REQUEST_TIMEOUT = int(os.getenv("UI_REQUEST_TIMEOUT", 600)) -def api_get(path: str, **kwargs): +def api_get(path: str, **kwargs: Any) -> Any: try: r = httpx.get(f"{API_BASE}{path}", timeout=30, **kwargs) r.raise_for_status() @@ -35,7 +36,7 @@ def api_get(path: str, **kwargs): return None -def api_post(path: str, json: dict[str, object | None] | dict[str, str]): +def api_post(path: str, json: dict[str, object | None] | dict[str, str]) -> Any: try: r = httpx.post(f"{API_BASE}{path}", json=json, timeout=REQUEST_TIMEOUT) r.raise_for_status() @@ -60,7 +61,9 @@ def api_post(path: str, json: dict[str, object | None] | dict[str, str]): return None -def poll_job(job_id: str, placeholder: st.delta_generator.DeltaGenerator) -> dict[str, object]: +def poll_job( + job_id: str, placeholder: st.delta_generator.DeltaGenerator +) -> dict[str, object]: """Poll ingestion job until done or error.""" for _ in range(300): # max ~5 minutes data = api_get(f"/ingest/{job_id}") @@ -75,7 +78,7 @@ def poll_job(job_id: str, placeholder: st.delta_generator.DeltaGenerator) -> dic return {} -def load_video_options(): +def load_video_options() -> tuple[dict[str, str | None], Any]: """Load available videos for selection dropdowns""" videos_data = api_get("/videos", params={"status": "indexed", "limit": 100}) video_options = {"All videos": None} @@ -87,7 +90,7 @@ def load_video_options(): return video_options, videos_data -def apply_custom_css(): +def apply_custom_css() -> None: """Apply custom CSS styles""" st.markdown( """ diff --git a/vector_store/__init__.py b/vector_store/__init__.py index 695176a..770fad6 100644 --- a/vector_store/__init__.py +++ b/vector_store/__init__.py @@ -20,4 +20,10 @@ def get_vector_store() -> BaseVectorStore: return _vector_store_instance -__all__ = ["BaseVectorStore", "ChromaVectorStore", "QdrantVectorStore", "SearchResult", "get_vector_store"] +__all__ = [ + "BaseVectorStore", + "ChromaVectorStore", + "QdrantVectorStore", + "SearchResult", + "get_vector_store", +] diff --git a/vector_store/base.py b/vector_store/base.py index 8355dd9..4715c00 100644 --- a/vector_store/base.py +++ b/vector_store/base.py @@ -32,7 +32,9 @@ def end_ts(self) -> str: class BaseVectorStore(ABC): @abstractmethod - def add_chunks(self, chunks: list[VideoChunk], embeddings: list[list[float]]) -> None: + def add_chunks( + self, chunks: list[VideoChunk], embeddings: list[list[float]] + ) -> None: """Store chunks and their embeddings.""" ... diff --git a/vector_store/chroma_store.py b/vector_store/chroma_store.py index 0e145b1..70debe7 100644 --- a/vector_store/chroma_store.py +++ b/vector_store/chroma_store.py @@ -20,7 +20,7 @@ class ChromaVectorStore(BaseVectorStore): - def __init__(self): + def __init__(self) -> None: self._client = chromadb.PersistentClient( path=settings.chroma_persist_dir, settings=ChromaSettings(anonymized_telemetry=False), @@ -29,9 +29,13 @@ def __init__(self): name=COLLECTION_NAME, metadata={"hnsw:space": "cosine"}, ) - logger.info(f"[Chroma] Connected — collection '{COLLECTION_NAME}', {self._collection.count()} chunks indexed") + logger.info( + f"[Chroma] Connected — collection '{COLLECTION_NAME}', {self._collection.count()} chunks indexed" + ) - def add_chunks(self, chunks: list[VideoChunk], embeddings: list[list[float]]) -> None: + def add_chunks( + self, chunks: list[VideoChunk], embeddings: list[list[float]] + ) -> None: if not chunks: return @@ -54,10 +58,14 @@ def search( top_k: int = 5, filter_video_id: str | None = None, ) -> list[SearchResult]: - where = cast(Where | None, {"video_id": filter_video_id} if filter_video_id else None) + where = cast( + Where | None, {"video_id": filter_video_id} if filter_video_id else None + ) results = self._collection.query( - query_embeddings=cast(Sequence[Sequence[float | int] | None], [query_vector]), + query_embeddings=cast( + Sequence[Sequence[float | int] | None], [query_vector] + ), n_results=min(top_k, self._collection.count() or 1), where=where, include=["documents", "metadatas", "distances"], @@ -69,9 +77,13 @@ def search( metas = results["metadatas"][0] if results["metadatas"] else [] distances = results["distances"][0] if results["distances"] else [] - for i, (doc_id, text, meta, dist) in enumerate(zip(ids, docs, metas, distances, strict=False)): + for i, (doc_id, text, meta, dist) in enumerate( + zip(ids, docs, metas, distances, strict=False) + ): if not meta or "video_id" not in meta: - logger.warning(f"[Chroma] Skipping result {doc_id!r} — missing metadata") + logger.warning( + f"[Chroma] Skipping result {doc_id!r} — missing metadata" + ) continue # Chroma returns cosine distance (0=identical, 2=opposite) # Convert to similarity score: 1 - dist/2 @@ -98,7 +110,9 @@ def delete_video(self, video_id: str) -> int: ids_to_delete = existing["ids"] if ids_to_delete: self._collection.delete(ids=ids_to_delete) - logger.info(f"[Chroma] Deleted {len(ids_to_delete)} chunks for video {video_id}") + logger.info( + f"[Chroma] Deleted {len(ids_to_delete)} chunks for video {video_id}" + ) return len(ids_to_delete) def count(self, video_id: str | None = None) -> int: diff --git a/vector_store/qdrant_store.py b/vector_store/qdrant_store.py index 5308e2b..cc2d3bd 100644 --- a/vector_store/qdrant_store.py +++ b/vector_store/qdrant_store.py @@ -41,7 +41,9 @@ def __init__(self) -> None: # Infer dimension from embedder self._dim = cast(Any, Embedder()).dimension self._ensure_collection() - logger.info(f"[Qdrant] Connected — collection '{self._collection}', dim={self._dim}") + logger.info( + f"[Qdrant] Connected — collection '{self._collection}', dim={self._dim}" + ) def _ensure_collection(self) -> None: """Create the collection if it doesn't exist yet.""" @@ -53,7 +55,9 @@ def _ensure_collection(self) -> None: ) logger.info(f"[Qdrant] Created collection '{self._collection}'") - def add_chunks(self, chunks: list[VideoChunk], embeddings: list[list[float]]) -> None: + def add_chunks( + self, chunks: list[VideoChunk], embeddings: list[list[float]] + ) -> None: if not chunks: return @@ -78,9 +82,17 @@ def search( try: query_filter = None if filter_video_id: - query_filter = Filter(must=[FieldCondition(key="video_id", match=MatchValue(value=filter_video_id))]) - - logger.debug(f"[Qdrant] Executing search with top_k={top_k}, filter={filter_video_id}") + query_filter = Filter( + must=[ + FieldCondition( + key="video_id", match=MatchValue(value=filter_video_id) + ) + ] + ) + + logger.debug( + f"[Qdrant] Executing search with top_k={top_k}, filter={filter_video_id}" + ) hits = self._client.search( collection_name=self._collection, @@ -108,15 +120,21 @@ def search( ) results.append(SearchResult(chunk=chunk, score=float(hit.score))) except Exception as hit_error: - logger.warning(f"[Qdrant] Failed to parse hit id={hit.id}: {hit_error}") + logger.warning( + f"[Qdrant] Failed to parse hit id={hit.id}: {hit_error}" + ) continue except Exception as e: logger.error(f"[Qdrant] Search failed: {e!s}") - logger.error(f"[Qdrant] Collection: {self._collection}, top_k: {top_k}, filter: {filter_video_id}") + logger.error( + f"[Qdrant] Collection: {self._collection}, top_k: {top_k}, filter: {filter_video_id}" + ) raise else: - logger.debug(f"[Qdrant] Search completed successfully, found {len(results)} valid results") + logger.debug( + f"[Qdrant] Search completed successfully, found {len(results)} valid results" + ) return results def delete_video(self, video_id: str) -> int: @@ -124,7 +142,9 @@ def delete_video(self, video_id: str) -> int: count_before = self.count(video_id) self._client.delete( collection_name=self._collection, - points_selector=Filter(must=[FieldCondition(key="video_id", match=MatchValue(value=video_id))]), + points_selector=Filter( + must=[FieldCondition(key="video_id", match=MatchValue(value=video_id))] + ), ) logger.info(f"[Qdrant] Deleted ~{count_before} chunks for video {video_id}") return count_before @@ -133,11 +153,17 @@ def count(self, video_id: str | None = None) -> int: if video_id: result = self._client.count( collection_name=self._collection, - count_filter=Filter(must=[FieldCondition(key="video_id", match=MatchValue(value=video_id))]), + count_filter=Filter( + must=[ + FieldCondition(key="video_id", match=MatchValue(value=video_id)) + ] + ), exact=True, ) return cast(int, result.count) - return cast(int, self._client.count(collection_name=self._collection, exact=True).count) + return cast( + int, self._client.count(collection_name=self._collection, exact=True).count + ) @staticmethod def _chunk_id_to_int(chunk_id: str) -> int: