Configurator.py

"""
Configuration Management Module

This module provides centralized configuration management for the WhatTheBot RAG system.
It handles loading configuration from keys.json, environment variable overrides,
and provides type-safe configuration objects for all system components.

Key Features:
- Centralized configuration loading from keys.json
- Environment variable overrides for all settings
- Type-safe configuration objects with dataclasses
- Comprehensive validation and error handling
- Support for all embedding models and chunking strategies

Author: Professional Development Team
Version: 1.0.0
"""

import json
import os
from enum import Enum
from dataclasses import dataclass, field
from typing import Dict, Optional, List


class ChunkStrategy(Enum):
    """
    Available text chunking strategies.
    
    Each strategy implements a different approach to splitting text into chunks
    for optimal embedding and retrieval performance.
    
    Attributes:
        FIXED: Split text by fixed character count
        OVERLAP: Split with overlapping chunks for context preservation
        SENTENCE: Group sentences until target token count
        PARAGRAPH: Split at paragraph boundaries
        SEMANTIC: LLM-guided semantic boundary detection (placeholder)
        HYBRID: Heading-based with token windows inside sections
        DOC_STRUCTURE: Markdown-aware structure preservation
        SLIDING_WINDOW: Token-based sliding window approach
    """
    FIXED = "FIXED"
    OVERLAP = "OVERLAP"
    SENTENCE = "SENTENCE"
    PARAGRAPH = "PARAGRAPH"
    SEMANTIC = "SEMANTIC"
    HYBRID = "HYBRID"
    DOC_STRUCTURE = "DOC_STRUCTURE"
    SLIDING_WINDOW = "SLIDING_WINDOW"


class EmbedModel(Enum):
    """
    Available Azure OpenAI embedding models.
    
    Each model offers different performance characteristics and vector dimensions.
    
    Attributes:
        ADA_002: Original embedding model, 1536 dimensions, cost-effective
        EMB_3_SMALL: Newer model, 1536 dimensions, balanced performance
        EMB_3_LARGE: Latest model, 3072 dimensions, highest quality
    """
    ADA_002 = "ADA_002"
    EMB_3_SMALL = "EMB_3_SMALL"
    EMB_3_LARGE = "EMB_3_LARGE"


class RerankMethod(Enum):
    """
    Available re-ranking methods for RAG retrieval.
    
    Re-ranking improves retrieval quality by reordering initial results
    based on different criteria.
    
    Attributes:
        MMR: Maximal Marginal Relevance - balances relevance and diversity
        CROSS_ENCODER: Cross-encoder model for precise relevance scoring
        LLM_RERANK: LLM-based re-ranking for complex relevance assessment
    """
    MMR = "MMR"
    CROSS_ENCODER = "CROSS_ENCODER"
    LLM_RERANK = "LLM_RERANK"


@dataclass
class ChunkConfig:
    """Configuration for text chunking."""
    strategy: ChunkStrategy = ChunkStrategy.FIXED
    max_chars: int = 1000
    overlap_chars: int = 100
    max_tokens: int = 250
    overlap_tokens: int = 25
    sentence_target_tokens: int = 200
    paragraph_target_tokens: int = 500
    sliding_window_tokens: int = 200
    sliding_step_tokens: int = 50
    semantic_hints: bool = False


@dataclass
class EmbedConfig:
    """Configuration for text embedding."""
    model: EmbedModel = EmbedModel.EMB_3_SMALL
    dimensions: Optional[int] = None


@dataclass
class LLMConfig:
    """Configuration for LLM (Chat/Completion) models."""
    model: str = "gpt-4o"
    max_tokens: int = 1000
    temperature: float = 0.7


@dataclass
class ConversationConfig:
    """Conversation tracking and summarization configuration."""
    conversation_mood: bool = False
    number_of_conversations_to_store: int = 10
    conversations: List[Dict] = field(default_factory=list)

@dataclass
class RAGConfig:
    """Configuration for RAG (Retrieval-Augmented Generation) pipeline."""
    top_k: int = 10
    rerank_method: RerankMethod = RerankMethod.MMR
    final_chunks: int = 5
    context_max_tokens: int = 1500
    system_prompt: str = "You are a retrieval-augmented assistant. Use only the provided context. Cite sources. If unsure, say you don't know."
    mmr_diversity_threshold: float = 0.7


@dataclass
class AppConfig:
    """Main application configuration."""
    knowledge_dir: str = "Knowledge"
    chunk: ChunkConfig = None
    embed: EmbedConfig = None
    llm: LLMConfig = None
    rag: RAGConfig = None
    conversation: ConversationConfig = None
    
    # Azure OpenAI configuration
    azure_endpoint: str = ""
    azure_api_key: str = ""
    azure_api_version: str = "2024-06-01"
    azure_embedding_deployment_map: Dict[EmbedModel, str] = None
    
    # Qdrant configuration
    qdrant_url: str = ""
    qdrant_api_key: str = ""
    qdrant_collection: str = ""
    
    def __post_init__(self):
        if self.chunk is None:
            self.chunk = ChunkConfig()
        if self.embed is None:
            self.embed = EmbedConfig()
        if self.llm is None:
            self.llm = LLMConfig()
        if self.rag is None:
            self.rag = RAGConfig()
        if self.azure_embedding_deployment_map is None:
            self.azure_embedding_deployment_map = {
                EmbedModel.ADA_002: "text-embedding-ada-002",
                EmbedModel.EMB_3_SMALL: "text-embedding-3-small",
                EmbedModel.EMB_3_LARGE: "text-embedding-3-large"
            }


def load_keys() -> Dict[str, str]:
    """
    Load configuration from secret.json and keys.json files.
    
    This function loads configuration from both secret.json (for sensitive data)
    and keys.json (for non-sensitive configuration). It prioritizes secret.json
    for sensitive credentials and falls back to keys.json for other settings.
    
    Returns:
        Dict[str, str]: Dictionary containing all keys from both files
        
    Raises:
        FileNotFoundError: If keys.json file doesn't exist
        KeyError: If required keys are missing
        json.JSONDecodeError: If JSON files are invalid
    """
    keys_file = "keys.json"
    
    if not os.path.exists(keys_file):
        raise FileNotFoundError(f"Keys file '{keys_file}' not found. Please create it with required credentials.")
    
    try:
        with open(keys_file, 'r', encoding='utf-8') as f:
            keys = json.load(f)
    except json.JSONDecodeError as e:
        raise json.JSONDecodeError(f"Invalid JSON in {keys_file}: {e}")
    
    # Load sensitive credentials from secret.json if it exists
    secret_file = "secret.json"
    if os.path.exists(secret_file):
        try:
            with open(secret_file, 'r', encoding='utf-8') as f:
                secret_keys = json.load(f)
            # Override keys.json with secret.json values
            keys.update(secret_keys)
        except json.JSONDecodeError as e:
            raise json.JSONDecodeError(f"Invalid JSON in {secret_file}: {e}")
    
    # Validate required keys
    required_keys = [
        "AZURE_OPENAI_ENDPOINT",
        "AZURE_OPENAI_API_KEY", 
        "QDRANT_URL",
        "QDRANT_API_KEY",
        "QDRANT_COLLECTION"
    ]
    
    missing_keys = [key for key in required_keys if key not in keys or not keys[key] or keys[key].startswith("your-")]
    if missing_keys:
        raise KeyError(f"Missing or placeholder values for required keys: {', '.join(missing_keys)}. Please add these to secret.json or update keys.json with real values.")
    
    return keys


def get_config() -> AppConfig:
    """
    Build and return AppConfig with values from keys.json and environment overrides.
    
    Returns:
        AppConfig: Complete application configuration
        
    Raises:
        FileNotFoundError: If keys.json is missing
        KeyError: If required keys are missing
        ValueError: If environment variable values are invalid
    """
    # Load base configuration from keys.json
    keys = load_keys()
    
    # Create base config
    config = AppConfig()
    
    # Initialize configuration objects
    config.chunk = ChunkConfig()
    config.embed = EmbedConfig()
    config.llm = LLMConfig()
    config.rag = RAGConfig()
    config.conversation = ConversationConfig()
    
    # Set Azure OpenAI configuration
    config.azure_endpoint = keys["AZURE_OPENAI_ENDPOINT"]
    config.azure_api_key = keys["AZURE_OPENAI_API_KEY"]
    config.azure_api_version = keys.get("AZURE_OPENAI_API_VERSION", "2024-06-01")
    
    # Set LLM configuration
    config.llm.model = keys.get("LLM_MODEL", "gpt-4o")
    config.llm.max_tokens = int(keys.get("LLM_MAX_TOKENS", "1000"))
    config.llm.temperature = float(keys.get("LLM_TEMPERATURE", "0.7"))
    
    # Set RAG configuration
    config.rag.top_k = int(keys.get("RAG_TOP_K", "10"))
    config.rag.final_chunks = int(keys.get("RAG_FINAL_CHUNKS", "5"))
    config.rag.context_max_tokens = int(keys.get("RAG_CONTEXT_MAX_TOKENS", "1500"))
    config.rag.system_prompt = keys.get("RAG_SYSTEM_PROMPT", "You are a retrieval-augmented assistant. Use only the provided context. Cite sources. If unsure, say you don't know.")
    config.rag.mmr_diversity_threshold = float(keys.get("RAG_MMR_DIVERSITY_THRESHOLD", "0.7"))
    
    # Set re-ranking method
    rerank_method = keys.get("RAG_RERANK_METHOD", "MMR")
    try:
        config.rag.rerank_method = RerankMethod(rerank_method)
    except ValueError:
        raise ValueError(f"Invalid RAG_RERANK_METHOD: {rerank_method}. Valid options: {[m.value for m in RerankMethod]}")
    
    # Set conversation configuration
    conversation_mood = keys.get("CONVERSATION_MOOD", "false")
    if isinstance(conversation_mood, bool):
        config.conversation.conversation_mood = conversation_mood
    else:
        config.conversation.conversation_mood = str(conversation_mood).lower() == "true"
    
    config.conversation.number_of_conversations_to_store = int(keys.get("NUMBER_OF_CONVERSATIONS_TO_STORE", "10"))
    config.conversation.conversations = keys.get("CONVERSATIONS", [])
    
    # Set Qdrant configuration
    config.qdrant_url = keys["QDRANT_URL"]
    config.qdrant_api_key = keys["QDRANT_API_KEY"]
    config.qdrant_collection = keys["QDRANT_COLLECTION"]
    
    # Apply environment variable overrides (including from keys.json)
    config.knowledge_dir = os.getenv("KNOWLEDGE_DIR", config.knowledge_dir)
    
    # Override chunk configuration (env vars take priority over keys.json)
    chunk_strategy = os.getenv("CHUNK_STRATEGY") or keys.get("CHUNK_STRATEGY")
    if chunk_strategy:
        try:
            config.chunk.strategy = ChunkStrategy(chunk_strategy)
        except ValueError:
            raise ValueError(f"Invalid CHUNK_STRATEGY: {chunk_strategy}. Valid options: {[s.value for s in ChunkStrategy]}")
    
    if max_chars := os.getenv("MAX_CHARS"):
        try:
            config.chunk.max_chars = int(max_chars)
        except ValueError:
            raise ValueError(f"Invalid MAX_CHARS: {max_chars}. Must be an integer.")
    
    if overlap_chars := os.getenv("OVERLAP_CHARS"):
        try:
            config.chunk.overlap_chars = int(overlap_chars)
        except ValueError:
            raise ValueError(f"Invalid OVERLAP_CHARS: {overlap_chars}. Must be an integer.")
    
    if max_tokens := os.getenv("MAX_TOKENS"):
        try:
            config.chunk.max_tokens = int(max_tokens)
        except ValueError:
            raise ValueError(f"Invalid MAX_TOKENS: {max_tokens}. Must be an integer.")
    
    if overlap_tokens := os.getenv("OVERLAP_TOKENS"):
        try:
            config.chunk.overlap_tokens = int(overlap_tokens)
        except ValueError:
            raise ValueError(f"Invalid OVERLAP_TOKENS: {overlap_tokens}. Must be an integer.")
    
    if sentence_target_tokens := os.getenv("SENTENCE_TARGET_TOKENS"):
        try:
            config.chunk.sentence_target_tokens = int(sentence_target_tokens)
        except ValueError:
            raise ValueError(f"Invalid SENTENCE_TARGET_TOKENS: {sentence_target_tokens}. Must be an integer.")
    
    if paragraph_target_tokens := os.getenv("PARAGRAPH_TARGET_TOKENS"):
        try:
            config.chunk.paragraph_target_tokens = int(paragraph_target_tokens)
        except ValueError:
            raise ValueError(f"Invalid PARAGRAPH_TARGET_TOKENS: {paragraph_target_tokens}. Must be an integer.")
    
    if sliding_window_tokens := os.getenv("SLIDING_WINDOW_TOKENS"):
        try:
            config.chunk.sliding_window_tokens = int(sliding_window_tokens)
        except ValueError:
            raise ValueError(f"Invalid SLIDING_WINDOW_TOKENS: {sliding_window_tokens}. Must be an integer.")
    
    if sliding_step_tokens := os.getenv("SLIDING_STEP_TOKENS"):
        try:
            config.chunk.sliding_step_tokens = int(sliding_step_tokens)
        except ValueError:
            raise ValueError(f"Invalid SLIDING_STEP_TOKENS: {sliding_step_tokens}. Must be an integer.")
    
    if semantic_hints := os.getenv("SEMANTIC_HINTS"):
        config.chunk.semantic_hints = semantic_hints.lower() in ("true", "1", "yes", "on")
    
    # Override embed configuration (env vars take priority over keys.json)
    embed_model = os.getenv("EMBED_MODEL") or keys.get("EMBED_MODEL")
    if embed_model:
        try:
            config.embed.model = EmbedModel(embed_model)
        except ValueError:
            raise ValueError(f"Invalid EMBED_MODEL: {embed_model}. Valid options: {[m.value for m in EmbedModel]}")
    
    if embed_dimensions := os.getenv("EMBED_DIMENSIONS"):
        try:
            config.embed.dimensions = int(embed_dimensions)
        except ValueError:
            raise ValueError(f"Invalid EMBED_DIMENSIONS: {embed_dimensions}. Must be an integer.")
    
    return config