improved_documentation_manager.py

#!/usr/bin/env python3
"""
Improved Documentation Manager
Fixes the critical issue of using only 203-character previews instead of full documentation
Ensures comprehensive documentation content for LLM code generation
"""

import asyncio
import json
import time
from typing import Dict, Optional, Tuple, List
import logging

logger = logging.getLogger(__name__)

class ImprovedDocumentationManager:
    """Manages documentation extraction with full content utilization"""
    
    def __init__(self):
        self.documentation_cache = {}
        self.min_documentation_length = 1000  # Minimum chars for quality documentation
        
    async def initialize(self):
        """Initialize documentation manager"""
        logger.info("Improved documentation manager initialized")
        
    async def get_full_documentation_content(self, api_name: str) -> Tuple[bool, str, int, str]:
        """
        Get FULL documentation content for API (not just preview)
        
        Returns:
            Tuple of (success, full_content, extraction_time_ms, source_used)
        """
        
        # First, try to load from our successful documentation extraction test
        try:
            with open('documentation_extraction_test_results.json', 'r') as f:
                doc_results = json.load(f)
                
            if api_name in doc_results['api_results']:
                api_data = doc_results['api_results'][api_name]
                
                if api_data['extraction_success']:
                    # Get the FULL content, not just preview
                    full_content = await self.extract_full_content_from_test_results(api_name, api_data)
                    
                    if len(full_content) >= self.min_documentation_length:
                        logger.info(f"Using full documentation for {api_name}: {len(full_content)} characters")
                        return True, full_content, api_data['extraction_time_ms'], 'test_results'
                    else:
                        logger.warning(f"Documentation for {api_name} too short: {len(full_content)} chars")
                        
        except FileNotFoundError:
            logger.warning("Documentation extraction test results not found")
            
        # Fallback: Re-extract using enhanced Playwright
        logger.info(f"Re-extracting documentation for {api_name} using enhanced Playwright")
        return await self.extract_fresh_documentation(api_name)
        
    async def extract_full_content_from_test_results(self, api_name: str, api_data: Dict) -> str:
        """Extract full content from test results (not just preview)"""
        
        # The test results only stored previews, so we need to re-extract
        # using our enhanced documentation extractor
        from enhanced_documentation_extractor import EnhancedDocumentationExtractor
        
        extractor = EnhancedDocumentationExtractor()
        
        try:
            await extractor.initialize()
            
            # Get the documentation URL from our API configs
            documentation_url = self.get_documentation_url_for_api(api_name)
            
            if documentation_url:
                success, content, time_ms = await extractor.extract_documentation_with_retry(
                    api_name, documentation_url, max_retries=2
                )
                
                if success and len(content) >= self.min_documentation_length:
                    # Cache the result
                    self.documentation_cache[api_name] = content
                    return content
                    
        except Exception as e:
            logger.error(f"Failed to extract full content for {api_name}: {e}")
            
        finally:
            await extractor.cleanup()
            
        # Return empty string if extraction failed
        return ""
        
    def get_documentation_url_for_api(self, api_name: str) -> Optional[str]:
        """Get documentation URL for API"""
        
        api_urls = {
            "OpenWeatherMap": "https://openweathermap.org/api/one-call-3",
            "Visual Crossing": "https://www.visualcrossing.com/resources/documentation/weather-api/timeline-weather-api/",
            "WeatherBit": "https://www.weatherbit.io/api/weather-current",
            "NewsAPI": "https://newsapi.org/docs/endpoints/top-headlines",
            "NewsData.io": "https://newsdata.io/documentation/#latest_news",
            "Guardian API": "https://open-platform.theguardian.com/documentation/",
            "Currents API": "https://currentsapi.services/en/docs/",
            "Mediastack": "https://mediastack.com/documentation",
            "ExchangeRate-API": "https://www.exchangerate-api.com/docs/overview",
            "Fixer.io": "https://fixer.io/documentation",
            "CurrencyAPI": "https://currencyapi.com/docs",
            "CurrencyLayer": "https://currencylayer.com/documentation",
            "Clerk": "https://clerk.com/docs/reference/backend-api",
            "Uploadcare": "https://uploadcare.com/docs/api_reference/rest/",
            "IPGeolocation": "https://ipgeolocation.io/documentation/ip-geolocation-api.html"
        }
        
        return api_urls.get(api_name)
        
    async def extract_fresh_documentation(self, api_name: str) -> Tuple[bool, str, int, str]:
        """Extract fresh documentation using enhanced extractor"""
        
        documentation_url = self.get_documentation_url_for_api(api_name)
        
        if not documentation_url:
            logger.error(f"No documentation URL found for {api_name}")
            return False, f"No documentation URL configured for {api_name}", 0, "none"
            
        from enhanced_documentation_extractor import EnhancedDocumentationExtractor
        
        extractor = EnhancedDocumentationExtractor()
        
        try:
            await extractor.initialize()
            
            success, content, time_ms = await extractor.extract_documentation_with_retry(
                api_name, documentation_url, max_retries=3
            )
            
            if success:
                if len(content) >= self.min_documentation_length:
                    # Cache the result
                    self.documentation_cache[api_name] = content
                    logger.info(f"Successfully extracted fresh documentation for {api_name}: {len(content)} characters")
                    return True, content, time_ms, "enhanced_playwright"
                else:
                    logger.warning(f"Fresh documentation for {api_name} too short: {len(content)} chars")
                    return False, f"Documentation too short: {len(content)} characters", time_ms, "enhanced_playwright"
            else:
                logger.error(f"Failed to extract fresh documentation for {api_name}: {content}")
                return False, content, time_ms, "enhanced_playwright"
                
        except Exception as e:
            logger.error(f"Exception during fresh documentation extraction for {api_name}: {e}")
            return False, f"Extraction exception: {str(e)}", 0, "none"
            
        finally:
            await extractor.cleanup()
            
    async def validate_documentation_quality(self, api_name: str, content: str) -> Dict[str, any]:
        """Validate documentation quality and completeness"""
        
        quality_metrics = {
            'length': len(content),
            'has_authentication': any(keyword in content.lower() for keyword in ['auth', 'api key', 'token', 'bearer']),
            'has_endpoints': any(keyword in content.lower() for keyword in ['endpoint', 'url', 'api/', 'http']),
            'has_examples': any(keyword in content.lower() for keyword in ['example', 'sample', 'curl', 'python']),
            'has_parameters': any(keyword in content.lower() for keyword in ['parameter', 'param', 'query', 'body']),
            'has_responses': any(keyword in content.lower() for keyword in ['response', 'return', 'json', 'xml']),
            'quality_score': 0
        }
        
        # Calculate quality score
        score = 0
        if quality_metrics['length'] >= 1000:
            score += 20
        if quality_metrics['length'] >= 3000:
            score += 20
        if quality_metrics['has_authentication']:
            score += 20
        if quality_metrics['has_endpoints']:
            score += 15
        if quality_metrics['has_examples']:
            score += 15
        if quality_metrics['has_parameters']:
            score += 5
        if quality_metrics['has_responses']:
            score += 5
            
        quality_metrics['quality_score'] = score
        
        logger.info(f"Documentation quality for {api_name}: {score}/100 points")
        
        return quality_metrics
        
    async def get_documentation_for_all_apis(self, api_names: List[str]) -> Dict[str, Dict]:
        """Get full documentation for all APIs with quality validation"""
        
        logger.info(f"Getting full documentation for {len(api_names)} APIs")
        
        results = {}
        
        for api_name in api_names:
            try:
                success, content, time_ms, source = await self.get_full_documentation_content(api_name)
                
                if success:
                    quality_metrics = await self.validate_documentation_quality(api_name, content)
                    
                    results[api_name] = {
                        'success': True,
                        'content': content,
                        'time_ms': time_ms,
                        'source': source,
                        'quality_metrics': quality_metrics
                    }
                    
                    logger.info(f"✅ {api_name}: {len(content)} chars, quality {quality_metrics['quality_score']}/100")
                else:
                    results[api_name] = {
                        'success': False,
                        'content': content,  # Error message
                        'time_ms': time_ms,
                        'source': source,
                        'quality_metrics': {'quality_score': 0}
                    }
                    
                    logger.warning(f"❌ {api_name}: {content}")
                    
            except Exception as e:
                logger.error(f"Exception getting documentation for {api_name}: {e}")
                results[api_name] = {
                    'success': False,
                    'content': f"Exception: {str(e)}",
                    'time_ms': 0,
                    'source': 'none',
                    'quality_metrics': {'quality_score': 0}
                }
                
        # Summary
        successful = len([r for r in results.values() if r['success']])
        avg_length = sum([len(r['content']) for r in results.values() if r['success']]) / max(successful, 1)
        avg_quality = sum([r['quality_metrics']['quality_score'] for r in results.values() if r['success']]) / max(successful, 1)
        
        logger.info(f"Documentation extraction summary:")
        logger.info(f"  Success rate: {successful}/{len(api_names)} ({successful/len(api_names)*100:.1f}%)")
        logger.info(f"  Average length: {avg_length:.0f} characters")
        logger.info(f"  Average quality: {avg_quality:.1f}/100")
        
        return results
        
    async def cleanup(self):
        """Clean up resources"""
        logger.info("Improved documentation manager cleaned up")

# Test function
async def test_improved_documentation_manager():
    """Test improved documentation manager"""
    manager = ImprovedDocumentationManager()
    
    try:
        await manager.initialize()
        
        # Test with a few APIs
        test_apis = ["OpenWeatherMap", "NewsAPI", "IPGeolocation"]
        
        results = await manager.get_documentation_for_all_apis(test_apis)
        
        print("\nImproved Documentation Manager Results:")
        print("=" * 60)
        
        for api_name, result in results.items():
            print(f"\n{api_name}:")
            print(f"  Success: {result['success']}")
            print(f"  Source: {result['source']}")
            print(f"  Time: {result['time_ms']}ms")
            print(f"  Content length: {len(result['content'])}")
            print(f"  Quality score: {result['quality_metrics']['quality_score']}/100")
            
            if result['success']:
                print(f"  Preview: {result['content'][:200]}...")
                
    finally:
        await manager.cleanup()

if __name__ == "__main__":
    asyncio.run(test_improved_documentation_manager())