url_client_examples.py

#!/usr/bin/env python3
"""
Text Extraction API Client Examples - URL-based Processing

Examples of how to use the URL-based endpoints of the Text Extraction API.
"""

import requests
import json
from typing import List, Dict, Any

# API Configuration
API_BASE_URL = "http://localhost:8001"

class URLTextExtractionClient:
    """Client for URL-based Text Extraction API."""

    def __init__(self, base_url: str = API_BASE_URL):
        self.base_url = base_url.rstrip('/')
        self.session = requests.Session()

    def health_check(self) -> Dict[str, Any]:
        """Check API health and configuration."""
        response = self.session.get(f"{self.base_url}/health")
        response.raise_for_status()
        return response.json()

    def extract_from_url(self, url: str, filename: str = None) -> Dict[str, Any]:
        """Extract text from a single URL."""
        payload = {"url": url}
        if filename:
            payload["filename"] = filename

        response = self.session.post(
            f"{self.base_url}/extract-url",
            json=payload,
            headers={"Content-Type": "application/json"}
        )
        response.raise_for_status()
        return response.json()

    def extract_from_urls(self, urls: List[str], filenames: List[str] = None) -> Dict[str, Any]:
        """Extract text from multiple URLs."""
        payload = {"urls": urls}
        if filenames:
            payload["filenames"] = filenames

        response = self.session.post(
            f"{self.base_url}/extract-batch-url",
            json=payload,
            headers={"Content-Type": "application/json"}
        )
        response.raise_for_status()
        return response.json()

def example_single_url():
    """Example: Extract text from a single URL."""
    print("=== Single URL Extraction Example ===")

    client = URLTextExtractionClient()

    # Example URLs (replace with actual public file URLs)
    test_urls = [
        "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf",
        "https://file-examples.com/storage/fef68e5c7a66dac75285c32/2017/10/file_example_JPG_100kB.jpg",
        "https://sample-videos.com/zip/10/mp4/360/SampleVideo_360x240_1mb.mp4"
    ]

    for url in test_urls:
        try:
            print(f"\n📄 Processing: {url}")
            result = client.extract_from_url(url)

            print(f"✅ File ID: {result['file_id']}")
            print(f"   Success: {result['success']}")
            print(f"   Processor: {result['processor_used']}")
            print(f"   Processing time: {result['processing_time']:.2f}s")
            print(f"   Text length: {result['text_length']} characters")
            print(f"   File info: {result['file_info']['name']} ({result['file_info']['size_mb']:.2f} MB)")

            if result['success']:
                print(f"\n   Extracted text preview:")
                text = result['extracted_text']
                preview = text[:200] + "..." if len(text) > 200 else text
                print(f"   {preview}")
            else:
                print(f"   Error: {result['error']}")

        except requests.exceptions.HTTPError as e:
            print(f"❌ HTTP Error: {e}")
            if hasattr(e.response, 'text'):
                print(f"   Details: {e.response.text}")
        except Exception as e:
            print(f"❌ Failed to process {url}: {e}")

def example_multiple_urls():
    """Example: Extract text from multiple URLs."""
    print("\n=== Multiple URLs Extraction Example ===")

    client = URLTextExtractionClient()

    # Example URLs for batch processing
    test_urls = [
        "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf",
        "https://raw.githubusercontent.com/microsoft/vscode/main/README.md"
    ]

    # Optional custom filenames
    custom_filenames = [
        "sample_document.pdf",
        "readme_file.md"
    ]

    try:
        print(f"📄 Processing {len(test_urls)} URLs...")
        result = client.extract_from_urls(test_urls, custom_filenames)

        print(f"✅ Batch ID: {result['batch_id']}")
        print(f"   Total files: {result['total_files']}")
        print(f"   Successful: {result['successful']}")
        print(f"   Failed: {result['failed']}")
        print(f"   Total processing time: {result['total_processing_time']:.2f}s")
        print(f"   Total characters extracted: {result['total_characters']}")

        print(f"\n   Individual results:")
        for i, file_result in enumerate(result['results']):
            status = "✅" if file_result['success'] else "❌"
            print(f"   {i+1}. {status} {file_result['file_info']['name']}")
            print(f"      URL: {test_urls[i]}")
            print(f"      Processor: {file_result['processor_used']}")
            print(f"      Text length: {file_result['text_length']} chars")
            if not file_result['success']:
                print(f"      Error: {file_result['error']}")
            print()

    except Exception as e:
        print(f"❌ Multiple URLs extraction failed: {e}")

def example_curl_commands():
    """Example: Show equivalent curl commands for URL endpoints."""
    print("\n=== Curl Command Examples for URLs ===")

    print("1. Health check:")
    print("   curl http://localhost:8001/health")

    print("\n2. Extract text from single URL:")
    print("   curl -X POST http://localhost:8001/extract-url \\")
    print("        -H 'Content-Type: application/json' \\")
    print("        -d '{\"url\": \"https://example.com/document.pdf\"}'")

    print("\n3. Extract text from single URL with custom filename:")
    print("   curl -X POST http://localhost:8001/extract-url \\")
    print("        -H 'Content-Type: application/json' \\")
    print("        -d '{\"url\": \"https://example.com/file\", \"filename\": \"document.pdf\"}'")

    print("\n4. Extract text from multiple URLs:")
    print("   curl -X POST http://localhost:8001/extract-batch-url \\")
    print("        -H 'Content-Type: application/json' \\")
    print("        -d '{\"urls\": [\"https://example.com/doc1.pdf\", \"https://example.com/doc2.txt\"]}'")

    print("\n5. Extract text from multiple URLs with custom filenames:")
    print("   curl -X POST http://localhost:8001/extract-batch-url \\")
    print("        -H 'Content-Type: application/json' \\")
    print("        -d '{\"urls\": [\"https://example.com/file1\", \"https://example.com/file2\"], \"filenames\": [\"doc1.pdf\", \"doc2.txt\"]}'")

    print("\n6. Extract text from WebM audio URL:")
    print("   curl -X POST http://localhost:8001/extract-url \\")
    print("        -H 'Content-Type: application/json' \\")
    print("        -d '{\"url\": \"https://example.com/audio.webm\"}'")

    print("\n7. Extract text from OGG audio URL:")
    print("   curl -X POST http://localhost:8001/extract-url \\")
    print("        -H 'Content-Type: application/json' \\")
    print("        -d '{\"url\": \"https://example.com/audio.ogg\"}'")

    print("\n8. API documentation:")
    print("   Open http://localhost:8001/docs in your browser")

def example_public_file_urls():
    """Example: Real public file URLs for testing."""
    print("\n=== Public File URLs for Testing ===")

    public_urls = {
        "PDF Document": "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf",
        "Text File": "https://raw.githubusercontent.com/microsoft/vscode/main/README.md",
        "Sample Video": "https://sample-videos.com/zip/10/mp4/360/SampleVideo_360x240_1mb.mp4",
        "Sample Audio (WAV)": "https://www.soundjay.com/misc/sounds/bell-ringing-05.wav",
        "Sample Audio (WebM)": "https://example.com/audio.webm",  # Replace with actual WebM URL
        "Sample Audio (OGG)": "https://example.com/audio.ogg",    # Replace with actual OGG URL
        "Word Document": "https://file-examples.com/storage/fef68e5c7a66dac75285c32/2017/02/file-sample_100kB.doc"
    }

    print("Here are some public file URLs you can test with:")
    for file_type, url in public_urls.items():
        print(f"\n{file_type}:")
        print(f"  {url}")

    print(f"\nUsage example:")
    print(f"  curl -X POST http://localhost:8001/extract-url \\")
    print(f"       -H 'Content-Type: application/json' \\")
    print(f"       -d '{{\"url\": \"{list(public_urls.values())[0]}\"}}'")

def example_audio_urls():
    """Example: Extract text from WebM and OGG audio files via URL."""
    print("\n=== Audio URL Extraction Example (WebM & OGG) ===")

    client = URLTextExtractionClient()

    # Example audio URLs for testing WebM and OGG support
    # Note: Replace these with actual public URLs to WebM/OGG files
    audio_test_urls = [
        # Example WebM audio URL (replace with actual URL)
        "https://example.com/audio.webm",
        # Example OGG audio URL (replace with actual URL)
        "https://example.com/audio.ogg",
        # Working sample audio for comparison
        "https://www.soundjay.com/misc/sounds/bell-ringing-05.wav"
    ]

    print("📡 Testing audio transcription from URLs...")
    print("   Supported formats: MP3, WAV, M4A, WebM, OGG")

    for url in audio_test_urls:
        try:
            print(f"\n🎵 Processing audio: {url}")
            result = client.extract_from_url(url)

            print(f"✅ File ID: {result['file_id']}")
            print(f"   Success: {result['success']}")
            print(f"   Processor: {result['processor_used']}")
            print(f"   Processing time: {result['processing_time']:.2f}s")
            print(f"   File info: {result['file_info']['name']} ({result['file_info']['size_mb']:.2f} MB)")

            if result['success']:
                print(f"   Transcript length: {result['text_length']} characters")
                text = result['extracted_text']
                preview = text[:300] + "..." if len(text) > 300 else text
                print(f"\n   Transcript preview:")
                print(f"   {preview}")
            else:
                print(f"   Error: {result['error']}")

        except requests.exceptions.HTTPError as e:
            print(f"❌ HTTP Error: {e}")
            if hasattr(e.response, 'text'):
                error_detail = json.loads(e.response.text) if e.response.text else {}
                print(f"   Details: {error_detail.get('detail', 'Unknown error')}")
        except Exception as e:
            print(f"❌ Failed to process {url}: {e}")

def main():
    """Run all URL-based examples."""
    print("Text Extraction API - URL-based Client Examples")
    print("=" * 60)

    # Check if API is running
    try:
        client = URLTextExtractionClient()
        health = client.health_check()
        print(f"✅ API Status: {health['status']}")
        print(f"   OpenAI configured: {health['api_keys_configured']['openai']}")
        print(f"   Gemini configured: {health['api_keys_configured']['gemini']}")
    except Exception as e:
        print(f"❌ Cannot connect to API: {e}")
        print("   Make sure the server is running with: python api_server.py")
        return

    # Run examples
    example_public_file_urls()
    example_curl_commands()
    example_audio_urls() # Added this line to call the new example

    # Uncomment to test with real URLs (be mindful of rate limits)
    # example_single_url()
    # example_multiple_urls()

    print("\n" + "=" * 60)
    print("URL-based examples completed!")
    print("\nTo start the API server, run:")
    print("  python api_server.py")
    print("\nThen test with URLs instead of file uploads!")

if __name__ == "__main__":
    main()