example_usage.py

"""Example usage of the data processing framework."""

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent))

from src.logging import setup_logging
from src.pipeline import PipelineOrchestrator


def example_1_process_single_source():
    """Example: Process data from a single source."""
    print("\n=== Example 1: Process Single Source ===\n")

    # Setup
    setup_logging(log_level="INFO")
    pipeline = PipelineOrchestrator(config_dir="config")

    # Process a specific source
    result = pipeline.process_source("job_api_single")

    print(f"Success: {result['success']}")
    if result['success']:
        print(f"Raw Data ID: {result['raw_data_id']}")
        print(f"Entries Created: {result['layer2'].get('num_entries', 0)}")

    pipeline.shutdown()


def example_2_process_with_custom_data():
    """Example: Process custom data without fetching from source."""
    print("\n=== Example 2: Process Custom Data ===\n")

    setup_logging(log_level="INFO")
    pipeline = PipelineOrchestrator(config_dir="config")

    # Custom data to process
    custom_data = {
        "title": "Marine Biology PhD Position",
        "organization": "Ocean Research Institute",
        "location": "California, USA",
        "description": "Seeking PhD student for kelp forest ecosystem research",
        "requirements": [
            "Masters degree in marine biology or related field",
            "Scuba diving certification",
            "Experience with underwater surveys",
        ],
        "deadline": "2025-03-31",
        "url": "https://example.com/positions/marine-phd",
    }

    # Process through the pipeline
    result = pipeline.process_source("job_api_single", data=custom_data)

    print(f"Success: {result['success']}")
    if result['success']:
        print(f"Entry created and will be evaluated for relevance")

    pipeline.shutdown()


def example_3_batch_relevance_filtering():
    """Example: Manually trigger Layer 3 batch processing."""
    print("\n=== Example 3: Batch Relevance Filtering ===\n")

    setup_logging(log_level="INFO")
    pipeline = PipelineOrchestrator(config_dir="config")

    # Run Layer 3 on pending entries
    result = pipeline.run_layer3_batch(batch_size=20)

    print(f"Success: {result['success']}")
    if result['success']:
        print(f"Processed: {result.get('processed', 0)}")
        print(f"Relevant: {result.get('relevant', 0)}")
        print(f"Not Relevant: {result.get('not_relevant', 0)}")
        print(f"Quarantined: {result.get('quarantined', 0)}")

    pipeline.shutdown()


def example_4_process_all_sources():
    """Example: Process all enabled sources."""
    print("\n=== Example 4: Process All Sources ===\n")

    setup_logging(log_level="INFO")
    pipeline = PipelineOrchestrator(config_dir="config")

    # Process all sources
    result = pipeline.process_all_sources()

    print(f"Total Sources: {result['total_sources']}")
    print(f"Successful: {result['successful']}")
    print(f"Failed: {result['failed']}")

    pipeline.shutdown()


def example_5_direct_api_integration():
    """Example: Direct integration with components."""
    print("\n=== Example 5: Direct Component Integration ===\n")

    setup_logging(log_level="DEBUG")

    from src.core import ConfigLoader, LLMClient, DatabaseManager
    from src.layer2 import PromptBuilder

    # Load configuration
    config = ConfigLoader("config")

    # Initialize database
    db_config = config.get_database_config()
    db = DatabaseManager(db_config)

    # Initialize LLM client
    llm = LLMClient()

    # Test LLM call with validation
    messages = [{"role": "user", "content": "What is marine biology?"}]

    validation_config = {
        "enabled": True,
        "parallel_calls": 3,
        "agreement_threshold": "majority",
    }

    llm_settings = {
        "endpoint": "http://localhost:10006",
        "intelligence_level": "low",
        "max_wait_seconds": 20,
        "options": {"max_tokens": 100, "temperature": 0.3},
    }

    result, metadata = llm.call_with_validation(
        messages=messages,
        validation_config=validation_config,
        llm_settings=llm_settings,
    )

    print(f"Success: {result.get('success')}")
    print(f"Validation Enabled: {metadata.get('validation_enabled')}")
    print(f"Agreement Achieved: {metadata.get('agreement_achieved')}")
    print(f"Response: {result.get('content', '')[:200]}...")


if __name__ == "__main__":
    print("=" * 60)
    print("Data Processing Framework - Usage Examples")
    print("=" * 60)

    # Run examples
    try:
        example_1_process_single_source()
        # example_2_process_with_custom_data()
        # example_3_batch_relevance_filtering()
        # example_4_process_all_sources()
        # example_5_direct_api_integration()

    except Exception as e:
        print(f"\nError running examples: {e}")
        import traceback
        traceback.print_exc()