run.py

#!/usr/bin/env python3
"""
Web Scraper API Server

This script runs the FastAPI web scraper application with configurable settings.
Use environment variables or command line arguments to configure the server.

For Render deployment:
- PORT is automatically set by Render
- Other environment variables can be set in render.yaml or the Render dashboard
"""
import argparse
import os
import logging
import uvicorn
from pathlib import Path

def parse_arguments():
    """Parse command line arguments for server configuration."""
    parser = argparse.ArgumentParser(
        description="Run the Web Scraper API server",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )

    parser.add_argument(
        "--host",
        default=os.getenv("HOST", "0.0.0.0"),
        help="Host interface to bind to"
    )
    parser.add_argument(
        "--port",
        type=int,
        # Render sets the PORT environment variable automatically
        default=int(os.getenv("PORT", "8000")),
        help="Port to bind to"
    )
    parser.add_argument(
        "--reload",
        action="store_true",
        default=os.getenv("RELOAD", "").lower() in ("true", "1", "yes"),
        help="Enable auto-reload on code changes (development mode)"
    )
    parser.add_argument(
        "--workers",
        type=int,
        default=int(os.getenv("WORKERS", "1")),
        help="Number of worker processes"
    )
    parser.add_argument(
        "--log-level",
        choices=["debug", "info", "warning", "error", "critical"],
        default=os.getenv("LOG_LEVEL", "info").lower(),
        help="Logging level"
    )

    return parser.parse_args()

def setup_logging(log_level):
    """Configure basic logging."""
    numeric_level = getattr(logging, log_level.upper(), None)
    logging.basicConfig(
        level=numeric_level,
        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
    )

def main():
    """Run the FastAPI application with Uvicorn."""
    # Parse command line arguments
    args = parse_arguments()

    # Setup logging
    setup_logging(args.log_level)

    # Check if running in cloud environment
    is_render = "RENDER" in os.environ
    environment = "Render" if is_render else "development"

    # Display startup message
    print(f"Starting Web Scraper API on http://{args.host}:{args.port} ({environment} environment)")
    print(f"API documentation available at http://{args.host}:{args.port}/docs")

    # In production environments, disable reload
    if is_render:
        args.reload = False
        # Adjust worker count based on dyno size
        if not os.getenv("WORKERS"):
            args.workers = 2  # Conservative default for Render free tier

    # Run the server
    uvicorn.run(
        "app.main:app",
        host=args.host,
        port=args.port,
        reload=args.reload,
        workers=args.workers,
        log_level=args.log_level
    )

if __name__ == "__main__":
    main()