diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5c78d44 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,34 @@ +# Git +.git +.gitignore + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +*.egg-info/ +dist/ +build/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# Logs +logs/ +*.log + +# OS +.DS_Store +Thumbs.db \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..292fbaa --- /dev/null +++ b/.env.example @@ -0,0 +1,42 @@ +# Environment Configuration +# Valid values: development, production, staging, testing +ENV=development +UVICORN_PORT=8000 + +# API Keys +CRYPTOPANIC_API_URL=https://cryptopanic.com/api/v1 +CRYPTOPANIC_API_KEY=your_cryptopanic_api_key +OPENAI_API_KEY=your_openai_api_key +GEMINI_API_KEY=your_gemini_api_key +BITQUERY_API_KEY=your_bitquery_api_key + +# Life Simulator API Keys +OPENROUTER_API_KEY=your_openrouter_api_key_here +OPENWEATHER_API_KEY=your_openweather_api_key_here +GOOGLE_MAPS_API_KEY=your_google_maps_api_key_here + +# Logging Configuration +LOG_LEVEL=INFO # DEBUG, INFO, WARNING, ERROR, CRITICAL +LOG_FILE=logs/app.log # Optional: Comment out to log to console only +#LOG_FORMAT=%(asctime)s - %(name)s - %(levelname)s - %(message)s +#LOG_DATE_FORMAT=%Y-%m-%d %H:%M:%S +#LOG_MAX_SIZE=10485760 # 10 MB +#LOG_BACKUP_COUNT=5 + + +# LLM Settings +LLM_TIMEOUT=10 # seconds +LLM_MAX_RETRIES=2 +LLM_MODEL=gpt-4o-mini # or other OpenAI model +LLM_MAX_TOKENS=4000 + +# Firecrawl Settings +FIRECRAWL_API_KEY=your_firecrawl_api_key +# FIRECRAWL_BASE_URL=optional_custom_firecrawl_url # Uncomment if using a custom Firecrawl instance + +# Supabase Configuration +SUPABASE_URL=your_supabase_project_url +SUPABASE_SERVICE_KEY=your_supabase_service_role_key + +# Admin Configuration +ADMIN_SECRET=your_admin_secret \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e782d89 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +venv/ +.env +__pycache__/ +.DS_Store +app/__pycache__/ +app/services/__pycache__/ +.venv/ +.pytest_cache/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9b7b72e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +# Use Python 3.11 slim image +FROM python:3.11-slim + +# Set working directory +WORKDIR /app + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + ENVIRONMENT=production + +# Install system dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements first to leverage Docker cache +COPY requirements.txt . + +# Set environment variables +ENV PLAYWRIGHT_BROWSERS_PATH=/app/ms-playwright + +# Install Playwright browsers and dependencies +RUN pip install playwright +RUN playwright install --with-deps chromium + +# Install Python dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Copy the rest of the application +COPY . . + +# Create logs directory +RUN mkdir -p logs + +# Expose port +EXPOSE 8000 + +# Command to run the application +CMD ["python", "run.py"] diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..30dee76 --- /dev/null +++ b/app/config.py @@ -0,0 +1,91 @@ +from pydantic_settings import BaseSettings +from typing import Optional +from app.utils.enums import Environment +import logging + +class Settings(BaseSettings): + # Environment + ENV: Environment = Environment.DEVELOPMENT + UVICORN_PORT: int = 8000 + + # admin key + ADMIN_SECRET: str + + # API Settings + CRYPTOPANIC_API_URL: str = "https://cryptopanic.com/api/v1" + CRYPTOPANIC_API_KEY: str + + # BitQuery Settings + BITQUERY_API_KEY: str + + # OpenAI Settings + OPENAI_API_KEY: str + + # Gemini Settings + GEMINI_API_KEY: str + + # Proxy Settings + PROXY_CONFIG: Optional[str] = None + + # Supabase Settings + SUPABASE_URL: str + SUPABASE_SERVICE_KEY: str + + # LLM Settings + LLM_TIMEOUT: int = 10 # seconds + LLM_MAX_RETRIES: int = 2 + LLM_MODEL: str = "gpt-4o-mini" + OPENAI_API_URL: str = "https://api.openai.com/v1/chat/completions" + LLM_MAX_TOKENS: int = 4000 + + # Logging Settings + LOG_LEVEL: str = "INFO" + LOG_FILE: Optional[str] = None # Path to log file (optional) + LOG_FORMAT: str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + LOG_DATE_FORMAT: str = "%Y-%m-%d %H:%M:%S" + LOG_MAX_SIZE: int = 10 * 1024 * 1024 # 10 MB + LOG_BACKUP_COUNT: int = 5 + + class Config: + env_file = ".env" + case_sensitive = True + extra = "ignore" + + def get_log_level(self) -> str: + """Get the appropriate log level based on environment""" + if self.ENV == Environment.DEVELOPMENT: + return "DEBUG" + return self.LOG_LEVEL + + @property + def is_production(self) -> bool: + """Check if running in production environment""" + return self.ENV == Environment.PRODUCTION + + @property + def is_development(self) -> bool: + """Check if running in development environment""" + return self.ENV == Environment.DEVELOPMENT + + def configure_logging(self) -> None: + """Configure logging based on environment""" + log_level = self.get_log_level() + logging.basicConfig( + level=log_level, + format=self.LOG_FORMAT, + datefmt=self.LOG_DATE_FORMAT + ) + + # Adjust third-party loggers + if self.is_production: + logging.getLogger("uvicorn").setLevel(logging.WARNING) + logging.getLogger("aiohttp").setLevel(logging.WARNING) + logging.getLogger("asyncio").setLevel(logging.WARNING) + else: + logging.getLogger("uvicorn").setLevel(logging.INFO) + logging.getLogger("aiohttp").setLevel(logging.INFO) + logging.getLogger("asyncio").setLevel(logging.INFO) + +# Initialize settings +settings = Settings() +settings.configure_logging() \ No newline at end of file diff --git a/app/dependencies.py b/app/dependencies.py new file mode 100644 index 0000000..b9a0346 --- /dev/null +++ b/app/dependencies.py @@ -0,0 +1,38 @@ +from fastapi import Header, HTTPException, status, Request +from typing import Optional +from app.services.supabase_service import SupabaseService +import logging + +logger = logging.getLogger(__name__) +supabase_service = SupabaseService() + +async def require_api_key( + request: Request, + x_api_key: Optional[str] = Header(None, convert_underscores=True) +): + """ + Global dependency for all non-admin endpoints. + - Checks if 'X-API-Key' is valid/active in Supabase. + - Resets daily usage if 'last_reset' < today's date. + - Stores user record in request.state.user for downstream usage. + """ + # Skip check for admin routes if needed + if request.url.path.startswith("/api/v1/admin"): + return + + if not x_api_key: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="X-API-Key header is required" + ) + + user = supabase_service.get_user_by_api_key(x_api_key) + if not user or not user.get("active", False): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or inactive API key" + ) + + # Reset daily usage if a new day + user = supabase_service.reset_daily_usage_if_needed(user) + request.state.user = user # store the user for later usage \ No newline at end of file diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..9e4d7e6 --- /dev/null +++ b/app/main.py @@ -0,0 +1,265 @@ +from fastapi import FastAPI, Depends +from fastapi.middleware.cors import CORSMiddleware +from app.routes import ( + life_simulator, news, router, scraper, math, + token_information, image_recognition, solana_dex_sales, + solana_dex_buys, membership, admin +) +from app.dependencies import require_api_key +from app.utils.logging_config import setup_logging, get_logger +from app.config import settings + +# Set up logging first +setup_logging(log_level=settings.LOG_LEVEL, log_file=settings.LOG_FILE) + +# Get logger for this module +logger = get_logger(__name__) + +description = """ +### State of Mika: Intelligent Context-Based Query Routing Service + +State of Mika is an advanced context-based routing service that intelligently handles diverse data silos and routes queries to the appropriate tools based on natural language understanding. + +## Key Features + +- **Universal Router**: A single, intelligent interface for understanding natural language queries and dynamically routing them to the relevant tools or services. +- **News Aggregation**: Provides comprehensive coverage of cryptocurrency and blockchain news from multiple sources, delivering the latest updates and trends. +- **Web Scraping**: Extracts and analyzes content from external websites to provide insights and summaries. +- **Real-Time Token Prices**: Retrieves up-to-date market data and token prices across multiple blockchain networks. +- **Mathematical Operations**: Handles complex mathematical expressions, including arithmetic, exponentiation, and percentage calculations. +- **Image Recognition**: Leverages AI vision to analyze and describe images, identifying objects and providing contextual insights. +- **Membership System**: Manages bots and their associated friends across multiple platforms with features like: + - Bot Management with secure API key authentication + - Friend tracking across multiple platforms (Discord, Telegram, Twitter, WhatsApp) + - Points and level progression system + - Platform-specific contact management + - Points history tracking + +## How It Works + +The **Universal Router** serves as the core of the system, processing user queries and determining the best tool or service to handle the request. Queries are automatically parsed, routed, and processed to generate accurate, context-aware responses. The intelligent router also manages post-processing of results, ensuring that responses are tailored to the user's needs without requiring additional instructions. + +### Examples of Supported Queries +- **Price Queries**: "What is the price of WBTC on Solana?" +- **Image Analysis**: "What objects are in this image?" +- **News Updates**: "Get me the latest cryptocurrency news." +- **Content Extraction**: "Summarize the key points of this article." +- **Mathematics**: "What is (5 + 3) * 2?" +- **Membership**: + - "Create a new bot for my community" + - "Add a friend with Discord ID 123456" + - "Update points for friend XYZ" + - "Get all friends of bot ABC" + - "Add Telegram contact for existing friend" + +## API Documentation + +### Membership API +The Membership API provides comprehensive endpoints for managing bots and their associated friends across different platforms: + +1. **Bot Management** + - Create new bots with secure API key authentication + - Each bot gets a unique API key for secure operations + +2. **Friend Management** + - Create and track friends across multiple platforms + - Update friend attributes (nickname, description) + - Manage points and levels + - View detailed friend information + +3. **Platform Integration** + - Support for multiple platforms: + - Discord + - Telegram + - Twitter + - WhatsApp + - Custom platforms + - Add multiple platform contacts per friend + - Unique platform identifiers per friend + +4. **Points & Levels** + - Track friend engagement through points + - Automatic level progression + - Points history with reason tracking + - Flexible point adjustment system + +## Rate Limits + +- **News**: Limited by CryptoPanic API key restrictions. +- **Real-Time Market Data**: 300 requests per minute for token price queries. +- **Web Scraping**: No specific limits, but usage should remain considerate. +- **Image Recognition**: Governed by Gemini API rate limits. +- **Mathematics**: Unlimited access. +- **Membership**: No specific rate limits. + +State of Mika ensures seamless interaction with diverse data silos, empowering users with intelligent query routing and post-processed, actionable responses. +""" + +tags_metadata = [ + { + "name": "router", + "description": """ + Universal router for processing and directing queries to appropriate tools. + + The router is the core component that intelligently processes natural language queries + and routes them to the appropriate service. + """ + }, + { + "name": "basic-tools", + "description": """ + Core utility tools for common operations. + + Includes: + * News Aggregation: Cryptocurrency and blockchain news + * Web Scraping: Content extraction and analysis + * Mathematics: Expression evaluation + * Image Recognition: AI-powered image analysis + """ + }, + { + "name": "tokens", + "description": """ + Token and cryptocurrency related operations. + + Includes: + * Token Price Information: Real-time price and market data + * Solana DEX Sales: Historical sales data for Solana tokens + * Solana DEX Buys: Buy order tracking for specific wallets + """ + }, + { + "name": "membership", + "description": """ + The Membership API provides endpoints for managing bots and their associated friends across different platforms. + + Key features: + * Bot Management: Create and manage bots with secure API key authentication + * Friend Management: Track friends across multiple platforms with customizable attributes + * Platform Contacts: Support for multiple platforms (Discord, Telegram, Twitter, WhatsApp, etc.) + * Points System: Track and update friend points with history tracking + * Level System: Manage friend levels and progression + + Authentication: + * All endpoints (except bot creation) require the bot's API key in the x-api-key header + * API keys are automatically generated when creating a new bot + """ + } +] + +# if production, remove docs and redoc: +if settings.is_production: + app = FastAPI( + title="State Of Mika", + description=description, + version="1.0.0", + contact={ + "name": "API Support", + "url": "" + }, + license_info={ + "name": "MIT", + }, + openapi_tags=tags_metadata + ) +else: + app = FastAPI( + title="State Of Mika", + description=description, + version="1.0.0", + docs_url="/docs", + redoc_url="/redoc", + contact={ + "name": "API Support", + "url": "https://github.com/ChasmNetwork/StateOfMika-Toolkit", + }, + license_info={ + "name": "MIT", + }, + swagger_ui_parameters={"defaultModelsExpandDepth": -1}, + openapi_tags=tags_metadata + ) + +# Configure CORS +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +@app.on_event("startup") +async def startup_event(): + logger.info(f"Starting State Of Mika API in {settings.ENV.value} mode...") + logger.info(f"Running on port: {settings.UVICORN_PORT}") + if settings.is_production: + logger.info("Production mode: Enhanced security and performance settings enabled") + elif settings.is_development: + logger.info("Development mode: Debug features and detailed logging enabled") + +@app.on_event("shutdown") +async def shutdown_event(): + logger.info("Shutting down State Of Mika API...") + +app.include_router(admin.router) + +app.include_router( + router.router, + prefix="/api/v1", + tags=["router"], + responses={404: {"description": "Not found"}}, + dependencies=[Depends(require_api_key)] +) +# Basic Tools +app.include_router( + news.router, + tags=["basic-tools"], + dependencies=[Depends(require_api_key)] +) +app.include_router( + scraper.router, + tags=["basic-tools"], + dependencies=[Depends(require_api_key)] +) +app.include_router( + math.router, + tags=["basic-tools"], + dependencies=[Depends(require_api_key)] +) +app.include_router( + image_recognition.router, + tags=["basic-tools"], + dependencies=[Depends(require_api_key)] +) + +# Token Operations +app.include_router( + token_information.router, + tags=["tokens"], + dependencies=[Depends(require_api_key)] +) +app.include_router( + solana_dex_sales.router, + tags=["tokens"], + dependencies=[Depends(require_api_key)] +) +app.include_router( + solana_dex_buys.router, + tags=["tokens"], + dependencies=[Depends(require_api_key)] +) + +# Life Simulator +app.include_router( + life_simulator.router, + tags=["life-simulator"], + dependencies=[Depends(require_api_key)] +) + +# Membership +app.include_router( + membership.router, + tags=["membership"], + dependencies=[Depends(require_api_key)] +) diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/models/admin.py b/app/models/admin.py new file mode 100644 index 0000000..242624b --- /dev/null +++ b/app/models/admin.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel +from typing import Dict + +class CreateAPIKeyRequest(BaseModel): + name: str + limits: Dict[str, int] diff --git a/app/models/image_recognition.py b/app/models/image_recognition.py new file mode 100644 index 0000000..aa34468 --- /dev/null +++ b/app/models/image_recognition.py @@ -0,0 +1,27 @@ +from pydantic import BaseModel, Field +from typing import List, Optional +from fastapi import UploadFile + +class ImageAnalysisResponse(BaseModel): + """Model for image analysis results""" + descriptions: List[str] = Field(..., description="List of image descriptions") + errors: List[str] = Field(default_factory=list, description="List of any errors encountered") + + class Config: + json_encoders = { + bytes: lambda v: None # Ignore binary data + } + +class ImageRecognitionRequest(BaseModel): + """Model for image recognition requests""" + query: Optional[str] = Field(default="Describe this image in detail", description="Instructions for image analysis") + images: List[UploadFile] = Field(..., description="List of image files") + + class Config: + arbitrary_types_allowed = True + +class ImageRecognitionResponse(BaseModel): + """Model for image recognition response""" + query: str = Field(..., description="Original query/instructions") + results: ImageAnalysisResponse = Field(..., description="Analysis results") + error: Optional[str] = None \ No newline at end of file diff --git a/app/models/life_simulator.py b/app/models/life_simulator.py new file mode 100644 index 0000000..0d8d52d --- /dev/null +++ b/app/models/life_simulator.py @@ -0,0 +1,71 @@ +from pydantic import BaseModel +from datetime import datetime +from typing import Dict, List, Optional, Any + +class Location(BaseModel): + name: str + lat: float + lon: float + type: str + role: Optional[str] = None # e.g., "home", "office", "shopping", etc. + country: Optional[str] = None + +class SimulationConfig(BaseModel): + residence: Location + occupation: str + office: Location + current_time: Optional[datetime] = None + available_locations: Dict[str, Location] + city: str = "Tokyo" + country: str = "Japan" + timezone: str = "Asia/Tokyo" + gender: str = "female" + age: int = 23 + name: str = "Mika" + +class Incident(BaseModel): + type: str + severity: int + description: str + impact_duration: int + affects_next_activity: bool + +class TransitInfo(BaseModel): + line_name: str + delay_minutes: int + crowding_level: str + next_departure: datetime + +class TransitRoute(BaseModel): + line_name: str + departure_time: datetime + arrival_time: datetime + duration_minutes: int + transfers: int + sections: List[str] + +class WeatherInfo(BaseModel): + condition: str + temperature: float + details: Dict + +class TransitDetails(BaseModel): + departures: List[Dict[str, Any]] + routes: List[Dict[str, Any]] + +class ActivityDetails(BaseModel): + main_action: str + location: Dict[str, str] + reason: str + narrative: str + details: Dict[str, Any] + +class ActivityResponse(BaseModel): + activity: ActivityDetails + incident: Optional[Dict[str, Any]] = None + + class Config: + extra = "allow" # Allow extra fields that might come from the simulation + +class SimulationRequest(BaseModel): + config: SimulationConfig diff --git a/app/models/math.py b/app/models/math.py new file mode 100644 index 0000000..3e4942c --- /dev/null +++ b/app/models/math.py @@ -0,0 +1,9 @@ +from pydantic import BaseModel, Field + +class MathRequest(BaseModel): + expression: str = Field(..., description="Mathematical expression to evaluate") + +class MathResponse(BaseModel): + expression: str + result: float + steps: list[str] = [] # To show the calculation steps \ No newline at end of file diff --git a/app/models/membership.py b/app/models/membership.py new file mode 100644 index 0000000..93e7a56 --- /dev/null +++ b/app/models/membership.py @@ -0,0 +1,81 @@ +from pydantic import BaseModel, UUID4, Field, constr +from typing import Optional, List +from datetime import datetime +from enum import Enum + +class PlatformType(str, Enum): + DISCORD = "discord" + TELEGRAM = "telegram" + TWITTER = "twitter" + WHATSAPP = "whatsapp" + CUSTOM = "custom" + +class BotCreate(BaseModel): + name: constr(min_length=1, max_length=100) + +class BotResponse(BaseModel): + bot_id: UUID4 + api_key: UUID4 + name: str + created_at: datetime + +class PlatformContact(BaseModel): + platform_type: PlatformType + platform_user_id: str = Field(..., max_length=100) + +class FriendCreate(BaseModel): + nickname: constr(min_length=1, max_length=100) + description: Optional[str] = None + platform_contact: PlatformContact + +class PlatformContactResponse(PlatformContact): + contact_id: UUID4 + friend_id: UUID4 + created_at: datetime + +class FriendResponse(BaseModel): + friend_id: UUID4 + is_newly_created: bool + nickname: str + level: int = 1 + points: int = 0 + description: Optional[str] = None + created_at: datetime + updated_at: datetime + platform_contacts: List[PlatformContactResponse] + +class FriendBasicResponse(BaseModel): + friend_id: UUID4 + nickname: str + level: int + points: int + +class FriendUpdate(BaseModel): + level: Optional[int] = None + points: Optional[int] = None + description: Optional[str] = None + reason: Optional[str] = None + +class FriendUpdateResponse(BaseModel): + success: bool + friend_id: UUID4 + level: int + points: int + description: Optional[str] + +class PointsHistoryEntry(BaseModel): + history_id: UUID4 + friend_id: UUID4 + timestamp: datetime + points_change: int + previous_points: int + current_points: int + reason: Optional[str] + created_at: datetime + +class FriendList(BaseModel): + friends: List[FriendBasicResponse] + +class PlatformContactCreate(BaseModel): + platform_type: PlatformType + platform_user_id: str = Field(..., max_length=100) diff --git a/app/models/news.py b/app/models/news.py new file mode 100644 index 0000000..7325fcb --- /dev/null +++ b/app/models/news.py @@ -0,0 +1,57 @@ +from pydantic import BaseModel, Field +from typing import List, Optional, Dict +from datetime import datetime + +class NewsSource(BaseModel): + title: str + region: str + domain: str + path: Optional[str] = None + type: str + url: Optional[str] = None + +class Currency(BaseModel): + code: str + title: str + slug: str + url: str + +class Votes(BaseModel): + negative: int = 0 + positive: int = 0 + important: int = 0 + liked: int = 0 + disliked: int = 0 + lol: int = 0 + toxic: int = 0 + saved: int = 0 + comments: int = 0 + +class NewsItem(BaseModel): + kind: str + domain: str + source: NewsSource + title: str + published_at: datetime + slug: str + id: int + url: str + created_at: datetime + votes: Votes + currencies: Optional[List[Currency]] = None + +class NewsResponse(BaseModel): + count: int + results: List[NewsItem] + has_next: bool = False + + @classmethod + def from_api_response(cls, response: Dict) -> "NewsResponse": + """ + Create NewsResponse from API response, safely handling pagination + """ + return cls( + count=response["count"], + results=response["results"], + has_next=bool(response.get("next")) + ) \ No newline at end of file diff --git a/app/models/scraper.py b/app/models/scraper.py new file mode 100644 index 0000000..f4a118a --- /dev/null +++ b/app/models/scraper.py @@ -0,0 +1,35 @@ +from pydantic import BaseModel, HttpUrl, validator +from typing import Optional, Dict, Any, List +from urllib.parse import urlparse +import re + +class ScraperRequest(BaseModel): + url: HttpUrl + instructions: Optional[str] = "Extract main content" + + @validator('url') + def validate_url(cls, v): + parsed = urlparse(str(v)) + # Check if URL is external (not localhost or internal) + if parsed.hostname in ['localhost', '127.0.0.1'] or \ + parsed.hostname.startswith('192.168.') or \ + parsed.hostname.startswith('10.') or \ + parsed.hostname.startswith('172.'): + raise ValueError("Internal URLs are not allowed") + return v + +class ProcessedContent(BaseModel): + url: HttpUrl + title: str + content: str + metadata: Dict[str, Any] + summary: Optional[str] = None + tags: List[str] = [] + word_count: int + timestamp: str + +class ScraperResponse(BaseModel): + original_url: HttpUrl + instructions: str + processed_content: ProcessedContent + error: Optional[str] = None \ No newline at end of file diff --git a/app/models/solana_dex_buys.py b/app/models/solana_dex_buys.py new file mode 100644 index 0000000..d412ace --- /dev/null +++ b/app/models/solana_dex_buys.py @@ -0,0 +1,39 @@ +from pydantic import BaseModel, Field +from typing import Optional, List +from datetime import datetime + +class TokenAccount(BaseModel): + owner: Optional[str] + +class Account(BaseModel): + token: Optional[TokenAccount] + address: str + owner: str + +class Currency(BaseModel): + name: str + mint_address: str + +class TradeAction(BaseModel): + amount: float + account: Account + currency: Currency + +class Trade(BaseModel): + buy: TradeAction + sell: TradeAction + +class Transaction(BaseModel): + signature: str + signer: str + +class DexTrade(BaseModel): + trade: Trade + transaction: Transaction + +class DexTradeResponse(BaseModel): + dex_trades: List[DexTrade] + +class DexBuysRequest(BaseModel): + mint_address: str = Field(..., description="Token mint address to query") + signer_address: str = Field(..., description="User's wallet address that signed the transactions") \ No newline at end of file diff --git a/app/models/solana_dex_sales.py b/app/models/solana_dex_sales.py new file mode 100644 index 0000000..7849214 --- /dev/null +++ b/app/models/solana_dex_sales.py @@ -0,0 +1,43 @@ +from pydantic import BaseModel, Field +from typing import Optional, List +from datetime import datetime + +class TokenAccount(BaseModel): + owner: Optional[str] + +class Account(BaseModel): + token: Optional[TokenAccount] + address: str + owner: str + +class Currency(BaseModel): + name: str + mint_address: str + +class TradeAction(BaseModel): + amount: float + account: Account + currency: Currency + +class Trade(BaseModel): + buy: TradeAction + sell: TradeAction + +class Transaction(BaseModel): + signature: str + signer: str + +class Block(BaseModel): + time: datetime + +class DexTrade(BaseModel): + trade: Trade + transaction: Transaction + block: Block + +class DexTradeResponse(BaseModel): + dex_trades: List[DexTrade] + +class DexSalesRequest(BaseModel): + mint_address: str + time_window: int = Field(..., description="Time window in minutes (e.g., 60 for last hour, 1440 for last day)") diff --git a/app/models/token_information.py b/app/models/token_information.py new file mode 100644 index 0000000..9e364c2 --- /dev/null +++ b/app/models/token_information.py @@ -0,0 +1,57 @@ +from pydantic import BaseModel, HttpUrl +from typing import List, Optional, Dict, Union, Any + +class SocialLink(BaseModel): + type: str + url: str + +class Website(BaseModel): + url: str + +class TokenInfo(BaseModel): + imageUrl: Optional[HttpUrl] = None + websites: Optional[List[Website]] = None + socials: Optional[List[SocialLink]] = None + +class Token(BaseModel): + address: str + name: str + symbol: str + +class Liquidity(BaseModel): + usd: float + base: float + quote: float + +class Boosts(BaseModel): + active: int + +class Pair(BaseModel): + chainId: str + dexId: str + url: HttpUrl + pairAddress: str + labels: Optional[List[str]] = None + baseToken: Token + quoteToken: Token + priceNative: str + priceUsd: str + liquidity: Optional[Liquidity] = None + fdv: Optional[float] = None + marketCap: Optional[float] = None + pairCreatedAt: Optional[int] = None # Made optional since some pairs don't have it + info: Optional[TokenInfo] = None + boosts: Optional[Boosts] = None + +class DexScreenerResponse(BaseModel): + schemaVersion: str + pairs: List[Pair] + +class TokenPriceRequest(BaseModel): + query: str + chain_id: Optional[str] = None + +class TokenPriceResponse(BaseModel): + query: str + chain_id: Optional[str] + result: DexScreenerResponse \ No newline at end of file diff --git a/app/models/web_search.py b/app/models/web_search.py new file mode 100644 index 0000000..6bcb7c8 --- /dev/null +++ b/app/models/web_search.py @@ -0,0 +1,23 @@ +from pydantic import BaseModel, Field +from typing import List, Optional, Dict, Any + +class SearchResult(BaseModel): + url: str = Field(..., description="URL of the search result") + title: str = Field(..., description="Title of the search result") + markdown: str = Field(..., description="Content of the search result in markdown format") + +class Learning(BaseModel): + content: str = Field(..., description="Learning content") + source_url: Optional[str] = Field(None, description="Source URL for the learning") + +class WebSearchRequest(BaseModel): + query: str = Field(..., description="Search query") + num_results: Optional[int] = Field(5, description="Number of search results to fetch", ge=1, le=20) + depth: Optional[int] = Field(1, description="Research depth (1-3)", ge=1, le=3) + +class WebSearchResponse(BaseModel): + query: str = Field(..., description="Original search query") + results: List[SearchResult] = Field(default_factory=list, description="Raw search results") + learnings: List[Learning] = Field(default_factory=list, description="Key learnings extracted from the results") + summary: str = Field("", description="Summary of the research findings") + visited_urls: List[str] = Field(default_factory=list, description="List of URLs that were visited") \ No newline at end of file diff --git a/app/routes/__init__.py b/app/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/routes/admin.py b/app/routes/admin.py new file mode 100644 index 0000000..c422e71 --- /dev/null +++ b/app/routes/admin.py @@ -0,0 +1,45 @@ +from fastapi import APIRouter, HTTPException, Header, status +from typing import Optional, Dict +from app.config import settings +from app.models.admin import CreateAPIKeyRequest +import logging + +from app.services.supabase_service import SupabaseService + +router = APIRouter(prefix="/api/v1/admin", tags=["admin"]) +supabase_service = SupabaseService() +logger = logging.getLogger(__name__) + +@router.post("/create-api-key") +def create_api_key( + request_data: CreateAPIKeyRequest, + x_admin_key: Optional[str] = Header(None, convert_underscores=True) +): + """ + Creates a new user API key with daily usage limits. + + Must provide the correct ADMIN_SECRET in 'X-Admin-Key' header. + Example cURL: + curl -X POST http://localhost:8000/api/v1/admin/create-api-key \ + -H 'X-Admin-Key: ' \ + -H 'Content-Type: application/json' \ + -d '{ + "name": "Test User", + "limits": {"news": 500, "image_recognition": 25} + }' + """ + # Validate admin key + admin_secret = settings.ADMIN_SECRET + if x_admin_key != admin_secret: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid admin credentials" + ) + + # Create the API key in Supabase + created_record = supabase_service.create_user_api_key(name=request_data.name, limits=request_data.limits) + return { + "message": "API key created successfully", + "api_key": created_record["api_key"], + "limits": created_record["limits"] + } diff --git a/app/routes/image_recognition.py b/app/routes/image_recognition.py new file mode 100644 index 0000000..c0863da --- /dev/null +++ b/app/routes/image_recognition.py @@ -0,0 +1,47 @@ +from fastapi import APIRouter, HTTPException, File, Form, UploadFile +from app.models.image_recognition import ImageRecognitionResponse, ImageAnalysisResponse +from app.services.image_recognition import ImageRecognitionService +from app.utils.logging_config import get_logger +from typing import List, Optional + +logger = get_logger(__name__) +router = APIRouter() +image_recognition_service = ImageRecognitionService() + +@router.post("/image-recognition", response_model=ImageRecognitionResponse) +async def analyze_images( + images: UploadFile = File(..., description="Image file to analyze"), + query: Optional[str] = Form(None, description="Instructions for analysis") +) -> ImageRecognitionResponse: + """ + Analyze images using Gemini vision model + + - Accept image files + - Provide optional analysis instructions + - Returns descriptions and any errors encountered + """ + try: + if not images: + raise HTTPException( + status_code=400, + detail="At least one image is required" + ) + + # Read image data + image_data = await images.read() + results = await image_recognition_service.analyze_images( + images=[image_data], + query=query + ) + + return ImageRecognitionResponse( + query=query or "Describe this image in detail", + results=ImageAnalysisResponse(**results) + ) + + except Exception as e: + logger.error(f"Error processing image recognition request: {str(e)}") + raise HTTPException( + status_code=500, + detail=f"Error processing images: {str(e)}" + ) \ No newline at end of file diff --git a/app/routes/life_simulator.py b/app/routes/life_simulator.py new file mode 100644 index 0000000..82d9153 --- /dev/null +++ b/app/routes/life_simulator.py @@ -0,0 +1,126 @@ +from fastapi import APIRouter, Depends, HTTPException +from typing import Dict +from app.services.life_simulator import LifeSimulator +from app.models.life_simulator import SimulationConfig, ActivityResponse +from pydantic import BaseModel +import logging +import os +from dotenv import load_dotenv + +load_dotenv() + +router = APIRouter() +logger = logging.getLogger(__name__) + +@router.post("/simulate", response_model=ActivityResponse) +async def simulate_activity(request: SimulationConfig) -> ActivityResponse: # Changed from config to request + """ + Simulate life activities based on provided configuration. + + Args: + config: Simulation configuration + + Returns: + Simulated activity details including weather, transit, and potential incidents + """ + try: + api_keys = { + "openrouter": os.getenv("OPENROUTER_API_KEY"), + "openweather": os.getenv("OPENWEATHER_API_KEY"), + "google": os.getenv("GOOGLE_MAPS_API_KEY") + } + + missing_keys = [key for key, value in api_keys.items() if not value] + if missing_keys: + raise HTTPException( + status_code=500, + detail=f"Missing environment variables for: {', '.join(missing_keys)}" + ) + + logger.info(f"Simulation request for location: {request.residence.name}") + simulator = LifeSimulator(api_keys) + result = await simulator.simulate_activity(request) + return ActivityResponse.model_validate(result) + except Exception as e: + logger.error(f"Simulation failed: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + + +""" +Life Simulator API endpoint for simulating daily activities based on provided configuration. +This endpoint processes simulation requests containing API keys and configuration parameters +to generate realistic life activity scenarios including weather conditions, transit information, +and potential incidents. + request (SimulationRequest): Request object containing: + api_keys (Dict[str, str]): Dictionary of required API keys including: + - openrouter: OpenRouter API key for AI services + - openweather: OpenWeather API key for weather data + - google: Google Maps API key for transit information + config (SimulationConfig): Simulation configuration including: + - residence: Location details of residence (name, coordinates, type, etc.) + - occupation: User's occupation + - office: Office location details + - current_time: Timestamp for simulation + - available_locations: Dictionary of available locations + - city: City name + - country: Country name + - timezone: Timezone string + - personal details (gender, age, name) + ActivityResponse: Object containing: + - weather_conditions: Current weather at location + - transit_info: Transit details between locations + - suggested_activity: AI-generated activity suggestion + - potential_incidents: Possible events that might occur +Example API Call: + ``` + curl -X 'POST' \ + 'http://localhost:8000/simulate' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -H 'X-API-Key: your-SoM-api-key' \ + -d '{ + "config": { + "residence": { + "name": "Daikanyama", + "lat": 35.648450, + "lon": 139.703305, + "type": "residential", + "role": "home", + "country": "Japan" + }, + "occupation": "Software Engineer", + "office": { + "name": "Shinjuku", + "lat": 35.689700, + "lon": 139.700400, + "type": "business", + "role": "office", + "country": "Japan" + }, + "current_time": "2024-03-15T14:30:00+09:00", + "available_locations": { + "Shibuya": { + "name": "Shibuya", + "lat": 35.658000, + "lon": 139.701600, + "type": "shopping", + "role": "entertainment", + "country": "Japan" + } + // ... other locations + }, + "city": "Tokyo", + "country": "Japan", + "timezone": "Asia/Tokyo", + "gender": "female", + "age": 23, + "name": "Mika" + } + }' + ``` +Raises: + HTTPException: + - 400: If required API keys are missing + - 500: If simulation fails due to internal error +""" \ No newline at end of file diff --git a/app/routes/math.py b/app/routes/math.py new file mode 100644 index 0000000..708fb1a --- /dev/null +++ b/app/routes/math.py @@ -0,0 +1,50 @@ +from fastapi import APIRouter, HTTPException +from app.models.math import MathRequest, MathResponse +from app.services.math import MathService +from app.utils.logging_config import get_logger + +logger = get_logger(__name__) +router = APIRouter() +math_service = MathService() + +@router.post("/math", response_model=MathResponse) +def evaluate_expression(request: MathRequest) -> MathResponse: + """ + Evaluate a mathematical expression. + + Examples of valid expressions: + - "2 + 2" + - "sin(pi/2)" + - "sqrt(16)" + - "2^3" + - "log(e)" + + Returns: + MathResponse containing: + - The original expression + - The calculated result + - Steps taken during calculation + """ + logger.info(f"Received math evaluation request: {request.expression}") + + try: + result, steps = math_service.evaluate(request.expression) + logger.info(f"Successfully evaluated expression: {result}") + + return MathResponse( + expression=request.expression, + result=result, + steps=steps + ) + except ValueError as e: + logger.error(f"Invalid expression: {str(e)}") + raise HTTPException( + status_code=400, + detail=f"Invalid mathematical expression: {str(e)}" + ) + except Exception as e: + logger.error(f"Error processing math request: {str(e)}", exc_info=True) + raise HTTPException( + status_code=500, + detail=f"Error processing mathematical expression: {str(e)}" + ) \ No newline at end of file diff --git a/app/routes/membership.py b/app/routes/membership.py new file mode 100644 index 0000000..df37909 --- /dev/null +++ b/app/routes/membership.py @@ -0,0 +1,241 @@ +from fastapi import APIRouter, Depends, HTTPException, Header +from typing import Optional, List +from uuid import UUID + +from app.models.membership import ( + BotCreate, BotResponse, FriendCreate, FriendResponse, + FriendBasicResponse, FriendUpdate, FriendUpdateResponse, FriendList, + PlatformContactCreate, PlatformContactResponse +) +from app.services.membership import MembershipService +from app.utils.logging_config import get_logger + +router = APIRouter( + prefix="/api/v1/membership", + tags=["membership"], + responses={401: {"description": "Invalid API Key"}} +) + +# Add tag metadata at the module level +membership_tag_metadata = { + "name": "membership", + "description": """ + The Membership API provides endpoints for managing bots and their associated friends across different platforms. + + Key features: + * Bot Management: Create and manage bots with secure API key authentication + * Friend Management: Track friends across multiple platforms with customizable attributes + * Platform Contacts: Support for multiple platforms (Discord, Telegram, Twitter, WhatsApp, etc.) + * Points System: Track and update friend points with history tracking + * Level System: Manage friend levels and progression + + Authentication: + * All endpoints (except bot creation) require the bot's API key in the x-api-key header + * API keys are automatically generated when creating a new bot + """ +} + +logger = get_logger(__name__) + +async def get_api_key(x_api_key: str = Header(...)) -> UUID: + try: + return UUID(x_api_key) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid API key format") + +async def validate_bot_access( + bot_id: UUID, + api_key: UUID = Depends(get_api_key), + membership_service: MembershipService = Depends() +) -> None: + if not await membership_service.validate_bot_api_key(bot_id, api_key): + raise HTTPException(status_code=403, detail="Invalid API key for this bot") + +@router.post("/bots", response_model=BotResponse) +async def create_bot( + bot_data: BotCreate, + membership_service: MembershipService = Depends() +) -> BotResponse: + """ + Create a new bot and get its API key. + + Parameters: + - **bot_data**: Bot creation data containing the name (1-100 characters) + + Returns: + - Bot details including bot_id, api_key, name, and created_at timestamp + + The generated API key should be stored securely as it's required for all other endpoints. + """ + try: + return await membership_service.create_bot(bot_data) + except Exception as e: + logger.error(f"Failed to create bot: {str(e)}") + raise HTTPException(status_code=500, detail="Failed to create bot") + +@router.post("/bots/{bot_id}/friends", response_model=FriendResponse) +async def create_or_get_friend( + bot_id: UUID, + friend_data: FriendCreate, + membership_service: MembershipService = Depends(), + _: None = Depends(validate_bot_access) +) -> FriendResponse: + """ + Create a new friend or get existing friend details. + + If a friend with the same platform contact already exists, returns the existing friend's details. + Otherwise, creates a new friend with the specified platform contact. + + Parameters: + - **bot_id**: UUID of the bot + - **friend_data**: Friend creation data including: + - nickname (1-100 characters) + - description (optional) + - platform_contact with platform_type and platform_user_id + + Returns: + - Friend details including all platform contacts + + Requires authentication via x-api-key header. + """ + try: + return await membership_service.create_or_get_friend(bot_id, friend_data) + except Exception as e: + logger.error(f"Failed to create/get friend: {str(e)}") + raise HTTPException(status_code=500, detail="Failed to create/get friend") + +@router.get("/bots/{bot_id}/friends", response_model=FriendList) +async def list_friends( + bot_id: UUID, + membership_service: MembershipService = Depends(), + _: None = Depends(validate_bot_access) +) -> FriendList: + """ + List all friends of a bot. + + Returns a basic overview of each friend including their ID, nickname, level, and points. + For detailed friend information, use the get_friend_details endpoint. + + Parameters: + - **bot_id**: UUID of the bot + + Returns: + - List of friends with basic information + + Requires authentication via x-api-key header. + """ + try: + friends = await membership_service.get_friends(bot_id) + return FriendList(friends=friends) + except Exception as e: + logger.error(f"Failed to list friends: {str(e)}") + raise HTTPException(status_code=500, detail="Failed to list friends") + +@router.get("/bots/{bot_id}/friends/{friend_id}", response_model=FriendResponse) +async def get_friend_details( + bot_id: UUID, + friend_id: UUID, + membership_service: MembershipService = Depends(), + _: None = Depends(validate_bot_access) +) -> FriendResponse: + """ + Get detailed information about a specific friend. + + Returns comprehensive friend details including all platform contacts. + + Parameters: + - **bot_id**: UUID of the bot + - **friend_id**: UUID of the friend + + Returns: + - Detailed friend information including: + - Basic details (nickname, level, points, description) + - Creation and update timestamps + - List of all platform contacts + + Requires authentication via x-api-key header. + """ + try: + friend = await membership_service.get_friend_details(bot_id, friend_id) + if not friend: + raise HTTPException(status_code=404, detail="Friend not found") + return friend + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to get friend details: {str(e)}") + raise HTTPException(status_code=500, detail="Failed to get friend details") + +@router.patch("/bots/{bot_id}/friends/{friend_id}", response_model=FriendUpdateResponse) +async def update_friend_attributes( + bot_id: UUID, + friend_id: UUID, + update_data: FriendUpdate, + membership_service: MembershipService = Depends(), + _: None = Depends(validate_bot_access) +) -> FriendUpdateResponse: + """ + Update friend attributes (level, points, description). + + All fields in the update request are optional. Only specified fields will be updated. + When updating points, you can provide a reason which will be recorded in the points history. + + Parameters: + - **bot_id**: UUID of the bot + - **friend_id**: UUID of the friend + - **update_data**: Update data containing any of: + - level (optional) + - points (optional) + - description (optional) + - reason (optional, used when updating points) + + Returns: + - Update status and current friend attributes + + Requires authentication via x-api-key header. + """ + try: + result = await membership_service.update_friend(bot_id, friend_id, update_data) + if not result: + raise HTTPException(status_code=404, detail="Friend not found") + return result + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to update friend: {str(e)}") + raise HTTPException(status_code=500, detail="Failed to update friend") + +@router.post("/bots/{bot_id}/friends/{friend_id}/contacts", response_model=PlatformContactResponse) +async def add_platform_contact( + bot_id: UUID, + friend_id: UUID, + contact_data: PlatformContactCreate, + membership_service: MembershipService = Depends(), + _: None = Depends(validate_bot_access) +) -> PlatformContactResponse: + """ + Add a new platform contact for a friend. + + Each friend can have multiple platform contacts, but only one contact per platform type. + Attempting to add a duplicate platform contact will result in an error. + + Parameters: + - **bot_id**: UUID of the bot + - **friend_id**: UUID of the friend + - **contact_data**: Platform contact data including: + - platform_type (discord, telegram, twitter, whatsapp, or custom) + - platform_user_id (max 100 characters) + + Returns: + - Created platform contact details + + Requires authentication via x-api-key header. + """ + try: + result = await membership_service.add_platform_contact(bot_id, friend_id, contact_data) + if not result: + raise HTTPException(status_code=404, detail="Friend not found") + return result + except Exception as e: + logger.error(f"Failed to add platform contact: {str(e)}") + raise HTTPException(status_code=500, detail="Failed to add platform contact") diff --git a/app/routes/news.py b/app/routes/news.py new file mode 100644 index 0000000..5e4e4e1 --- /dev/null +++ b/app/routes/news.py @@ -0,0 +1,74 @@ +from fastapi import APIRouter, Query, HTTPException +from typing import Optional, List +from app.services.news import NewsService, NewsFilter, NewsKind +from app.models.news import NewsResponse + +router = APIRouter() +news_service = NewsService() + +@router.get("/news", response_model=NewsResponse) +async def get_news( + public: bool = Query(False, description="Use public API"), + filter: Optional[NewsFilter] = Query( + None, + description="Filter type (rising, hot, bullish, bearish, important, saved, lol)" + ), + currencies: Optional[str] = Query( + None, + description="Comma-separated list of currency codes (max 50)", + regex="^[A-Z,]*$" + ), + regions: Optional[str] = Query( + None, + description="Comma-separated list of region codes (en,de,nl,es,fr,it,pt,ru,tr,ar,cn,jp,ko)" + ), + kind: Optional[NewsKind] = Query( + NewsKind.ALL, + description="Content type (news or media)" + ), + page: Optional[int] = Query( + None, + description="Page number for pagination", + gt=0 + ) +) -> NewsResponse: + """ + Get latest crypto news with various filtering options. + + - Use `public=true` for public API access + - Filter by type using `filter` parameter + - Filter by currencies using comma-separated currency codes (max 50) + - Filter by regions using comma-separated region codes + - Filter by content type using `kind` parameter + - Use `page` for pagination + """ + try: + # Process currency codes + currency_list = ( + [c.strip() for c in currencies.upper().split(",")] + if currencies else None + ) + + # Process region codes + region_list = ( + [r.strip().lower() for r in regions.split(",")] + if regions else None + ) + + return await news_service.get_latest_news( + public=public, + filter_type=filter, + currencies=currency_list, + regions=region_list, + kind=kind, + page=page + ) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + +@router.get("/news/supported-currencies", response_model=List[str]) +async def get_supported_currencies() -> List[str]: + """ + Get list of supported currency codes for filtering + """ + return await news_service.get_supported_currencies() \ No newline at end of file diff --git a/app/routes/scraper.py b/app/routes/scraper.py new file mode 100644 index 0000000..a464627 --- /dev/null +++ b/app/routes/scraper.py @@ -0,0 +1,41 @@ +from fastapi import APIRouter, HTTPException +from app.models.scraper import ScraperRequest, ScraperResponse +from app.services.scraper import ScraperService +from app.utils.logging_config import get_logger + + +router = APIRouter() +logger = get_logger(__name__) +scraper_service = ScraperService() + +@router.post("/scraper", response_model=ScraperResponse, summary="Scrape and analyze web content") +async def scrape_url(request: ScraperRequest) -> ScraperResponse: + """ + Scrape and process content from a given URL + + Args: + - url: External URL to scrape (must not be localhost or internal network) + - instructions: Optional processing instructions for content analysis + + Returns: + - Processed content with summary, tags, and analysis + + Example: + ```json + { + "url": "https://example.com/article", + "instructions": "Summarize and extract key points" + } + ``` + """ + try: + return await scraper_service.scrape_url( + url=str(request.url), + instructions=request.instructions + ) + except Exception as e: + logger.error(f"Error processing scrape request: {str(e)}", exc_info=True) + raise HTTPException( + status_code=500, + detail=f"Error processing scrape request: {str(e)}" + ) \ No newline at end of file diff --git a/app/routes/solana_dex_buys.py b/app/routes/solana_dex_buys.py new file mode 100644 index 0000000..7e13576 --- /dev/null +++ b/app/routes/solana_dex_buys.py @@ -0,0 +1,29 @@ +from fastapi import APIRouter, Depends +from app.services.solana_dex_buys import SolanaDexBuysService +from app.models.solana_dex_buys import DexBuysRequest, DexTradeResponse +from app.utils.logging_config import get_logger + +router = APIRouter( + prefix="/solana/dex", + responses={404: {"description": "Not found"}}, +) + +logger = get_logger(__name__) + +@router.post("/buys", response_model=DexTradeResponse) +async def get_dex_buys( + request: DexBuysRequest, + service: SolanaDexBuysService = Depends(SolanaDexBuysService) +) -> DexTradeResponse: + """ + Get DEX buy orders for a specific token by a specific user + + Args: + request (DexBuysRequest): Request containing mint address and signer address + service (SolanaDexBuysService): Injected service instance + + Returns: + DexTradeResponse: DEX trades data + """ + logger.info(f"Fetching DEX buys for mint {request.mint_address} by signer {request.signer_address}") + return await service.get_dex_buys(request.mint_address, request.signer_address) \ No newline at end of file diff --git a/app/routes/solana_dex_sales.py b/app/routes/solana_dex_sales.py new file mode 100644 index 0000000..3fe85e0 --- /dev/null +++ b/app/routes/solana_dex_sales.py @@ -0,0 +1,29 @@ +from fastapi import APIRouter, Depends +from app.services.solana_dex_sales import SolanaDexSalesService +from app.models.solana_dex_sales import DexSalesRequest, DexTradeResponse +from app.utils.logging_config import get_logger + +router = APIRouter( + prefix="/solana/dex", + responses={404: {"description": "Not found"}}, +) + +logger = get_logger(__name__) + +@router.post("/sales", response_model=DexTradeResponse) +async def get_dex_sales( + request: DexSalesRequest, + service: SolanaDexSalesService = Depends(SolanaDexSalesService) +) -> DexTradeResponse: + """ + Get DEX sales for a specific mint address within the specified time window + + Args: + request (DexSalesRequest): Request containing mint address and time window + service (SolanaDexSalesService): Injected service instance + + Returns: + DexTradeResponse: DEX sales data + """ + logger.info(f"Fetching DEX sales for mint {request.mint_address} in last {request.time_window} hours") + return await service.get_dex_sales(request.mint_address, request.time_window) \ No newline at end of file diff --git a/app/routes/token_information.py b/app/routes/token_information.py new file mode 100644 index 0000000..3c65327 --- /dev/null +++ b/app/routes/token_information.py @@ -0,0 +1,27 @@ +from fastapi import APIRouter, Query +from typing import Optional +from app.services.token_information import TokenInformationService +from app.models.token_information import DexScreenerResponse, TokenPriceRequest +import logging + +router = APIRouter() +token_information_service = TokenInformationService() +logger = logging.getLogger(__name__) + +@router.get("/token-price", response_model=DexScreenerResponse) +async def get_token_information( + query: str = Query(..., description="Token address or name to search for"), + chain_id: Optional[str] = Query(None, description="Optional blockchain ID") +) -> DexScreenerResponse: + """ + Get token price information from DexScreener. + + Args: + query: Token address or name to search for + chain_id: Optional blockchain ID (e.g., 'ethereum', 'bsc', 'solana') + + Returns: + Token pair information including price, liquidity, and market data + """ + logger.info(f"Token price request - query: {query}, chain: {chain_id}") + return await token_information_service.get_token_information(query, chain_id) \ No newline at end of file diff --git a/app/routes/web_search.py b/app/routes/web_search.py new file mode 100644 index 0000000..8448899 --- /dev/null +++ b/app/routes/web_search.py @@ -0,0 +1,55 @@ +from fastapi import APIRouter, HTTPException, Depends +from app.models.web_search import WebSearchRequest, WebSearchResponse +from app.services.web_search import WebSearchService +from app.utils.logging_config import get_logger + +logger = get_logger(__name__) +router = APIRouter() + +@router.post("/web-search", response_model=WebSearchResponse) +async def perform_web_search( + request: WebSearchRequest, + service: WebSearchService = Depends(WebSearchService) +) -> WebSearchResponse: + """ + Perform web search and deep research on a query + + Args: + request: WebSearchRequest containing query and optional parameters + service: WebSearchService instance (injected) + + Returns: + WebSearchResponse with search results, learnings, and summary + """ + try: + logger.info(f"Web search request for query: {request.query}") + + # Validate inputs + if not request.query.strip(): + raise HTTPException( + status_code=400, + detail="Search query cannot be empty" + ) + + # Constrain num_results to reasonable values + num_results = max(1, min(20, request.num_results)) + + # Constrain depth to reasonable values + depth = max(1, min(3, request.depth)) + + # Perform the research + response = await service.research( + query=request.query, + num_results=num_results, + depth=depth + ) + + logger.info(f"Web search completed for '{request.query}' with {len(response.results)} results and {len(response.learnings)} learnings") + return response + + except Exception as e: + logger.error(f"Error in web search: {str(e)}", exc_info=True) + raise HTTPException( + status_code=500, + detail=f"Error performing web search: {str(e)}" + ) \ No newline at end of file diff --git a/app/services/__init__.py b/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/image_recognition.py b/app/services/image_recognition.py new file mode 100644 index 0000000..aef669a --- /dev/null +++ b/app/services/image_recognition.py @@ -0,0 +1,159 @@ +from app.services.llm import LLMService +from typing import List, Optional, Dict +import logging +import json +import base64 + +logger = logging.getLogger(__name__) + +class ImageRecognitionService: + def __init__(self): + self.llm_service = LLMService() + + async def analyze_images(self, images: List[bytes], query: Optional[str] = None) -> Dict[str, List[str]]: + """ + Analyze images using Gemini model through LLM service + + Args: + images: List of image binary data + query: Optional specific instructions for analysis + + Returns: + Dict containing list of descriptions and any errors + """ + try: + if not images: + logger.warning("No images provided for analysis") + return { + "descriptions": ["No images provided for analysis"], + "errors": [] + } + + descriptions = [] + errors = [] + query = query or "Describe this image in detail" + + # Process each image through Gemini + for idx, image in enumerate(images): + try: + if not image or not self._validate_image_data(image): + msg = f"Invalid or empty image data at index {idx}" + logger.warning(msg) + errors.append(msg) + descriptions.append("Invalid or corrupt image data") + continue + + # Call Gemini with proper error handling + result = await self.llm_service._call_gemini_with_images( + system_prompt="""You are an expert image analyzer. + Analyze and describe the image in detail, including: + - Main subjects and objects in the scene + - Colors, lighting, and overall composition + - Any actions or activities taking place + - The setting and context + - Notable details, features, or points of interest + - Any text or recognizable symbols + - Overall mood or atmosphere + + Provide a clear, detailed description that gives a complete picture of what's in the image.""", + user_content=query, + images=[image] + ) + + # Check if result is a string (normal response) or error + if isinstance(result, str): + try: + # Check if it's a JSON error message + error_data = json.loads(result) + if isinstance(error_data, dict) and "error" in error_data: + error_msg = f"Error analyzing image {idx}: {error_data['error']}" + logger.error(error_msg) + errors.append(error_msg) + descriptions.append("Error analyzing image") + else: + # It's valid JSON but not an error - treat as description + descriptions.append(str(error_data)) + except json.JSONDecodeError: + # Not JSON - it's a normal text description + cleaned_result = result.strip() + if cleaned_result: + descriptions.append(cleaned_result) + else: + msg = f"Empty description generated for image {idx}" + logger.warning(msg) + descriptions.append("No description generated") + errors.append(msg) + else: + error_msg = f"Unexpected response type for image {idx}: {type(result)}" + logger.error(error_msg) + descriptions.append("Error: Unexpected response format") + errors.append(error_msg) + + except Exception as img_error: + error_msg = f"Error processing image {idx}: {str(img_error)}" + logger.error(error_msg, exc_info=True) + descriptions.append("Error processing image") + errors.append(error_msg) + + if not descriptions: + return { + "descriptions": ["No valid descriptions generated"], + "errors": errors if errors else ["Unknown error occurred"] + } + + return { + "descriptions": descriptions, + "errors": errors + } + + except Exception as e: + error_msg = f"Error in image analysis: {str(e)}" + logger.error(error_msg, exc_info=True) + # return { + # "descriptions": [error_msg] * (len(images) if images else 1), + # "errors": [error_msg] + # } + # At the end of the analyze_images method + return { + "descriptions": descriptions, + "errors": errors # Make sure this is a plain list of strings + } + + + def _validate_image_data(self, image: bytes) -> bool: + """ + Validate image data + + Args: + image: Binary image data + + Returns: + bool: True if image data appears valid + """ + try: + # Check for minimum viable image size + if not image or len(image) < 100: + return False + + # Check for common image headers + headers = { + 'jpeg': [0xFF, 0xD8, 0xFF], + 'png': [0x89, 0x50, 0x4E, 0x47], + 'gif': [0x47, 0x49, 0x46, 0x38] + } + + # Get the first few bytes of the image + image_bytes = bytes(image[:8]) + + # Check against known image format headers + for format_name, header in headers.items(): + if all(image_bytes[i] == header[i] for i in range(len(header))): + logger.debug(f"Valid {format_name} image detected") + return True + + logger.warning("Image format not recognized or supported") + return False + + except Exception as e: + logger.error(f"Error validating image data: {str(e)}") + return False \ No newline at end of file diff --git a/app/services/life_simulator.py b/app/services/life_simulator.py new file mode 100644 index 0000000..7cd31fc --- /dev/null +++ b/app/services/life_simulator.py @@ -0,0 +1,788 @@ +import traceback +from typing import Dict, List, Optional +import httpx +import asyncio +import random +import json +import logging +import sys +from logging.handlers import RotatingFileHandler +import pytz +from datetime import datetime, timezone, timedelta +from app.models.life_simulator import ( + Location, + SimulationConfig, + Incident, + TransitInfo, + TransitRoute, + ActivityResponse +) + +# Set up logging +logger = logging.getLogger('life_simulator') +logger.setLevel(logging.INFO) + +# Console handler +console_handler = logging.StreamHandler(sys.stdout) +console_handler.setLevel(logging.INFO) +console_format = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +console_handler.setFormatter(console_format) + +# File handler with rotation +file_handler = RotatingFileHandler( + 'life_simulator.log', + maxBytes=1024*1024, # 1MB + backupCount=5 +) +file_handler.setLevel(logging.DEBUG) +file_format = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s - %(context)s' +) +file_handler.setFormatter(file_format) + +logger.addHandler(console_handler) +logger.addHandler(file_handler) + +class LifeSimulator: + def __init__(self, api_keys: Dict[str, str]): + self.openrouter_key = api_keys['openrouter'] + self.weather_api_key = api_keys['openweather'] + self.google_api_key = api_keys['google'] + + # Remove hardcoded city reference from train_lines + self.train_lines = {} + + logger.info("LifeSimulator initialized", extra={'context': 'initialization'}) + + async def generate_incident(self, context: Dict) -> Optional[Incident]: + """Generate a contextually appropriate incident using LLM""" + if random.randint(1, 23) != 1: + return None + + logger.debug("Generating incident", extra={'context': context}) + incident_type = "positive" if random.random() < 0.6 else "negative" + + prompt = f"""Given the following context for {context.get('name', 'the person')} in {context.get('city', 'Tokyo')}: +- Time: {context['current_time'].strftime('%H:%M on %A')} +- Location: {context['location']} +- Weather: {context['weather'].get('condition', 'Unknown')}, {context['weather'].get('temp', 20)}°C +- Current activity: {context.get('main_action', 'Unknown')} +- Departure: {context.get('departure', 'Unknown')} +- Destination: {context.get('destination', 'Unknown')} +- Neighborhood: {context.get('neighborhood', 'Unknown')} + +Generate a realistic {incident_type} incident that could occur in this situation. Consider: +1. Local culture and social norms +2. Time-appropriate events +3. Weather impact +4. Location-specific possibilities +5. Current activity context +6. Realistic probabilities + +Return a JSON object with: +{ + "description": "Detailed description of the incident", + "severity": "Number 1-5 (1: minor, 5: major)", + "impact_duration": "Estimated duration in minutes", + "category": "Type of incident (social/environmental/circumstantial/etc)", + "affects_next_activity": "true/false - whether this impacts next activities", + "cultural_context": "Brief note on how this relates to {context.get('city', 'Tokyo')} life" +}""" + + try: + async with httpx.AsyncClient() as client: + response = await client.post( + "https://openrouter.ai/api/v1/chat/completions", + headers={ + "Authorization": f"Bearer {self.openrouter_key}", + "HTTP-Referer": "https://github.com/liauroufan/SoM-tool-collection", + "X-Title": "State of Mika" + }, + json={ + "model": "anthropic/claude-3-opus-20240229", + "messages": [{"role": "user", "content": prompt}], + "max_tokens": 500 + } + ) + result = response.json() + incident_data = json.loads(result['choices'][0]['message']['content']) + + logger.info( + "Incident generated successfully", + extra={ + 'context': { + 'incident_type': incident_type, + 'severity': incident_data['severity'], + 'category': incident_data['category'] + } + } + ) + + return Incident( + type=incident_type, + severity=int(incident_data['severity']), + description=incident_data['description'], + impact_duration=int(incident_data['impact_duration']), + affects_next_activity=bool(incident_data['affects_next_activity']) + ) + except Exception as e: + logger.error(f"Error generating incident: {str(e)}", extra={'context': 'incident_generation'}) + return None + + async def _get_station_id(self, client: httpx.AsyncClient, station_name: str) -> str: + """Get HERE station ID from station name""" + try: + response = await client.get( + "https://transit.hereapi.com/v8/stations", + params={ + "apiKey": self.here_api_key, + "in": f"{self.transit_stations[station_name]['lat']},{self.transit_stations[station_name]['lon']}", + "radius": 500, + "return": "transport" + } + ) + stations = response.json() + if stations['stations']: + return stations['stations'][0]['place']['id'] + raise ValueError(f"No station found for {station_name}") + except Exception as e: + logger.error(f"Error getting station ID: {str(e)}", extra={'context': 'here_api'}) + raise + + async def _get_transit_routes(self, client: httpx.AsyncClient, from_station: str, to_station: str, available_locations: Dict[str, Location]) -> List[TransitRoute]: + """Get available transit routes between stations using Google Routes API""" + try: + # Use available_locations instead of self.transit_stations + origin = available_locations[from_station] + destination = available_locations[to_station] + + print("Origin:", origin) + print("Destination:", destination) + + url = "https://routes.googleapis.com/directions/v2:computeRoutes" + + headers = { + "Content-Type": "application/json", + "X-Goog-Api-Key": self.google_api_key, + "X-Goog-FieldMask": "routes.legs.steps.transitDetails" + } + + body = { + "origin": { + "location": { + "latLng": { + "latitude": origin.lat, # Update to use Location object + "longitude": origin.lon + } + } + }, + "destination": { + "location": { + "latLng": { + "latitude": destination.lat, # Update to use Location object + "longitude": destination.lon + } + } + }, + "travelMode": "TRANSIT", + "transitPreferences": { + "routingPreference": "LESS_WALKING", + "allowedTravelModes": ["TRAIN", "BUS"] + }, + "computeAlternativeRoutes": True, + "languageCode": "en-US", + "units": "METRIC" + } + + response = await client.post(url, headers=headers, json=body) + data = response.json() + + routes = [] + for route in data.get('routes', []): + sections = [] + departure_time = None + arrival_time = None + + for leg in route.get('legs', []): + for step in leg.get('steps', []): + if step.get('transitDetails'): + transit = step['transitDetails'] + # Updated line name extraction + line_name = transit.get('transitLine', {}).get('name', 'Unknown Line') + sections.append(line_name) + + # Updated datetime parsing + if not departure_time: + departure_time = datetime.fromisoformat( + transit['stopDetails']['departureTime'].replace('Z', '+00:00') + ) + arrival_time = datetime.fromisoformat( + transit['stopDetails']['arrivalTime'].replace('Z', '+00:00') + ) + elif step.get('walkingDetails'): + sections.append('Walk') + + if departure_time and arrival_time: + duration = (arrival_time - departure_time).total_seconds() / 60 + + routes.append(TransitRoute( + line_name=" → ".join(sections), + departure_time=departure_time, + arrival_time=arrival_time, + duration_minutes=int(duration), + transfers=len([s for s in sections if s != 'Walk']) - 1, + sections=sections + )) + + return routes + + except Exception as e: + logger.error(f"Error getting transit routes: {str(e)}", extra={'context': 'google_routes_api'}) + return [] + + async def get_transit_status_llm(self, from_station: str, to_station: str, available_locations: Dict[str, Location]) -> Dict[str, List]: + """Generate transit information using LLM for Japan""" + logger.debug( + "Generating Japan transit status with LLM", + extra={'context': {'from': from_station, 'to': to_station}} + ) + + origin = available_locations[from_station] + destination = available_locations[to_station] + + current_time = datetime.now(pytz.timezone('Asia/Tokyo')) + + prompt = f"""Given these locations in Japan: +Origin: {from_station} at {origin.lat}, {origin.lon} +Destination: {to_station} at {destination.lat}, {destination.lon} +Current time: {current_time.strftime('%H:%M')} on {current_time.strftime('%A')} + +You MUST respond with ONLY a valid JSON object. No other text, explanations, or markdown. +The response MUST follow this EXACT format: + +{{ + "departures": [ + {{ + "line_name": "string", + "delay_minutes": number, + "crowding_level": "Low|Medium|High", + "next_departure": "ISO-8601 datetime" + }} + ], + "routes": [ + {{ + "line_name": "string", + "departure_time": "ISO-8601 datetime", + "arrival_time": "ISO-8601 datetime", + "duration_minutes": number, + "transfers": number, + "sections": ["string"] + }} + ] +}} + +Consider: +1. Available train lines between locations +2. Transfer stations if needed +3. Realistic departure times based on current time +4. Typical journey duration +5. Crowding levels based on time of day +6. Common delays for Japanese trains""" + + try: + async with httpx.AsyncClient() as client: + response = await client.post( + "https://openrouter.ai/api/v1/chat/completions", + headers={ + "Authorization": f"Bearer {self.openrouter_key}", + "HTTP-Referer": "https://github.com/liauroufan/SoM-tool-collection", + "X-Title": "State of Mika" + }, + json={ + "model": "anthropic/claude-3-opus-20240229", + "messages": [{"role": "user", "content": prompt}], + "response_format": {"type": "json_object"}, + "max_tokens": 500 + } + ) + + result = response.json() + transit_data = json.loads(result['choices'][0]['message']['content']) + return transit_data + + except Exception as e: + logger.error(f"Error generating Japan transit data: {str(e)}", extra={'context': 'llm_transit_generation'}) + return { + "departures": [], + "routes": [] + } + + async def get_transit_status(self, from_station: str, to_station: str, available_locations: Dict[str, Location]) -> Dict[str, List]: + """Get transit information using appropriate method based on country""" + logger.debug( + "Fetching transit status", + extra={'context': {'from': from_station, 'to': to_station}} + ) + + try: + # Check if we're dealing with Japanese locations + if any(loc.name == from_station for loc in available_locations.values() if getattr(loc, 'country', '') == 'Japan') or \ + any(loc.name == to_station for loc in available_locations.values() if getattr(loc, 'country', '') == 'Japan'): + return await self.get_transit_status_llm(from_station, to_station, available_locations) + + # For other countries, use the existing Google Routes implementation + async with httpx.AsyncClient() as client: + # Use available_locations to _get_transit_routes + routes = await self._get_transit_routes(client, from_station, to_station, available_locations) + + # Since Google Routes API doesn't provide real-time departure board info, + # we'll create synthetic departures based on the route information + departures = [] + for route in routes[:3]: # Limit to 3 departures + # Estimate crowding based on time of day + hour = route.departure_time.hour + if 7 <= hour <= 9 or 17 <= hour <= 19: + crowding = "High" + elif 9 < hour < 17: + crowding = "Medium" + else: + crowding = "Low" + + departures.append({ + "line_name": route.line_name, + "delay_minutes": random.randint(0, 5), # Simulate random delays + "crowding_level": crowding, + "next_departure": route.departure_time.isoformat() + }) + + return { + "departures": departures, + "routes": [ + { + "line_name": route.line_name, + "departure_time": route.departure_time.isoformat(), + "arrival_time": route.arrival_time.isoformat(), + "duration_minutes": route.duration_minutes, + "transfers": route.transfers, + "sections": route.sections + } + for route in routes + ] + } + + except Exception as e: + logger.error(f"Failed to get transit data: {str(e)}", extra={'context': 'google_routes_api'}) + return { + "departures": [], + "routes": [] + } + + async def get_weather(self, lat: float, lon: float) -> Dict: + """Get real-time weather data""" + logger.debug( + "Fetching weather data", + extra={'context': {'lat': lat, 'lon': lon}} + ) + + async with httpx.AsyncClient() as client: + try: + response = await client.get( + "https://api.openweathermap.org/data/2.5/weather", + params={ + "lat": lat, + "lon": lon, + "units": "metric", + "appid": self.weather_api_key + } + ) + data = response.json() + print(data) + logger.info( + "Weather data retrieved", + extra={'context': { + 'condition': data['weather'][0]['main'], + 'temp': data['main']['temp'] + }} + ) + return data + except Exception as e: + logger.error(f"Error fetching weather: {str(e)}", extra={'context': 'weather_api'}) + raise + + async def simulate_activity(self, config: SimulationConfig) -> Dict: + """Generate a realistic activity with potential random incidents""" + # Use provided current_time directly without timezone conversion + current_time = config.current_time + + logger.info( + "Starting activity simulation", + extra={'context': { + 'location': config.residence.name, + 'occupation': config.occupation, + 'time': current_time, + 'city': config.city + }} + ) + + print("Simulating Activity**") + + try: + # Get real-time data + weather = await self.get_weather( + config.residence.lat, + config.residence.lon + ) + + print("Checking Real-Time & Real-Life Weather**", weather) + + # Update to pass available_locations + transit_status = await self.get_transit_status( + config.residence.name, + config.office.name, + config.available_locations + ) + + print("Checking Real-Time & Real-Life Transit Status**", transit_status) + + # Build context for incident generation + current_time = config.current_time + context = { + 'current_time': current_time, + 'location': config.residence.name, + 'weather': weather, + 'neighborhood': config.residence.name.split()[0], # First word of location + 'name': config.name, + 'city': config.city + } + + # Create prompt for activity generation + incident_context = "" + + # Modified prompt to generate activity type and locations with roles + location_prompt = f"""Given the following context for {config.name} in {config.city}, {config.country}: +- Current time: {current_time.strftime('%Y-%m-%d %H:%M')} ({current_time.strftime('%A')}) {current_time.tzname()} +- Weather: {weather['weather'][0]['main']}, {weather['main']['temp']}°C +- Person: {config.name}, {config.age} year old {config.gender} +- Occupation: {config.occupation} +- Home location: {config.residence.name} (role: {config.residence.role}) +- Office location: {config.office.name} (role: {config.office.role}) +- Available locations: +{chr(10).join([f" - {name}: {loc.type} ({loc.role if loc.role else 'general'})" for name, loc in config.available_locations.items()])} + +Time-based context for {config.city}: +- Early morning (5-8 AM): Typically preparing for work, commuting to office +- Morning (8-11 AM): Working at office, meetings +- Lunch (11 AM-2 PM): Dining, casual meetings +- Afternoon (2-5 PM): Working, errands +- Evening (5-8 PM): Commuting home, dinner, shopping +- Night (8-11 PM): Entertainment, relaxation, groceries +- Late night (11 PM-5 AM): Home activities, sleep + +You must respond ONLY with a valid JSON object in exactly this format (no additional text or explanations): +{{ + "activity": {{ + "main_action": "A specific activity like 'Working', 'Having lunch', 'Shopping'", + "departure": "Name from available locations list", + "destination": "Different name from available locations list", + "reason": "Brief explanation of why this activity makes sense now" + }} +}} + +Consider: +1. Time of day ({current_time.strftime('%H:%M')}) and typical activities for this hour +2. Weather conditions affecting choices +3. Typical {config.city} urban lifestyle +4. The person's home and office locations +5. Available venue types and their operating hours +6. Realistic travel patterns for this time of day""" + + # Get activity suggestion from LLM with strict formatting + async with httpx.AsyncClient() as client: + activity_response = await client.post( + "https://openrouter.ai/api/v1/chat/completions", + headers={ + "Authorization": f"Bearer {self.openrouter_key}", + "HTTP-Referer": "https://github.com/liauroufan/SoM-tool-collection", + "X-Title": "State of Mika" + }, + json={ + "model": "anthropic/claude-3-opus-20240229", + "messages": [ + {"role": "system", "content": "You are a JSON-only response bot. You must return only valid JSON matching the exact schema requested, with no additional text or formatting."}, + {"role": "user", "content": location_prompt} + ], + "response_format": {"type": "json_object"}, + "max_tokens": 500 + } + ) + print("location prompt", location_prompt) + + activity_result = activity_response.json() + print("LLM Response:", activity_result['choices'][0]['message']['content']) + + try: + # Clean the string content and parse JSON + content = activity_result['choices'][0]['message']['content'].strip() + parsed_data = json.loads(content) + + # Validate the response format + if not isinstance(parsed_data, dict) or 'activity' not in parsed_data: + logger.error("Invalid JSON structure from LLM", extra={ + 'context': {'content': content} + }) + raise ValueError("Invalid response format from LLM") + + activity_data = parsed_data['activity'] + required_fields = ['main_action', 'departure', 'destination', 'reason'] + missing_fields = [field for field in required_fields if field not in activity_data] + + if missing_fields: + logger.error("Missing fields in LLM response", extra={ + 'context': {'missing': missing_fields} + }) + raise ValueError(f"Missing required fields in LLM response: {missing_fields}") + + # Validate locations exist + if activity_data['departure'] not in config.available_locations: + raise ValueError(f"Invalid departure location: {activity_data['departure']}") + if activity_data['destination'] not in config.available_locations: + raise ValueError(f"Invalid destination location: {activity_data['destination']}") + + except json.JSONDecodeError as e: + logger.error("Failed to parse LLM response as JSON", extra={ + 'context': {'error': str(e), 'content': content} + }) + raise ValueError("Invalid JSON in LLM response") + + # Continue with the rest of the code... + departure_loc = config.available_locations[activity_data['departure']] + destination_loc = config.available_locations[activity_data['destination']] + + weather = await self.get_weather( + departure_loc.lat, + departure_loc.lon + ) + + print("Getting transit status") + print("Departure Location:", activity_data['departure']) + print("Destination Location:", activity_data['destination']) + + + + # Later in the method, update the transit_status call: + transit_status = await self.get_transit_status( + activity_data['departure'], + activity_data['destination'], + config.available_locations + ) + + print("Transit Status**:", transit_status) + + # Update context with activity data for incident generation + context.update({ + 'main_action': activity_data['main_action'], + 'departure': activity_data['departure'], + 'destination': activity_data['destination'] + }) + + # Generate potential incident with updated context + incident = await self.generate_incident(context) + + # Create incident context after generating the incident + if (incident): + incident_context = f""" +An incident has occurred: {incident.description} +Type: {incident.type} +Severity: {incident.severity}/5 +Impact Duration: {incident.impact_duration} minutes +This incident {'' if incident.affects_next_activity else 'does not '}affects the next activity. +""" + + # Create narrative prompt + narrative_prompt = f"""Given the following context for {config.name} in {config.city}, {config.country}: +- Time: {current_time.strftime('%Y-%m-%d %H:%M')} ({current_time.strftime('%A')}) {current_time.tzname()} +- Person: {config.name}, {config.age} year old {config.gender} +- Weather: {weather['weather'][0]['main']}, {weather['main']['temp']}°C +- Activity: {activity_data['main_action']} +- From: {activity_data['departure']} +- To: {activity_data['destination']} +- Reason: {activity_data['reason']} +- Transit Info: {self._format_transit_status(transit_status)} + +{incident_context if incident else ''} + +Generate a detailed narrative of this person's current activity in {config.city}. Include: +1. Description of the current situation +2. Environmental details +3. Social context +4. Cultural elements +5. Emotional state +6. Movement and transportation details""" + + # Generate narrative + narrative_response = await client.post( + "https://openrouter.ai/api/v1/chat/completions", + headers={ + "Authorization": f"Bearer {self.openrouter_key}", + "HTTP-Referer": "https://github.com/liauroufan/SoM-tool-collection", + "X-Title": "State of Mika" + }, + json={ + "model": "anthropic/claude-3-opus-20240229", + "messages": [{"role": "user", "content": narrative_prompt}], + "max_tokens": 1000 + } + ) + + narrative_result = narrative_response.json() + narrative = narrative_result['choices'][0]['message']['content'] + + # Create structured activity data + activity_result = { + "activity": { + "main_action": activity_data['main_action'], + "location": { + "current": activity_data['departure'], + "destination": activity_data['destination'] + }, + "reason": activity_data['reason'], + "narrative": narrative, + "details": { + "time": current_time.strftime('%Y-%m-%d %H:%M'), + "weather": { + "condition": weather['weather'][0]['main'], + "temperature": weather['main']['temp'], + "details": weather + }, + "transit_info": transit_status + } + } + } + + if incident: + activity_result["incident"] = { + "type": incident.type, + "severity": incident.severity, + "description": incident.description, + "impact_duration": incident.impact_duration, + "affects_next_activity": incident.affects_next_activity + } + + return activity_result # Return the dictionary directly instead of wrapping in ActivityResponse + + except Exception as e: + logger.error(f"Failed to simulate activity: {str(e)}") + # Return a basic valid response in case of error + return { + "activity": { + "main_action": "Unable to determine activity", + "location": { + "current": config.residence.name, + "destination": config.residence.name + }, + "reason": "Error occurred", + "narrative": "Error generating activity details", + "details": { + "time": current_time.strftime('%Y-%m-%d %H:%M'), + "weather": { + "condition": "unknown", + "temperature": 0.0, + "details": {} + }, + "transit_info": {"departures": [], "routes": []} + } + } + } + + def _format_transit_status(self, status: Dict[str, List]) -> str: + """Format transit status for LLM consumption""" + parts = [] + + if status['departures']: + parts.append("Departures: " + "; ".join([ + f"{info['line_name']}: {info['delay_minutes']}min delay, {info['crowding_level']} crowding, " + f"next train at {info['next_departure']}" + for info in status['departures'] + ])) + + if status['routes']: + parts.append("Route options: " + "; ".join([ + f"{route['line_name']}: {route['duration_minutes']}min, " + f"{route['transfers']} transfer(s), depart {route['departure_time']}" + for route in status['routes'] + ])) + + return "\n".join(parts) + +# Example usage: +if __name__ == "__main__": + async def main(): + api_keys = { + "openrouter": "your-openrouter-key", + "openweather": "your-key", + "google": "your-google-key" + } + + # Example available locations for Tokyo + available_locations = { + 'Shibuya': Location( + name="Shibuya", + lat=35.658000, + lon=139.701600, + type="shopping", + role="entertainment", + country="Japan" + ), + 'Daikanyama': Location( + name="Daikanyama", + lat=35.648450, + lon=139.703305, + type="residential", + role="home", + country="Japan" + ), + 'Shinjuku': Location( + name="Shinjuku", + lat=35.689700, + lon=139.700400, + type="business", + role="office", + country="Japan" + ), + 'Tsukiji': Location( + name="Tsukiji", + lat=35.665400, + lon=139.770700, + type="dining", + role="food", + country="Japan" + ), + 'Ginza': Location( + name="Ginza", + lat=35.672100, + lon=139.763600, + type="shopping", + role="shopping", + country="Japan" + ) + } + + simulator = LifeSimulator(api_keys) + config = SimulationConfig( + residence=available_locations['Daikanyama'], + occupation="Software Engineer", + office=available_locations['Shinjuku'], + current_time=datetime.now(timezone.utc).astimezone(timezone(timedelta(hours=9))), # UTC+9 for Tokyo + available_locations=available_locations, + city="Tokyo", + country="Japan", + timezone="Asia/Tokyo", + gender="female", + age=23, + name="Mika" + ) + + result = await simulator.simulate_activity(config) + print(json.dumps(result, indent=2, ensure_ascii=False)) + + asyncio.run(main()) \ No newline at end of file diff --git a/app/services/llm.py b/app/services/llm.py new file mode 100644 index 0000000..4d49508 --- /dev/null +++ b/app/services/llm.py @@ -0,0 +1,440 @@ +import aiohttp +import logging +from typing import Dict, Any, Optional, List, Union +from app.config import settings +import json +import google.generativeai as genai +import base64 +from enum import Enum +import asyncio + +logger = logging.getLogger(__name__) + +class ModelProvider(str, Enum): + OPENAI = "openai" + GEMINI = "gemini" + +class LLMService: + def __init__(self): + self.api_url = settings.OPENAI_API_URL + self.model = settings.LLM_MODEL + self.MAX_RETRIES = 2 + + # Configure Gemini if API key is available + if settings.GEMINI_API_KEY: + try: + genai.configure(api_key=settings.GEMINI_API_KEY) + self.gemini_model = genai.GenerativeModel("gemini-2.0-flash-exp") + logger.info("Gemini API configured successfully") + except Exception as e: + logger.error(f"Failed to initialize Gemini: {str(e)}") + self.gemini_model = None + else: + logger.info("No Gemini API key provided, image processing features will be limited") + self.gemini_model = None + + async def analyze_intent( + self, + query: str, + tools: Dict[str, Any], + context: Optional[Dict[str, Any]] = None, + images: Optional[List[bytes]] = None + ) -> Dict[str, Any]: + """ + Analyze user query using LLM to determine intent and extract parameters. + Now supports image analysis with Gemini. + """ + tools_description = "\n".join([ + f"Tool: {name}\nDescription: {tool.description}\n" + f"Parameters: {tool.parameters}\nExample queries: {tool.example_queries}" + for name, tool in tools.items() + ]) + + system_prompt = f"""You are an AI tool router. Your task is to: +1. Analyze the user's query and any provided images +2. Determine the most appropriate tool from the available options +3. Extract relevant parameters for the chosen tool +4. Provide a confidence score (0-1) for your decision + +Available tools: +{tools_description} + +You must respond with valid JSON in exactly this format: +{{ + "tool": "tool_name", + "confidence": 0.0-1.0, + "parameters": {{}}, + "reasoning": "Brief explanation of your choice" +}}""" + + try: + # Clean context to prevent serialization issues + clean_context = {} + if context: + clean_context = {k: str(v) for k, v in context.items() if k != 'images'} + + result = await self._call_gemini_with_images(system_prompt, query, images) if images and self.gemini_model else await self._call_llm(system_prompt, query, clean_context, force_json=True) + return self._process_llm_response(result) + except Exception as e: + logger.error(f"Error in analyze_intent: {str(e)}", exc_info=True) + return self._get_default_routing_response(str(e)) + + async def generate_post_processing_instructions( + self, query: str, tool_name: str + ) -> str: + """ + Generate post-processing instructions for the given query and tool. + + Args: + query (str): The user query from `analyze_intent`. + tool_name (str): The name of the tool selected in `analyze_intent`. + + Returns: + str: Post-processing instructions generated by the LLM. + """ + try: + system_prompt = f"""You are an AI assistant tasked with generating minimal, direct post-processing instructions for a tool’s response. Your goal is to provide only the essential output for the given query, with no extra text or explanation. + +Tool: {tool_name} +Query: {query} +Please provide brief, actionable instructions for how to present the tool’s result in the simplest possible form. For instance, if the query is just “1 + 1,” instruct the LLM to return only “2” with nothing else. + +Deliverable: A short set of post-processing instructions that align with the query’s purpose, prioritize brevity, and avoid superfluous details.""" + logger.debug(f"Generating post-processing instructions for query: {query}") + result = await self._call_llm( + system_prompt=system_prompt, + user_content=f"Generate post-processing instructions for the above query and tool: {tool_name}", + temperature=0.7, + force_json=False + ) + + if isinstance(result, str): + return result.strip() + else: + logger.warning("Unexpected LLM response format, returning default instructions.") + return "Ensure the response aligns with the query intent. Perform necessary transformations, extractions, or summarizations as specified." + + except Exception as e: + logger.error(f"Error generating post-processing instructions: {str(e)}", exc_info=True) + return "Error generating post-processing instructions. Ensure the response aligns with the query intent." + + async def post_process_response( + self, + response: Any, + instructions: str, + tool_name: str, + images: Optional[List[bytes]] = None + ) -> Dict[str, Any]: + """Post-process tool response using LLM based on user instructions.""" + try: + response = self._make_json_serializable(response) + response_str = json.dumps(response) if isinstance(response, (dict, list)) else str(response) + + system_prompt = f"""Task: Process the response from {tool_name} based on the instructions provided. + +Instructions: {instructions} + +Response: {response_str} + +Requirements: + +Extract or transform the data strictly as instructed. +Ensure the output matches the requested fields and format precisely.""" + + result = await self._call_gemini_with_images(system_prompt, response_str, images) if images and self.gemini_model else await self._call_llm( + system_prompt=system_prompt, + user_content="Process the above response according to the given instructions.", + temperature=0.1, + force_json=False + ) + + processed_result = self._process_post_processing_response(result, response) + return json.loads(json.dumps(processed_result)) # Ensure serializable + + except Exception as e: + logger.error(f"Error in post-processing response: {str(e)}", exc_info=True) + return { + "processed_response": { + "error": "Post-processing failed", + "message": str(e), + "original_response": str(response) + } + } + + async def process_web_content( + self, + content: str, + metadata: Dict[str, Any], + instructions: Optional[str] = None, + url: Optional[str] = None, + images: Optional[List[bytes]] = None + ) -> Dict[str, Any]: + """Process web content using LLM""" + try: + # Create context information with only serializable values + context_info = [] + if url: + context_info.append(f"URL: {url}") + for key, value in metadata.items(): + if value and isinstance(value, (str, int, float, bool)): + context_info.append(f"{key}: {str(value)}") + + context_str = "\n".join(context_info) + + default_instructions = """ +1. Summarize the main points +2. Identify key topics/tags +3. Extract relevant metadata +4. Assess content quality and relevance""" + + user_instructions = instructions if instructions else default_instructions + + system_prompt = f"""You are an expert content analyzer. Respond with valid JSON in this format: +{{ + "summary": "A concise summary of the main points", + "tags": ["list", "of", "relevant", "tags"], + "key_insights": ["list", "of", "important", "takeaways"], + "metadata": {{ + "content_type": "article/blog/news/etc", + "topic_category": "main topic category", + "reading_time_minutes": estimated reading time, + "content_quality": "high/medium/low" + }}, + "analysis": "Brief analysis of content quality and relevance" +}} + +Context Information: +{context_str} + +Instructions: +{user_instructions}""" + + # Use Gemini for image analysis, fall back to OpenAI for text + if images and self.gemini_model: + result = await self._call_gemini_with_images(system_prompt, content, images) + else: + result = await self._call_llm(system_prompt, content, force_json=True) + + # Handle error response + if isinstance(result, dict) and "error" in result: + logger.error(f"LLM API error: {result['error']}") + return self._get_default_content_response(content) + + # Parse response if it's a string + if isinstance(result, str): + try: + result = json.loads(result) + except json.JSONDecodeError as e: + logger.error(f"Failed to parse LLM response as JSON: {e}") + return self._get_default_content_response(content) + + # Ensure response is serializable + return json.loads(json.dumps(result)) + + except Exception as e: + logger.error(f"Error in process_web_content: {str(e)}", exc_info=True) + return self._get_default_content_response(content) + + async def _call_gemini_with_images( + self, + system_prompt: str, + user_content: str, + images: Optional[List[bytes]] = None + ) -> str: + """Call Gemini API with text and optional images.""" + try: + if not self.gemini_model: + raise ValueError("Gemini API not configured") + + # Prepare content parts + content_parts = [] + + # Add system prompt and user content + content_parts.append(system_prompt + "\n\n" + user_content) + + # Add images if provided + if images: + for img in images: + try: + content_parts.append({ + 'mime_type': 'image/jpeg', + 'data': base64.b64encode(img).decode('utf-8') + }) + except Exception as e: + logger.error(f"Error processing image data: {str(e)}") + continue + + # Run Gemini generation in thread pool since it's not async + loop = asyncio.get_event_loop() + response = await loop.run_in_executor( + None, + lambda: self.gemini_model.generate_content(content_parts) + ) + + # Extract text from response + if hasattr(response, 'text'): + return response.text + elif isinstance(response, dict) and 'candidates' in response: + return response['candidates'][0]['content']['parts'][0]['text'] + else: + error_msg = "Unexpected Gemini response format" + logger.error(error_msg) + return json.dumps({"error": error_msg}) + + except Exception as e: + error_msg = f"Error in Gemini API call: {str(e)}" + logger.error(error_msg, exc_info=True) + return json.dumps({"error": error_msg}) + + async def _call_llm( + self, + system_prompt: str, + user_content: str, + context: Optional[Dict[str, Any]] = None, + temperature: float = 0.1, + force_json: bool = False + ) -> Dict[str, Any]: + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_content} + ] + + if context: + context_str = "\nContext: " + "\n".join(f"{k}: {v}" for k, v in context.items()) + messages[1]["content"] += context_str + + try: + request_body = { + "model": self.model, + "messages": messages, + "temperature": temperature, + "max_tokens": settings.LLM_MAX_TOKENS, + } + + if force_json: + request_body["response_format"] = {"type": "json_object"} + + async with aiohttp.ClientSession() as session: + async with session.post( + self.api_url, + headers={ + "Authorization": f"Bearer {settings.OPENAI_API_KEY}", + "Content-Type": "application/json" + }, + json=request_body, + timeout=30 + ) as response: + response.raise_for_status() + data = await response.json() + + if not data or "choices" not in data or not data["choices"]: + logger.error("Invalid response structure from OpenAI API") + return {"error": "Invalid API response structure"} + + return data["choices"][0]["message"]["content"] + + except aiohttp.ClientError as e: + logger.error(f"HTTP error in LLM call: {str(e)}") + return {"error": f"HTTP error: {str(e)}"} + except Exception as e: + logger.error(f"Error in LLM call: {str(e)}", exc_info=True) + return {"error": str(e)} + + def _process_llm_response(self, result: Union[str, Dict]) -> Dict[str, Any]: + """Process and validate LLM response""" + if isinstance(result, dict) and "error" in result: + logger.error(f"LLM API error: {result['error']}") + return self._get_default_routing_response("Error in LLM API call") + + if isinstance(result, str): + try: + result = json.loads(result) + except json.JSONDecodeError as e: + logger.error(f"Failed to parse LLM response as JSON: {e}") + return self._get_default_routing_response("Invalid JSON response") + + required_fields = ["tool", "confidence", "parameters", "reasoning"] + if not all(field in result for field in required_fields): + logger.error(f"Missing required fields in LLM response. Got: {list(result.keys())}") + return self._get_default_routing_response("Missing required fields") + + try: + result["confidence"] = float(result["confidence"]) + result["confidence"] = max(0.0, min(1.0, result["confidence"])) + except (TypeError, ValueError): + logger.warning("Invalid confidence value, defaulting to 0.5") + result["confidence"] = 0.5 + + if not isinstance(result["parameters"], dict): + logger.warning("Invalid parameters format, defaulting to empty dict") + result["parameters"] = {} + + return result + + def _make_json_serializable(self, data): + """ + Convert non-serializable objects in the data to JSON-serializable formats. + """ + if isinstance(data, dict): + return {k: self._make_json_serializable(v) for k, v in data.items()} + elif isinstance(data, list): + return [self._make_json_serializable(item) for item in data] + else: + return str(data) + + def _process_post_processing_response(self, result: Union[str, Dict], original_response: Any) -> Dict[str, Any]: + """ + Process post-processing response and ensure JSON serializability. + """ + try: + if isinstance(result, str): + if result.strip().startswith('{') or result.strip().startswith('['): + try: + result = json.loads(result) + except json.JSONDecodeError: + pass + else: + result = result.strip() + + # Ensure the response is JSON serializable + processed_result = self._make_json_serializable(result) + original_serializable = self._make_json_serializable(original_response) + + return { + "processed_response": processed_result, + "original_response": original_serializable + } + + except Exception as e: + logger.error(f"Error in processing post-processing response: {str(e)}", exc_info=True) + return { + "processed_response": { + "error": "Post-processing failed", + "message": str(e), + "original_response": str(original_response) + } + } + + def _get_default_routing_response(self, error_reason: str) -> Dict[str, Any]: + """Get a default routing response for error cases""" + return { + "tool": "news", + "confidence": 0.3, + "parameters": {}, + "reasoning": f"Default routing due to error: {error_reason}" + } + + def _get_default_content_response(self, content: str) -> Dict[str, Any]: + """Get a default content analysis response for error cases""" + return { + "summary": "Error processing content", + "tags": [], + "key_insights": [], + "metadata": { + "content_type": "unknown", + "topic_category": "unknown", + "reading_time_minutes": len(content.split()) // 200, + "content_quality": "unknown" + }, + "analysis": "Error during content analysis" + } \ No newline at end of file diff --git a/app/services/math.py b/app/services/math.py new file mode 100644 index 0000000..18cf730 --- /dev/null +++ b/app/services/math.py @@ -0,0 +1,113 @@ +import math +from typing import Tuple, List +import sympy +from sympy import ( + sin, cos, tan, asin, acos, atan, exp, log, sqrt, + symbols, sympify, SympifyError, pi, E +) +from sympy.parsing.sympy_parser import parse_expr, standard_transformations, implicit_multiplication_application +from app.utils.logging_config import get_logger + +logger = get_logger(__name__) + +class MathService: + """ + A service class to evaluate advanced mathematical/scientific expressions + using Sympy for parsing and evaluation. + """ + + def __init__(self): + # Define which names/functions are allowed in the parsing environment + self.allowed_functions = { + # Common math constants + "pi": pi, # 3.14159... + "e": E, # 2.71828... + + # Basic arithmetic + "abs": abs, + "round": round, + "pow": pow, + "max": max, + "min": min, + "sum": sum, + + # Trigonometric functions + "sin": sin, + "cos": cos, + "tan": tan, + "asin": asin, + "acos": acos, + "atan": atan, + + # Exponential and log + "exp": exp, + "log": log, # Natural log by default + + # Root + "sqrt": sqrt, + } + + def evaluate(self, expression: str) -> Tuple[float, List[str]]: + """ + Evaluate a mathematical expression and return the numeric result with steps. + + Args: + expression (str): A string containing the mathematical expression. + + Returns: + Tuple[float, List[str]]: A tuple with the numeric result of the expression + and a list of "steps" describing the processing stages. + + Raises: + ValueError: If the expression is invalid or cannot be evaluated. + """ + logger.info(f"Evaluating mathematical expression: {expression}") + steps = [f"Received expression: '{expression}'"] + + # Basic validation + if not expression or not expression.strip(): + logger.error("Empty expression received") + raise ValueError("Expression cannot be empty") + + try: + # Set up parsing transformations + transformations = (standard_transformations + + (implicit_multiplication_application,)) + + # Step 1: Parse the expression into a Sympy object + logger.debug(f"Attempting to parse expression: {expression}") + parsed_expr = parse_expr( + expression, + local_dict=self.allowed_functions, + transformations=transformations, + evaluate=False + ) + steps.append(f"Parsed expression (Sympy form): {parsed_expr}") + logger.debug(f"Successfully parsed expression to: {parsed_expr}") + + # Step 2: Simplify the expression + logger.debug("Attempting to simplify expression") + simplified_expr = sympy.simplify(parsed_expr) + steps.append(f"Simplified expression: {simplified_expr}") + logger.debug(f"Simplified to: {simplified_expr}") + + # Step 3: Evaluate numerically + logger.debug("Attempting numerical evaluation") + evaluated_value = float(simplified_expr.evalf()) + steps.append(f"Final numeric result: {evaluated_value}") + logger.info(f"Successfully evaluated expression to: {evaluated_value}") + + return evaluated_value, steps + + except SympifyError as e: + error_msg = f"Invalid mathematical expression: {str(e)}" + logger.error(error_msg) + raise ValueError(error_msg) + except (TypeError, ValueError) as e: + error_msg = f"Error evaluating expression: {str(e)}" + logger.error(error_msg) + raise ValueError(error_msg) + except Exception as e: + error_msg = f"Unexpected error during evaluation: {str(e)}" + logger.error(error_msg, exc_info=True) + raise ValueError(error_msg) diff --git a/app/services/membership.py b/app/services/membership.py new file mode 100644 index 0000000..dbd76a5 --- /dev/null +++ b/app/services/membership.py @@ -0,0 +1,188 @@ +from supabase import create_client, Client +from uuid import UUID +from datetime import datetime +from typing import Optional, List +from app.config import settings +from app.models.membership import ( + BotCreate, BotResponse, FriendCreate, FriendResponse, + FriendBasicResponse, FriendUpdate, FriendUpdateResponse, + PlatformContactResponse, PlatformContactCreate, PointsHistoryEntry +) +from app.utils.logging_config import get_logger + +logger = get_logger(__name__) + +class MembershipService: + def __init__(self): + self.supabase: Client = create_client( + settings.SUPABASE_URL, + settings.SUPABASE_SERVICE_KEY + ) + + async def create_bot(self, bot_data: BotCreate) -> BotResponse: + """Create a new bot with generated bot_id and api_key""" + data = { + "name": bot_data.name + } + + result = self.supabase.table("bots").insert(data).execute() + + if not result.data: + raise Exception("Failed to create bot") + + return BotResponse(**result.data[0]) + + async def validate_bot_api_key(self, bot_id: UUID, api_key: UUID) -> bool: + """Validate if the provided api_key matches the bot_id""" + result = self.supabase.table("bots").select("bot_id").eq("bot_id", str(bot_id)).eq("api_key", str(api_key)).execute() + return bool(result.data) + + async def create_or_get_friend(self, bot_id: UUID, friend_data: FriendCreate) -> FriendResponse: + """Create a new friend or get existing friend details""" + # Check if friend already exists for this bot and platform + contact_query = self.supabase.table("friend_platform_contacts")\ + .select("friend_id")\ + .eq("platform_type", friend_data.platform_contact.platform_type.value)\ + .eq("platform_user_id", friend_data.platform_contact.platform_user_id)\ + .execute() + + now = datetime.utcnow() + + if contact_query.data: + friend_id = contact_query.data[0]["friend_id"] + return await self.get_friend_details(bot_id, UUID(friend_id)) + + # Create new friend + friend_data_dict = { + "bot_id": str(bot_id), + "nickname": friend_data.nickname, + "description": friend_data.description, + "level": 1, + "points": 0, + "created_at": now.isoformat(), + "updated_at": now.isoformat() + } + + friend_result = self.supabase.table("friends").insert(friend_data_dict).execute() + + if not friend_result.data: + raise Exception("Failed to create friend") + + friend_id = friend_result.data[0]["friend_id"] + + # Create platform contact + contact_data = { + "friend_id": friend_id, + "platform_type": friend_data.platform_contact.platform_type.value, + "platform_user_id": friend_data.platform_contact.platform_user_id, + "created_at": now.isoformat() + } + + contact_result = self.supabase.table("friend_platform_contacts").insert(contact_data).execute() + + if not contact_result.data: + # Rollback friend creation + self.supabase.table("friends").delete().eq("friend_id", friend_id).execute() + raise Exception("Failed to create platform contact") + + return await self.get_friend_details(bot_id, UUID(friend_id)) + + async def get_friends(self, bot_id: UUID) -> List[FriendBasicResponse]: + """Get list of all friends for a bot""" + result = self.supabase.table("friends").select("friend_id,nickname,level,points")\ + .eq("bot_id", str(bot_id)).execute() + + return [FriendBasicResponse(**friend) for friend in result.data] + + async def get_friend_details(self, bot_id: UUID, friend_id: UUID) -> Optional[FriendResponse]: + """Get detailed information about a specific friend""" + friend_result = self.supabase.table("friends").select("*")\ + .eq("bot_id", str(bot_id))\ + .eq("friend_id", str(friend_id)).execute() + + if not friend_result.data: + return None + + contacts_result = self.supabase.table("friend_platform_contacts").select("*")\ + .eq("friend_id", str(friend_id)).execute() + + platform_contacts = [ + PlatformContactResponse(**contact) + for contact in contacts_result.data + ] + + return FriendResponse( + is_newly_created=False, + platform_contacts=platform_contacts, + **friend_result.data[0] + ) + + async def update_friend(self, bot_id: UUID, friend_id: UUID, update_data: FriendUpdate) -> Optional[FriendUpdateResponse]: + """Update friend attributes and record points history if points changed""" + # Get current friend data + current = await self.get_friend_details(bot_id, friend_id) + if not current: + return None + + # Prepare update data + update_dict = update_data.dict(exclude={"reason"}, exclude_unset=True) + if update_dict: + update_dict["updated_at"] = datetime.utcnow().isoformat() + + result = self.supabase.table("friends").update(update_dict)\ + .eq("bot_id", str(bot_id))\ + .eq("friend_id", str(friend_id)).execute() + + if not result.data: + raise Exception("Failed to update friend") + + # Record points history if points changed + if "points" in update_dict: + points_change = update_dict["points"] - current.points + if points_change != 0: + history_data = { + "friend_id": str(friend_id), + "points_change": points_change, + "previous_points": current.points, + "current_points": update_dict["points"], + "reason": update_data.reason, + "timestamp": update_dict["updated_at"] + } + self.supabase.table("friend_points_history").insert(history_data).execute() + + return FriendUpdateResponse( + success=True, + friend_id=friend_id, + level=result.data[0]["level"], + points=result.data[0]["points"], + description=result.data[0].get("description") + ) + + return FriendUpdateResponse( + success=False, + friend_id=friend_id, + level=current.level, + points=current.points, + description=current.description + ) + + async def add_platform_contact(self, bot_id: UUID, friend_id: UUID, contact_data: PlatformContactCreate) -> Optional[PlatformContactResponse]: + """Add a new platform contact for a friend""" + # Verify friend belongs to bot + friend = await self.get_friend_details(bot_id, friend_id) + if not friend: + return None + + data = { + "friend_id": str(friend_id), + "platform_type": contact_data.platform_type.value, + "platform_user_id": contact_data.platform_user_id, + "created_at": datetime.utcnow().isoformat() + } + + result = self.supabase.table("friend_platform_contacts").insert(data).execute() + + if not result.data: + raise Exception("Failed to create platform contact") + + return PlatformContactResponse(**result.data[0]) diff --git a/app/services/news.py b/app/services/news.py new file mode 100644 index 0000000..15e0ada --- /dev/null +++ b/app/services/news.py @@ -0,0 +1,142 @@ +from typing import Optional, List, Dict +from app.utils.http_client import HTTPClient +from app.models.news import NewsResponse +from app.config import settings +from enum import Enum +import logging + +logger = logging.getLogger(__name__) + +class NewsFilter(str, Enum): + RISING = "rising" + HOT = "hot" + BULLISH = "bullish" + BEARISH = "bearish" + IMPORTANT = "important" + SAVED = "saved" + LOL = "lol" + +class NewsKind(str, Enum): + ALL = "all" + NEWS = "news" + MEDIA = "media" + +class NewsService: + # Available region codes as per API documentation + SUPPORTED_REGIONS = { + 'en': 'English', + 'de': 'Deutsch', + 'nl': 'Dutch', + 'es': 'Español', + 'fr': 'Français', + 'it': 'Italiano', + 'pt': 'Português', + 'ru': 'Русский', + 'tr': 'Türkçe', + 'ar': 'عربي', + 'cn': '中國人', + 'jp': '日本', + 'ko': '한국인' + } + + MAX_CURRENCIES = 50 + DEFAULT_REGION = 'en' + + def __init__(self): + self.http_client = HTTPClient() + self.base_url = settings.CRYPTOPANIC_API_URL + self.api_key = settings.CRYPTOPANIC_API_KEY + + async def get_latest_news( + self, + public: bool = False, + filter: Optional[str] = None, + currencies: Optional[List[str]] = None, + regions: Optional[List[str]] = None, + kind: Optional[str] = None, + page: Optional[int] = None, + **kwargs # Handle any extra parameters + ) -> NewsResponse: + """ + Fetch news from CryptoPanic API with various filters. + + Args: + public: Whether to use public API + filter: Filter type (rising, hot, bullish, bearish, important, saved, lol) + currencies: List of currency codes (max 50) + regions: List of region codes (en, de, nl, es, fr, it, pt, ru, tr, ar, cn, jp, ko) + kind: Content type (news or media) + page: Page number for pagination + """ + # Validate inputs + if regions: + if isinstance(regions, str): + regions = [regions] + + # Validate region codes + invalid_regions = set(region.lower() for region in regions) - set(self.SUPPORTED_REGIONS.keys()) + if invalid_regions: + error_msg = (f"Invalid region(s): {invalid_regions}. " + f"Supported regions are: {', '.join(self.SUPPORTED_REGIONS.keys())}") + logger.error(error_msg) + raise ValueError(error_msg) + + regions = [region.lower() for region in regions] + else: + # Default to English if no region specified + regions = [self.DEFAULT_REGION] + + if currencies: + if isinstance(currencies, str): + currencies = [currencies] + if len(currencies) > self.MAX_CURRENCIES: + raise ValueError(f"Maximum {self.MAX_CURRENCIES} currencies allowed") + currencies = [curr.upper() for curr in currencies] + + if filter and filter not in NewsFilter.__members__.values(): + raise ValueError(f"Invalid filter. Must be one of: {', '.join(f.value for f in NewsFilter)}") + + if kind and kind not in NewsKind.__members__.values(): + raise ValueError(f"Invalid kind. Must be one of: {', '.join(f.value for f in NewsKind)}") + + # Build parameters + params = { + "auth_token": self.api_key + } + + if public: + params["public"] = "true" + + if filter: + params["filter"] = filter + + if currencies: + params["currencies"] = ",".join(currencies) + + if regions: + params["regions"] = ",".join(regions) + + if kind and kind != NewsKind.ALL.value: + params["kind"] = kind + + if page: + params["page"] = str(page) + + try: + # Make API request + response = await self.http_client.get( + f"{self.base_url}/posts/", + params=params + ) + + return NewsResponse.from_api_response(response) + + except Exception as e: + logger.error(f"Error fetching news: {str(e)}") + raise + + async def get_supported_currencies(self) -> List[str]: + """ + Get list of supported currency codes. + """ + return ["BTC", "ETH", "SOL", "DOGE", "XRP", "ADA", "DOT", "MATIC"] \ No newline at end of file diff --git a/app/services/scraper.py b/app/services/scraper.py new file mode 100644 index 0000000..3908d56 --- /dev/null +++ b/app/services/scraper.py @@ -0,0 +1,241 @@ +from typing import Optional, Dict, Any +from bs4 import BeautifulSoup +import aiohttp +import datetime +import logging +import asyncio +from urllib.parse import urlparse +import re + +from trafilatura import extract, extract_metadata + +from app.models.scraper import ScraperResponse, ProcessedContent +from app.services.llm import LLMService +from app.config import settings + +from playwright.async_api import async_playwright + +logger = logging.getLogger(__name__) + +class ScraperService: + def __init__(self): + self.llm_service = LLMService() + # We no longer need a persistent aiohttp session if we're using Pyppeteer: + self.session = None + self.headers = { + 'User-Agent': ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/91.0.4472.124 Safari/537.36" + ) + } + + async def scrape_url(self, url: str, instructions: Optional[str] = None) -> ScraperResponse: + """ + Scrape and process content from a given URL using a headless browser (Pyppeteer) + for dynamic rendering, then analyze content with the LLM. + """ + try: + # Fetch HTML content via Pyppeteer + # Note: We're using a proxy here, but we should remove if not needed + html_content = await self._fetch_with_playwright(url, proxy=None) + + # Extract content using trafilatura + extracted_text = extract(html_content) + raw_metadata = extract_metadata(html_content) + + # Convert metadata to dictionary if it's not already + metadata = {} + if raw_metadata: + if hasattr(raw_metadata, '__dict__'): + metadata = raw_metadata.__dict__ + elif hasattr(raw_metadata, 'items'): + metadata = dict(raw_metadata) + elif isinstance(raw_metadata, dict): + metadata = raw_metadata + + # Clean up metadata: remove None values and internal attributes + metadata = { + k: v for k, v in metadata.items() + if v is not None and not k.startswith('_') + } + + # If trafilatura couldn't extract text, use BeautifulSoup as a fallback + if not extracted_text: + logger.warning(f"No content could be extracted by trafilatura from {url}. Falling back to BeautifulSoup.") + + soup = BeautifulSoup(html_content, 'html.parser') + extracted_text = self._extract_main_content(soup) + + # Extract minimal metadata if still empty + if not metadata: + title_tag = soup.find('title') + description_tag = soup.find('meta', attrs={'name': 'description'}) + metadata = { + 'title': title_tag.get_text() if title_tag else None, + 'description': description_tag.get('content') if description_tag else None, + } + # Remove None values + metadata = {k: v for k, v in metadata.items() if v is not None} + + if not extracted_text: + logger.warning(f"No content could be extracted from {url} after all attempts.") + extracted_text = "No content could be extracted from the webpage." + + # Process content with LLM + processed_content = await self._process_content_with_llm( + url=url, + raw_content=extracted_text, + metadata=metadata, + instructions=instructions + ) + + return ScraperResponse( + original_url=url, + instructions=instructions or "Extract main content", + processed_content=processed_content + ) + + except Exception as e: + logger.error(f"Error scraping URL {url}: {str(e)}", exc_info=True) + return ScraperResponse( + original_url=url, + instructions=instructions or "Extract main content", + processed_content=ProcessedContent( + url=url, + title="Error", + content="", + metadata={}, + error=str(e), + word_count=0, + timestamp=datetime.datetime.now().isoformat() + ), + error=str(e) + ) + + def _extract_main_content(self, soup: BeautifulSoup) -> str: + """ + Extract main content from HTML using BeautifulSoup (basic fallback). + """ + # Remove unwanted elements + for element in soup(['script', 'style', 'nav', 'header', 'footer', 'aside']): + element.decompose() + + # Get text content + text = soup.get_text(separator='\n', strip=True) + + # Clean up text + lines = [line.strip() for line in text.splitlines() if line.strip()] + return '\n'.join(lines) + + async def _process_content_with_llm( + self, + url: str, + raw_content: str, + metadata: Dict[str, Any], + instructions: Optional[str] + ) -> ProcessedContent: + """ + Process raw content using the LLM service for summaries, key insights, etc. + """ + # Basic content cleanup + content = re.sub(r'\s+', ' ', raw_content).strip() + + # Call the LLM to process and analyze the content + llm_analysis = await self.llm_service.process_web_content( + content=content[:settings.LLM_MAX_TOKENS], # Limit content length for the LLM + metadata=metadata, + instructions=instructions, + url=url + ) + + # Extract word count + word_count = len(content.split()) + + # Combine metadata with the LLM analysis + combined_metadata = { + **metadata, + **llm_analysis.get('metadata', {}), + 'content_analysis': llm_analysis.get('analysis', ''), + 'key_insights': llm_analysis.get('key_insights', []) + } + + return ProcessedContent( + url=url, + title=metadata.get('title', 'Unknown Title'), + content=content, + metadata=combined_metadata, + summary=llm_analysis.get('summary', ''), + tags=llm_analysis.get('tags', []), + word_count=word_count, + timestamp=datetime.datetime.now().isoformat() + ) + + async def _fetch_with_playwright(self, url: str, proxy: str = None) -> str: + """ + Use Playwright to launch a headless browser, navigate to the URL, + and retrieve the final rendered HTML content after JavaScript execution. + Optionally uses a proxy if provided. + + :param url: URL to navigate to. + :param proxy: Proxy in the format IP:Port:Username:Password (optional). + :return: Rendered HTML content of the page. + """ + # Default launch args + launch_args = [ + "--no-sandbox", + "--disable-setuid-sandbox", + "--disable-infobars", + "--disable-extensions", + "--disable-dev-shm-usage", + "--disable-gpu" + ] + proxy_config = None + + # Parse proxy if provided + if proxy: + try: + proxy_parts = proxy.split(':') + proxy_ip_port = f"{proxy_parts[0]}:{proxy_parts[1]}" + proxy_username = proxy_parts[2] + proxy_password = proxy_parts[3] + proxy_config = { + "server": f"http://{proxy_ip_port}", + "username": proxy_username, + "password": proxy_password + } + except (IndexError, ValueError): + raise ValueError("Proxy must be in the format IP:Port:Username:Password") + + async with async_playwright() as p: + # You can switch to p.firefox or p.webkit if needed + browser = await p.chromium.launch( + headless=True, + args=launch_args, + proxy=proxy_config + ) + try: + # Create a browser context so we can set user agent + context = await browser.new_context( + user_agent=self.headers.get('User-Agent', 'Mozilla/5.0') + ) + page = await context.new_page() + + # Navigate to the URL and wait for network to be idle + # 'networkidle' is the closest equivalent to Pyppeteer’s "networkidle2" + await page.goto(url, wait_until="networkidle", timeout=30000) + + # Get the full HTML after JS has run + content = await page.content() + finally: + await browser.close() + + return content + + async def close(self): + """ + Currently a no-op since we aren't storing a persistent session for Playwright. + If you had other connections you wanted to close, you'd do so here. + """ + if self.session and not self.session.closed: + await self.session.close() diff --git a/app/services/solana_dex_buys.py b/app/services/solana_dex_buys.py new file mode 100644 index 0000000..98e3ee5 --- /dev/null +++ b/app/services/solana_dex_buys.py @@ -0,0 +1,232 @@ +import json +from datetime import datetime +from typing import List + +import requests +from fastapi import HTTPException + +from app.config import settings +from app.models.solana_dex_buys import DexTradeResponse, DexTrade, Trade, Transaction +from app.utils.logging_config import get_logger + +logger = get_logger(__name__) + +class SolanaDexBuysService: + def __init__(self): + self.graphql_url = "https://streaming.bitquery.io/eap" + self.headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {settings.BITQUERY_API_KEY}' + } + + async def get_dex_buys(self, mint_address: str, signer_address: str) -> DexTradeResponse: + """ + Fetch DEX buy orders for a specific token by a specific user + + Args: + mint_address (str): The token's mint address to query + signer_address (str): The user's wallet address that signed the transactions + + Returns: + DexTradeResponse: Processed DEX trades data + """ + try: + # Construct GraphQL query + query = self._build_query(mint_address, signer_address) + variables = {} + + payload = { + 'query': query, + 'variables': json.dumps(variables) + } + + logger.info(f"Making request to BitQuery API - Mint: {mint_address}, Signer: {signer_address}") + logger.debug(f"Full GraphQL query: {query}") + + # Make request + response = requests.post( + self.graphql_url, + headers=self.headers, + json=payload + ) + + # Log raw response + logger.info(f"BitQuery API Response Status: {response.status_code}") + logger.debug(f"BitQuery API Raw Response: {response.text}") + + if response.status_code != 200: + logger.error(f"Failed to fetch DEX buys: {response.text}") + raise HTTPException( + status_code=response.status_code, + detail=f"BitQuery API error: {response.text}" + ) + + # Process response + data = response.json() + logger.debug(f"Parsed JSON response: {json.dumps(data, indent=2)}") + + if 'errors' in data: + logger.error(f"GraphQL errors: {json.dumps(data['errors'], indent=2)}") + raise HTTPException( + status_code=400, + detail=f"BitQuery GraphQL error: {data['errors']}" + ) + + if 'data' in data: + return self._process_response(data['data']) + else: + logger.error("No 'data' field in response") + return DexTradeResponse(dex_trades=[]) + + except Exception as e: + logger.error(f"Error fetching DEX buys: {str(e)}", exc_info=True) + raise HTTPException( + status_code=500, + detail=f"Internal server error: {str(e)}" + ) + + def _build_query(self, mint_address: str, signer_address: str) -> str: + """ + Build GraphQL query for DEX buys + """ + return f''' + query SolanaDEXBuys {{ + Solana {{ + DEXTrades( + orderBy: {{descending: Block_Height}} + where: {{ + Trade: {{ + Buy: {{ + Currency: {{ + MintAddress: {{ + is: "{mint_address}" + }} + }} + }} + }}, + Transaction: {{ + Signer: {{ + is: "{signer_address}" + }} + }} + }} + limit: {{count: 100}} + ) {{ + Trade {{ + Sell {{ + Amount + Account {{ + Token {{ + Owner + }} + Address + Owner + }} + Currency {{ + Name + MintAddress + }} + }} + Buy {{ + Amount + Account {{ + Token {{ + Owner + }} + Address + Owner + }} + Currency {{ + MintAddress + Name + }} + }} + }} + Transaction {{ + Signature + Signer + }} + }} + }} + }} + ''' + + def _process_response(self, response_data: dict) -> DexTradeResponse: + """ + Process and validate the response data + """ + try: + logger.debug(f"Processing response data: {json.dumps(response_data, indent=2)}") + + # Handle empty or invalid response + if not response_data: + logger.warning("Empty response data, returning empty result") + return DexTradeResponse(dex_trades=[]) + + solana_data = response_data.get('Solana') + if not solana_data: + logger.warning("No 'Solana' field in response, returning empty result") + return DexTradeResponse(dex_trades=[]) + + dex_trades_data = solana_data.get('DEXTrades', []) + logger.info(f"Found {len(dex_trades_data)} DEX trades") + + dex_trades = [] + for raw_trade in dex_trades_data: + if not raw_trade: + logger.warning("Skipping null trade data") + continue + + logger.debug(f"Processing trade: {json.dumps(raw_trade, indent=2)}") + trade_data = raw_trade.get('Trade', {}) + transaction_data = raw_trade.get('Transaction', {}) + + try: + dex_trade = DexTrade( + trade=Trade( + buy=self._process_trade_action(trade_data.get('Buy', {})), + sell=self._process_trade_action(trade_data.get('Sell', {})) + ), + transaction=Transaction( + signature=transaction_data.get('Signature', ''), + signer=transaction_data.get('Signer', '') + ) + ) + dex_trades.append(dex_trade) + except Exception as e: + logger.error(f"Error processing individual trade: {str(e)}") + continue # Skip this trade but continue processing others + + return DexTradeResponse(dex_trades=dex_trades) + + except Exception as e: + logger.error(f"Error processing response: {str(e)}", exc_info=True) + raise ValueError(f"Error processing response: {str(e)}") + + def _process_trade_action(self, trade_action_data: dict) -> 'TradeAction': + """Helper method to process trade action data""" + from app.models.solana_dex_buys import TradeAction, Account, TokenAccount, Currency + + if not trade_action_data: + logger.error("Received empty trade action data") + raise ValueError("Empty trade action data") + + logger.debug(f"Processing trade action: {json.dumps(trade_action_data, indent=2)}") + + account_data = trade_action_data.get('Account', {}) + currency_data = trade_action_data.get('Currency', {}) + + return TradeAction( + amount=float(trade_action_data.get('Amount', 0)), + account=Account( + token=TokenAccount( + owner=account_data.get('Token', {}).get('Owner') + ), + address=account_data.get('Address', ''), + owner=account_data.get('Owner', '') + ), + currency=Currency( + name=currency_data.get('Name', ''), + mint_address=currency_data.get('MintAddress', '') + ) + ) \ No newline at end of file diff --git a/app/services/solana_dex_sales.py b/app/services/solana_dex_sales.py new file mode 100644 index 0000000..671bab6 --- /dev/null +++ b/app/services/solana_dex_sales.py @@ -0,0 +1,245 @@ +import json +from datetime import datetime, timedelta +from typing import List + +import requests +from fastapi import HTTPException + +from app.config import settings +from app.models.solana_dex_sales import DexTradeResponse, DexTrade, Trade, Transaction, Block +from app.utils.logging_config import get_logger + +logger = get_logger(__name__) + +class SolanaDexSalesService: + def __init__(self): + self.graphql_url = "https://streaming.bitquery.io/eap" # Updated endpoint + self.headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {settings.BITQUERY_API_KEY}' + } + + async def get_dex_sales(self, mint_address: str, time_window: int) -> DexTradeResponse: + """ + Fetch DEX sales for a specific mint address within the specified time window + + Args: + mint_address (str): The mint address to query + time_window (int): Time window in minutes + + Returns: + DexTradeResponse: Processed DEX sales data + """ + try: + # Calculate time range + end_time = datetime.utcnow() + start_time = end_time - timedelta(minutes=time_window) + + # Construct GraphQL query + query = self._build_query(mint_address, start_time) + variables = {} + + payload = { + 'query': query, + 'variables': json.dumps(variables) + } + + logger.info(f"Making request to BitQuery API with mint address: {mint_address}") + logger.info(f"Time range: {start_time.isoformat()}Z to {end_time.isoformat()}Z") + logger.info(f"Full GraphQL query: {query}") + + # Make request + response = requests.post( + self.graphql_url, + headers=self.headers, + json=payload + ) + + # Log raw response + logger.info(f"BitQuery API Response Status: {response.status_code}") + logger.info(f"BitQuery API Raw Response: {response.text}") + + if response.status_code != 200: + logger.error(f"Failed to fetch DEX sales: {response.text}") + raise HTTPException( + status_code=response.status_code, + detail=f"BitQuery API error: {response.text}" + ) + + # Process response + data = response.json() + logger.info(f"Parsed JSON response: {json.dumps(data, indent=2)}") + + if 'errors' in data: + logger.error(f"GraphQL errors: {json.dumps(data['errors'], indent=2)}") + raise HTTPException( + status_code=400, + detail=f"BitQuery GraphQL error: {data['errors']}" + ) + + if 'data' in data: + return self._process_response(data['data']) # Note: changed to use data['data'] + else: + logger.error("No 'data' field in response") + return DexTradeResponse(dex_trades=[]) + + except Exception as e: + logger.error(f"Error fetching DEX sales: {str(e)}", exc_info=True) + raise HTTPException( + status_code=500, + detail=f"Internal server error: {str(e)}" + ) + + def _build_query(self, mint_address: str, start_time: datetime) -> str: + """ + Build GraphQL query for DEX sales + """ + return f''' + query SolanaDEXTrades {{ + Solana {{ + DEXTrades( + where: {{ + Trade: {{ + Sell: {{ + Currency: {{ + MintAddress: {{ + is: "{mint_address}" + }} + }} + }} + }}, + Block: {{ + Time: {{ + after: "{start_time.isoformat()}Z" + }} + }} + }} + orderBy: {{descending: Block_Height}} + limit: {{count: 100}} + ) {{ + Trade {{ + Sell {{ + Amount + Account {{ + Token {{ + Owner + }} + Address + Owner + }} + Currency {{ + Name + MintAddress + }} + }} + Buy {{ + Amount + Account {{ + Token {{ + Owner + }} + Address + Owner + }} + Currency {{ + MintAddress + Name + }} + }} + }} + Transaction {{ + Signature + Signer + }} + Block {{ + Time + }} + }} + }} + }} + ''' + + def _process_response(self, response_data: dict) -> DexTradeResponse: + """ + Process and validate the response data + """ + try: + logger.debug(f"Processing response data: {json.dumps(response_data, indent=2)}") + + # Handle empty or invalid response + if not response_data: + logger.warning("Empty response data, returning empty result") + return DexTradeResponse(dex_trades=[]) + + # The response has Solana at the top level + solana_data = response_data.get('Solana') + if not solana_data: + logger.warning("No 'Solana' field in response, returning empty result") + return DexTradeResponse(dex_trades=[]) + + dex_trades_data = solana_data.get('DEXTrades', []) + logger.info(f"Found {len(dex_trades_data)} DEX trades") + + dex_trades = [] + for raw_trade in dex_trades_data: + if not raw_trade: + logger.warning("Skipping null trade data") + continue + + logger.debug(f"Processing trade: {json.dumps(raw_trade, indent=2)}") + trade_data = raw_trade.get('Trade', {}) + transaction_data = raw_trade.get('Transaction', {}) + block_data = raw_trade.get('Block', {}) + + try: + dex_trade = DexTrade( + trade=Trade( + buy=self._process_trade_action(trade_data.get('Buy', {})), + sell=self._process_trade_action(trade_data.get('Sell', {})) + ), + transaction=Transaction( + signature=transaction_data.get('Signature', ''), + signer=transaction_data.get('Signer', '') + ), + block=Block( + time=datetime.fromisoformat(block_data.get('Time', datetime.utcnow().isoformat()).replace('Z', '+00:00')) + ) + ) + dex_trades.append(dex_trade) + except Exception as e: + logger.error(f"Error processing individual trade: {str(e)}") + continue # Skip this trade but continue processing others + + return DexTradeResponse(dex_trades=dex_trades) + + except Exception as e: + logger.error(f"Error processing response: {str(e)}", exc_info=True) + raise ValueError(f"Error processing response: {str(e)}") + + def _process_trade_action(self, trade_action_data: dict) -> 'TradeAction': + """Helper method to process trade action data""" + from app.models.solana_dex_sales import TradeAction, Account, TokenAccount, Currency + + if not trade_action_data: + logger.error("Received empty trade action data") + raise ValueError("Empty trade action data") + + logger.debug(f"Processing trade action: {json.dumps(trade_action_data, indent=2)}") + + account_data = trade_action_data.get('Account', {}) + currency_data = trade_action_data.get('Currency', {}) + + return TradeAction( + amount=float(trade_action_data.get('Amount', 0)), + account=Account( + token=TokenAccount( + owner=account_data.get('Token', {}).get('Owner') + ), + address=account_data.get('Address', ''), + owner=account_data.get('Owner', '') + ), + currency=Currency( + name=currency_data.get('Name', ''), + mint_address=currency_data.get('MintAddress', '') + ) + ) diff --git a/app/services/supabase_service.py b/app/services/supabase_service.py new file mode 100644 index 0000000..7a8e292 --- /dev/null +++ b/app/services/supabase_service.py @@ -0,0 +1,155 @@ +import logging +import uuid +from datetime import date, datetime +from typing import Optional, Dict, Any + +from fastapi import HTTPException, status +from supabase import create_client, Client + +from app.config import settings + +logger = logging.getLogger(__name__) + + +class SupabaseService: + def __init__(self): + supabase_url = settings.SUPABASE_URL + supabase_service_key = settings.SUPABASE_SERVICE_KEY + if not supabase_url or not supabase_service_key: + raise ValueError("Missing Supabase configuration (SUPABASE_URL / SUPABASE_SERVICE_KEY).") + + # Initialize the client + self.client: Client = create_client(supabase_url, supabase_service_key) + # Adjust table name to match your Supabase table + self.table_name = "api_keys" + + def create_user_api_key(self, name: str, limits: Dict[str, int], active: bool = True) -> Dict[str, Any]: + """Creates a new user API key record in Supabase.""" + api_key = str(uuid.uuid4()) + usage = {tool: 0 for tool in limits.keys()} + + data = { + "name": name, + "api_key": api_key, + "limits": limits, + "usage": usage, + "active": active, + "last_reset": str(date.today()) + } + + try: + res = self.client.table(self.table_name).insert(data).execute() + # Check if data is returned + if not res.data: + logger.error( + f"Failed to create user API key record. " + f"Supabase returned status_code={res.status_code}, data={res.data}" + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to create user API key in Supabase." + ) + + created_record = res.data[0] + return created_record + + except Exception as e: + logger.error(f"SupabaseException when creating user API key: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Error communicating with Supabase." + ) + + def get_user_by_api_key(self, api_key: str) -> Optional[Dict[str, Any]]: + """ + Retrieve a user row from Supabase by their api_key. Returns None if not found. + """ + try: + res = self.client.table(self.table_name).select("*").eq("api_key", api_key).execute() + logger.info(f"Supabase response: {res}") + + # If the query returns nothing, user doesn't exist: + if not res.data: + return None + + return res.data[0] + + except Exception as e: + logger.error(f"Exception fetching user by API key: {str(e)}") + # Depending on desired behavior, could raise an HTTPException or just return None + return None + + def update_usage(self, user_id: int, usage: Dict[str, int]) -> None: + """ + Update the usage JSON column for a user. + 'usage' is a dict (tool -> usage_count). + """ + try: + res = self.client.table(self.table_name).update({"usage": usage}).eq("id", user_id).execute() + # If no rows are updated, it's typically returned as empty data. + if not res.data: + logger.error( + f"No rows updated for usage. Supabase returned status_code={res.status_code}, data={res.data}" + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Could not update usage in Supabase." + ) + except Exception as e: + logger.error(f"Error updating usage for user {user_id}: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Could not update usage in Supabase." + ) + + def update_last_reset(self, user_id: int, new_date: str) -> None: + """ + Update the 'last_reset' date for a user. + """ + try: + res = self.client.table(self.table_name).update({"last_reset": new_date}).eq("id", user_id).execute() + if not res.data: + logger.error( + f"No rows updated for last_reset. Supabase returned status_code={res.status_code}, data={res.data}" + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Could not update last_reset in Supabase." + ) + except Exception as e: + logger.error(f"Error updating last_reset for user {user_id}: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Could not update last_reset in Supabase." + ) + + def reset_daily_usage_if_needed(self, user_record: Dict[str, Any]) -> Dict[str, Any]: + """ + If today's date is different from user_record['last_reset'], + reset usage to 0 for all tools and update 'last_reset' to today in Supabase. + + Returns an updated user_record (in-memory) after the reset if it happens. + """ + user_id = user_record["id"] + last_reset_str = user_record.get("last_reset") + limits = user_record.get("limits", {}) + usage = user_record.get("usage", {}) + + # Handle edge case if last_reset is None or invalid + if not last_reset_str: + last_reset_str = "1970-01-01" + + last_reset_date = datetime.strptime(last_reset_str, "%Y-%m-%d").date() + today = date.today() + + if last_reset_date < today: + # It's a new day, so reset usage + new_usage = {tool: 0 for tool in limits.keys()} + self.update_usage(user_id, new_usage) + self.update_last_reset(user_id, str(today)) + + # Update record in-memory + user_record["usage"] = new_usage + user_record["last_reset"] = str(today) + + return user_record diff --git a/app/services/token_information.py b/app/services/token_information.py new file mode 100644 index 0000000..478f500 --- /dev/null +++ b/app/services/token_information.py @@ -0,0 +1,52 @@ +from typing import Optional +from app.utils.http_client import HTTPClient +from app.models.token_information import DexScreenerResponse +import logging +import re + +logger = logging.getLogger(__name__) + +class TokenInformationService: + BASE_URL = "https://api.dexscreener.com/latest/dex" + + def __init__(self): + self.http_client = HTTPClient() + + async def get_token_information(self, query: str, chain_id: Optional[str] = None) -> DexScreenerResponse: + """ + Get token price information either by address or name. + + Args: + query: Token address or name + chain_id: Optional blockchain ID for specific chain lookup + + Returns: + DexScreenerResponse with token pair information + """ + logger.info(f"Getting token price for {query} on chain {chain_id}") + + # Check if query looks like an address (basic validation) + is_address = bool(re.match(r'^0x[a-fA-F0-9]{40}$|^[1-9A-HJ-NP-Za-km-z]{32,44}$', query)) + + try: + if is_address and chain_id: + # If we have both chain ID and address, use the pairs endpoint + logger.debug(f"Using pairs endpoint with chain {chain_id} and address {query}") + url = f"{self.BASE_URL}/pairs/{chain_id}/{query}" + else: + # Otherwise use the search endpoint + logger.debug(f"Using search endpoint with query {query}") + url = f"{self.BASE_URL}/search" + if not is_address: + url += f"?q={query}" + else: + url += f"?q={query}" + if chain_id: + url += f"&chain={chain_id}" + + response = await self.http_client.get(url) + return DexScreenerResponse(**response) + + except Exception as e: + logger.error(f"Error getting token price: {str(e)}", exc_info=True) + raise \ No newline at end of file diff --git a/app/services/web_search.py b/app/services/web_search.py new file mode 100644 index 0000000..41336b6 --- /dev/null +++ b/app/services/web_search.py @@ -0,0 +1,381 @@ +import os +import json +from typing import List, Dict, Any, Optional +import aiohttp +import logging +from firecrawl import FirecrawlApp +from app.models.web_search import SearchResult, Learning, WebSearchResponse +from app.services.llm import LLMService +from app.config import settings + +logger = logging.getLogger(__name__) + +class WebSearchService: + """Service for web search using Firecrawl and research inspired by deep-research-py""" + + def __init__(self): + self.firecrawl_api_key = settings.FIRECRAWL_API_KEY + self.firecrawl_base_url = getattr(settings, 'FIRECRAWL_BASE_URL', None) + self.llm_service = LLMService() + # Initialize Firecrawl app + self.firecrawl = FirecrawlApp( + api_key=self.firecrawl_api_key, + api_url=self.firecrawl_base_url + ) + + async def search(self, query: str, num_results: int = 5) -> Dict[str, List[Dict[str, str]]]: + """ + Search using Firecrawl SDK + + Args: + query: Search query + num_results: Number of results to fetch + + Returns: + Dict containing search results + """ + try: + # Log the search request + logger.info(f"Searching with Firecrawl, query: '{query}', api_key_length: {len(self.firecrawl_api_key) if self.firecrawl_api_key else 0}") + + # Since FirecrawlApp is synchronous, run it in an event loop executor + import asyncio + + # Call Firecrawl search + logger.debug(f"Calling Firecrawl search with query: {query}") + response = await asyncio.get_event_loop().run_in_executor( + None, + lambda: self.firecrawl.search( + query=query, + ), + ) + + # Log the raw response for debugging + logger.info(f"Raw Firecrawl response type: {type(response)}") + logger.info(f"Raw Firecrawl response: {response}") + + # Process and limit the results after receiving them + if isinstance(response, dict) and "data" in response: + # Response is already in the right format + data = response["data"][:num_results] + logger.info(f"Got {len(data)} results from Firecrawl (dict with data format)") + return {"data": data} + elif isinstance(response, dict) and "success" in response: + # Response is in documented format + data = response.get("data", [])[:num_results] + logger.info(f"Got {len(data)} results from Firecrawl (success format)") + return {"data": data} + elif isinstance(response, list): + # Response is a list of results (limited to num_results) + formatted_data = [] + for item in response[:num_results]: + if isinstance(item, dict): + formatted_data.append(item) + else: + # Handle non-dict items (like objects) + formatted_data.append({ + "url": getattr(item, "url", ""), + "markdown": getattr(item, "markdown", "") or getattr(item, "content", ""), + "title": getattr(item, "title", "") or getattr(item, "metadata", {}).get("title", ""), + }) + logger.info(f"Got {len(formatted_data)} results from Firecrawl (list format)") + return {"data": formatted_data} + else: + logger.error(f"Unexpected response format from Firecrawl: {type(response)}") + # Log more details about the unexpected response + logger.error(f"Response content: {str(response)[:200]}...") + return {"data": []} + + except Exception as e: + logger.error(f"Error searching with Firecrawl: {str(e)}", exc_info=True) + return {"data": []} + + async def research(self, query: str, num_results: int = 5, depth: int = 1) -> WebSearchResponse: + """ + Perform deep research on a topic using Firecrawl and LLM analysis + + Args: + query: Research query + num_results: Number of search results to fetch per query + depth: Research depth (1-3) + + Returns: + WebSearchResponse with search results, learnings, and summary + """ + try: + # Start with initial search + logger.info(f"Starting web search for query: {query}") + search_results = await self.search(query, num_results) + logger.debug(f"Search results: {search_results}") + + # Prepare response object + response = WebSearchResponse( + query=query, + results=[], + learnings=[], + summary="", + visited_urls=[] + ) + + # Extract content from search results + contents = [] + urls = [] + + for item in search_results.get("data", []): + logger.debug(f"Processing item: {item}") + + # Check if we have either markdown or description + has_content = item.get("markdown") or item.get("description") + has_url = item.get("url") + + if has_url and has_content: + # Use description field if markdown is not available + content = item.get("markdown", "") or item.get("description", "") + + # Truncate content if too long + if len(content) > 25000: + content = content[:25000] + "... [content truncated]" + + result = SearchResult( + url=item.get("url", ""), + title=item.get("title", "Unknown Title"), + # Use description as markdown if markdown is not available + markdown=content + ) + response.results.append(result) + contents.append(content) + urls.append(item.get("url")) + + # Record visited URLs + response.visited_urls = urls + + if not contents: + logger.warning(f"No valid search results found for query: {query}") + response.summary = "No research results found. Please try a different query." + return response + + # Process search results to extract learnings + learnings = await self._extract_learnings(query, contents, urls) + response.learnings = learnings + + # For depth > 1, perform follow-up research (simplified version of deep-research) + if depth > 1 and learnings: + # Generate follow-up queries based on learnings + follow_up_queries = await self._generate_follow_up_queries( + query, + [learning.content for learning in learnings], + min(3, depth) # Cap at 3 follow-up queries + ) + + # Perform follow-up searches + all_follow_up_learnings = [] + for follow_up_query in follow_up_queries: + follow_up_results = await self.search(follow_up_query, num_results=3) + + follow_up_contents = [] + follow_up_urls = [] + + for item in follow_up_results.get("data", []): + if item.get("markdown") and item.get("url"): + follow_up_contents.append(item.get("markdown", "")) + follow_up_urls.append(item.get("url")) + # Add to visited URLs if not already there + if item.get("url") not in response.visited_urls: + response.visited_urls.append(item.get("url")) + + if follow_up_contents: + follow_up_learnings = await self._extract_learnings( + follow_up_query, + follow_up_contents, + follow_up_urls + ) + all_follow_up_learnings.extend(follow_up_learnings) + + # Add unique follow-up learnings + existing_contents = [learning.content for learning in response.learnings] + for learning in all_follow_up_learnings: + if learning.content not in existing_contents: + response.learnings.append(learning) + existing_contents.append(learning.content) + + # Generate final summary + response.summary = await self._generate_summary( + query, + [learning.content for learning in response.learnings] + ) + + return response + + except Exception as e: + logger.error(f"Error in web research: {str(e)}", exc_info=True) + return WebSearchResponse( + query=query, + summary=f"Error performing research: {str(e)}" + ) + + async def _extract_learnings(self, query: str, contents: List[str], urls: List[str]) -> List[Learning]: + """ + Extract key learnings from search result contents using LLM + + Args: + query: The original search query + contents: List of content from search results + urls: Corresponding URLs for each content + + Returns: + List of Learning objects + """ + try: + # Create the prompt for the LLM + contents_str = "\n\n".join([f"CONTENT: {content}" for content in contents]) + + system_prompt = f"""You are an expert research assistant. Extract key learnings from the provided search results. +Focus on facts, data points, insights, and unique information related to the query. +Avoid generic statements and prioritize information-dense findings. + +QUERY: {query} + +Analyze the following search results and extract 5-7 key learnings. Each learning should be: +1. Directly relevant to the query +2. Specific and information-dense (include entities, metrics, dates, etc. when available) +3. Factual and objective +4. Independent from other learnings (avoid redundancy) +5. Cited properly to the source URL when possible + +FORMAT YOUR RESPONSE AS JSON with a "learnings" array, where each learning has: +- "content": The factual learning as a concise statement +- "source_url": The URL source of this information (use null if unclear)""" + + # Call the LLM with the system prompt + response = await self.llm_service._call_llm( + system_prompt=system_prompt, + user_content=contents_str, + force_json=True + ) + + # Parse response as JSON + if isinstance(response, str): + try: + parsed_response = json.loads(response) + if "learnings" in parsed_response and isinstance(parsed_response["learnings"], list): + learnings = [] + for item in parsed_response["learnings"]: + if isinstance(item, dict) and "content" in item: + learning = Learning( + content=item["content"], + source_url=item.get("source_url") + ) + learnings.append(learning) + return learnings + except json.JSONDecodeError: + logger.error("Failed to parse LLM response as JSON") + + # Fallback if JSON parsing fails + return [Learning(content="Failed to extract learnings from search results", source_url=None)] + + except Exception as e: + logger.error(f"Error extracting learnings: {str(e)}", exc_info=True) + return [Learning(content=f"Error extracting learnings: {str(e)}", source_url=None)] + + async def _generate_follow_up_queries(self, original_query: str, learnings: List[str], count: int = 3) -> List[str]: + """ + Generate follow-up queries based on learnings to deepen the research + + Args: + original_query: The original search query + learnings: List of learning content + count: Number of follow-up queries to generate + + Returns: + List of follow-up query strings + """ + try: + # Create prompt for the LLM + learnings_str = "\n".join([f"- {learning}" for learning in learnings]) + + system_prompt = f"""You are an expert research assistant. Generate follow-up search queries based on the original query and learnings. +The follow-up queries should explore different aspects or deeper details related to the original query. + +ORIGINAL QUERY: {original_query} + +KEY LEARNINGS: +{learnings_str} + +Generate {count} follow-up search queries that: +1. Explore specific aspects mentioned in the learnings +2. Fill knowledge gaps not covered in the existing learnings +3. Dive deeper into the most important aspects of the topic +4. Are specific and focused enough to yield useful search results + +FORMAT YOUR RESPONSE AS JSON with a "queries" array of strings.""" + + # Call the LLM with the system prompt + response = await self.llm_service._call_llm( + system_prompt=system_prompt, + user_content="", + force_json=True + ) + + # Parse response as JSON + if isinstance(response, str): + try: + parsed_response = json.loads(response) + if "queries" in parsed_response and isinstance(parsed_response["queries"], list): + # Return only the requested number of queries + return parsed_response["queries"][:count] + except json.JSONDecodeError: + logger.error("Failed to parse LLM response as JSON") + + # Fallback + return [f"{original_query} details", f"{original_query} examples", f"{original_query} application"] + + except Exception as e: + logger.error(f"Error generating follow-up queries: {str(e)}", exc_info=True) + return [original_query] + + async def _generate_summary(self, query: str, learnings: List[str]) -> str: + """ + Generate a comprehensive summary based on the collected learnings + + Args: + query: The original search query + learnings: List of learning content + + Returns: + Summary text + """ + try: + # Create prompt for the LLM + learnings_str = "\n".join([f"- {learning}" for learning in learnings]) + + system_prompt = f"""You are an expert research assistant. Create a comprehensive summary based on the learnings from research. +The summary should synthesize the key points into a coherent, well-structured response to the original query. + +QUERY: {query} + +LEARNINGS: +{learnings_str} + +Create a well-structured summary that: +1. Directly addresses the original query +2. Incorporates the most important learnings +3. Organizes information logically with proper flow +4. Highlights connections between different learnings +5. Is comprehensive but concise (300-500 words)""" + + # Call the LLM with the system prompt + response = await self.llm_service._call_llm( + system_prompt=system_prompt, + user_content="", + force_json=False + ) + + if isinstance(response, str): + return response.strip() + + # Fallback + return "Failed to generate summary from research findings." + + except Exception as e: + logger.error(f"Error generating summary: {str(e)}", exc_info=True) + return f"Error generating summary: {str(e)}" \ No newline at end of file diff --git a/app/utils/__init__.py b/app/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/utils/enums.py b/app/utils/enums.py new file mode 100644 index 0000000..7f8f92a --- /dev/null +++ b/app/utils/enums.py @@ -0,0 +1,7 @@ +from enum import Enum + +class Environment(str, Enum): + DEVELOPMENT = "development" + PRODUCTION = "production" + STAGING = "staging" + TESTING = "testing" \ No newline at end of file diff --git a/app/utils/http_client.py b/app/utils/http_client.py new file mode 100644 index 0000000..bf275d0 --- /dev/null +++ b/app/utils/http_client.py @@ -0,0 +1,17 @@ +import aiohttp +from typing import Dict, Any, Optional + +class HTTPClient: + async def get( + self, + url: str, + params: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None + ) -> Dict: + """ + Perform GET request + """ + async with aiohttp.ClientSession() as session: + async with session.get(url, params=params, headers=headers) as response: + response.raise_for_status() + return await response.json() \ No newline at end of file diff --git a/app/utils/logging_config.py b/app/utils/logging_config.py new file mode 100644 index 0000000..e87d5a5 --- /dev/null +++ b/app/utils/logging_config.py @@ -0,0 +1,71 @@ +import logging +import sys +from typing import Optional +from logging.handlers import RotatingFileHandler + +def setup_logging( + log_level: str = "INFO", + log_file: Optional[str] = None, + max_file_size: int = 10 * 1024 * 1024, # 10 MB + backup_count: int = 5 +) -> None: + """ + Configure global logging settings for the application. + + Args: + log_level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + log_file: Optional path to log file. If None, logs only to console + max_file_size: Maximum size of each log file in bytes + backup_count: Number of backup files to keep + """ + # Convert string log level to logging constant + numeric_level = getattr(logging, log_level.upper(), logging.INFO) + + # Create formatter + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + # Configure root logger + root_logger = logging.getLogger() + root_logger.setLevel(numeric_level) + + # Clear existing handlers + root_logger.handlers = [] + + # Console handler + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setFormatter(formatter) + root_logger.addHandler(console_handler) + + # File handler (if log file specified) + if log_file: + file_handler = RotatingFileHandler( + log_file, + maxBytes=max_file_size, + backupCount=backup_count + ) + file_handler.setFormatter(formatter) + root_logger.addHandler(file_handler) + + # Suppress unnecessary logs from other libraries + logging.getLogger('urllib3').setLevel(logging.WARNING) + logging.getLogger('aiohttp').setLevel(logging.WARNING) + + # Log configuration details + root_logger.debug(f"Logging configured with level: {log_level}") + if log_file: + root_logger.debug(f"Log file: {log_file}") + +def get_logger(name: str) -> logging.Logger: + """ + Get a logger instance for a specific module. + + Args: + name: Usually __name__ of the module + + Returns: + Logger instance configured with global settings + """ + return logging.getLogger(name) \ No newline at end of file diff --git a/create_key.py b/create_key.py new file mode 100644 index 0000000..01f798e --- /dev/null +++ b/create_key.py @@ -0,0 +1,40 @@ +import requests +import os +from dotenv import load_dotenv +import json + +# Load environment variables from .env +load_dotenv() + +# Get admin secret from .env +ADMIN_SECRET = os.getenv('ADMIN_SECRET') +if not ADMIN_SECRET: + raise ValueError("ADMIN_SECRET not found in .env file") + +print(f"Using admin secret: {ADMIN_SECRET}") + +# Create an API key +response = requests.post( + "http://localhost:8000/api/v1/admin/create-api-key", + headers={"X-Admin-Key": ADMIN_SECRET}, + json={ + "name": "Test User", + "limits": { + "news": 500, + "image_recognition": 25, + "token_information": 1000, + "math": 1000, + "scraper": 100, + "solana_dex_sales": 1000, + "solana_dex_buys": 1000 + } + } +) + +if response.status_code == 200: + result = response.json() + print("\nAPI Key created successfully!") + print(f"API Key: {result.get('api_key')}") + print(f"Limits: {json.dumps(result.get('limits'), indent=2)}") +else: + print(f"\nError creating API key: {response.text}") \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..813993e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,24 @@ +services: + api: + build: . + container_name: som-api + ports: + - "8000:8000" + volumes: + - ./logs:/app/logs + - ./.env:/app/.env + environment: + - ENV=development + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/docs"] + interval: 30s + timeout: 10s + retries: 3 + + networks: + - stateofmika_net + +networks: + stateofmika_net: + driver: bridge \ No newline at end of file diff --git a/example.py b/example.py new file mode 100644 index 0000000..0681e98 --- /dev/null +++ b/example.py @@ -0,0 +1,148 @@ +""" +State of Mika API Examples +------------------------- +This file demonstrates the usage of each tool available in the State of Mika API. +Each example includes: +- Feature description +- Example API request +- Expected response + +All requests are routed through the universal router, which intelligently +directs queries to the appropriate tool. +""" + +import requests +import json +from typing import Dict, Any + +BASE_URL = "http://localhost:8000/api/v1" +API_KEY = "your_api_key_here" + +def route_query(query: str) -> Dict: + """Helper function to route all queries through the universal router""" + print("\n=== Starting API Request ===") + + # Set up headers for multipart/form-data + headers = { + "X-API-Key": API_KEY, + "accept": "application/json" + } + + # Set up form data + form_data = { + 'query': (None, query), + 'tool': (None, ''), + 'parameters_str': (None, ''), + 'file': (None, '') + } + + url = f"{BASE_URL}/" + print(f"URL: {url}") + print(f"Headers: {json.dumps(headers, indent=2)}") + print(f"Form data: {form_data}") + + try: + print("\nMaking request...") + response = requests.post( + url, + headers=headers, + files=form_data # Use files parameter for multipart/form-data + ) + print(f"Status code: {response.status_code}") + + if response.status_code != 200: + print(f"Error response: {response.text}") + else: + print("Request successful!") + + return response.json() + except Exception as e: + print(f"Error making request: {str(e)}") + return {"error": str(e)} + +def test_connection(): + """Test basic connectivity with a simple query""" + print("\nTesting API connection...") + result = route_query("What is 2 + 2?") + print("\nFull response:") + print(json.dumps(result, indent=2)) + return result + +def test_news(): + """Test news endpoint""" + print("\nTesting news endpoint...") + result = route_query("Show me the latest crypto news about Bitcoin and Ethereum") + print("\nFull response:") + print(json.dumps(result, indent=2)) + return result + +def test_math(): + """Test math endpoint""" + print("\nTesting math endpoint...") + result = route_query("Calculate 15% of 150 and add 500") + print("\nFull response:") + print(json.dumps(result, indent=2)) + return result + +def test_token_price(): + """Test token price endpoint""" + print("\nTesting token price endpoint...") + result = route_query("What is the current price of Solana?") + print("\nFull response:") + print(json.dumps(result, indent=2)) + return result + +def test_scraper(): + """Test scraper endpoint""" + print("\nTesting scraper endpoint...") + result = route_query("Summarize the article at https://example.com/crypto-article") + print("\nFull response:") + print(json.dumps(result, indent=2)) + return result + +def test_dex_sales(): + """Test DEX sales endpoint""" + print("\nTesting DEX sales endpoint...") + result = route_query("Show me SOL token sales in the last hour for address SOL_TOKEN_MINT_ADDRESS") + print("\nFull response:") + print(json.dumps(result, indent=2)) + return result + +def test_dex_buys(): + """Test DEX buys endpoint""" + print("\nTesting DEX buys endpoint...") + result = route_query("Show me all SOL purchases by wallet WALLET_ADDRESS") + print("\nFull response:") + print(json.dumps(result, indent=2)) + return result + +def test_web_search(): + """Test web search endpoint""" + print("\nTesting web search endpoint...") + result = route_query("Research the latest advancements in quantum computing") + print("\nFull response:") + print(json.dumps(result, indent=2)) + return result + +def run_all_tests(): + """Run all test functions""" + tests = [ + test_connection, + test_news, + test_math, + test_token_price, + test_scraper, + test_dex_sales, + test_dex_buys, + test_web_search + ] + + for test in tests: + print(f"\n{'='*50}") + print(f"Running {test.__name__}") + print('='*50) + test() + +if __name__ == "__main__": + print("Run test_connection() to test basic connectivity") + print("Run run_all_tests() to test all endpoints") \ No newline at end of file diff --git a/migrations/20240131_create_membership_tables.sql b/migrations/20240131_create_membership_tables.sql new file mode 100644 index 0000000..5afa1ef --- /dev/null +++ b/migrations/20240131_create_membership_tables.sql @@ -0,0 +1,55 @@ +-- Create extension if not exists +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +-- Create enum type for platform types +CREATE TYPE platform_type AS ENUM ('discord', 'telegram', 'twitter', 'whatsapp', 'custom'); + +-- Create bots table +CREATE TABLE IF NOT EXISTS bots ( + bot_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + api_key UUID NOT NULL UNIQUE DEFAULT uuid_generate_v4(), + name VARCHAR(100) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +-- Create friends table +CREATE TABLE IF NOT EXISTS friends ( + friend_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + bot_id UUID NOT NULL REFERENCES bots(bot_id) ON DELETE CASCADE, + nickname VARCHAR(100) NOT NULL, + level INTEGER NOT NULL DEFAULT 1, + points INTEGER NOT NULL DEFAULT 0, + description TEXT, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +-- Create friend platform contacts table +CREATE TABLE IF NOT EXISTS friend_platform_contacts ( + contact_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + friend_id UUID NOT NULL REFERENCES friends(friend_id) ON DELETE CASCADE, + platform_type platform_type NOT NULL, + platform_user_id VARCHAR(100) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(friend_id, platform_type, platform_user_id) +); + +-- Create friend points history table +CREATE TABLE IF NOT EXISTS friend_points_history ( + history_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + friend_id UUID NOT NULL REFERENCES friends(friend_id) ON DELETE CASCADE, + timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + points_change INTEGER NOT NULL, + previous_points INTEGER NOT NULL, + current_points INTEGER NOT NULL, + reason VARCHAR(255), + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +-- Create indexes +CREATE INDEX idx_friends_bot_id ON friends(bot_id); +CREATE INDEX idx_friends_nickname ON friends(nickname); +CREATE INDEX idx_platform_contacts_friend_id ON friend_platform_contacts(friend_id); +CREATE INDEX idx_platform_contacts_platform ON friend_platform_contacts(platform_type, platform_user_id); +CREATE INDEX idx_points_history_friend_id ON friend_points_history(friend_id); +CREATE INDEX idx_points_history_timestamp ON friend_points_history(timestamp); diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..aaf786f --- /dev/null +++ b/readme.md @@ -0,0 +1,454 @@ +# State of Mika + +An advanced context-based routing service for intelligently handling diverse data silos and providing tailored, **daily usage–limited** responses. Now with **Solana DEX** query capabilities and **Supabase-based** API key management! + +--- + +## Project Structure + +Below is an **updated** structure reflecting newly added files for Supabase integration, admin routes, and Solana DEX endpoints: + +``` +SoM-tool-collection/ +├── app/ +│ ├── __init__.py +│ ├── main.py # Application entry (FastAPI) +│ ├── config.py # Pydantic-based settings +│ ├── dependencies.py # [NEW] Global dependency for API key validation +│ ├── routes/ +│ │ ├── __init__.py +│ │ ├── admin.py # [NEW] Admin routes for creating user API keys +│ │ ├── news.py +│ │ ├── token_information.py +│ │ ├── math.py +│ │ ├── image_recognition.py +│ │ ├── scraper.py +│ │ ├── solana_dex_buys.py +│ │ ├── solana_dex_sales.py +│ │ └── router.py # Universal router +│ ├── models/ +│ │ ├── __init__.py +│ │ ├── news.py +│ │ ├── token_information.py +│ │ ├── math.py +│ │ ├── image_recognition.py +│ │ ├── scraper.py +│ │ ├── router.py +│ │ ├── solana_dex_buys.py +│ │ ├── solana_dex_sales.py +│ │ └── ... +│ ├── services/ +│ │ ├── __init__.py +│ │ ├── news.py +│ │ ├── token_information.py +│ │ ├── math.py +│ │ ├── router.py +│ │ ├── scraper.py +│ │ ├── llm.py +│ │ ├── image_recognition.py +│ │ ├── solana_dex_buys.py +│ │ ├── solana_dex_sales.py +│ │ └── supabase_service.py # [NEW] Interacts with Supabase for API key usage gating +│ └── utils/ +│ ├── __init__.py +│ ├── http_client.py +│ └── logging_config.py +├── requirements.txt +├── run.py +└── README.md (this file) +``` + +--- + +## Setup + +### 1. Create and Activate a Virtual Environment + +```bash +python -m venv venv +source venv/bin/activate # On Windows: .\venv\Scripts\activate +``` + +### 2. Install Dependencies + +```bash +pip install -r requirements.txt +``` + +In addition to the original dependencies, you now need **supabase-py** for database integration. + +### 3. Configure Environment Variables in `.env` + +Below are **required** environment variables for the new setup: + +```env +# --- Basic/Existing --- +CRYPTOPANIC_API_URL=https://cryptopanic.com/api/v1 +CRYPTOPANIC_API_KEY=your_cryptopanic_api_key +OPENAI_API_KEY=your_openai_api_key +BITQUERY_API_KEY=your_bitquery_api_key +UVICORN_PORT=8000 + +# --- Gemini AI (Image Recognition) --- +# Optional if you use Gemini for image analysis +GEMINI_API_KEY=your_gemini_api_key + +# --- Supabase for API Key Management --- +SUPABASE_URL=https://.supabase.co +SUPABASE_SERVICE_KEY= + +# --- Admin Secret --- +# Used to protect admin route for creating new keys +ADMIN_SECRET= + +# (You may also configure logging, environment, etc. in .env or in config.py) +``` + +### 4. Run the Application + +```bash +# Recommended: +python run.py + +# Or directly via uvicorn (if you want): +uvicorn app.main:app --host 0.0.0.0 --port 8000 +``` + +The API will be available at `http://localhost:` with interactive docs at `http://localhost:/docs`. +`` defaults to 8000 if not overridden by `UVICORN_PORT`. + +--- + +## Features + +1. **Universal Router** + A single endpoint that interprets user queries (text or images), decides which tool to invoke, and returns structured results. + +2. **News Aggregation** + Comprehensive crypto/blockchain news via CryptoPanic with filtering for regions, currencies, and content type. + +3. **Web Scraping** + Dynamically fetches and processes external webpages using a headless browser (Pyppeteer), returning summaries and insights. + +4. **Real-Time Token Prices** + Fetches up-to-date market data and token prices (via DexScreener) for multiple blockchain networks. + +5. **Mathematical Operations** + Evaluates complex math expressions (via Sympy), including arithmetic, functions, and symbolic manipulation. + +6. **Image Recognition** + Uses Gemini/OpenAI-like image capabilities (if configured) to describe or analyze images. + +7. **Solana DEX** + New endpoints to query **Solana DEX buys and sales** using BitQuery data: + - `POST /solana/dex/buys` + - `POST /solana/dex/sales` + Each returns trade information (token addresses, amounts, block time, etc.). + +--- + +## Daily Usage Limits & API Key Management + +### API Key Gating + +- **All non-admin endpoints** require a valid `X-API-Key` header. +- Each user’s **daily usage** is tracked in Supabase. Once a user’s usage for a specific tool meets their daily limit, calls to that tool return `429 Too Many Requests`. + +### Admin Endpoint + +An **admin-only** endpoint allows creating new user API keys: + +``` +POST /api/v1/admin/create-api-key +Headers: + X-Admin-Key: +Body: + { + "name": "Test User", + "limits": { + "news": 500, + "image_recognition": 25, + "math": 1000, + ... + } + } +``` + +When valid, it returns a **fresh** `api_key` that the user can use. +Storing and enforcing usage occurs in the `api_keys` table on Supabase. + +### Daily Reset + +- The system stores a `last_reset` date for each API key. +- On each request, if `last_reset < today`, usage is automatically reset to **0** for all tools, and `last_reset` is set to the current date. + +Hence, each user’s usage counters refresh **once per day**. + +--- + +## How It Works (Detailed) + +1. **User Request** + The user calls the **universal route** (or any dedicated route like `/news`), providing `X-API-Key`. + +2. **API Key Validation** + A global dependency (in `app/dependencies.py`) fetches the user record from Supabase by `api_key`, checks if **active**. + - If `last_reset` < today, all usage counters are reset to 0 in Supabase. + +3. **Router Service** + The query is analyzed: + - **If** a tool is explicitly provided, or if the LLM-based analysis detects a tool, we proceed with that tool. + - The user’s usage for that tool is checked. If usage >= limit, we return `429`. + - Otherwise, the tool is invoked. + +4. **Response & Usage Increment** + After a successful tool invocation, the usage counter for that tool is incremented in Supabase. + +5. **Next Day**: usage resets automatically on the next request after midnight (or whenever `date.today()` changes). + +--- + +## Examples of Supported Queries + +- **Price Queries** + “What is the price of WBTC on Solana?” + “Get market info for USDC on Ethereum.” + +- **Image Analysis** + “What objects are in this image?” (uploads an image file) + +- **News Updates** + “Get me the latest cryptocurrency news for BTC and ETH.” + +- **Content Extraction** + “Scrape this URL and summarize: https://example.com/article.” + +- **Mathematics** + “Evaluate (5 + 3) * 2.” + +- **Solana DEX** + - `POST /solana/dex/buys` + ```json + { + "mint_address": "SomeSolanaMint...", + "signer_address": "SomeSolanaAddress..." + } + ``` + - `POST /solana/dex/sales` + ```json + { + "mint_address": "SomeSolanaMint...", + "time_window": 60 // In minutes + } + ``` + +--- + +## Rate Limits & Daily Usage + +Thanks to Supabase-based usage gating, each user can have custom daily limits per tool. When the limit is reached for a day, calls to that tool return `429 Too Many Requests`. + +--- + +## Error Handling + +The API provides robust error handling: + +- Missing or invalid `X-API-Key` → `401 Unauthorized`. +- Exceeding daily usage limit → `429 Too Many Requests`. +- Invalid request parameters → `400 Bad Request`. +- Internal or external service errors → `500 Internal Server Error`. +- Unsupported or unrecognized query → fallback routing or `404` in extreme cases. + +--- + +## Membership API + +The Membership API provides endpoints for managing bots and their associated friends, including platform-specific contacts and points tracking. + +### Authentication + +All endpoints (except bot creation) require an `x-api-key` header containing the bot's API key. + +### Endpoints + +#### Bots + +- **Create Bot** + ``` + POST /api/v1/membership/bots + ``` + Create a new bot and receive its API key. + + Request body: + ```json + { + "name": "bot_name" // Required, 1-100 characters + } + ``` + + Response: + ```json + { + "bot_id": "uuid", + "api_key": "uuid", + "name": "bot_name", + "created_at": "timestamp" + } + ``` + +#### Friends + +- **Create/Get Friend** + ``` + POST /api/v1/membership/bots/{bot_id}/friends + ``` + Create a new friend or get existing friend details if the platform contact already exists. + + Request body: + ```json + { + "nickname": "friend_name", // Required, 1-100 characters + "description": "friend description", // Optional + "platform_contact": { + "platform_type": "discord|telegram|twitter|whatsapp|custom", + "platform_user_id": "platform_specific_id" // Max 100 characters + } + } + ``` + +- **List Friends** + ``` + GET /api/v1/membership/bots/{bot_id}/friends + ``` + Get a list of all friends for a bot. + +- **Get Friend Details** + ``` + GET /api/v1/membership/bots/{bot_id}/friends/{friend_id} + ``` + Get detailed information about a specific friend. + +- **Update Friend** + ``` + PATCH /api/v1/membership/bots/{bot_id}/friends/{friend_id} + ``` + Update friend attributes. All fields are optional. + + Request body: + ```json + { + "level": 2, // Optional + "points": 100, // Optional + "description": "new description", // Optional + "reason": "reason for points change" // Optional, used when updating points + } + ``` + +#### Platform Contacts + +- **Add Platform Contact** + ``` + POST /api/v1/membership/bots/{bot_id}/friends/{friend_id}/contacts + ``` + Add a new platform contact for an existing friend. + + Request body: + ```json + { + "platform_type": "discord|telegram|twitter|whatsapp|custom", + "platform_user_id": "platform_specific_id" // Max 100 characters + } + ``` + +### Database Schema + +The API uses the following tables: + +- **bots**: Stores bot information + - `bot_id` (UUID, Primary Key) + - `api_key` (UUID, Unique) + - `name` (VARCHAR(100)) + - `created_at` (TIMESTAMP WITH TIME ZONE) + +- **friends**: Stores friend information + - `friend_id` (UUID, Primary Key) + - `bot_id` (UUID, Foreign Key) + - `nickname` (VARCHAR(100)) + - `level` (INTEGER) + - `points` (INTEGER) + - `description` (TEXT) + - `created_at` (TIMESTAMP WITH TIME ZONE) + - `updated_at` (TIMESTAMP WITH TIME ZONE) + +- **friend_platform_contacts**: Stores platform-specific contact information + - `contact_id` (UUID, Primary Key) + - `friend_id` (UUID, Foreign Key) + - `platform_type` (ENUM) + - `platform_user_id` (VARCHAR(100)) + - `created_at` (TIMESTAMP WITH TIME ZONE) + - Unique constraint on (friend_id, platform_type, platform_user_id) + +- **friend_points_history**: Tracks points changes + - `history_id` (UUID, Primary Key) + - `friend_id` (UUID, Foreign Key) + - `timestamp` (TIMESTAMP WITH TIME ZONE) + - `points_change` (INTEGER) + - `previous_points` (INTEGER) + - `current_points` (INTEGER) + - `reason` (VARCHAR(255)) + - `created_at` (TIMESTAMP WITH TIME ZONE) + +## Logging + +Comprehensive logging is in place, including: + +- Query parsing and routing decisions. +- Post-processing attempts. +- LLM service interactions and fallback routes. +- Parameter and usage validations. +- Web scraping activities and statuses. +- Error conditions at various stages. + +Configure via environment or modify `app/config.py` and `app/utils/logging_config.py` as needed. + +### Error Handling + +- 400: Invalid request (e.g., invalid API key format) +- 403: Invalid API key for the bot +- 404: Friend not found +- 500: Internal server error with details in the response + + +## Contributing + +1. Fork or clone this repo. +2. Make changes in feature branches. +3. Submit pull requests for review. + +**Important**: Keep API keys and secrets out of the code base, using `.env` or environment variables instead. + +--- +## License + +MIT License + +Copyright (c) 2025 Chasm + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0e612ee --- /dev/null +++ b/requirements.txt @@ -0,0 +1,39 @@ +# Core dependencies +fastapi==0.109.0 +uvicorn==0.27.0 +pydantic>=2.7.4,<3.0.0 +pydantic-core>=2.18.2,<3.0.0 +pydantic-settings>=2.4.0,<3.0.0 + +# HTTP clients +aiohttp>=3.11.11,<4.0.0 +httpx>=0.27.0,<0.28.0 +lxml_html_clean>=0.4.1 + +# Environment variables +python-dotenv + +# Web scraping and content processing +beautifulsoup4==4.12.3 +trafilatura==1.6.1 +python-dateutil==2.8.2 +playwright + +# Other utilities +requests +typing-extensions>=4.12.2,<5.0.0 # <-- Updated to satisfy Pydantic/FASTAPI + +# Database +protobuf==5.29.3 + +# math +sympy + +# gemini +google-generativeai +python-multipart + +supabase>=2.12.0 + +semantic-router==0.1.0.dev3 +firecrawl \ No newline at end of file diff --git a/run.py b/run.py new file mode 100644 index 0000000..193568b --- /dev/null +++ b/run.py @@ -0,0 +1,10 @@ +import uvicorn +from app.config import settings + +if __name__ == "__main__": + uvicorn.run( + "app.main:app", + host="0.0.0.0", + port=settings.UVICORN_PORT, + reload=settings.is_development + ) \ No newline at end of file diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..6873db8 --- /dev/null +++ b/setup.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Create logs directory if it doesn't exist +mkdir -p logs + +# Install dependencies +pip install -r requirements.txt + +# Create .env file from example if it doesn't exist +if [ ! -f .env ]; then + cp .env.example .env + echo "Created .env file from .env.example. Please update it with your API keys." +fi + +echo "Setup complete! Please update your .env file with your API keys before running the service." \ No newline at end of file