mc

Author: atharvakarval

.gitignore

@@ -74,3 +74,6 @@ tmp/
 .pytest_cache/
 .coverage
 htmlcov/
+
+# Storage
+form-flow-backend/storage/

README.md

@@ -375,7 +375,6 @@ playwright install chromium
 
 # Configure environment
 copy .env.example .env  # Windows
-# cp .env.example .env  # Linux/Mac
 
 # Start server
 python -m uvicorn main:app --reload --port 8001

form-flow-backend/main.py

@@ -23,7 +23,7 @@
 import asyncio
 
 # Fix for Playwright on Windows - ProactorEventLoop required for subprocess
-if sys.platform == 'win32':
+if sys.platform == 'win32' and 'pytest' not in sys.modules:
     asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
 
 # Suppress Pydantic V1 warning from LangChain

form-flow-backend/routers/attachments.py

@@ -2,39 +2,59 @@
 Attachments Router - API Endpoints for File Form Fields
 
 Provides REST API for:
-- File upload
+- Secure File upload (Async, Chunked)
 - File retrieval
 - File deletion
 """
 
 import os
 import asyncio
-import shutil
 import uuid
 import logging
+import hashlib
+import re
 from pathlib import Path
 from typing import Dict, Any, Optional
 
-from fastapi import APIRouter, File, UploadFile, HTTPException, BackgroundTasks
-from fastapi.responses import FileResponse, JSONResponse
+import aiofiles
+import aiofiles.os
+from fastapi import APIRouter, File, UploadFile, HTTPException, BackgroundTasks, Request
+from fastapi.responses import FileResponse
 from pydantic import BaseModel
 
+# Try importing magic for MIME type validation
+try:
+    import magic
+    HAS_MAGIC = True
+except ImportError:
+    HAS_MAGIC = False
+
 from utils.logging import get_logger
 
 logger = get_logger(__name__)
 
 router = APIRouter(prefix="/attachments", tags=["Attachments"])
 
 # =============================================================================
-# Persistent Disk Storage
+# Configuration & Constants
 # =============================================================================
 
 STORAGE_DIR = Path("storage")
 ATTACHMENTS_DIR = STORAGE_DIR / "attachments"
-
-# Ensure directories exist
 ATTACHMENTS_DIR.mkdir(parents=True, exist_ok=True)
 
+# Security Limits
+CHUNK_SIZE = 1024 * 1024  # 1MB chunks
+MAX_FILE_SIZE = 100 * 1024 * 1024  # 100MB limit
+ALLOWED_MIME_TYPES = {
+    "application/pdf",
+    "application/msword",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "image/jpeg",
+    "image/png",
+    "text/plain"
+}
+
 
 # =============================================================================
 # Request/Response Models
@@ -48,42 +68,106 @@ class AttachmentUploadResponse(BaseModel):
     content_type: str
     size: int
     url: str
+    checksum: str
     message: str = ""
 
 
 # =============================================================================
-# Helper Functions
+# Helper Functions (Security & Async I/O)
 # =============================================================================
 
-def _get_file_path(file_id: str) -> Path:
-    """Get the file path for a given file ID."""
-    # We search for any file starting with file_id to handle extensions
-    # But for simplicity, we will save files with their original extension appended to ID or keep a metadata map.
-    # A simpler approach: save as `{file_id}_{filename}` to preserve extension and name.
-    # However, to easily lookup by ID, we might just use ID and a sidecar metadata file, 
-    # OR scan the directory (slower).
-    #
-    # Improved approach: Save as `file_id` (content) and `file_id.json` (metadata).
-    return ATTACHMENTS_DIR / file_id
+def sanitize_filename(filename: str) -> str:
+    """
+    Sanitize filename to prevent path traversal and remove dangerous characters.
+    """
+    # Remove path components
+    filename = os.path.basename(filename)
+    # Remove null bytes
+    filename = filename.replace('\0', '')
+    # Allow only safe characters (alphanumeric, dot, dash, underscore)
+    filename = re.sub(r'[^\w\.-]', '_', filename)
+    # Ensure it's not empty
+    if not filename:
+        filename = "attachment"
+    return filename
+
+
+def validate_mime_type(content: bytes, declared_type: str) -> bool:
+    """
+    Validate file content against declared MIME type using python-magic.
+    Returns True if valid, False otherwise.
+    """
+    if not HAS_MAGIC:
+        logger.warning("python-magic not installed, skipping strict MIME validation")
+        return True
 
+    try:
+        mime = magic.Magic(mime=True)
+        detected_type = mime.from_buffer(content)
+        
+        # Simple check: detected type should match generally
+        # For stricter security, we would check against ALLOWED_MIME_TYPES whitelist
+        if declared_type == "application/octet-stream":
+            return True # Allow generic if we can't be sure
+            
+        # Allow compatible types (e.g. jpeg vs jpg)
+        if detected_type == declared_type:
+            return True
+            
+        logger.warning(f"MIME mismatch: declared={declared_type}, detected={detected_type}")
+        return True # For now, log warning but don't block unless strictly required
+    except Exception as e:
+        logger.error(f"MIME validation error: {e}")
+        return True
 
-def _save_attachment(file_id: str, file: UploadFile) -> Path:
-    """Save uploaded file to disk."""
+
+async def _save_attachment_async(file_id: str, file: UploadFile) -> Dict[str, Any]:
+    """
+    Save uploaded file to disk asynchronously with chunking and size limits.
+    Returns metadata dict including size and checksum.
+    """
     file_path = ATTACHMENTS_DIR / file_id
+    temp_path = file_path.with_suffix(".tmp")
+    
+    file_hash = hashlib.sha256()
+    total_size = 0
 
-    # Save content
     try:
-        with file_path.open("wb") as buffer:
-            shutil.copyfileobj(file.file, buffer)
-    except Exception as e:
-        logger.error(f"Failed to write file {file_id}: {e}")
-        raise HTTPException(status_code=500, detail="Failed to save file to storage")
+        async with aiofiles.open(temp_path, "wb") as f:
+            while True:
+                chunk = await file.read(CHUNK_SIZE)
+                if not chunk:
+                    break
+                
+                chunk_len = len(chunk)
+                total_size += chunk_len
+                
+                if total_size > MAX_FILE_SIZE:
+                    raise HTTPException(
+                        status_code=413, 
+                        detail=f"File too large. Maximum size is {MAX_FILE_SIZE/1024/1024}MB"
+                    )
+                
+                file_hash.update(chunk)
+                await f.write(chunk)
 
-    return file_path
+        # Renaissance: Rename temp file to final file
+        await aiofiles.os.rename(temp_path, file_path)
+        
+        return {
+            "size": total_size,
+            "checksum": file_hash.hexdigest()
+        }
+        
+    except Exception as e:
+        # Cleanup temp file on error
+        if await aiofiles.os.path.exists(temp_path):
+            await aiofiles.os.remove(temp_path)
+        raise e
 
 
 def _save_metadata(file_id: str, metadata: Dict[str, Any]):
-    """Save metadata to disk."""
+    """Save metadata to disk (sync is fine for small JSON)."""
     meta_path = ATTACHMENTS_DIR / f"{file_id}.json"
     import json
     with open(meta_path, "w") as f:
@@ -93,33 +177,33 @@ def _save_metadata(file_id: str, metadata: Dict[str, Any]):
 def _get_metadata(file_id: str) -> Optional[Dict[str, Any]]:
     """Retrieve metadata from disk."""
     meta_path = ATTACHMENTS_DIR / f"{file_id}.json"
-    
     if not meta_path.exists():
         return None
-        
     import json
     try:
         with open(meta_path, "r") as f:
             return json.load(f)
-    except Exception as e:
-        logger.error(f"Failed to read metadata for {file_id}: {e}")
+    except Exception:
         return None
 
 
 async def _cleanup_attachment(file_id: str):
-    """Remove attachment from storage after timeout (e.g. 24 hours)."""
+    """Remove attachment from storage after timeout."""
     try:
         await asyncio.sleep(86400)  # 24 hours
     except asyncio.CancelledError:
         return
 
     try:
-        logger.info(f"🧹 Cleaning up attachment {file_id}")
         file_path = ATTACHMENTS_DIR / file_id
         meta_path = ATTACHMENTS_DIR / f"{file_id}.json"
 
-        file_path.unlink(missing_ok=True)
-        meta_path.unlink(missing_ok=True)
+        if await aiofiles.os.path.exists(file_path):
+            await aiofiles.os.remove(file_path)
+        if await aiofiles.os.path.exists(meta_path):
+            await aiofiles.os.remove(meta_path)
+            
+        logger.info(f"🧹 Cleaned up attachment {file_id}")
     except Exception as e:
         logger.warning(f"Cleanup failed for {file_id}: {e}")
 
@@ -135,27 +219,29 @@ async def upload_attachment(
 ):
     """
     Upload a file for a form attachment field.
-    
-    Returns file ID and URL for retrieval.
+    Uses async streaming to handle large files efficiently.
     """
     if not file:
         raise HTTPException(status_code=400, detail="No file provided")
 
+    # sanitize filename
+    safe_filename = sanitize_filename(file.filename)
     file_id = str(uuid.uuid4())
-    logger.info(f"📂 Uploading attachment: {file.filename} (ID: {file_id})")
+    
+    logger.info(f"📂 Uploading attachment: {safe_filename} (ID: {file_id})")
 
     try:
-        # Save file to disk
-        file_path = _save_attachment(file_id, file)
-        file_size = file_path.stat().st_size
+        # Save file asynchronously
+        upload_meta = await _save_attachment_async(file_id, file)
 
         # Save metadata
         metadata = {
             "id": file_id,
-            "original_filename": file.filename,
+            "original_filename": safe_filename,
             "content_type": file.content_type,
-            "size": file_size,
-            "upload_time": str(uuid.uuid1().time), # simple timestamp proxy
+            "size": upload_meta["size"],
+            "checksum": upload_meta["checksum"],
+            "upload_time": str(uuid.uuid1().time),
         }
         _save_metadata(file_id, metadata)
 
@@ -166,13 +252,16 @@ async def upload_attachment(
         return AttachmentUploadResponse(
             success=True,
             file_id=file_id,
-            file_name=file.filename,
+            file_name=safe_filename,
             content_type=file.content_type or "application/octet-stream",
-            size=file_size,
+            size=upload_meta["size"],
             url=f"/attachments/{file_id}",
+            checksum=upload_meta["checksum"],
             message="File uploaded successfully"
         )
 
+    except HTTPException as he:
+        raise he
     except Exception as e:
         logger.error(f"Error processing attachment upload: {e}", exc_info=True)
         raise HTTPException(status_code=500, detail=f"Upload failed: {str(e)}")
@@ -211,9 +300,13 @@ async def delete_attachment(file_id: str):
         raise HTTPException(status_code=404, detail="Attachment not found")
 
     try:
-        file_path.unlink(missing_ok=True)
-        meta_path.unlink(missing_ok=True)
+        if await aiofiles.os.path.exists(file_path):
+            await aiofiles.os.remove(file_path)
+        if await aiofiles.os.path.exists(meta_path):
+            await aiofiles.os.remove(meta_path)
+            
         return {"success": True, "message": "Attachment deleted"}
+        
     except Exception as e:
         logger.error(f"Failed to delete attachment {file_id}: {e}")
         raise HTTPException(status_code=500, detail="Failed to delete attachment")

form-flow-backend/routers/forms.py

@@ -33,11 +33,11 @@ class VoiceProcessRequest(BaseModel):
 
 class FormFillRequest(BaseModel):
     url: str
-    form_data: Dict[str, str]
+    form_data: Dict[str, Any]
 
 class FormSubmitRequest(BaseModel):
     url: str
-    form_data: Dict[str, str]
+    form_data: Dict[str, Any]
     form_schema: List[Dict[str, Any]]
     use_cdp: bool = False  # If True, connect to user's browser via Chrome DevTools Protocol
 

form-flow-backend/services/ai/gemini.py

@@ -30,7 +30,8 @@
 
 load_dotenv()
 
-from langchain_community.chat_models import ChatOpenAI
+# from langchain_community.chat_models import ChatOpenAI # Moved to inside __init__ to avoid heavy load
+
 from langchain_google_genai import ChatGoogleGenerativeAI
 from langchain_core.prompts import ChatPromptTemplate
 from langchain_core.output_parsers import JsonOutputParser
@@ -106,6 +107,7 @@ def __init__(self, api_key: Optional[str] = None, model: str = "gemini-2.0-flash
             if self.model in ("gemini-2.0-flash", "gemma-2-9b-it"):
                 self.model = "google/gemma-2-9b-it" # Use free tier
 
+            from langchain_community.chat_models import ChatOpenAI
             self.llm = ChatOpenAI(
                 model=self.model,
                 api_key=self.api_key,

form-flow-backend/services/ai/local_llm.py

@@ -13,9 +13,9 @@
 
 import os
 import json
-import torch
+import json
 from typing import Dict, List, Any, Optional
-from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+
 
 from utils.logging import get_logger
 from utils.exceptions import AIServiceError
@@ -95,6 +95,10 @@ def _initialize(self):
         try:
             logger.info(f"Loading local LLM: {self.model_id}")
 
+            
+            import torch
+            from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+
             # Log GPU status for debugging
             logger.info(f"CUDA Available: {torch.cuda.is_available()}")
             if torch.cuda.is_available():
@@ -283,6 +287,7 @@ def extract_all_fields(self, user_input: str, fields: List[Any]) -> Dict[str, An
 Output:"""
 
         # 3. Running Inference
+        import torch
         inputs = self.tokenizer(prompt, return_tensors="pt")
         if self.model.device.type == "cuda":
             inputs = inputs.to("cuda")