Lead ingestion + API-first lead management UI (CSV/TXT/JSON/XLSX, normalization, CORS)

README.md

@@ -1,20 +1,21 @@
 # Midas
 
-Midas is a production-oriented MVP for automated cold outreach, follow-up orchestration, and reply handling. It combines:
+Midas is a production-oriented platform for autonomous cold outreach operations. This iteration ships a robust **lead ingestion and storage foundation** with a modern UX and API-first backend.
 
-- **Lead ingestion** (CSV/TXT/JSON)
-- **De-duplication** against prior contacts and opt-outs
-- **Template generation/optimization agents**
-- **Outreach sending and follow-up sequencing**
-- **Reply sentiment analysis + draft response generation**
-- **Dashboard** with campaign metrics, alerts, and controls
-- **Provider/model failover** (API key/model rotation)
+## What this base includes
+
+- Multi-format lead upload: **CSV / TXT / JSON / XLSX**
+- Field normalization for common header variations (`full_name`, `email_address`, `job_title`, etc.)
+- Database-backed deduplication and opt-out safeguards
+- Lead inventory browsing with search + status filters
+- API-first backend designed for separate frontend/backend deployments
+- Standalone frontend bundle (`/frontend`) that can run on a different host from the API
 
 ## Stack
 
-- FastAPI + Jinja dashboard
-- SQLAlchemy + SQLite (swap-ready for PostgreSQL)
-- Agent layer with pluggable providers (Google ADK/Gemini-ready interfaces)
+- Backend: FastAPI + SQLAlchemy
+- Database: SQLite by default (PostgreSQL-ready)
+- Frontend: static app (HTML/CSS/JS) with API integration
 
 ## Quickstart
 
@@ -25,19 +26,26 @@ pip install -e .[dev]
 uvicorn app.main:app --reload
 ```
 
-Open `http://127.0.0.1:8000`.
+Open `http://127.0.0.1:8000` for the integrated dashboard.
 
-## Environment
+To host frontend separately:
+
+1. Serve `frontend/` on any static host.
+2. Set `window.MIDAS_API_BASE` in `frontend/index.html` to your backend URL.
+3. Configure backend CORS via `MIDAS_CORS_ALLOWED_ORIGINS`.
+
+## API endpoints (lead foundation)
 
-Set optional env vars:
+- `GET /api/v1/health`
+- `GET /api/v1/dashboard`
+- `GET /api/v1/leads?status=&search=&limit=&offset=`
+- `POST /api/v1/leads/import` (multipart `file`)
+
+## Environment
 
 - `MIDAS_DB_URL` (default: `sqlite:///./midas.db`)
+- `MIDAS_CORS_ALLOWED_ORIGINS` (default: `*`, comma-separated)
 - `MIDAS_SENDER_EMAIL` (default: `hello@midas.local`)
 - `MIDAS_DAILY_SEND_LIMIT_PER_MAILBOX` (default: `80`)
 - `MIDAS_REPLY_AUTO_SEND_DELAY_MINUTES` (default: `60`)
-- `MIDAS_MODEL_CONFIG` JSON list for model/key rotation (see `app/core/config.py`)
-
-## Notes
-
-- Email sending and inbound sync use adapter interfaces with a safe local logger implementation by default.
-- Replace adapters in `app/services/email_gateway.py` and `app/services/inbox_sync.py` for SMTP/IMAP, Gmail API, SES, etc.
+- `MIDAS_MODEL_CONFIG` JSON list for model/key rotation

app/api/routes.py

@@ -1,47 +1,130 @@
 from __future__ import annotations
 
-from fastapi import APIRouter, Depends, File, Form, Request, UploadFile
+from datetime import datetime
+
+from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile
 from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.templating import Jinja2Templates
-from sqlalchemy import select
+from sqlalchemy import func, select
 from sqlalchemy.orm import Session
 
 from app.db.session import get_db
-from app.models.entities import Alert, Lead, ReplyMessage
-from app.models.schemas import IncomingReply
+from app.models.entities import Alert, Lead, LeadStatus, ReplyMessage
+from app.models.schemas import IncomingReply, LeadImportResult, LeadListResponse, LeadOut
 from app.services.campaign_service import CampaignService
 from app.services.lead_importer import LeadImporter
 
 router = APIRouter()
+api_router = APIRouter(prefix="/api/v1", tags=["api"])
 templates = Jinja2Templates(directory="app/templates")
 
 
 @router.get("/", response_class=HTMLResponse)
-def dashboard(request: Request, db: Session = Depends(get_db)):
+def dashboard(request: Request):
+    return templates.TemplateResponse(request, "dashboard.html", {})
+
+
+@router.post("/leads/import")
+async def import_leads(file: UploadFile = File(...), db: Session = Depends(get_db)):
+    payload = await file.read()
+    importer = LeadImporter(db)
+    rows = importer.parse(file.filename or "upload", payload)
+    _ = importer.import_rows(rows)
+    return RedirectResponse(url="/", status_code=303)
+
+
+@api_router.get("/health")
+def health_check() -> dict[str, str]:
+    return {"status": "ok"}
+
+
+@api_router.get("/dashboard")
+def dashboard_data(db: Session = Depends(get_db)):
     service = CampaignService(db)
     metrics = service.metrics()
-    leads = db.scalars(select(Lead).order_by(Lead.created_at.desc()).limit(20)).all()
     alerts = db.scalars(select(Alert).order_by(Alert.created_at.desc()).limit(10)).all()
     replies = db.scalars(select(ReplyMessage).order_by(ReplyMessage.received_at.desc()).limit(10)).all()
-    return templates.TemplateResponse(
-        request,
-        "dashboard.html",
-        {
-            "metrics": metrics,
-            "leads": leads,
-            "alerts": alerts,
-            "replies": replies,
-        },
+    return {
+        "metrics": metrics.model_dump(),
+        "alerts": [
+            {
+                "id": item.id,
+                "severity": item.severity,
+                "message": item.message,
+                "created_at": item.created_at.isoformat(),
+            }
+            for item in alerts
+        ],
+        "replies": [
+            {
+                "id": reply.id,
+                "lead_id": reply.lead_id,
+                "sentiment": reply.sentiment.value,
+                "received_at": reply.received_at.isoformat(),
+            }
+            for reply in replies
+        ],
+    }
+
+
+@api_router.get("/leads", response_model=LeadListResponse)
+def list_leads(
+    db: Session = Depends(get_db),
+    status: LeadStatus | None = Query(default=None),
+    search: str = Query(default="", min_length=0, max_length=255),
+    limit: int = Query(default=50, ge=1, le=200),
+    offset: int = Query(default=0, ge=0),
+):
+    query = select(Lead)
+    count_query = select(func.count()).select_from(Lead)
+
+    if status:
+        query = query.where(Lead.status == status)
+        count_query = count_query.where(Lead.status == status)
+
+    search_value = search.strip()
+    if search_value:
+        pattern = f"%{search_value}%"
+        query = query.where((Lead.email.ilike(pattern)) | (Lead.name.ilike(pattern)) | (Lead.company.ilike(pattern)))
+        count_query = count_query.where((Lead.email.ilike(pattern)) | (Lead.name.ilike(pattern)) | (Lead.company.ilike(pattern)))
+
+    total = db.scalar(count_query) or 0
+    leads = db.scalars(query.order_by(Lead.created_at.desc()).offset(offset).limit(limit)).all()
+
+    return LeadListResponse(
+        total=total,
+        leads=[
+            LeadOut(
+                id=lead.id,
+                name=lead.name,
+                email=lead.email,
+                company=lead.company,
+                position=lead.position,
+                niche=lead.niche,
+                status=lead.status.value,
+                created_at=lead.created_at.isoformat(),
+            )
+            for lead in leads
+        ],
     )
 
 
-@router.post("/leads/import")
-async def import_leads(file: UploadFile = File(...), db: Session = Depends(get_db)):
+@api_router.post("/leads/import", response_model=LeadImportResult)
+async def import_leads_api(file: UploadFile = File(...), db: Session = Depends(get_db)):
+    if not file.filename:
+        raise HTTPException(status_code=400, detail="Uploaded file must include a filename.")
+
     payload = await file.read()
+    if not payload:
+        raise HTTPException(status_code=400, detail="Uploaded file is empty.")
+
     importer = LeadImporter(db)
-    rows = importer.parse(file.filename, payload)
-    _ = importer.import_rows(rows)
-    return RedirectResponse(url="/", status_code=303)
+    try:
+        rows = importer.parse(file.filename, payload)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    return importer.import_rows(rows)
 
 
 @router.post("/templates/generate")
@@ -69,7 +152,7 @@ def send_followups(db: Session = Depends(get_db)):
 def ingest_reply(payload: IncomingReply, db: Session = Depends(get_db)):
     service = CampaignService(db)
     service.process_incoming_reply(payload.lead_email, payload.raw_body)
-    return {"status": "ok"}
+    return {"status": "ok", "received_at": datetime.utcnow().isoformat()}
 
 
 @router.post("/reply/{lead_id}/approve")

app/core/config.py

@@ -25,6 +25,13 @@ class Settings:
     reply_auto_send_delay_minutes: int = int(
         os.getenv("MIDAS_REPLY_AUTO_SEND_DELAY_MINUTES", "60")
     )
+    cors_allowed_origins: list[str] = field(
+        default_factory=lambda: [
+            origin.strip()
+            for origin in os.getenv("MIDAS_CORS_ALLOWED_ORIGINS", "*").split(",")
+            if origin.strip()
+        ]
+    )
     model_config_raw: str = os.getenv(
         "MIDAS_MODEL_CONFIG",
         json.dumps(

app/main.py

@@ -1,14 +1,24 @@
 from __future__ import annotations
 
 from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
 
-from app.api.routes import router
+from app.api.routes import api_router, router
+from app.core.config import settings
 from app.db.session import init_db
 
 app = FastAPI(title="Midas")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.cors_allowed_origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
 app.mount("/static", StaticFiles(directory="app/static"), name="static")
 app.include_router(router)
+app.include_router(api_router)
 
 
 @app.on_event("startup")

app/models/schemas.py

@@ -17,6 +17,23 @@ class LeadImportResult(BaseModel):
     inserted: int
     skipped_existing: int
     skipped_opted_out: int
+    invalid_rows: int
+
+
+class LeadOut(BaseModel):
+    id: int
+    name: str
+    email: EmailStr
+    company: str | None
+    position: str | None
+    niche: str | None
+    status: str
+    created_at: str
+
+
+class LeadListResponse(BaseModel):
+    total: int
+    leads: list[LeadOut]
 
 
 class DraftEmail(BaseModel):