feat: resilient background job retry & monitoring

README.md

@@ -65,7 +65,72 @@ OpenAPI: `backend/app/openapi.yaml`
 - Expenses: CRUD `/expenses`
 - Bills: CRUD `/bills`, pay/mark `/bills/{id}/pay`
 - Reminders: CRUD `/reminders`, trigger `/reminders/run`
+  - Admin monitoring: `/admin/reminders/metrics`, `/admin/reminders/failures`
 - Insights: `/insights/monthly`, `/insights/budget-suggestion`
+- Digest: `/digest/weekly`, `/digest/weekly/send`, `/digest/weekly/preview`
+
+## Weekly Financial Digest
+
+FinMind provides a **Smart Weekly Financial Summary** that delivers actionable insights directly to users' inboxes.
+
+### Features
+- **Comprehensive Overview**: Total income, expenses, net flow, and savings rate for the week
+- **Week-over-Week Trends**: Compare spending and income changes from the previous week
+- **Category Breakdown**: Detailed spending analysis by category with percentages
+- **Notable Transactions**: Highlights of significant expenses and income
+- **Upcoming Bills**: Preview of bills due in the next 7 days
+- **Smart Insights**: AI-generated financial tips and recommendations
+
+### Endpoints
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `/digest/weekly` | GET | Get weekly digest data (JSON) |
+| `/digest/weekly/send` | POST | Send digest email to current user |
+| `/digest/weekly/preview` | GET | Preview email subject and body |
+
+### Scheduled Delivery
+The weekly digest is automatically generated and sent to all users every **Sunday at 9:00 AM UTC** via APScheduler.
+
+### Manual Trigger (CLI)
+```bash
+# Generate digest for all users
+flask generate-digest
+
+# Generate for specific user with email
+flask generate-digest --user-id 1 --send-email
+```
+
+### Scheduler Status
+Check scheduler status at `/scheduler/status` endpoint.
+
+## Resilient Reminder Delivery & Monitoring
+
+FinMind's reminder system includes robust retry logic with exponential backoff to ensure reliable delivery even during transient failures.
+
+### Retry Mechanism
+- Each reminder tracks retry attempts with `retry_count`, `last_retry_at`, `next_retry_at`, `failure_reason`, and `status`.
+- When a reminder fails to send, it is automatically rescheduled with exponential backoff:
+  - Base delay: 5 minutes
+  - Multiplier: 2^(retry_count - 1)
+  - Maximum retries: 5
+- After max retries, the reminder status becomes `failed` and no further attempts are made.
+- Successful delivery clears all retry tracking fields and marks the reminder as `sent`.
+
+### Admin Monitoring Endpoints
+Administrators can monitor reminder health and failures via:
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `/admin/reminders/metrics` | GET | Aggregate metrics: total, counts by status, pending retry count, average retries for failed, recent failure details |
+| `/admin/reminders/failures` | GET | List failed reminders (use `?limit=` to control page size) |
+
+These endpoints require admin role (JWT with `role=ADMIN`).
+
+### Reminder Status Values
+- `pending`: Not yet sent, waiting for scheduled time
+- `sent`: Successfully delivered
+- `retrying`: Failed, waiting for next retry attempt
+- `failed`: Permanent failure after exhausting all retries
 
 ## MVP UI/UX Plan
 - Auth screens: register/login.
@@ -104,11 +169,14 @@ finmind/
         bills.py
         reminders.py
         insights.py
+        digest.py
       services/
         __init__.py
         ai.py
         cache.py
         reminders.py
+        digest.py
+        scheduler.py
       db/
         schema.sql
       openapi.yaml

packages/backend/app/__init__.py

@@ -14,6 +14,14 @@
 import logging
 from datetime import timedelta
 
+# Scheduler imports
+from .services.scheduler import (
+    init_scheduler,
+    start_scheduler,
+    shutdown_scheduler,
+    get_scheduler_status,
+)
+
 
 def create_app(settings: Settings | None = None) -> Flask:
     app = Flask(__name__)
@@ -56,6 +64,12 @@ def create_app(settings: Settings | None = None) -> Flask:
     with app.app_context():
         _ensure_schema_compatibility(app)
 
+    # Initialize scheduler (but don't start automatically in testing)
+    if not app.config.get("TESTING"):
+        init_scheduler(app)
+        start_scheduler()
+        logger.info("Scheduler initialized and started")
+
     @app.before_request
     def _before_request():
         init_request_context()
@@ -73,6 +87,11 @@ def metrics():
         obs = app.extensions["observability"]
         return obs.metrics_response()
 
+    @app.get("/scheduler/status")
+    def scheduler_status():
+        """Get the current status of scheduled jobs."""
+        return jsonify(get_scheduler_status()), 200
+
     @app.errorhandler(500)
     def internal_error(_error):
         return jsonify(error="internal server error"), 500
@@ -93,6 +112,28 @@ def init_db():
             finally:
                 conn.close()
 
+    @app.cli.command("generate-digest")
+    @click.option("--user-id", type=int, default=None, help="Generate for specific user")
+    @click.option("--send-email", is_flag=True, help="Send email to user")
+    def generate_digest(user_id, send_email):
+        """Generate weekly digest for testing."""
+        from .services.digest import WeeklyDigestService
+
+        with app.app_context():
+            if user_id:
+                week_start, week_end = WeeklyDigestService.get_week_bounds()
+                summary = WeeklyDigestService.generate_weekly_summary(
+                    user_id, week_start, week_end
+                )
+                if send_email:
+                    success = WeeklyDigestService.send_digest_email(user_id, summary)
+                    click.echo(f"Digest sent: {success}")
+                else:
+                    click.echo(f"Digest generated: {summary}")
+            else:
+                results = WeeklyDigestService.generate_and_send_all_digests()
+                click.echo(f"Batch results: {results}")
+
     return app
 
 
@@ -103,17 +144,29 @@ def _ensure_schema_compatibility(app: Flask) -> None:
     conn = db.engine.raw_connection()
     try:
         cur = conn.cursor()
+        # Add users.preferred_currency if missing
         cur.execute(
             """
             ALTER TABLE users
             ADD COLUMN IF NOT EXISTS preferred_currency VARCHAR(10)
             NOT NULL DEFAULT 'INR'
             """
         )
+        # Add reminders retry tracking columns if missing
+        cur.execute(
+            """
+            ALTER TABLE reminders
+            ADD COLUMN IF NOT EXISTS retry_count INT NOT NULL DEFAULT 0,
+            ADD COLUMN IF NOT EXISTS last_retry_at TIMESTAMP,
+            ADD COLUMN IF NOT EXISTS next_retry_at TIMESTAMP,
+            ADD COLUMN IF NOT EXISTS failure_reason VARCHAR(500),
+            ADD COLUMN IF NOT EXISTS status VARCHAR(20) NOT NULL DEFAULT 'pending'
+            """
+        )
         conn.commit()
     except Exception:
         app.logger.exception(
-            "Schema compatibility patch failed for users.preferred_currency"
+            "Schema compatibility patch failed"
         )
         conn.rollback()
     finally:

packages/backend/app/db/schema.sql

@@ -91,7 +91,12 @@ CREATE TABLE IF NOT EXISTS reminders (
   message VARCHAR(500) NOT NULL,
   send_at TIMESTAMP NOT NULL,
   sent BOOLEAN NOT NULL DEFAULT FALSE,
-  channel VARCHAR(20) NOT NULL DEFAULT 'email'
+  channel VARCHAR(20) NOT NULL DEFAULT 'email',
+  retry_count INT NOT NULL DEFAULT 0,
+  last_retry_at TIMESTAMP,
+  next_retry_at TIMESTAMP,
+  failure_reason VARCHAR(500),
+  status VARCHAR(20) NOT NULL DEFAULT 'pending'
 );
 CREATE INDEX IF NOT EXISTS idx_reminders_due ON reminders(user_id, sent, send_at);
 

packages/backend/app/models.py

@@ -98,6 +98,16 @@ class Reminder(db.Model):
     send_at = db.Column(db.DateTime, nullable=False)
     sent = db.Column(db.Boolean, default=False, nullable=False)
     channel = db.Column(db.String(20), default="email", nullable=False)
+    # Retry tracking fields
+    retry_count = db.Column(db.Integer, default=0, nullable=False)
+    last_retry_at = db.Column(db.DateTime, nullable=True)
+    next_retry_at = db.Column(db.DateTime, nullable=True)
+    failure_reason = db.Column(db.String(500), nullable=True)
+    status = db.Column(
+        db.String(20),
+        default="pending",
+        nullable=False,
+    )  # pending, sent, failed, retrying
 
 
 class AdImpression(db.Model):

packages/backend/app/openapi.yaml

@@ -12,6 +12,7 @@ tags:
   - name: Bills
   - name: Reminders
   - name: Insights
+  - name: Admin
 paths:
   /auth/register:
     post:
@@ -444,6 +445,55 @@ paths:
                 properties:
                   created: { type: integer }
 
+  /admin/reminders/metrics:
+    get:
+      summary: Get reminder processing metrics (admin only)
+      tags: [Admin]
+      security: [{ bearerAuth: [] }]
+      responses:
+        '200':
+          description: Metrics summary
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  total: { type: integer }
+                  by_status: { type: object, additionalProperties: { type: integer } }
+                  pending_retry: { type: integer }
+                  avg_retries_for_failed: { type: number }
+                  recent_failures: { type: array, items: { $ref: '#/components/schemas/Failure' } }
+        '403':
+          description: Admin access required
+          content:
+            application/json:
+              schema: { $ref: '#/components/schemas/Error' }
+
+  /admin/reminders/failures:
+    get:
+      summary: List failed reminders (admin only)
+      tags: [Admin]
+      security: [{ bearerAuth: [] }]
+      parameters:
+        - in: query
+          name: limit
+          required: false
+          schema: { type: integer, default: 100 }
+      responses:
+        '200':
+          description: List of failed reminders
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/FailureDetail'
+        '403':
+          description: Admin access required
+          content:
+            application/json:
+              schema: { $ref: '#/components/schemas/Error' }
+
   /insights/budget-suggestion:
     get:
       summary: Get monthly budget suggestion
@@ -580,10 +630,36 @@ components:
         send_at: { type: string, format: date-time }
         sent: { type: boolean }
         channel: { type: string, enum: [email, whatsapp] }
+        status: { type: string, enum: [pending, sent, failed, retrying] }
+        retry_count: { type: integer }
+        last_retry_at: { type: string, format: date-time, nullable: true }
+        next_retry_at: { type: string, format: date-time, nullable: true }
+        failure_reason: { type: string, nullable: true }
     NewReminder:
       type: object
       required: [message, send_at]
       properties:
         message: { type: string }
         send_at: { type: string, format: date-time }
         channel: { type: string, enum: [email, whatsapp], default: email }
+    Failure:
+      type: object
+      properties:
+        id: { type: integer }
+        user_id: { type: integer }
+        message: { type: string }
+        retry_count: { type: integer }
+        last_retry_at: { type: string, format: date-time, nullable: true }
+        failure_reason: { type: string, nullable: true }
+    FailureDetail:
+      type: object
+      properties:
+        id: { type: integer }
+        user_id: { type: integer }
+        bill_id: { type: integer, nullable: true }
+        message: { type: string }
+        channel: { type: string }
+        send_at: { type: string, format: date-time }
+        retry_count: { type: integer }
+        last_retry_at: { type: string, format: date-time, nullable: true }
+        failure_reason: { type: string, nullable: true }

packages/backend/app/routes/__init__.py

@@ -7,6 +7,8 @@
 from .categories import bp as categories_bp
 from .docs import bp as docs_bp
 from .dashboard import bp as dashboard_bp
+from .digest import bp as digest_bp
+from .admin import bp as admin_bp
 
 
 def register_routes(app: Flask):
@@ -18,3 +20,5 @@ def register_routes(app: Flask):
     app.register_blueprint(categories_bp, url_prefix="/categories")
     app.register_blueprint(docs_bp, url_prefix="/docs")
     app.register_blueprint(dashboard_bp, url_prefix="/dashboard")
+    app.register_blueprint(digest_bp, url_prefix="/digest")
+    app.register_blueprint(admin_bp, url_prefix="/admin")