Celery+redis+prom+grafana

.gitignore

@@ -8,8 +8,11 @@ plans
 .claude*
 CLAUDE.md
 temp*
-.next_features
-.envv
+!tempo
+!tempo/
+!**/tempo.yaml
+!**/tempo.yml
+.env*
 .infisical.json
 .task*
 task*

alertmanager/alertmanager.yml

@@ -0,0 +1,40 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  group_by: ['alertname', 'severity']
+  # Wait 30s after first alert fires before sending - collects related alerts into one message.
+  group_wait: 30s
+  # After the initial message, wait 5m before sending an update if new alerts join the group.
+  group_interval: 5m
+  # If an alert is still firing, re-notify every 4h so it doesn't silently stay broken.
+  repeat_interval: 4h
+  receiver: slack-alerts
+
+receivers:
+  - name: slack-alerts
+    slack_configs:
+      # Webhook URL is written to this file by the container entrypoint from $SLACK_WEBHOOK_URL.
+      # Same pattern as Prometheus reading the metrics bearer token from /run/metrics_token.
+      - api_url_file: /tmp/slack_webhook_url
+        channel: '#spendhound-alerts'
+        username: SpendHound
+        send_resolved: true
+        color: '{{ if eq .CommonLabels.severity "critical" }}danger{{ else }}warning{{ end }}'
+        title: '{{ if eq .Status "firing" }}[FIRING]{{ else }}[RESOLVED]{{ end }} {{ .CommonLabels.alertname }}'
+        text: |
+          {{ range .Alerts -}}
+          *{{ .Annotations.summary }}*
+          Severity: {{ .Labels.severity }}
+          {{ .Annotations.description }}
+          {{ if eq .Status "firing" }}Since: {{ .StartsAt.Format "2006-01-02 15:04 UTC" }}{{ else }}Resolved: {{ .EndsAt.Format "2006-01-02 15:04 UTC" }}{{ end }}
+          {{ end }}
+
+# Suppress downstream alerts when the root cause is already firing.
+# If BackendDown fires, HighErrorRate is a symptom - no need for two messages.
+inhibit_rules:
+  - source_match:
+      alertname: BackendDown
+    target_match:
+      alertname: HighErrorRate
+    equal: []

backend/app/celery_app.py

@@ -17,9 +17,31 @@
 
 from celery import Celery
 from celery.schedules import crontab
+from celery.signals import worker_init
 
 from app.config import settings
 
+if settings.sentry_dsn:
+    import sentry_sdk
+    from sentry_sdk.integrations.celery import CeleryIntegration
+
+    sentry_sdk.init(
+        dsn=settings.sentry_dsn,
+        environment=settings.sentry_environment,
+        traces_sample_rate=0.0,
+        send_default_pii=False,
+        integrations=[CeleryIntegration()],
+    )
+
+
+@worker_init.connect
+def _on_worker_init(**kwargs: object) -> None:
+    if settings.otel_endpoint:
+        from app.services.tracing import setup_tracing
+
+        setup_tracing(settings.otel_service_name, settings.otel_endpoint)
+
+
 celery_app = Celery(
     "spendhound",
     broker=settings.redis_url,
@@ -51,11 +73,17 @@
 )
 
 # Beat schedule — runs when `celery beat` is started (see docker-compose celery_beat service).
-# Wall-clock crontab (UTC :00 and :30) so the reset time is predictable and the
-# frontend can derive an accurate countdown without any server-side state.
 celery_app.conf.beat_schedule = {
+    # Wall-clock crontab (UTC :00 and :30) so the reset time is predictable and the
+    # frontend can derive an accurate countdown without any server-side state.
     "reset-demo-user-on-the-half-hour": {
         "task": "app.tasks.demo_tasks.reset_demo_user",
         "schedule": crontab(minute="0,30"),
     },
+    # 1st of each month at 00:20 UTC - matches the former systemd timer schedule.
+    # Fans out one deliver_monthly_report task per eligible user.
+    "dispatch-monthly-reports": {
+        "task": "app.tasks.report_tasks.dispatch_monthly_reports",
+        "schedule": crontab(hour=0, minute=20, day_of_month=1),
+    },
 }

backend/app/config.py

@@ -152,6 +152,22 @@ class Settings(BaseSettings):
         default="",
         description="Bearer token required to scrape /metrics. Empty = endpoint blocked. Set in Infisical for prod.",
     )
+    sentry_dsn: str = Field(
+        default="",
+        description="Sentry DSN for error tracking. Empty = Sentry disabled. Set in Infisical for prod.",
+    )
+    sentry_environment: str = Field(
+        default="production",
+        description="Sentry environment tag sent with every event (production / development).",
+    )
+    otel_endpoint: str = Field(
+        default="",
+        description="OTLP HTTP base URL for OpenTelemetry traces (e.g. http://tempo:4318). Empty = tracing disabled.",
+    )
+    otel_service_name: str = Field(
+        default="spendhound",
+        description="OTel service name shown in Grafana Tempo. Set per container: spendhound-api, spendhound-worker.",
+    )
 
     debug: bool = Field(default=False, description="Enable debug mode")
     cors_origins: list[str] = Field(

backend/app/main.py

@@ -21,6 +21,28 @@
 from app.services.cache import close_redis, get_celery_queue_depth, init_redis
 from app.services.metrics import RATE_LIMIT_HITS_TOTAL, RECEIPT_QUEUE_DEPTH, classify_limit_type
 
+if settings.sentry_dsn:
+    import sentry_sdk
+    from sentry_sdk.integrations.fastapi import FastApiIntegration
+    from sentry_sdk.integrations.starlette import StarletteIntegration
+
+    sentry_sdk.init(
+        dsn=settings.sentry_dsn,
+        environment=settings.sentry_environment,
+        traces_sample_rate=0.0,
+        send_default_pii=False,
+        integrations=[StarletteIntegration(), FastApiIntegration()],
+    )
+
+if settings.otel_endpoint:
+    from app.services.tracing import setup_tracing
+
+    setup_tracing(settings.otel_service_name, settings.otel_endpoint)
+    # FastAPI instrumentation is deferred to create_app() via instrument_fastapi()
+    # because the FastAPI app must exist before middleware can be attached.
+    # Calling FastAPIInstrumentor().instrument() here (before app creation) patches
+    # FastAPI.__init__ globally and conflicts with the Prometheus Instrumentator.
+
 logger = structlog.get_logger(__name__)
 
 
@@ -85,6 +107,11 @@ async def _rate_limit_handler(request: Request, exc: Exception) -> Response:
 
     app.add_exception_handler(RateLimitExceeded, _rate_limit_handler)
 
+    if settings.otel_endpoint:
+        from app.services.tracing import instrument_fastapi
+
+        instrument_fastapi(app)
+
     app.add_middleware(
         CORSMiddleware,
         allow_origins=settings.cors_origins,
@@ -93,6 +120,27 @@ async def _rate_limit_handler(request: Request, exc: Exception) -> Response:
         allow_headers=["*"],
     )
 
+    if settings.sentry_dsn:
+
+        @app.middleware("http")
+        async def _attach_sentry_user(request: Request, call_next):  # type: ignore[no-untyped-def]
+            auth = request.headers.get("authorization", "")
+            if auth.startswith("Bearer "):
+                try:
+                    from jose import jwt as _jwt
+
+                    payload = _jwt.decode(
+                        auth.removeprefix("Bearer "),
+                        settings.jwt_secret,
+                        algorithms=[settings.jwt_algorithm],
+                    )
+                    user_id = payload.get("sub")
+                    if user_id:
+                        sentry_sdk.set_user({"id": user_id})
+                except Exception:
+                    pass
+            return await call_next(request)
+
     # Starlette 0.40+ adds _IncludedRouter objects to app.routes when include_router()
     # is used. prometheus-fastapi-instrumentator unconditionally accesses route.path on
     # every route, but _IncludedRouter has no .path → AttributeError on every request.

backend/app/services/metrics.py

@@ -17,6 +17,7 @@
 import time
 from collections.abc import AsyncGenerator
 
+from opentelemetry import trace as _otel_trace
 from prometheus_client import Counter, Gauge, Histogram
 
 from app.services.llm.base import BaseLLMProvider, LLMConfig, Message
@@ -58,13 +59,22 @@ def __init__(self, inner: BaseLLMProvider, provider_name: str) -> None:
         self._provider_name = provider_name
 
     async def complete(self, messages: list[Message], config: LLMConfig | None = None) -> str:
-        start = time.monotonic()
-        try:
-            return await self._inner.complete(messages, config)
-        finally:
-            LLM_RESPONSE_SECONDS.labels(provider=self._provider_name).observe(
-                time.monotonic() - start
-            )
+        tracer = _otel_trace.get_tracer(__name__)
+        with tracer.start_as_current_span(
+            "llm.complete",
+            attributes={"gen_ai.system": self._provider_name},
+        ) as span:
+            start = time.monotonic()
+            try:
+                return await self._inner.complete(messages, config)
+            except Exception as exc:
+                span.record_exception(exc)
+                span.set_status(_otel_trace.Status(_otel_trace.StatusCode.ERROR))
+                raise
+            finally:
+                LLM_RESPONSE_SECONDS.labels(provider=self._provider_name).observe(
+                    time.monotonic() - start
+                )
 
     async def stream(
         self, messages: list[Message], config: LLMConfig | None = None

backend/app/services/tracing.py

@@ -0,0 +1,73 @@
+"""OpenTelemetry tracing setup for SpendHound.
+
+Call setup_tracing() once at process startup. No-op when otel_endpoint is empty.
+Idempotent - safe to call multiple times; only the first call takes effect.
+
+Instrumented libraries (auto):
+  FastAPI/Starlette  - HTTP request spans with route, method, status
+  SQLAlchemy         - every DB query as a child span
+  HTTPX              - outbound HTTP calls (LLM provider APIs, Puppeteer)
+  Redis              - cache get/set/delete spans
+  Celery             - task publish (in API process) + task execute (in worker)
+
+Manual spans are added in MeteredLLMProvider.complete() with provider attributes.
+"""
+
+from __future__ import annotations
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+_initialized = False
+
+
+def setup_tracing(service_name: str, endpoint: str) -> None:
+    """Configure the global TracerProvider and auto-instrument common libraries.
+
+    service_name: label in Grafana Tempo service map (e.g. spendhound-api)
+    endpoint:     OTLP HTTP base URL for Grafana Tempo (e.g. http://tempo:4318)
+    """
+    global _initialized
+    if _initialized or not endpoint:
+        return
+    _initialized = True
+
+    from opentelemetry import trace
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+    from opentelemetry.instrumentation.celery import CeleryInstrumentor
+    from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
+    from opentelemetry.instrumentation.redis import RedisInstrumentor
+    from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
+    from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+    resource = Resource(attributes={SERVICE_NAME: service_name})
+    provider = TracerProvider(resource=resource)
+    provider.add_span_processor(
+        BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces"))
+    )
+    trace.set_tracer_provider(provider)
+
+    SQLAlchemyInstrumentor().instrument()
+    HTTPXClientInstrumentor().instrument()
+    RedisInstrumentor().instrument()
+    CeleryInstrumentor().instrument()
+
+    logger.info("tracing.initialized", service=service_name, endpoint=endpoint)
+
+
+def instrument_fastapi(app: object) -> None:
+    """Add OTel HTTP tracing middleware to an already-created FastAPI app.
+
+    Must be called after setup_tracing() and after the FastAPI app instance
+    exists. Calling FastAPIInstrumentor().instrument() without an app patches
+    FastAPI.__init__ globally and conflicts with the Prometheus Instrumentator
+    middleware added later in create_app(), producing an unpredictable
+    middleware stack that causes intermittent request failures.
+    """
+    from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+
+    FastAPIInstrumentor.instrument_app(app)  # type: ignore[arg-type]
+    logger.info("tracing.fastapi_instrumented")

backend/pyproject.toml

@@ -45,6 +45,15 @@ dependencies = [
     "celery[redis]>=5.3",
     "prometheus-client>=0.20.0",
     "prometheus-fastapi-instrumentator==7.1.0",
+    "sentry-sdk[fastapi]>=2.0.0",
+    # OpenTelemetry - distributed tracing to Grafana Tempo
+    "opentelemetry-sdk>=1.24.0",
+    "opentelemetry-exporter-otlp-proto-http>=1.24.0",
+    "opentelemetry-instrumentation-fastapi>=0.45b0",
+    "opentelemetry-instrumentation-sqlalchemy>=0.45b0",
+    "opentelemetry-instrumentation-httpx>=0.45b0",
+    "opentelemetry-instrumentation-redis>=0.45b0",
+    "opentelemetry-instrumentation-celery>=0.45b0",
 ]
 
 [project.optional-dependencies]