New Job Monitor, Faster Exports, and UI Fixes

backend/alembic/versions/0020_user_activity_seen.py

@@ -0,0 +1,33 @@
+"""Add users.activity_seen_at for the job monitor's unseen-error badge.
+
+Nullable timezone-aware timestamp: when this user last marked the activity
+error feed as seen (POST /api/activity/seen). NULL means never marked, which
+the frontend treats as "every fetched error is unseen".
+
+Plain add_column with no existence guard: guards are reserved for revisions
+with a specific stated reason (see 0018's autocommit backfill), and installs
+bootstrapped by create_all-then-stamp are no longer supported
+(MIN_UPGRADE_FROM_DATA_VERSION = 13).
+
+No index: the column is only ever read for the requesting user's own row,
+which the primary key already serves.
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "0020"
+down_revision = "0019"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "users",
+        sa.Column("activity_seen_at", sa.DateTime(timezone=True), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("users", "activity_seen_at")

backend/alembic/versions/0021_arcsec_per_pixel.py

@@ -0,0 +1,41 @@
+"""Add images.arcsec_per_pixel, the materialized plate scale.
+
+Nullable float: 206.265 * XPIXSZ(um) / FOCALLEN(mm), computed at ingest by the
+scanner via units.arcsec_per_pixel. NULL when the headers carry no usable pixel
+size or focal length. Materialized because deriving the plate scale per-row
+from raw_headers JSONB (regex-guarded text casts, units.sql_arcsec_per_pixel)
+made cold /api/stats aggregations take tens of seconds.
+
+Plain add_column with no existence guard: guards are reserved for revisions
+with a specific stated reason (see 0018's autocommit backfill), and installs
+bootstrapped by create_all-then-stamp are no longer supported
+(MIN_UPGRADE_FROM_DATA_VERSION = 13).
+
+Schema only. Backfilling existing rows is inference from raw_headers, not a
+structural DDL step, so it runs through app.services.data_migrations as data
+migration v15, where raw-headers re-derivation already lives and where the
+version gate makes it run exactly once (same split as 0019 / v14).
+
+No index: the column is aggregated over full scans (multiplied into HFR/FWHM
+conversions), never used as a filter, so a btree would cost every ingest write
+for no read. Add one when a query needs it.
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "0021"
+down_revision = "0020"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "images",
+        sa.Column("arcsec_per_pixel", sa.Float(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("images", "arcsec_per_pixel")

backend/app/api/activity.py

@@ -3,7 +3,7 @@
 import base64
 import json
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 
 from fastapi import APIRouter, Depends, Query
 from sqlalchemy import delete, func, select
@@ -13,7 +13,11 @@
 from app.database import get_session
 from app.models.activity_event import ActivityEvent
 from app.models.user import User
-from app.schemas.activity import ActivityItem, PaginatedActivityResponse
+from app.schemas.activity import (
+    ActivityItem,
+    ActivitySeenResponse,
+    PaginatedActivityResponse,
+)
 from app.schemas.common import StatusResponse
 
 logger = logging.getLogger(__name__)
@@ -138,3 +142,21 @@ async def clear_activity(
     await session.execute(delete(ActivityEvent))
     await session.commit()
     return {"status": "cleared"}
+
+
+@router.post("/seen", response_model=ActivitySeenResponse)
+async def mark_activity_seen(
+    session: AsyncSession = Depends(get_session),
+    user: User = Depends(get_current_user),
+):
+    """Record that the current user has seen the activity error feed.
+
+    Any authenticated role may call this: it mutates only the caller's own
+    row. get_current_user and this endpoint share the request's get_session
+    dependency (FastAPI caches per-request), so mutating the loaded user and
+    committing persists it, the same pattern change_password uses in auth.py.
+    """
+    now = datetime.now(timezone.utc)
+    user.activity_seen_at = now
+    await session.commit()
+    return ActivitySeenResponse(activity_seen_at=now)

backend/app/api/analysis.py

@@ -30,7 +30,6 @@
     TrendLine,
 )
 from app.services.normalization import load_alias_maps, expand_canonical, normalize_equipment, normalize_filter
-from app.services.units import sql_arcsec_per_pixel
 from app.api.auth import get_current_user
 
 router = APIRouter(prefix="/analysis", tags=["analysis"])
@@ -764,11 +763,12 @@ async def get_compare(
     # Pixel-domain metrics cannot be compared across optical trains directly:
     # the same star measures a different pixel HFR on a different focal length
     # or pixel size. For these, each frame is also converted to arcsec via its
-    # plate scale (XPIXSZ/FOCALLEN raw headers) and the verdict runs on the
-    # arcsec medians. Frames without a derivable plate scale contribute to the
-    # px box plots (backward compatible) but not to the arcsec comparison.
+    # materialized plate scale (arcsec_per_pixel, NULL when the headers lacked
+    # a derivable XPIXSZ/FOCALLEN) and the verdict runs on the arcsec medians.
+    # Frames without a plate scale contribute to the px box plots (backward
+    # compatible) but not to the arcsec comparison.
     is_pixel_metric = metric in _PIXEL_METRICS
-    scale_expr = sql_arcsec_per_pixel(Image.raw_headers)
+    scale_expr = Image.arcsec_per_pixel
 
     async def _fetch_values(group: str) -> tuple[list[float], list[float]]:
         q = (

backend/app/api/auth.py

@@ -235,7 +235,12 @@ async def logout(
 
 @router.get("/me", response_model=MeResponse)
 async def me(user: User = Depends(get_current_user)):
-    return MeResponse(id=user.id, username=user.username, role=user.role.value)
+    return MeResponse(
+        id=user.id,
+        username=user.username,
+        role=user.role.value,
+        activity_seen_at=user.activity_seen_at,
+    )
 
 
 @router.put("/password", response_model=StatusResponse)

backend/app/api/stats.py

@@ -15,7 +15,6 @@
 from app.models.user import User
 from app.models import Image, Target
 from app.services.normalization import load_alias_maps, normalize_filter, normalize_equipment
-from app.services.units import sql_arcsec_per_pixel
 
 from app.schemas.stats import (
     StatsResponse, OverviewStats, EquipmentStats, EquipmentItem,
@@ -278,12 +277,12 @@ async def _query_overview():
         async with async_session() as s:
             return (await s.execute(q)).one()
 
-    # Plate scale (arcsec/px) derived per row from the XPIXSZ/FOCALLEN raw
-    # headers; NULL when underivable, so multiplying a pixel metric by it
-    # yields NULL and percentile_cont skips the frame. This keeps the pooled
-    # arcsec medians unit-consistent across optical trains, unlike the raw
-    # pixel medians beside them.
-    _scale = sql_arcsec_per_pixel(Image.raw_headers)
+    # Plate scale (arcsec/px) materialized at scan time from the
+    # XPIXSZ/FOCALLEN raw headers; NULL when underivable, so multiplying a
+    # pixel metric by it yields NULL and percentile_cont skips the frame.
+    # This keeps the pooled arcsec medians unit-consistent across optical
+    # trains, unlike the raw pixel medians beside them.
+    _scale = Image.arcsec_per_pixel
 
     async def _query_cameras():
         q = select(

backend/app/models/image.py

@@ -70,6 +70,14 @@ class Image(Base):
     # derived from RA/Dec + time + site.
     altitude_deg: Mapped[float | None] = mapped_column(Float, nullable=True)
 
+    # Plate scale in arcseconds per pixel, materialized at ingest from the
+    # XPIXSZ/FOCALLEN headers via units.arcsec_per_pixel (206.265 * um / mm).
+    # Stored as a column because deriving it per-row from raw_headers JSONB
+    # (regex-guarded text casts, see units.sql_arcsec_per_pixel) is what made
+    # cold /api/stats aggregations take tens of seconds. NULL when the headers
+    # carry no usable pixel size or focal length.
+    arcsec_per_pixel: Mapped[float | None] = mapped_column(Float, nullable=True)
+
     # --- CSV metrics (N.I.N.A. ImageMetaData) ---
     # Quality
     hfr_stdev: Mapped[float | None] = mapped_column(Float, nullable=True)

backend/app/models/user.py

@@ -22,5 +22,9 @@ class User(Base):
     password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
     role: Mapped[UserRole] = mapped_column(Enum(UserRole, name="user_role"), nullable=False)
     is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
+    # When this user last marked the activity/error feed as seen (job-monitor
+    # unseen badge). NULL means never marked; the frontend then treats every
+    # fetched error event as unseen.
+    activity_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=False)
     updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False)

backend/app/schemas/activity.py

@@ -41,3 +41,7 @@ class ActivityFilterParams(BaseModel):
     @classmethod
     def cap_limit(cls, v: int) -> int:
         return min(int(v), 200)
+
+
+class ActivitySeenResponse(BaseModel):
+    activity_seen_at: datetime

backend/app/schemas/auth.py

@@ -19,6 +19,7 @@ class MeResponse(BaseModel):
     id: uuid.UUID
     username: str
     role: str
+    activity_seen_at: datetime | None = None
 
 
 class PasswordChangeRequest(BaseModel):

backend/app/services/data_migrations.py

@@ -23,7 +23,7 @@
 
 # Current data version - bump this and add a migration function when
 # code changes affect how stored target data is derived.
-DATA_VERSION = 14
+DATA_VERSION = 15
 
 # Migrations whose per-target loops make external network calls (VizieR, Gaia,
 # SAC, HyperLEDA) with pacing sleeps commit their work every this many queried
@@ -812,6 +812,51 @@ def _migrate_v14_eccentricity_provenance(session: Session) -> str:
     return "; ".join(parts) if parts else "No changes needed"
 
 
+def _migrate_v15_arcsec_per_pixel(session: Session) -> str:
+    """Backfill the materialized plate scale (arcsec_per_pixel) from raw_headers.
+
+    Re-derives an image-level value from raw_headers after a schema change,
+    exactly as v12/v14 did: alembic 0021 added images.arcsec_per_pixel so
+    /api/stats aggregations no longer have to derive the plate scale per-row
+    from JSONB (regex-guarded text casts, units.sql_arcsec_per_pixel), which
+    is what made the cold path take tens of seconds. The scanner now sets the
+    column at ingest; existing rows carry XPIXSZ/FOCALLEN in raw_headers, so
+    the value is backfilled here rather than left to wait on a full re-ingest.
+
+    Uses units.plate_scale_from_headers, the Python counterpart of the SQL
+    expression, so the backfilled values match what the scanner writes and
+    what the SQL derivation produced (same keyword semantics, None for
+    missing/non-numeric/non-positive inputs).
+
+    Idempotent: only rows still NULL in arcsec_per_pixel are selected, and the
+    derivation is a pure function of raw_headers, so replaying is a no-op.
+    """
+    from app.models import Image
+    from app.services.units import plate_scale_from_headers
+
+    rows = session.execute(
+        select(Image.id, Image.raw_headers).where(
+            Image.arcsec_per_pixel.is_(None),
+            Image.raw_headers.has_key("XPIXSZ"),    # noqa: W601 (JSONB ?)
+            Image.raw_headers.has_key("FOCALLEN"),  # noqa: W601
+        )
+    ).all()
+
+    updates: list[dict] = []
+    for img_id, headers in rows:
+        scale = plate_scale_from_headers(headers)
+        if scale is not None:
+            updates.append({"id": img_id, "arcsec_per_pixel": scale})
+
+    if updates:
+        session.bulk_update_mappings(Image, updates)
+        session.flush()
+
+    if updates:
+        return f"{len(updates)} plate scales backfilled from raw_headers"
+    return "No changes needed"
+
+
 def reference_catalogs_are_empty(session: Session) -> bool:
     """Return True if the static OpenNGC catalog table has no rows.
 
@@ -879,6 +924,7 @@ def load_reference_catalogs(session: Session) -> str:
     12: ("Backfill exposure_time from EXPOSURE keyword and split FWHM out of median_hfr", _migrate_v12_exposure_and_metric_split),
     13: ("Backfill per-target enrichment (constellation, memberships, Gaia, HyperLEDA) for targets created since v11", _migrate_v13_enrich_created_targets),
     14: ("Recover eccentricity provenance and backfill frame altitude from raw_headers", _migrate_v14_eccentricity_provenance),
+    15: ("Backfill materialized plate scale (arcsec_per_pixel) from raw_headers", _migrate_v15_arcsec_per_pixel),
 }
 
 

backend/app/services/scanner.py

@@ -10,6 +10,7 @@
 
 from app.services.csv_metadata import get_csv_metrics, merge_csv_metrics
 from app.services.scan_filters import ScanFilterConfig
+from app.services.units import arcsec_per_pixel
 
 logger = logging.getLogger(__name__)
 
@@ -236,6 +237,13 @@ def extract_metadata(fits_path: Path, header=None) -> dict[str, Any]:
         # OBJCTALT (MaxIm-style) first, CENTALT second; both are written by
         # common capture software and agree when both are present.
         "altitude_deg": _first_float(header, "OBJCTALT", "CENTALT"),
+        # Plate scale materialized at ingest (206.265 * XPIXSZ / FOCALLEN) so
+        # aggregations never have to re-derive it per-row from raw_headers
+        # JSONB. Same helper semantics as units.sql_arcsec_per_pixel: None
+        # when either keyword is missing, non-numeric, or non-positive.
+        "arcsec_per_pixel": arcsec_per_pixel(
+            header.get("XPIXSZ"), header.get("FOCALLEN")
+        ),
         "capture_date": capture_date,
         "raw_headers": raw_headers,
     }

backend/app/services/xisf_parser.py

@@ -21,6 +21,7 @@
 
 from app.services.csv_metadata import get_csv_metrics, merge_csv_metrics
 from app.services.stretch import normalize_to_unit, resize_array, stretch_channel
+from app.services.units import arcsec_per_pixel
 
 logger = logging.getLogger(__name__)
 
@@ -213,6 +214,11 @@ def extract_xisf_metadata(xisf_path: Path) -> dict[str, Any]:
         # Frame-centre altitude, for the same atmospheric-dispersion reason as
         # the FITS path (see scanner.extract_metadata).
         "altitude_deg": _first_float(merged.get("OBJCTALT"), merged.get("CENTALT")),
+        # Plate scale materialized at ingest, same as the FITS path (see
+        # scanner.extract_metadata).
+        "arcsec_per_pixel": arcsec_per_pixel(
+            merged.get("XPIXSZ"), merged.get("FOCALLEN")
+        ),
         "capture_date": _parse_capture_date(merged.get("DATE-OBS"), xisf_path),
         "raw_headers": raw_headers,
     }

backend/app/worker/tasks_scan.py

@@ -476,6 +476,7 @@ def _do_ingest(fits_path: str, include_calibration: bool = True) -> dict:
                 eccentricity=meta.get("eccentricity"),
                 eccentricity_source=meta.get("eccentricity_source"),
                 altitude_deg=meta.get("altitude_deg"),
+                arcsec_per_pixel=meta.get("arcsec_per_pixel"),
                 raw_headers=meta.get("raw_headers", {}),
                 # CSV metrics (N.I.N.A. Session Metadata)
                 hfr_stdev=meta.get("hfr_stdev"),

backend/tests/conftest.py

@@ -50,6 +50,7 @@ def admin_user():
     user.role = UserRole.admin
     user.is_active = True
     user.password_hash = "hashed"
+    user.activity_seen_at = None
     return user
 
 
@@ -62,4 +63,5 @@ def viewer_user():
     user.role = UserRole.viewer
     user.is_active = True
     user.password_hash = "hashed"
+    user.activity_seen_at = None
     return user