[05] Retire the database: db-less pipeline + /v3 + /v3/site (+ BACKEND_URL env config)

backend/main.py

@@ -72,10 +72,13 @@ async def read_root():
 
 @router.get("/info")
 def get_info():
+    from src.data import main as data_main
+
     return {
         "PROD": PROD,
         "CONN_STR": "REDACTED" if PROD else CONN_STR,
         "PYTHON_VERSION": platform.python_version(),
+        "DB_LESS_SEED_INCOMPLETE": data_main.db_less_seed_incomplete,
     }
 
 

backend/src/api/backend.py

@@ -1,6 +1,8 @@
 import os
 
-if os.environ.get("API_BACKEND", "").lower() == "duckdb":
+from src.constants import DISABLE_DB
+
+if DISABLE_DB or os.environ.get("API_BACKEND", "").lower() == "duckdb":
     from src.db_duckdb import (
         get_event,
         get_events,

backend/src/constants.py

@@ -6,7 +6,15 @@
 PROD = os.getenv("PROD", "False") == "True"
 
 # 8001 emulates the data server
-BACKEND_URL = "https://api.statbotics.io" if PROD else "http://localhost:8001"
+# The ETL self-trigger (update_curr_year_background and the freshness-ping probe)
+# HTTP-calls BACKEND_URL to reach the data router. When the data router runs as a
+# separate service the default is right, but a single-container deploy (e.g. a
+# staging mirror serving all routers from one service) must point BACKEND_URL at
+# its OWN backend, or the self-trigger hits the wrong host and ingestion never
+# runs. Read it from the environment, falling back to the original default.
+BACKEND_URL = os.getenv("BACKEND_URL") or (
+    "https://api.statbotics.io" if PROD else "http://localhost:8001"
+)
 
 # DB
 
@@ -36,6 +44,7 @@
 
 CURR_YEAR = 2026
 DISABLE_GCS = False
+DISABLE_DB = os.getenv("DISABLE_DB", "False") == "True"
 HIST_EPOCH = 1
 
 # MISC

backend/src/data/backend.py

@@ -0,0 +1,23 @@
+from src.constants import DISABLE_DB
+
+if DISABLE_DB:
+    from src.db_duckdb import (
+        get_events,
+        get_team_events,
+        get_team_years,
+        get_teams,
+    )
+else:
+    from src.db.read import (
+        get_events,
+        get_team_events,
+        get_team_years,
+        get_teams,
+    )
+
+__all__ = [
+    "get_events",
+    "get_team_events",
+    "get_team_years",
+    "get_teams",
+]

backend/src/data/main.py

@@ -3,7 +3,7 @@
 import traceback
 from typing import Dict, List, Optional, Tuple
 
-from src.constants import CURR_YEAR, DISABLE_GCS
+from src.constants import CURR_YEAR, DISABLE_DB, DISABLE_GCS
 from src.data.avg import process_year as process_year_avg
 from src.data.epa.main import (
     post_process as post_process_epa,
@@ -27,13 +27,13 @@
     process_year as process_year_wins,
     winrate,
 )
-from src.db.main import clean_db
-from src.db.models import Team, TeamYear
-from src.db.read import (
+from src.data.backend import (
     get_team_events as get_team_events_db,
     get_team_years as get_team_years_db,
     get_teams as get_teams_db,
 )
+from src.db.main import clean_db
+from src.db.models import Team, TeamYear
 from src.db.write.main import (
     update_team_events as update_team_events_db,
     update_team_years as update_team_years_db,
@@ -43,6 +43,10 @@
 from src.google.snapshot import read_snapshot, write_snapshot
 from src.google.storage import write_objs as write_objs_storage
 
+# Set true when a db-less cross-season EPA seed found no prior-year data (a
+# partial/forgotten Parquet backfill); surfaced via /info so it is not silent.
+db_less_seed_incomplete = False
+
 
 def process_year(
     year_num: int,
@@ -64,6 +68,14 @@ def process_year(
                     all_team_years[ty.year][ty.team] = ty
         except Exception:
             traceback.print_exc()
+        if DISABLE_DB and year_num > 2002 and not all_team_years:
+            global db_less_seed_incomplete
+            db_less_seed_incomplete = True
+            print(
+                "WARNING: db-less mode found no prior-year team_years for "
+                f"{year_num}; every team will regress to the rookie mean. Back-fill "
+                "prior years to Parquet BEFORE running db-less or EPA seeds are wrong."
+            )
 
     new_teams, objs = process_year_tba(year_num, teams, objs, tba_partial, cache)
     teams += new_teams
@@ -88,19 +100,23 @@ def process_year(
         write_objs_storage(objs, orig_objs if partial else None, teams, parquet_uploads)
         timer.print(str(year_num) + " Write Storage")
 
-        try:
-            db_orig = read_objs_db(year_num) if partial else None
-            write_objs_db(year_num, objs, db_orig, not partial)
-        except Exception:
-            traceback.print_exc()
-        timer.print(str(year_num) + " Write DB")
-    else:
+        if not DISABLE_DB:
+            try:
+                db_orig = read_objs_db(year_num) if partial else None
+                write_objs_db(year_num, objs, db_orig, not partial)
+            except Exception:
+                traceback.print_exc()
+            timer.print(str(year_num) + " Write DB")
+    elif not DISABLE_DB:
         write_objs_db(year_num, objs, orig_objs if partial else None, not partial)
         timer.print(str(year_num) + " Write DB")
 
-        if not DISABLE_GCS:
-            write_parquet(year_num, objs, teams)
-            timer.print(str(year_num) + " Write Parquet")
+    # Current-year parquet is folded into the site manifest above; historical years
+    # publish parquet on their own manifest write (independent of the DB, so db-less
+    # backfill still produces parquet).
+    if not curr_year_gcs and not DISABLE_GCS:
+        write_parquet(year_num, objs, teams)
+        timer.print(str(year_num) + " Write Parquet")
 
     return teams
 
@@ -122,11 +138,12 @@ def post_process(
     teams = post_process_epa(teams, all_team_years)
     timer.print("Post EPA")
 
-    update_teams_db(teams)
-    timer.print("Update DB")
+    if not DISABLE_DB:
+        update_teams_db(teams)
+        timer.print("Update DB")
 
-    post_process_tba()  # updates DB directly
-    timer.print("Post TBA")
+        post_process_tba()  # updates DB directly
+        timer.print("Post TBA")
 
 
 def reset_all_years():
@@ -135,8 +152,9 @@ def reset_all_years():
     start_year = 2002
     end_year = CURR_YEAR
 
-    clean_db()
-    timer.print("Clean DB")
+    if not DISABLE_DB:
+        clean_db()
+        timer.print("Clean DB")
 
     teams = load_teams_tba(cache=True)
     timer.print("Load Teams")
@@ -168,14 +186,19 @@ def update_curr_year(partial: bool, tba_partial: bool):
             timer.print("Read Snapshot")
 
     if objs is None or teams is None:
-        teams = get_teams_db()
-        timer.print("Load Teams")
-        if partial:
-            objs = read_objs_db(year)
-            timer.print("Read Objs")
-        else:
+        if DISABLE_DB:
+            teams = load_teams_tba(cache=True)
             objs = create_objs(year)
-            timer.print("Create Objs")
+            timer.print("Load Teams (TBA)")
+        else:
+            teams = get_teams_db()
+            timer.print("Load Teams")
+            if partial:
+                objs = read_objs_db(year)
+                timer.print("Read Objs")
+            else:
+                objs = create_objs(year)
+                timer.print("Create Objs")
 
     teams = process_year(
         year, partial, tba_partial, year < CURR_YEAR, teams, objs, None
@@ -188,6 +211,9 @@ def update_curr_year(partial: bool, tba_partial: bool):
 
 def refresh_teams() -> Dict[str, int]:
     """Refresh all stale team fields from TBA and recompute win records."""
+    if DISABLE_DB:
+        return {"teams_updated": 0}
+
     timer = Timer()
 
     fresh_teams = load_teams_tba(cache=False)

backend/src/data/router.py

@@ -1,10 +1,11 @@
 import requests
 from fastapi import APIRouter, BackgroundTasks
 
-from src.constants import BACKEND_URL, CURR_YEAR
+from src.constants import BACKEND_URL, CURR_YEAR, DISABLE_DB
 from src.data.main import refresh_teams, update_curr_year, reset_all_years
 from src.data.tba import check_year_partial as check_year_partial_tba
 from src.db.read import get_etags as get_etags_db, get_events as get_events_db
+from src.google.snapshot import read_snapshot
 from src.google.storage import GC_GRACE_HOURS, gc_versioned_blobs
 
 data_router = APIRouter()
@@ -60,8 +61,13 @@ def update_curr_year_background():
 
 @site_router.get("/update_curr_year")
 async def update_curr_year_site_endpoint(background_tasks: BackgroundTasks):
-    event_objs = get_events_db(year=CURR_YEAR)
-    etags = get_etags_db(CURR_YEAR)
+    if DISABLE_DB:
+        loaded = read_snapshot(CURR_YEAR)
+        event_objs = list(loaded[0][2].values()) if loaded else []
+        etags = list(loaded[0][5].values()) if loaded else []
+    else:
+        event_objs = get_events_db(year=CURR_YEAR)
+        etags = get_etags_db(CURR_YEAR)
     is_new_data = check_year_partial_tba(CURR_YEAR, event_objs, etags)
     if not is_new_data:
         return {"status": "skipped"}

backend/src/db/main.py

@@ -1,11 +1,11 @@
 from sqlalchemy import create_engine
 from sqlalchemy.orm import DeclarativeBase, MappedAsDataclass, sessionmaker
 
-from src.constants import CONN_STR
+from src.constants import CONN_STR, DISABLE_DB
 
-engine = create_engine(CONN_STR)
+engine = None if DISABLE_DB else create_engine(CONN_STR)
 
-Session = sessionmaker(bind=engine)
+Session = None if DISABLE_DB else sessionmaker(bind=engine)
 
 
 # Only for type hints, doesn't enable slots

backend/src/db_duckdb/__init__.py

@@ -3,12 +3,14 @@
     get_events,
     get_match,
     get_matches,
+    get_noteworthy_matches,
     get_team,
     get_team_event,
     get_team_events,
     get_team_year,
     get_team_years,
     get_teams,
+    get_upcoming_matches,
     get_year,
     get_years,
 )
@@ -18,12 +20,14 @@
     "get_events",
     "get_match",
     "get_matches",
+    "get_noteworthy_matches",
     "get_team_event",
     "get_team_events",
     "get_team_year",
     "get_team_years",
     "get_team",
     "get_teams",
+    "get_upcoming_matches",
     "get_year",
     "get_years",
 ]