diff --git a/backend/main.py b/backend/main.py index 33530155..6cb029f2 100644 --- a/backend/main.py +++ b/backend/main.py @@ -72,10 +72,13 @@ async def read_root(): @router.get("/info") def get_info(): + from src.data import main as data_main + return { "PROD": PROD, "CONN_STR": "REDACTED" if PROD else CONN_STR, "PYTHON_VERSION": platform.python_version(), + "DB_LESS_SEED_INCOMPLETE": data_main.db_less_seed_incomplete, } diff --git a/backend/src/api/backend.py b/backend/src/api/backend.py index 0d96f2a7..6866582b 100644 --- a/backend/src/api/backend.py +++ b/backend/src/api/backend.py @@ -1,6 +1,8 @@ import os -if os.environ.get("API_BACKEND", "").lower() == "duckdb": +from src.constants import DISABLE_DB + +if DISABLE_DB or os.environ.get("API_BACKEND", "").lower() == "duckdb": from src.db_duckdb import ( get_event, get_events, diff --git a/backend/src/constants.py b/backend/src/constants.py index 34ac62f6..627b43b6 100644 --- a/backend/src/constants.py +++ b/backend/src/constants.py @@ -6,7 +6,15 @@ PROD = os.getenv("PROD", "False") == "True" # 8001 emulates the data server -BACKEND_URL = "https://api.statbotics.io" if PROD else "http://localhost:8001" +# The ETL self-trigger (update_curr_year_background and the freshness-ping probe) +# HTTP-calls BACKEND_URL to reach the data router. When the data router runs as a +# separate service the default is right, but a single-container deploy (e.g. a +# staging mirror serving all routers from one service) must point BACKEND_URL at +# its OWN backend, or the self-trigger hits the wrong host and ingestion never +# runs. Read it from the environment, falling back to the original default. +BACKEND_URL = os.getenv("BACKEND_URL") or ( + "https://api.statbotics.io" if PROD else "http://localhost:8001" +) # DB @@ -36,6 +44,7 @@ CURR_YEAR = 2026 DISABLE_GCS = False +DISABLE_DB = os.getenv("DISABLE_DB", "False") == "True" HIST_EPOCH = 1 # MISC diff --git a/backend/src/data/backend.py b/backend/src/data/backend.py new file mode 100644 index 00000000..4c2bcb74 --- /dev/null +++ b/backend/src/data/backend.py @@ -0,0 +1,23 @@ +from src.constants import DISABLE_DB + +if DISABLE_DB: + from src.db_duckdb import ( + get_events, + get_team_events, + get_team_years, + get_teams, + ) +else: + from src.db.read import ( + get_events, + get_team_events, + get_team_years, + get_teams, + ) + +__all__ = [ + "get_events", + "get_team_events", + "get_team_years", + "get_teams", +] diff --git a/backend/src/data/main.py b/backend/src/data/main.py index c0129385..45187043 100644 --- a/backend/src/data/main.py +++ b/backend/src/data/main.py @@ -3,7 +3,7 @@ import traceback from typing import Dict, List, Optional, Tuple -from src.constants import CURR_YEAR, DISABLE_GCS +from src.constants import CURR_YEAR, DISABLE_DB, DISABLE_GCS from src.data.avg import process_year as process_year_avg from src.data.epa.main import ( post_process as post_process_epa, @@ -27,13 +27,13 @@ process_year as process_year_wins, winrate, ) -from src.db.main import clean_db -from src.db.models import Team, TeamYear -from src.db.read import ( +from src.data.backend import ( get_team_events as get_team_events_db, get_team_years as get_team_years_db, get_teams as get_teams_db, ) +from src.db.main import clean_db +from src.db.models import Team, TeamYear from src.db.write.main import ( update_team_events as update_team_events_db, update_team_years as update_team_years_db, @@ -43,6 +43,10 @@ from src.google.snapshot import read_snapshot, write_snapshot from src.google.storage import write_objs as write_objs_storage +# Set true when a db-less cross-season EPA seed found no prior-year data (a +# partial/forgotten Parquet backfill); surfaced via /info so it is not silent. +db_less_seed_incomplete = False + def process_year( year_num: int, @@ -64,6 +68,14 @@ def process_year( all_team_years[ty.year][ty.team] = ty except Exception: traceback.print_exc() + if DISABLE_DB and year_num > 2002 and not all_team_years: + global db_less_seed_incomplete + db_less_seed_incomplete = True + print( + "WARNING: db-less mode found no prior-year team_years for " + f"{year_num}; every team will regress to the rookie mean. Back-fill " + "prior years to Parquet BEFORE running db-less or EPA seeds are wrong." + ) new_teams, objs = process_year_tba(year_num, teams, objs, tba_partial, cache) teams += new_teams @@ -88,19 +100,23 @@ def process_year( write_objs_storage(objs, orig_objs if partial else None, teams, parquet_uploads) timer.print(str(year_num) + " Write Storage") - try: - db_orig = read_objs_db(year_num) if partial else None - write_objs_db(year_num, objs, db_orig, not partial) - except Exception: - traceback.print_exc() - timer.print(str(year_num) + " Write DB") - else: + if not DISABLE_DB: + try: + db_orig = read_objs_db(year_num) if partial else None + write_objs_db(year_num, objs, db_orig, not partial) + except Exception: + traceback.print_exc() + timer.print(str(year_num) + " Write DB") + elif not DISABLE_DB: write_objs_db(year_num, objs, orig_objs if partial else None, not partial) timer.print(str(year_num) + " Write DB") - if not DISABLE_GCS: - write_parquet(year_num, objs, teams) - timer.print(str(year_num) + " Write Parquet") + # Current-year parquet is folded into the site manifest above; historical years + # publish parquet on their own manifest write (independent of the DB, so db-less + # backfill still produces parquet). + if not curr_year_gcs and not DISABLE_GCS: + write_parquet(year_num, objs, teams) + timer.print(str(year_num) + " Write Parquet") return teams @@ -122,11 +138,12 @@ def post_process( teams = post_process_epa(teams, all_team_years) timer.print("Post EPA") - update_teams_db(teams) - timer.print("Update DB") + if not DISABLE_DB: + update_teams_db(teams) + timer.print("Update DB") - post_process_tba() # updates DB directly - timer.print("Post TBA") + post_process_tba() # updates DB directly + timer.print("Post TBA") def reset_all_years(): @@ -135,8 +152,9 @@ def reset_all_years(): start_year = 2002 end_year = CURR_YEAR - clean_db() - timer.print("Clean DB") + if not DISABLE_DB: + clean_db() + timer.print("Clean DB") teams = load_teams_tba(cache=True) timer.print("Load Teams") @@ -168,14 +186,19 @@ def update_curr_year(partial: bool, tba_partial: bool): timer.print("Read Snapshot") if objs is None or teams is None: - teams = get_teams_db() - timer.print("Load Teams") - if partial: - objs = read_objs_db(year) - timer.print("Read Objs") - else: + if DISABLE_DB: + teams = load_teams_tba(cache=True) objs = create_objs(year) - timer.print("Create Objs") + timer.print("Load Teams (TBA)") + else: + teams = get_teams_db() + timer.print("Load Teams") + if partial: + objs = read_objs_db(year) + timer.print("Read Objs") + else: + objs = create_objs(year) + timer.print("Create Objs") teams = process_year( year, partial, tba_partial, year < CURR_YEAR, teams, objs, None @@ -188,6 +211,9 @@ def update_curr_year(partial: bool, tba_partial: bool): def refresh_teams() -> Dict[str, int]: """Refresh all stale team fields from TBA and recompute win records.""" + if DISABLE_DB: + return {"teams_updated": 0} + timer = Timer() fresh_teams = load_teams_tba(cache=False) diff --git a/backend/src/data/router.py b/backend/src/data/router.py index 511002c3..e011a9f0 100644 --- a/backend/src/data/router.py +++ b/backend/src/data/router.py @@ -1,10 +1,11 @@ import requests from fastapi import APIRouter, BackgroundTasks -from src.constants import BACKEND_URL, CURR_YEAR +from src.constants import BACKEND_URL, CURR_YEAR, DISABLE_DB from src.data.main import refresh_teams, update_curr_year, reset_all_years from src.data.tba import check_year_partial as check_year_partial_tba from src.db.read import get_etags as get_etags_db, get_events as get_events_db +from src.google.snapshot import read_snapshot from src.google.storage import GC_GRACE_HOURS, gc_versioned_blobs data_router = APIRouter() @@ -60,8 +61,13 @@ def update_curr_year_background(): @site_router.get("/update_curr_year") async def update_curr_year_site_endpoint(background_tasks: BackgroundTasks): - event_objs = get_events_db(year=CURR_YEAR) - etags = get_etags_db(CURR_YEAR) + if DISABLE_DB: + loaded = read_snapshot(CURR_YEAR) + event_objs = list(loaded[0][2].values()) if loaded else [] + etags = list(loaded[0][5].values()) if loaded else [] + else: + event_objs = get_events_db(year=CURR_YEAR) + etags = get_etags_db(CURR_YEAR) is_new_data = check_year_partial_tba(CURR_YEAR, event_objs, etags) if not is_new_data: return {"status": "skipped"} diff --git a/backend/src/db/main.py b/backend/src/db/main.py index 4f0719af..628f6d40 100644 --- a/backend/src/db/main.py +++ b/backend/src/db/main.py @@ -1,11 +1,11 @@ from sqlalchemy import create_engine from sqlalchemy.orm import DeclarativeBase, MappedAsDataclass, sessionmaker -from src.constants import CONN_STR +from src.constants import CONN_STR, DISABLE_DB -engine = create_engine(CONN_STR) +engine = None if DISABLE_DB else create_engine(CONN_STR) -Session = sessionmaker(bind=engine) +Session = None if DISABLE_DB else sessionmaker(bind=engine) # Only for type hints, doesn't enable slots diff --git a/backend/src/db_duckdb/__init__.py b/backend/src/db_duckdb/__init__.py index e6c1f310..7c9d629e 100644 --- a/backend/src/db_duckdb/__init__.py +++ b/backend/src/db_duckdb/__init__.py @@ -3,12 +3,14 @@ get_events, get_match, get_matches, + get_noteworthy_matches, get_team, get_team_event, get_team_events, get_team_year, get_team_years, get_teams, + get_upcoming_matches, get_year, get_years, ) @@ -18,12 +20,14 @@ "get_events", "get_match", "get_matches", + "get_noteworthy_matches", "get_team_event", "get_team_events", "get_team_year", "get_team_years", "get_team", "get_teams", + "get_upcoming_matches", "get_year", "get_years", ] diff --git a/backend/src/db_duckdb/main.py b/backend/src/db_duckdb/main.py index 1809b7a5..73a34209 100644 --- a/backend/src/db_duckdb/main.py +++ b/backend/src/db_duckdb/main.py @@ -3,12 +3,16 @@ import tempfile import threading import time +from datetime import datetime from typing import Any, Dict, List, Optional, Tuple, Type import duckdb +from src.constants import CURR_YEAR from src.db.models.main import Model, ModelORM +from src.db.models.match import Match from src.db_duckdb.schema import PARQUET_PREFIX, SPECS, columns, from_row +from src.types.enums import EventType, MatchStatus SYNC_TTL = float(os.environ.get("DUCKDB_SYNC_TTL", "30")) @@ -140,7 +144,12 @@ def _query( sql += f" OFFSET {int(offset)}" cursor = _connection().cursor() - cursor.execute(sql, params) + try: + cursor.execute(sql, params) + except duckdb.IOException as e: + if "No files found" in str(e): + return [] + raise names = [d[0] for d in cursor.description] return [ from_row(model_cls, cols, dict(zip(names, row))) for row in cursor.fetchall() @@ -395,3 +404,149 @@ def _eq(filters: Dict[str, Any]) -> Tuple[List[str], List[Any]]: where.append(f'"{name}" = ?') params.append(value) return where, params + + +def _event_filters( + country: Optional[str], state: Optional[str], district: Optional[str] +) -> Tuple[List[str], List[Any]]: + where: List[str] = [] + params: List[Any] = [] + if country is not None: + where.append("e.country = ?") + params.append(country) + if state is not None: + where.append("e.state = ?") + params.append(state) + if district == "regionals": + where.append("e.district IS NULL") + elif district is not None: + where.append("e.district = ?") + params.append(district) + return where, params + + +def _joined_matches( + base: str, year: int, where: List[str], params: List[Any], order: str, limit: int +) -> List[Model]: + model, orm = SPECS["matches"] + cols = columns(orm) + sql = ( + f"SELECT m.* FROM {_source(base, 'matches', year)} AS m " + f"JOIN {_source(base, 'events', year)} AS e ON m.event = e.key " + f"WHERE {' AND '.join(where)} ORDER BY {order} LIMIT {int(limit)}" + ) + cursor = _connection().cursor() + cursor.execute(sql, params) + names = [d[0] for d in cursor.description] + return [from_row(model, cols, dict(zip(names, row))) for row in cursor.fetchall()] + + +def get_noteworthy_matches( + year: int, + country: Optional[str], + state: Optional[str], + district: Optional[str], + elim: Optional[bool], + week: Optional[int], +) -> Dict[str, List[Match]]: + where = ["m.year = ?", "m.status = ?", "e.type != ?"] + params: List[Any] = [year, MatchStatus.COMPLETED.value, EventType.OFFSEASON.value] + e_where, e_params = _event_filters(country, state, district) + where += e_where + params += e_params + if elim is not None: + where.append("m.elim = ?") + params.append(elim) + if week is not None: + where.append("e.week = ?") + params.append(week) + + red = "m.red_score" if year < 2016 else "m.red_no_foul" + blue = "m.blue_score" if year < 2016 else "m.blue_no_foul" + + base = _sync() + + def top(order: str) -> List[Match]: + return _joined_matches(base, year, where, params, order, 30) # type: ignore + + out: Dict[str, List[Match]] = { + "high_score": top(f"greatest({red}, {blue}) DESC, m.time ASC"), + "combined_score": top(f"({red} + {blue}) DESC, m.time ASC"), + "losing_score": top("least(m.red_score, m.blue_score) DESC, m.time ASC"), + } + if year >= 2016: + out["high_auto_score"] = top( + "greatest(m.red_auto, m.blue_auto) DESC, m.time ASC" + ) + out["high_teleop_score"] = top( + "greatest(m.red_teleop, m.blue_teleop) DESC, m.time ASC" + ) + out["high_endgame_score"] = top( + "greatest(m.red_endgame, m.blue_endgame) DESC, m.time ASC" + ) + return out + + +def get_upcoming_matches( + country: Optional[str], + state: Optional[str], + district: Optional[str], + elim: Optional[bool], + minutes: int, + limit: int, + metric: str, +) -> List[Tuple[Match, str]]: + curr_timestamp = int(datetime.now().timestamp()) - 60 * 5 + if minutes == -1: + minutes = 60 * 24 * 7 + + where = [ + "m.year = ?", + "m.status = ?", + "m.predicted_time > ?", + "m.predicted_time < ?", + "m.event = e.key", + ] + params: List[Any] = [ + CURR_YEAR, + MatchStatus.UPCOMING.value, + curr_timestamp, + curr_timestamp + 60 * minutes, + ] + e_where, e_params = _event_filters(country, state, district) + where += e_where + params += e_params + if elim is not None: + where.append("m.elim = ?") + params.append(elim) + + exprs = { + "max_epa": "greatest(m.epa_red_score_pred, m.epa_blue_score_pred)", + "sum_epa": "(m.epa_red_score_pred + m.epa_blue_score_pred)", + "diff_epa": "abs(m.epa_red_score_pred - m.epa_blue_score_pred)", + "time": "m.time", + } + order = "" + if metric in ("max_epa", "sum_epa"): + order = f" ORDER BY {exprs[metric]} DESC" + elif metric in ("time", "diff_epa"): + order = f" ORDER BY {exprs[metric]} ASC" + + model, orm = SPECS["matches"] + cols = columns(orm) + base = _sync() + sql = ( + f"SELECT m.*, e.name AS event_name FROM {_source(base, 'matches', CURR_YEAR)} " + f"AS m, {_source(base, 'events', CURR_YEAR)} AS e " + f"WHERE {' AND '.join(where)}{order} LIMIT {int(limit)}" + ) + cursor = _connection().cursor() + cursor.execute(sql, params) + names = [d[0] for d in cursor.description] + return [ + ( + from_row(model, cols, dict(zip(names, row))), + dict(zip(names, row))["event_name"], + ) + for row in cursor.fetchall() + ] diff --git a/backend/src/google/storage.py b/backend/src/google/storage.py index 9f75e9ad..945ef1e7 100644 --- a/backend/src/google/storage.py +++ b/backend/src/google/storage.py @@ -9,12 +9,14 @@ from google.cloud import storage from src.constants import CURR_YEAR, HIST_EPOCH, PROD +from src.data.backend import ( + get_events as get_events_db, + get_team_years as get_team_years_db, + get_teams as get_teams_db, +) from src.data.utils import nan_safe_eq, objs_type -from src.db.functions import get_noteworthy_matches, get_upcoming_matches from src.db.models.team import Team -from src.db.read.event import get_events as get_events_db -from src.db.read.team import get_teams as get_teams_db -from src.db.read.team_year import get_team_years as get_team_years_db +from src.site.backend import get_noteworthy_matches, get_upcoming_matches from src.google.publish import ( MANIFEST_OBJECT, VERSION_PREFIX, diff --git a/backend/src/site/backend.py b/backend/src/site/backend.py new file mode 100644 index 00000000..f8f654d8 --- /dev/null +++ b/backend/src/site/backend.py @@ -0,0 +1,11 @@ +from src.constants import DISABLE_DB + +if DISABLE_DB: + from src.db_duckdb import get_noteworthy_matches, get_upcoming_matches +else: + from src.db.functions import get_noteworthy_matches, get_upcoming_matches + +__all__ = [ + "get_noteworthy_matches", + "get_upcoming_matches", +] diff --git a/backend/src/site/match.py b/backend/src/site/match.py index d8d6e3c6..3f29f000 100644 --- a/backend/src/site/match.py +++ b/backend/src/site/match.py @@ -9,7 +9,7 @@ get_team_events_cached, get_year_cached, ) -from src.db.functions import get_noteworthy_matches, get_upcoming_matches +from src.site.backend import get_noteworthy_matches, get_upcoming_matches from src.db.models import Event, Match, TeamEvent, Year # from src.site.helper import compress