[06] Postgres compatibility (dialect, BigInteger timestamps, DATABASE_URL, pool_pre_ping)

backend/src/constants.py

@@ -14,7 +14,11 @@
 CRDB_PWD = os.getenv("CRDB_PWD", "")
 CRDB_HOST = os.getenv("CRDB_HOST", "")
 
-CONN_STR = (
+# DATABASE_URL, if set, overrides the CockroachDB connection string entirely.
+# Used to run the backend against plain PostgreSQL (e.g. Cloud SQL staging),
+# e.g. "postgresql+psycopg2://user:pwd@host:5432/statbotics3". The transaction
+# helper (src/db/transaction.py) selects retry behavior from the engine dialect.
+CONN_STR = os.getenv("DATABASE_URL") or (
     (
         "cockroachdb://"
         + CRDB_USER

backend/src/db/functions/clear_year.py

@@ -1,5 +1,4 @@
 from sqlalchemy.orm import Session as SessionType
-from sqlalchemy_cockroachdb import run_transaction  # type: ignore
 
 from src.db.main import Session
 from src.db.models.etag import ETagORM
@@ -8,6 +7,7 @@
 from src.db.models.team_event import TeamEventORM
 from src.db.models.team_year import TeamYearORM
 from src.db.models.year import YearORM
+from src.db.transaction import run_transaction
 
 
 def clear_year(year: int) -> None:

backend/src/db/functions/noteworthy_matches.py

@@ -2,11 +2,11 @@
 
 from sqlalchemy import asc, desc, func
 from sqlalchemy.orm import Session as SessionType
-from sqlalchemy_cockroachdb import run_transaction  # type: ignore
 
 from src.db.main import Session
 from src.db.models.event import EventORM
 from src.db.models.match import Match, MatchORM
+from src.db.transaction import run_transaction
 from src.types.enums import MatchStatus
 
 

backend/src/db/functions/remove_teams_no_events.py

@@ -1,13 +1,13 @@
 from typing import List
 
 from sqlalchemy.orm import Session as SessionType
-from sqlalchemy_cockroachdb import run_transaction  # type: ignore
 
 from src.constants import CURR_YEAR
 from src.db.main import Session
 from src.db.models.team import TeamORM
 from src.db.models.team_event import TeamEventORM
 from src.db.models.team_year import TeamYearORM
+from src.db.transaction import run_transaction
 
 
 def remove_teams_with_no_events() -> None:

backend/src/db/functions/upcoming_matches.py

@@ -3,12 +3,12 @@
 
 from sqlalchemy import func, text
 from sqlalchemy.orm import Session as SessionType
-from sqlalchemy_cockroachdb import run_transaction  # type: ignore
 
 from src.constants import CURR_YEAR
 from src.db.main import Session
 from src.db.models.event import EventORM
 from src.db.models.match import Match, MatchORM
+from src.db.transaction import run_transaction
 from src.types.enums import EventStatus
 
 

backend/src/db/functions/update_teams.py

@@ -1,9 +1,9 @@
 from sqlalchemy.orm import Session as SessionType
-from sqlalchemy_cockroachdb import run_transaction  # type: ignore
 
 from src.db.main import Session
 from src.db.models.team import TeamORM
 from src.db.models.team_year import TeamYearORM
+from src.db.transaction import run_transaction
 
 
 def update_team_districts() -> None:

backend/src/db/main.py

@@ -3,7 +3,12 @@
 
 from src.constants import CONN_STR
 
-engine = create_engine(CONN_STR)
+# pool_pre_ping: a pooled connection idle across the hourly-cron gap goes stale
+# (Cloud SQL / db-f1-micro / the Cloud SQL proxy reap idle connections), and the
+# next query raises "server closed the connection unexpectedly", 500ing the ETL
+# trigger and stalling ingestion. pre_ping reconnects transparently; recycle
+# drops connections older than 30 min proactively.
+engine = create_engine(CONN_STR, pool_pre_ping=True, pool_recycle=1800)
 
 Session = sessionmaker(bind=engine)
 

backend/src/db/models/event.py

@@ -1,6 +1,6 @@
 from typing import Any, Dict
 
-from sqlalchemy import Enum, Float, Integer, String
+from sqlalchemy import BigInteger, Enum, Float, Integer, String
 from sqlalchemy.orm import Mapped, mapped_column
 from sqlalchemy.sql.schema import ForeignKeyConstraint, PrimaryKeyConstraint
 
@@ -23,7 +23,7 @@ class EventORM(Base, ModelORM):
 
     """GENERAL"""
     name: MS = mapped_column(String(100))
-    time: MI = mapped_column(Integer)
+    time: MI = mapped_column(BigInteger)  # Unix timestamp; see match.py note
     country: MOS = mapped_column(String(30), nullable=True)
     state: MOS = mapped_column(String(10), nullable=True)
     district: MOS = mapped_column(String(10), nullable=True)

backend/src/db/models/match.py

@@ -1,7 +1,7 @@
 from typing import Any, Dict, List, Optional, Tuple
 
 import numpy as np
-from sqlalchemy import Boolean, Enum, Float, Integer, JSON, String
+from sqlalchemy import BigInteger, Boolean, Enum, Float, Integer, JSON, String
 from sqlalchemy.orm import Mapped, mapped_column
 from sqlalchemy.sql.schema import ForeignKeyConstraint, PrimaryKeyConstraint
 
@@ -35,8 +35,12 @@ class MatchORM(Base, ModelORM):
     set_number: MI = mapped_column(Integer)
     match_number: MI = mapped_column(Integer)
 
-    time: MI = mapped_column(Integer)  # Enforces ordering
-    predicted_time: MOI = mapped_column(Integer, nullable=True)  # For display
+    # BigInteger: Unix timestamps. TBA returns a ~1900 placeholder
+    # (-2,208,988,800) for matches with unknown time, which underflows Postgres
+    # int32; CockroachDB INT is 64-bit so this never surfaced in prod. BigInteger
+    # matches that width and is also future-proof past the 2038 int32 limit.
+    time: MI = mapped_column(BigInteger)  # Enforces ordering
+    predicted_time: MOI = mapped_column(BigInteger, nullable=True)  # For display
 
     status: Mapped[MatchStatus] = mapped_column(
         Enum(MatchStatus, values_callable=values_callable)

backend/src/db/models/team_event.py

@@ -1,6 +1,6 @@
 from typing import Any, Dict, Tuple
 
-from sqlalchemy import Boolean, Enum, Float, Integer, String
+from sqlalchemy import BigInteger, Boolean, Enum, Float, Integer, String
 from sqlalchemy.orm import Mapped, mapped_column
 from sqlalchemy.sql.schema import ForeignKeyConstraint, PrimaryKeyConstraint
 
@@ -27,7 +27,7 @@ class TeamEventORM(Base, ModelORM):
     ForeignKeyConstraint(["event"], ["events.key"])
 
     """GENERAL"""
-    time: MI = mapped_column(Integer)
+    time: MI = mapped_column(BigInteger)  # Unix timestamp; see match.py note
 
     """API COMPLETENESS"""
     team_name: MS = mapped_column(String(100))

backend/src/db/read/etag.py

@@ -1,10 +1,10 @@
 from typing import List, Optional
 
 from sqlalchemy.orm.session import Session as SessionType
-from sqlalchemy_cockroachdb import run_transaction  # type: ignore
 
 from src.db.main import Session
 from src.db.models.etag import ETag, ETagORM
+from src.db.transaction import run_transaction
 
 
 def get_etags(year: Optional[int] = None, path: Optional[str] = None) -> List[ETag]:

backend/src/db/read/event.py

@@ -1,11 +1,11 @@
 from typing import List, Optional
 
 from sqlalchemy.orm.session import Session as SessionType
-from sqlalchemy_cockroachdb import run_transaction  # type: ignore
 
 from src.db.main import Session
 from src.db.models.event import Event, EventORM
 from src.db.read.main import common_filters
+from src.db.transaction import run_transaction
 
 
 def get_event(event_id: str) -> Optional[Event]:

backend/src/db/read/match.py

@@ -1,11 +1,11 @@
 from typing import List, Optional
 
 from sqlalchemy.orm.session import Session as SessionType
-from sqlalchemy_cockroachdb import run_transaction  # type: ignore
 
 from src.db.main import Session
 from src.db.models.match import Match, MatchORM
 from src.db.read.main import common_filters
+from src.db.transaction import run_transaction
 
 
 def get_match(match: str) -> Optional[Match]:

backend/src/db/read/team.py

@@ -1,11 +1,11 @@
 from typing import List, Optional
 
 from sqlalchemy.orm.session import Session as SessionType
-from sqlalchemy_cockroachdb import run_transaction  # type: ignore
 
 from src.db.main import Session
 from src.db.models.team import Team, TeamORM
 from src.db.read.main import common_filters
+from src.db.transaction import run_transaction
 
 
 def get_team(team: int) -> Optional[Team]:

backend/src/db/read/team_event.py

@@ -1,11 +1,11 @@
 from typing import List, Optional
 
 from sqlalchemy.orm.session import Session as SessionType
-from sqlalchemy_cockroachdb import run_transaction  # type: ignore
 
 from src.db.main import Session
 from src.db.models.team_event import TeamEvent, TeamEventORM
 from src.db.read.main import common_filters
+from src.db.transaction import run_transaction
 
 
 def get_team_event(team: int, event: str) -> Optional[TeamEvent]:

backend/src/db/read/team_year.py

@@ -1,11 +1,11 @@
 from typing import List, Optional
 
 from sqlalchemy.orm.session import Session as SessionType
-from sqlalchemy_cockroachdb import run_transaction  # type: ignore
 
 from src.db.main import Session
 from src.db.models.team_year import TeamYear, TeamYearORM
 from src.db.read.main import common_filters
+from src.db.transaction import run_transaction
 
 
 def get_team_year(team: int, year: int) -> Optional[TeamYear]:

backend/src/db/read/year.py

@@ -1,11 +1,11 @@
 from typing import List, Optional
 
 from sqlalchemy.orm.session import Session as SessionType
-from sqlalchemy_cockroachdb import run_transaction  # type: ignore
 
 from src.db.main import Session
 from src.db.models.year import Year, YearORM
 from src.db.read.main import common_filters
+from src.db.transaction import run_transaction
 
 
 def get_year(year: int) -> Optional[Year]:

backend/src/db/transaction.py

@@ -0,0 +1,75 @@
+"""Database transaction helper.
+
+Provides a single ``run_transaction`` entry point used across ``src/db``.
+
+Historically the codebase imported ``run_transaction`` directly from
+``sqlalchemy_cockroachdb``. That helper is CockroachDB-specific (it rewrites
+savepoint names for CRDB's transaction-retry protocol). To let the backend run
+against plain PostgreSQL (e.g. Cloud SQL) as well as CockroachDB, this wrapper
+dispatches on the engine dialect:
+
+* CockroachDB  -> delegate to ``sqlalchemy_cockroachdb.run_transaction`` so
+  production behavior is byte-for-byte unchanged (imported lazily so a
+  Postgres-only deployment need not install the CRDB dialect).
+* everything else (PostgreSQL) -> a plain SQLAlchemy transaction with the same
+  retry-on-serialization-failure semantics (SQLSTATE 40001).
+
+Call signature matches the original: ``run_transaction(Session, callback)``
+where ``Session`` is a ``sessionmaker`` and ``callback(session)`` performs the
+work and returns a value. ``callback`` must not commit or roll back; it may be
+invoked more than once and so must be free of non-DB side effects.
+"""
+from typing import Any, Callable, Optional
+
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm.session import Session as SessionType
+
+from src.db.main import engine
+
+# PostgreSQL / CockroachDB serialization failure (retryable).
+SERIALIZATION_FAILURE = "40001"
+
+# Default retry budget for the plain-Postgres path (CRDB helper defaults to
+# unbounded; a small bounded budget is friendlier for a single-writer pipeline).
+DEFAULT_MAX_RETRIES = 3
+
+
+def _run_plain(
+    sessionmaker: Any,
+    callback: Callable[[SessionType], Any],
+    max_retries: int,
+) -> Any:
+    retry_count = 0
+    while True:
+        session = sessionmaker()
+        try:
+            with session.begin():
+                return callback(session)
+        except DBAPIError as exc:
+            retryable = getattr(exc.orig, "pgcode", None) == SERIALIZATION_FAILURE
+            if retryable and retry_count < max_retries:
+                retry_count += 1
+                continue
+            raise
+        finally:
+            session.close()
+
+
+def run_transaction(
+    transactor: Any,
+    callback: Callable[[SessionType], Any],
+    max_retries: Optional[int] = None,
+    max_backoff: int = 0,
+) -> Any:
+    if engine.dialect.name == "cockroachdb":
+        import sqlalchemy_cockroachdb
+
+        return sqlalchemy_cockroachdb.run_transaction(
+            transactor, callback, max_retries=max_retries, max_backoff=max_backoff
+        )
+
+    return _run_plain(
+        transactor,
+        callback,
+        DEFAULT_MAX_RETRIES if max_retries is None else max_retries,
+    )

backend/src/db/write/template.py

@@ -3,7 +3,6 @@
 import attr
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.orm.session import Session as SessionType
-from sqlalchemy_cockroachdb import run_transaction  # type: ignore
 
 from src.db.main import Session
 from src.db.models.etag import ETagORM
@@ -14,6 +13,7 @@
 from src.db.models.team_event import TeamEventORM
 from src.db.models.team_year import TeamYearORM
 from src.db.models.year import YearORM
+from src.db.transaction import run_transaction
 
 CUTOFF = 1000