docs(DEPLOY-004): Postgres worked guide + pool_pre_ping for server DBs

README.md

@@ -470,19 +470,19 @@ local prefetch/relaunch wrapper at container boot.
 Production containers default to fail-closed settings (`APP_ENV=production`,
 `AUTH_REQUIRED=true`, `DATA_DIR=/data`). Supply the same runtime environment the
 non-container app expects, mount a persistent `/data` volume, and provide
-Streamlit's Google OIDC secrets file. The inline `-e` values below are
-placeholders for a manual run; prefer your host's managed secret/environment
-injection for real deployments:
+Streamlit's Google OIDC secrets file. Put `DATABASE_URL`, `DHAN_CLIENT_ID`, and
+`DHAN_ACCESS_TOKEN` in the already-ignored `Dependencies/.env`, run
+`chmod 600 Dependencies/.env`, and load it with `--env-file` so credentials do
+not enter shell history or process arguments. Prefer your host's managed secret
+injection for long-lived deployments:
 
 ```bash
 docker run -d --name streamlit-scanner-app \
   -p 8501:8501 \
+  --env-file Dependencies/.env \
   -e APP_ENV=production \
   -e AUTH_REQUIRED=true \
   -e DATA_DIR=/data \
-  -e DATABASE_URL=postgresql+psycopg://scanner:<password>@db-host:5432/scanner \
-  -e DHAN_CLIENT_ID=your-dhan-client-id \
-  -e DHAN_ACCESS_TOKEN=your-dhan-access-token \
   -e ALLOWED_EMAILS=you@gmail.com \
   -e ADMIN_EMAILS=you@gmail.com \
   -e LOG_FORMAT=json \

backend/storage/database.py

@@ -21,6 +21,7 @@
 import threading
 from collections.abc import Iterator
 from contextlib import contextmanager
+from typing import Any
 
 from sqlalchemy import create_engine, event, inspect
 from sqlalchemy.engine import Engine
@@ -67,16 +68,28 @@ def _make_engine(url: str | None = None) -> Engine:
     """
     database_url = url or get_database_url()
     connect_args = {}
+    engine_kwargs: dict[str, Any] = {}
     if database_url.startswith("sqlite"):
         # Streamlit can re-run the app and use worker threads while the same
         # module-level engine stays imported. SQLite defaults to one-thread-only
         # connections, so we relax that guard and keep sessions short-lived.
         connect_args["check_same_thread"] = False
+    else:
+        # DEPLOY-004: server databases sit behind infrastructure that silently
+        # drops idle TCP connections (managed Postgres proxies, PgBouncer,
+        # cloud NAT). A pooled connection can therefore be dead by the next
+        # morning's first scan, which would fail with "server closed the
+        # connection unexpectedly". ``pool_pre_ping`` issues a lightweight
+        # liveness probe on checkout and transparently replaces dead
+        # connections. SQLite is a local file with no server to lose, so its
+        # engine arguments stay exactly as they were.
+        engine_kwargs["pool_pre_ping"] = True
 
     created_engine = create_engine(
         database_url,
         connect_args=connect_args,
         future=True,
+        **engine_kwargs,
     )
 
     if database_url.startswith("sqlite"):

docs/operations.md

@@ -336,7 +336,7 @@ Move to Postgres when the daily job and the UI run on different machines, or
 when more than one person reads scan history:
 
 ```env
-DATABASE_URL=postgresql+psycopg://scanner:<password>@db-host:5432/scanner
+DATABASE_URL=postgresql+psycopg://scanner:<percent-encoded-password>@db-host:5432/scanner
 ```
 
 The normal pinned setup installs `psycopg[binary]`, which supplies the psycopg 3
@@ -347,6 +347,190 @@ The schema is managed by Alembic; both the app and the daily job run
 `alembic upgrade head` equivalent automatically at startup. To pre-provision
 or debug: `python -m alembic upgrade head`.
 
+### Worked example: self-hosted Postgres, end to end
+
+The steps below take a machine with nothing on it to a scanner reading and
+writing shared Postgres history. Substitute a managed instance (Render, RDS,
+Cloud SQL) from step 3 onward — the app-side steps are identical.
+
+1. **Provision Postgres** (any 15+ works; the container route is the fastest
+   self-hosted path). Create the container environment file outside the
+   repository, restrict it to your account, and edit it with your normal
+   secret-aware editor:
+
+   ```bash
+   touch postgres.env
+   chmod 600 postgres.env
+   ${EDITOR:-vi} postgres.env
+   ```
+
+   Put the values below in that protected file. Generate a real strong
+   password; do not paste it into a command line.
+
+   ```env
+   POSTGRES_USER=scanner
+   POSTGRES_PASSWORD=<generate and paste a strong password here>
+   POSTGRES_DB=scanner
+   ```
+
+   Docker reads the values from the file, so the password does not enter shell
+   history or the process argument list:
+
+   ```bash
+   docker run -d --name scanner-postgres \
+     --env-file postgres.env \
+     -p 5432:5432 \
+     -v scanner-pgdata:/var/lib/postgresql/data \
+     postgres:16
+   ```
+
+   Never commit `postgres.env`; use the host's secret manager when one is
+   available. For a package-managed server, create the role/database as an
+   administrator and use psql's hidden password prompt instead of embedding a
+   password in SQL history:
+
+   ```sql
+   CREATE ROLE scanner WITH LOGIN;
+   CREATE DATABASE scanner OWNER scanner;
+   \password scanner
+   ```
+
+   The app needs no superuser rights: Alembic creates every table as the
+   `scanner` role, so database ownership is the only privilege required.
+
+2. **Point the app at it.** Keep the URL in the already-ignored
+   `Dependencies/.env`, protect that file before editing it, and use a
+   deployment secret manager in production:
+
+   ```bash
+   touch Dependencies/.env
+   chmod 600 Dependencies/.env
+   ${EDITOR:-vi} Dependencies/.env
+   ```
+
+   ```env
+   DATABASE_URL=postgresql+psycopg://scanner:<percent-encoded-password>@db-host:5432/scanner
+   ```
+
+   A URL password is not plain text with delimiters: percent-encode reserved
+   characters such as `@`, `:`, `/`, `?`, `#`, and `%` (`@` becomes `%40`).
+   This prompt reads the password without echoing it or placing it in command
+   history/process arguments; paste the encoded result into the protected env
+   file, not into a shell command:
+
+   ```bash
+   python -c "from getpass import getpass; from urllib.parse import quote; print(quote(getpass('Database password: '), safe=''))"
+   ```
+
+   Production deployments (`APP_ENV=production`) must also set `DATA_DIR`
+   explicitly — startup validation refuses implicit local fallbacks.
+
+3. **Apply the schema.** Nothing to script: the Streamlit app and the headless
+   daily job both run the migration pass automatically at startup. To
+   pre-provision from a workstation instead (e.g. before first deploy):
+
+   ```bash
+   python -m alembic upgrade head
+   ```
+
+   The settings loader reads `Dependencies/.env`, so no credential needs to
+   be repeated on the command line.
+
+4. **Verify.** `psql` should show the Alembic version and the scanner tables:
+
+   ```bash
+   psql -h db-host -U scanner -d scanner -W \
+     -c "SELECT version_num FROM alembic_version;" -c "\dt"
+   ```
+
+   `-W` asks for the password interactively rather than exposing it in process
+   arguments.
+
+   Then run one scan and confirm it lands: the **Scan history** view lists the
+   run, and **Admin health** shows the database as reachable. From the shell:
+   `SELECT id, screener_key, status FROM scan_runs ORDER BY id DESC LIMIT 3;`.
+
+### Moving existing SQLite history into Postgres
+
+Switching `DATABASE_URL` starts you with an **empty** Postgres history; the
+old runs stay in `data/scanner.db` unless you copy them. The recipe below
+moves everything (runs, results, audit rows, IPO evidence) in dependency
+order.
+
+1. **Stop every writer** — the Streamlit app and any scheduled jobs. A copy
+   taken while a scan is writing is torn.
+2. **Build the empty schema on Postgres first** (step 3 above). The copy
+   script inserts into tables Alembic created, so both sides agree on shape,
+   and `alembic_version` is already correct on the target — do not copy that
+   table.
+3. **Copy rows in dependency order.** One-off operator script, run from the
+   repo root (it reuses the ORM metadata, so new tables are picked up
+   automatically):
+
+   ```python
+   import os
+
+   from sqlalchemy import create_engine, select, text
+
+   from backend.storage.models import Base
+
+   source = create_engine("sqlite:///data/scanner.db")
+   target = create_engine(os.environ["DATABASE_URL"])
+
+   with source.connect() as read, target.begin() as write:
+       # SQLite stored these timestamps as naive UTC. Fix the session
+       # timezone so Postgres does not reinterpret them as local time.
+       write.execute(text("SET TIME ZONE 'UTC'"))
+       # sorted_tables yields parents before children, so foreign keys
+       # (scan_results -> scan_runs, IPO children -> ipo_issues) are safe.
+       for table in Base.metadata.sorted_tables:
+           rows = [dict(row._mapping) for row in read.execute(select(table))]
+           if rows:
+               write.execute(table.insert(), rows)
+
+   with target.begin() as write:
+       # The rows arrived with their original primary keys, so each serial
+       # sequence still says "next id = 1". Bump every sequence past the
+       # copied maximum or the first new scan fails with a duplicate key.
+       for table in Base.metadata.sorted_tables:
+           if "id" not in table.columns:
+               continue
+           sequence = write.execute(
+               text(f"SELECT pg_get_serial_sequence('{table.name}', 'id')")
+           ).scalar()
+           if sequence:
+               write.execute(
+                   text(
+                       f"SELECT setval('{sequence}', "
+                       f"(SELECT COALESCE(MAX(id), 1) FROM {table.name}))"
+                   )
+               )
+   ```
+
+4. **Verify counts, then flip.** Compare `SELECT COUNT(*)` for `scan_runs`,
+   `scan_results`, and `audit_logs` on both sides; open the Scan history page
+   against Postgres and spot-check an old run's details. Only then make the
+   new `DATABASE_URL` permanent. Keep `data/scanner.db` (and its `-wal`/`-shm`
+   siblings) as the rollback copy until the first few Postgres-backed scans
+   look healthy.
+
+### Connection-pool behavior and guidance
+
+- The engine enables **`pool_pre_ping` automatically for any non-SQLite URL**
+  (DEPLOY-004): managed Postgres proxies, PgBouncer, and cloud NAT silently
+  drop idle TCP connections, and without the ping the first scan after a
+  quiet stretch inherits a dead pooled connection and fails with "server
+  closed the connection unexpectedly". The ping is one lightweight round-trip
+  per checkout; SQLite behavior is unchanged.
+- SQLAlchemy's defaults (pool of 5, overflow 10) are ample here: the UI holds
+  sessions only for short transactions, and the daily job is a single writer.
+  Size the *database's* `max_connections` for the number of app instances,
+  not the other way around.
+- If you front Postgres with **PgBouncer**, use *session* pooling. The app's
+  short `session_scope()` transactions are compatible with transaction
+  pooling too, but session pooling avoids surprises with any
+  connection-scoped state and is the conservative default.
+
 ---
 
 ## Docker / container deployment
@@ -451,18 +635,19 @@ docker run --rm \
 ```
 
 For production, keep `/data` on persistent storage and point `DATABASE_URL` at
-Postgres. The inline `-e` values are placeholders for a manual run; prefer the
-host platform's managed secret/environment injection for long-lived deployments:
+Postgres. Put `DATABASE_URL`, `DHAN_CLIENT_ID`, and `DHAN_ACCESS_TOKEN` in the
+already-ignored `Dependencies/.env` file described above, then protect it with
+`chmod 600 Dependencies/.env`. Docker's `--env-file` option keeps those values
+out of shell history and the process arguments. Prefer the host platform's
+managed secret/environment injection for long-lived deployments:
 
 ```bash
 docker run -d --name streamlit-scanner-app \
   -p 8501:8501 \
+  --env-file Dependencies/.env \
   -e APP_ENV=production \
   -e AUTH_REQUIRED=true \
   -e DATA_DIR=/data \
-  -e DATABASE_URL=postgresql+psycopg://scanner:<password>@db-host:5432/scanner \
-  -e DHAN_CLIENT_ID=<client-id> \
-  -e DHAN_ACCESS_TOKEN=<access-token> \
   -e ALLOWED_EMAILS=you@gmail.com \
   -e ADMIN_EMAILS=you@gmail.com \
   -e LOG_FORMAT=json \
@@ -487,12 +672,10 @@ Run the headless daily job with the same image and runtime configuration:
 ```bash
 docker run --rm \
   --entrypoint python \
+  --env-file Dependencies/.env \
   -e APP_ENV=production \
   -e AUTH_REQUIRED=true \
   -e DATA_DIR=/data \
-  -e DATABASE_URL=postgresql+psycopg://scanner:<password>@db-host:5432/scanner \
-  -e DHAN_CLIENT_ID=<client-id> \
-  -e DHAN_ACCESS_TOKEN=<access-token> \
   -v streamlit-scanner-data:/data \
   -v /absolute/path/secrets.toml:/app/.streamlit/secrets.toml:ro \
   streamlit-scanner-app \

migrations/env.py

@@ -54,7 +54,13 @@ def run_migrations_online() -> None:
     command time so tests can point migrations at a temporary SQLite file and
     deployments can point them at Postgres.
     """
-    config.set_main_option("sqlalchemy.url", get_database_url())
+    # Alembic stores option values through ConfigParser, where ``%`` starts an
+    # interpolation expression. Database passwords commonly contain URL escapes
+    # such as ``%40`` for ``@``; doubling the percent sign protects that literal
+    # value inside ConfigParser. Reading the option gives SQLAlchemy the original
+    # URL again, while malformed interpolation can no longer echo the full secret
+    # URL in an early traceback.
+    config.set_main_option("sqlalchemy.url", get_database_url().replace("%", "%%"))
     connectable = engine_from_config(
         config.get_section(config.config_ini_section, {}),
         prefix="sqlalchemy.",

tests/test_docker_artifacts.py

@@ -125,7 +125,9 @@ def test_readme_and_operations_document_docker_runtime() -> None:
     assert "AUTH_REQUIRED=false" in readme
     assert "APP_ENV=production" in readme
     assert "DATA_DIR=/data" in readme
-    assert "DATABASE_URL=postgresql+psycopg://" in readme
+    assert "--env-file Dependencies/.env" in readme
+    assert "-e DATABASE_URL=" not in readme
+    assert "-e DHAN_ACCESS_TOKEN=" not in readme
     assert ".streamlit/secrets.toml" in readme
 
     assert "Docker / container deployment" in operations

tests/test_scan_storage_database.py

@@ -124,6 +124,32 @@ def test_make_engine_applies_sqlite_concurrency_pragmas(tmp_path: Path):
         engine.dispose()
 
 
+def test_make_engine_enables_pool_pre_ping_only_for_server_databases(tmp_path: Path):
+    """DEPLOY-004: dead pooled Postgres connections must be replaced on checkout.
+
+    Managed Postgres proxies and PgBouncer idle-kill TCP connections, so the
+    first scan after a quiet stretch used to inherit a dead pooled connection.
+    ``pool_pre_ping`` probes on checkout and reconnects transparently. SQLite
+    is a local file with no server to lose, so its engine must stay exactly as
+    before (no pre-ping). Building the Postgres engine is safe without a
+    server: SQLAlchemy connects lazily, and the pinned psycopg driver only has
+    to import.
+    """
+    from backend.storage.database import _make_engine
+
+    sqlite_engine = _make_engine(f"sqlite:///{(tmp_path / 'pre-ping.db').as_posix()}")
+    try:
+        assert sqlite_engine.pool._pre_ping is False
+    finally:
+        sqlite_engine.dispose()
+
+    postgres_engine = _make_engine("postgresql+psycopg://scanner:secret@localhost:5432/scanner")
+    try:
+        assert postgres_engine.pool._pre_ping is True
+    finally:
+        postgres_engine.dispose()
+
+
 def test_missing_expected_tables_empty_when_schema_complete(tmp_path: Path):
     """A fully built database reports no missing tables (the healthy path)."""
     from backend.storage.database import _make_engine, _missing_expected_tables