Feat dashboard tracking

alembic/versions/3f4dce7266b1_create_linker_table.py

@@ -0,0 +1,38 @@
+"""create_linker_table
+
+Revision ID: 3f4dce7266b1
+Revises:
+Create Date: 2026-02-20 09:23:02.062061
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql.json import JSONB
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "3f4dce7266b1"
+down_revision: Union[str, Sequence[str], None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    op.create_table(
+        "link_history",
+        sa.Column("link_id", sa.String(255), nullable=False, primary_key=True),
+        sa.Column("user", sa.String(255), nullable=False),
+        sa.Column("link_dict", JSONB),
+        sa.Column("link_timestamp", sa.DateTime(timezone=True), default=sa.func.now()),
+        sa.Column("decision", sa.Boolean),
+        sa.Column("decision_timestamp", sa.DateTime(timezone=True)),
+    )
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    op.drop_table("link_history")

code/create_users.sh

@@ -0,0 +1,10 @@
+set -a
+source ./.env
+set +a
+
+# https://stackoverflow.com/questions/78251247/how-to-use-systems-environnement-variables-in-sql-script
+if [ $1 == "local" ]; then
+    psql -U $dbusername -p $dbport -h 127.0.0.1 -d pir -f <(envsubst < $2)
+else
+    psql -U $DB_USER -p $DB_PORT -h $DB_HOST -d $DB_NAME -f <(envsubst < $2)
+fi

code/serve_qa_dashboard.sh

@@ -1,5 +1,19 @@
-cd ~/repos/ACF-pir-data/
-source .venv/bin/activate
-export LOADING_DASHBOARD="True"
-google-chrome http://localhost:8080 &>/dev/null &
-waitress-serve --host 127.0.0.1 --call "pir_pipeline.dashboard:create_app"
+source ./application/.venv/bin/activate
+
+USER_NUMBER=$(id -u)
+USER_NAME=$(id -un)
+USER_PORT=$((5000 + $USER_NUMBER % 1000))
+
+export DB_ENV="production"
+
+if [ $USER_NAME == "reggie.gilliard" ]; then
+    export DB_PROFILE="approver"
+elif [ $USER_NAME == "emily.kowall" ]; then
+    export DB_PROFILE="approver"
+elif [ $USER_NAME == "alecsandra.velez" ]; then
+    export DB_PROFILE="approver"
+else
+    export DB_PROFILE="reviewer"
+fi
+
+waitress-serve --host 127.0.0.1 --port=${USER_PORT} --call "pir_pipeline.dashboard:create_app" 

code/sql/user_local_approver.sql

@@ -0,0 +1,10 @@
+CREATE USER approver
+WITH PASSWORD '$approver_user_password';
+
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO approver;
+
+GRANT ALL PRIVILEGES ON proposed_changes, uqid_changelog, uqid_changelog_id_seq, link_history TO approver;
+GRANT INSERT, SELECT, UPDATE ON question TO approver;
+
+ALTER DEFAULT PRIVILEGES IN SCHEMA public 
+GRANT SELECT ON TABLES TO approver;

code/sql/user_local_reviewer.sql

@@ -0,0 +1,9 @@
+CREATE USER reviewer
+WITH PASSWORD '$reviewer_user_password';
+
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO reviewer;
+
+GRANT ALL PRIVILEGES ON proposed_changes, link_history TO reviewer;
+
+ALTER DEFAULT PRIVILEGES IN SCHEMA public 
+GRANT SELECT ON TABLES TO reviewer;

code/sql/user_prod_approver.sql

@@ -0,0 +1,10 @@
+CREATE USER approver
+WITH PASSWORD '$USER_PASSWORD_APPROVER';
+
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO approver;
+
+GRANT ALL PRIVILEGES ON proposed_changes, uqid_changelog, uqid_changelog_id_seq, link_history TO approver;
+GRANT INSERT, SELECT, UPDATE ON question TO approver;
+
+ALTER DEFAULT PRIVILEGES IN SCHEMA public 
+GRANT SELECT ON TABLES TO approver;

code/sql/user_prod_reviewer.sql

@@ -0,0 +1,9 @@
+CREATE USER reviewer
+WITH PASSWORD '$USER_PASSWORD_REVIEWER';
+
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO reviewer;
+
+GRANT ALL PRIVILEGES ON proposed_changes, link_history TO reviewer;
+
+ALTER DEFAULT PRIVILEGES IN SCHEMA public 
+GRANT SELECT ON TABLES TO reviewer;

src/pir_pipeline/dashboard/__init__.py

@@ -48,13 +48,14 @@ def create_app(test_config: dict = None, **kwargs) -> Flask:
 
     db.init_app(app)
 
-    from pir_pipeline.dashboard import finalize, index, review, search
+    from pir_pipeline.dashboard import finalize, index, metrics, review, search
 
     app.register_blueprint(index.bp)
     app.add_url_rule("/", endpoint="index")
 
     app.register_blueprint(search.bp)
     app.register_blueprint(review.bp)
     app.register_blueprint(finalize.bp)
+    app.register_blueprint(metrics.bp)
 
     return app

src/pir_pipeline/dashboard/finalize.py

@@ -7,6 +7,7 @@
 from sqlalchemy import select, text
 
 from pir_pipeline.dashboard.db import get_db
+from pir_pipeline.dashboard.utils import administrator
 
 bp = Blueprint("finalize", __name__, url_prefix="/finalize")
 
@@ -80,6 +81,7 @@ def get_page(number_displayed: Optional[int] = None) -> WrappedList:
 
 
 @bp.route("/", methods=["GET"])
+@administrator
 def index():
     session["finalize_page"] = 0
     session["number_displayed"] = session.get("number_displayed", DEFAULT_DISPLAYED)

src/pir_pipeline/dashboard/metrics.py

@@ -0,0 +1,193 @@
+"""Routes and logic for the metrics page"""
+
+from datetime import datetime, timedelta
+
+import pandas as pd
+from flask import Blueprint, render_template
+from sqlalchemy import func, select
+
+from pir_pipeline.dashboard.db import get_db
+from pir_pipeline.dashboard.utils import administrator
+
+bp = Blueprint("metrics", __name__, url_prefix="/metrics")
+
+
+def get_confirmed_dict(by: list[str] = []):
+    db = get_db()
+
+    uqid_changelog = db.tables["uqid_changelog"]
+    confirmed = db.tables["confirmed"]
+
+    # Confirmed uqids
+    uqids = (
+        select(
+            uqid_changelog.c["original_uqid"],
+            func.min(uqid_changelog.c["timestamp"]).label("timestamp"),
+        )
+        .group_by(uqid_changelog.c["original_uqid"])
+        .having(func.max(uqid_changelog.c["complete_series_flag"]) == 1)
+        .subquery()
+    )
+
+    # Number of confirmed questions
+    confirmed = select(confirmed, uqids.c["timestamp"]).join(
+        uqids,
+        onclause=uqids.c["original_uqid"] == confirmed.c["uqid"],
+    )
+    confirmed_df = db.get_records(confirmed)
+    confirmed_df["date"] = confirmed_df["timestamp"].map(
+        lambda date: date.date().strftime("%Y-%m-%d")
+    )
+    confirmed_dict = (
+        confirmed_df.groupby(by=["date"] + by)
+        .size()
+        .sort_index(ascending=False)
+        .to_dict()
+    )
+
+    return confirmed_dict
+
+
+@bp.route("/")
+@administrator
+def index():
+    db = get_db()
+    df = db.get_records("SELECT * FROM link_history")
+
+    approval_by_user = (
+        df.groupby("user")
+        .aggregate(percent_approved=("decision", "mean"))
+        .reset_index()
+        .to_dict(orient="index")
+    )
+
+    return render_template("metrics/metrics.html", approval_by_user=approval_by_user)
+
+
+@bp.route("/daily_links")
+@administrator
+def daily_links():
+    confirmed_dict = get_confirmed_dict()
+
+    return render_template("metrics/daily_links.html", confirmed_dict=confirmed_dict)
+
+
+@bp.route("/projections")
+@administrator
+def projections():
+    def get_projections(
+        average: int | pd.DataFrame, remaining_days: int, remaining: int | pd.DataFrame
+    ):
+        if isinstance(average, pd.DataFrame):
+            merged = average.join(remaining)
+            merged["year_end"] = merged["average"].map(
+                lambda x: round(x * remaining_days)
+            )
+            merged["difference"] = merged["count"] - merged["year_end"]
+            merged["shortfall"] = merged["difference"].map(lambda x: x if x > 0 else 0)
+            merged["average"] = merged["average"].round(1)
+            out_dict = (
+                merged.reset_index()[["year", "average", "year_end", "shortfall"]]
+                .sort_values("year", ascending=False)
+                .to_dict(orient="index")
+            )
+        else:
+            projection = round(average * remaining_days)
+            difference = remaining - projection
+            shortfall = difference if difference > 0 else 0
+
+            out_dict = {
+                "average": round(average, 1),
+                "remaining_days": remaining_days,
+                "year_end": projection,
+                "shortfall": shortfall,
+            }
+
+        return out_dict
+
+    db = get_db()
+    confirmed_dict = get_confirmed_dict()
+    confirmed_df = pd.DataFrame.from_dict(
+        confirmed_dict, orient="index", columns=["count"]
+    ).reset_index(names=["date"])
+
+    confirmed_df["date"] = confirmed_df["date"].map(datetime.fromisoformat)
+    unconfirmed = db.tables["unconfirmed"]
+
+    remaining_days = (datetime.fromisoformat("2026-10-01") - datetime.today()).days
+    remaining_questions = db.get_scalar(
+        select(func.count(unconfirmed.table_valued())), {}
+    )
+    remaining_questions_by_year = db.get_records(
+        select(
+            func.count(unconfirmed.c["question_id"]), unconfirmed.c["year"]
+        ).group_by(unconfirmed.c["year"])
+    ).set_index("year")
+
+    # Full data set projections
+    weekly_average = (
+        confirmed_df[
+            confirmed_df["date"].map(
+                lambda date: date >= datetime.today() - timedelta(days=7)
+            )
+        ]["count"].sum()
+        / 7
+    )
+    overall_average = (
+        confirmed_df["count"].sum()
+        / (datetime.today() - confirmed_df["date"].min()).days
+    )
+
+    overall_projection = {
+        "Weekly": get_projections(weekly_average, remaining_days, remaining_questions),
+        "Overall": get_projections(
+            overall_average, remaining_days, remaining_questions
+        ),
+    }
+
+    # Projections by year
+    confirmed_dict = get_confirmed_dict(["year"])
+    confirmed_df = pd.DataFrame.from_dict(
+        confirmed_dict, orient="index", columns=["count"]
+    ).reset_index(names=["date", "year"])
+
+    confirmed_df["date"], confirmed_df["year"] = zip(
+        *confirmed_df["date"].map(lambda x: (x[0], x[1]))
+    )
+    confirmed_df["date"] = confirmed_df["date"].map(datetime.fromisoformat)
+
+    weekly_df = confirmed_df[
+        confirmed_df["date"].map(
+            lambda date: date >= datetime.today() - timedelta(days=7)
+        )
+    ]
+    weekly_average_df = (
+        (weekly_df.groupby("year")["count"].sum() / 7)
+        .reset_index()
+        .rename(columns={"count": "average"})
+        .set_index("year")
+    )
+
+    overall_average_df = (
+        (
+            confirmed_df.groupby("year")["count"].sum()
+            / (datetime.today() - confirmed_df["date"].min()).days
+        )
+        .reset_index()
+        .rename(columns={"count": "average"})
+        .set_index("year")
+    )
+
+    weekly_projections_by_year = get_projections(
+        weekly_average_df, remaining_days, remaining_questions_by_year
+    )
+    overall_projections_by_year = get_projections(
+        overall_average_df, remaining_days, remaining_questions_by_year
+    )
+
+    return render_template(
+        "metrics/projections.html",
+        overall_projection=overall_projection,
+        weekly_by_year=weekly_projections_by_year,
+        overall_by_year=overall_projections_by_year,
+    )