From aba58fda7dcfa7239dc843674436d08cd5f5e078 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Fri, 20 Feb 2026 10:37:09 -0500 Subject: [PATCH 01/25] Add link_history table --- .../3f4dce7266b1_create_linker_table.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 alembic/versions/3f4dce7266b1_create_linker_table.py diff --git a/alembic/versions/3f4dce7266b1_create_linker_table.py b/alembic/versions/3f4dce7266b1_create_linker_table.py new file mode 100644 index 0000000..1bac696 --- /dev/null +++ b/alembic/versions/3f4dce7266b1_create_linker_table.py @@ -0,0 +1,38 @@ +"""create_linker_table + +Revision ID: 3f4dce7266b1 +Revises: +Create Date: 2026-02-20 09:23:02.062061 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql.json import JSONB + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "3f4dce7266b1" +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "link_history", + sa.Column("link_id", sa.String(255), nullable=False, primary_key=True), + sa.Column("user", sa.String(255), nullable=False), + sa.Column("link_dict", JSONB), + sa.Column("link_timestamp", sa.DateTime(timezone=True), default=sa.func.now()), + sa.Column("confirmed", sa.Boolean), + sa.Column("confirmed_timestamp", sa.DateTime(timezone=True)), + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_table("link_history") From 6b61a4dd796b0286764d9977e0d51ac54b25f9a2 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Fri, 20 Feb 2026 12:27:04 -0500 Subject: [PATCH 02/25] Dashboard: Track proposed links Track proposed links in link_history table, including who proposed the link and the decision on the proposal (confirm/deny) --- .../3f4dce7266b1_create_linker_table.py | 4 +- src/pir_pipeline/dashboard/review.py | 53 +++++++++++++++++-- src/pir_pipeline/models/pir_sql_models.py | 12 +++++ src/pir_pipeline/utils/SQLAlchemyUtils.py | 8 ++- tests/dashboard/test_review.py | 23 ++++++++ 5 files changed, 93 insertions(+), 7 deletions(-) diff --git a/alembic/versions/3f4dce7266b1_create_linker_table.py b/alembic/versions/3f4dce7266b1_create_linker_table.py index 1bac696..55acbe0 100644 --- a/alembic/versions/3f4dce7266b1_create_linker_table.py +++ b/alembic/versions/3f4dce7266b1_create_linker_table.py @@ -28,8 +28,8 @@ def upgrade() -> None: sa.Column("user", sa.String(255), nullable=False), sa.Column("link_dict", JSONB), sa.Column("link_timestamp", sa.DateTime(timezone=True), default=sa.func.now()), - sa.Column("confirmed", sa.Boolean), - sa.Column("confirmed_timestamp", sa.DateTime(timezone=True)), + sa.Column("decision", sa.Boolean), + sa.Column("decision_timestamp", sa.DateTime(timezone=True)), ) diff --git a/src/pir_pipeline/dashboard/review.py b/src/pir_pipeline/dashboard/review.py index e439b6e..91d42c8 100644 --- a/src/pir_pipeline/dashboard/review.py +++ b/src/pir_pipeline/dashboard/review.py @@ -2,12 +2,15 @@ import json from collections import OrderedDict +from datetime import datetime +from getpass import getuser from hashlib import sha1 from flask import Blueprint, render_template, request, session from sqlalchemy import bindparam, delete, func, select from pir_pipeline.dashboard.db import get_db +from pir_pipeline.models.pir_sql_models import link_history from pir_pipeline.utils.dashboard_utils import ( QuestionLinker, get_matches, @@ -175,32 +178,76 @@ def link(): ], "proposed_changes", ) + db.insert_records( + [ + { + "link_id": proposed_id, + "user": getuser(), + "link_dict": list(link_dict.values()), + } + ], + "link_history", + ) del session["link_dict"] message = f"Record {link_dict} written to proposed changes." # Execute all linking actions elif action == "confirm": + link_id = payload["id"] proposed_changes = db.tables["proposed_changes"] link_dict_query = select(proposed_changes.c["link_dict"]).where( proposed_changes.c["id"] == bindparam("id") ) - link_dict = db.get_scalar(link_dict_query, {"id": payload["id"]}) + link_dict = db.get_scalar(link_dict_query, {"id": link_id}) QuestionLinker(link_dict, db).update_links() delete_query = delete(proposed_changes).where( - proposed_changes.c["id"] == payload["id"] + proposed_changes.c["id"] == link_id ) with db.engine.begin() as conn: conn.execute(delete_query) + db.update_records( + link_history, + { + "decision": bindparam("decision"), + "decision_timestamp": bindparam("decision_timestamp"), + }, + link_history.c["link_id"] == bindparam("b_link_id"), + [ + { + "b_link_id": link_id, + "decision": True, + "decision_timestamp": datetime.now(), + } + ], + ) + message = "Links Updated!" elif action == "deny": + link_id = payload["id"] proposed_changes = db.tables["proposed_changes"] delete_query = delete(proposed_changes).where( - proposed_changes.c["id"] == payload["id"] + proposed_changes.c["id"] == link_id ) with db.engine.begin() as conn: conn.execute(delete_query) + db.update_records( + link_history, + { + "decision": bindparam("decision"), + "decision_timestamp": bindparam("decision_timestamp"), + }, + link_history.c["link_id"] == bindparam("b_link_id"), + [ + { + "b_link_id": link_id, + "decision": False, + "decision_timestamp": datetime.now(), + } + ], + ) + message = f"Removed link associated with id: {payload["id"]}" return {"message": message} diff --git a/src/pir_pipeline/models/pir_sql_models.py b/src/pir_pipeline/models/pir_sql_models.py index 18f3407..953353d 100644 --- a/src/pir_pipeline/models/pir_sql_models.py +++ b/src/pir_pipeline/models/pir_sql_models.py @@ -1,6 +1,7 @@ """SQLAlchemy models for creating and interacting with the PIR database""" from sqlalchemy import ( + Boolean, Column, DateTime, Float, @@ -115,6 +116,17 @@ Column("html", Text), ) +link_history = Table( + "link_history", + sql_metadata, + Column("link_id", String(255), nullable=False, primary_key=True), + Column("user", String(255), nullable=False), + Column("link_dict", JSONB), + Column("link_timestamp", DateTime(timezone=True), default=func.now()), + Column("decision", Boolean), + Column("decision_timestamp", DateTime(timezone=True)), +) + # Here to proposed_ids definition written with GPT dictionaries = ( func.jsonb_array_elements(proposed_changes.c.link_dict) diff --git a/src/pir_pipeline/utils/SQLAlchemyUtils.py b/src/pir_pipeline/utils/SQLAlchemyUtils.py index 4bf972a..6ced086 100644 --- a/src/pir_pipeline/utils/SQLAlchemyUtils.py +++ b/src/pir_pipeline/utils/SQLAlchemyUtils.py @@ -13,6 +13,7 @@ from pir_pipeline.models.pir_sql_models import ( confirmed, flashcard, + link_history, linked, program, proposed_changes, @@ -71,6 +72,7 @@ def __init__(self, user: str, password: str, host: str, port: int, database: str "program": program, "proposed_changes": proposed_changes, "linked": linked, + "link_history": link_history, "unlinked": unlinked, "uqid_changelog": uqid_changelog, "confirmed": confirmed, @@ -169,11 +171,13 @@ def get_columns(self, table: str, where: str = "") -> list[str]: elif self._dialect == "postgresql": table_schema = "table_catalog" - query = text(f""" + query = text( + f""" SELECT column_name FROM information_schema.columns WHERE table_name = :table AND {table_schema} = :schema {where} - """) + """ + ) with self._engine.connect() as conn: result = conn.execute( query, {"table": table, "schema": self._database, "where": where} diff --git a/tests/dashboard/test_review.py b/tests/dashboard/test_review.py index 89ffb3c..5c056ad 100644 --- a/tests/dashboard/test_review.py +++ b/tests/dashboard/test_review.py @@ -136,21 +136,40 @@ def test_post_link(self, client, sql_utils, error_message_constructor): result = conn.execute(select(sql_utils.tables["proposed_changes"])) records = result.all() + result = conn.execute( + select(sql_utils.tables["link_history"]).where( + sql_utils.tables["link_history"].c["link_id"] == deny_id + ) + ) + decision = result.one()[-2] + expected = 1 got = len(records) assert expected == got, error_message_constructor( "Incorrect number of records", expected, got ) + assert decision is False, error_message_constructor( + "Decision should be false", expected, got + ) + confirm_id = records[0][0] client.post("/review/link", json={"action": "confirm", "id": confirm_id}) with sql_utils.engine.connect() as conn: result = conn.execute(select(sql_utils.tables["proposed_changes"])) proposed_records = result.all() + result = conn.execute(select(sql_utils.tables["uqid_changelog"])) confirmed_records = result.all() + result = conn.execute( + select(sql_utils.tables["link_history"]).where( + sql_utils.tables["link_history"].c["link_id"] == confirm_id + ) + ) + decision = result.one()[-2] + expected = 0 got = len(proposed_records) assert expected == got, error_message_constructor( @@ -169,6 +188,10 @@ def test_post_link(self, client, sql_utils, error_message_constructor): "Incorrect record in db", expected, got ) + assert decision is True, error_message_constructor( + "Decision should be true", expected, got + ) + if __name__ == "__main__": pytest.main([__file__, "-sk", "test_post_link"]) From ed3e29a4dff42f52e24198e7e96cdfd044e9e636 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Sun, 22 Feb 2026 16:05:39 -0500 Subject: [PATCH 03/25] Dashboard: Light authentication for finalize page --- src/pir_pipeline/dashboard/finalize.py | 2 ++ src/pir_pipeline/dashboard/utils.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 src/pir_pipeline/dashboard/utils.py diff --git a/src/pir_pipeline/dashboard/finalize.py b/src/pir_pipeline/dashboard/finalize.py index 2d203cb..0b05483 100644 --- a/src/pir_pipeline/dashboard/finalize.py +++ b/src/pir_pipeline/dashboard/finalize.py @@ -7,6 +7,7 @@ from sqlalchemy import select, text from pir_pipeline.dashboard.db import get_db +from pir_pipeline.dashboard.utils import administrator bp = Blueprint("finalize", __name__, url_prefix="/finalize") @@ -80,6 +81,7 @@ def get_page(number_displayed: Optional[int] = None) -> WrappedList: @bp.route("/", methods=["GET"]) +@administrator def index(): session["finalize_page"] = 0 session["number_displayed"] = session.get("number_displayed", DEFAULT_DISPLAYED) diff --git a/src/pir_pipeline/dashboard/utils.py b/src/pir_pipeline/dashboard/utils.py new file mode 100644 index 0000000..b543bc4 --- /dev/null +++ b/src/pir_pipeline/dashboard/utils.py @@ -0,0 +1,17 @@ +from functools import wraps +from getpass import getuser + +from flask import flash, redirect, url_for + + +# https://flask.palletsprojects.com/en/stable/tutorial/views/ +def administrator(view): + @wraps(view) + def wrapped_view(**kwargs): + if getuser() not in ["jesse.escobar", "emily.kowall", "reggie.gilliard"]: + flash("You are not authorized to access this page.") + return redirect(url_for("index")) + + return view(**kwargs) + + return wrapped_view From bd200004d7abd87fa023ef4fa89b367f16df6b35 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Mon, 23 Feb 2026 07:37:01 -0500 Subject: [PATCH 04/25] SQL Models: Remove subquery from proposed_uqid_query --- src/pir_pipeline/models/pir_sql_models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pir_pipeline/models/pir_sql_models.py b/src/pir_pipeline/models/pir_sql_models.py index 953353d..65869aa 100644 --- a/src/pir_pipeline/models/pir_sql_models.py +++ b/src/pir_pipeline/models/pir_sql_models.py @@ -195,7 +195,6 @@ select(question.c.uqid) .where(and_(question.c.question_id.in_(proposed_ids), question.c.uqid != null())) .distinct() - .subquery() ) query = ( From 1566adbd6279305adab8711fb80c69d3b897fef5 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Mon, 23 Feb 2026 08:53:28 -0500 Subject: [PATCH 05/25] Metrics: Begin adding metrics page --- src/pir_pipeline/dashboard/__init__.py | 3 ++- src/pir_pipeline/dashboard/metrics.py | 23 +++++++++++++++++++ .../dashboard/templates/base.html | 3 +++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 src/pir_pipeline/dashboard/metrics.py diff --git a/src/pir_pipeline/dashboard/__init__.py b/src/pir_pipeline/dashboard/__init__.py index 6e1996d..b4b1b9b 100644 --- a/src/pir_pipeline/dashboard/__init__.py +++ b/src/pir_pipeline/dashboard/__init__.py @@ -48,7 +48,7 @@ def create_app(test_config: dict = None, **kwargs) -> Flask: db.init_app(app) - from pir_pipeline.dashboard import finalize, index, review, search + from pir_pipeline.dashboard import finalize, index, metrics, review, search app.register_blueprint(index.bp) app.add_url_rule("/", endpoint="index") @@ -56,5 +56,6 @@ def create_app(test_config: dict = None, **kwargs) -> Flask: app.register_blueprint(search.bp) app.register_blueprint(review.bp) app.register_blueprint(finalize.bp) + app.register_blueprint(metrics.bp) return app diff --git a/src/pir_pipeline/dashboard/metrics.py b/src/pir_pipeline/dashboard/metrics.py new file mode 100644 index 0000000..f015899 --- /dev/null +++ b/src/pir_pipeline/dashboard/metrics.py @@ -0,0 +1,23 @@ +"""Routes and logic for the home page""" + +from flask import Blueprint, render_template + +from pir_pipeline.dashboard.db import get_db +from pir_pipeline.dashboard.utils import administrator + +bp = Blueprint("metrics", __name__, url_prefix="/metrics") + + +@bp.route("/") +@administrator +def index(): + db = get_db() + df = db.get_records("SELECT * FROM link_history") + approval_by_user = ( + df.groupby("user") + .aggregate(percent_approved=("decision", "mean")) + .reset_index() + .to_dict(orient="index") + ) + + return render_template("metrics/metrics.html", approval_by_user=approval_by_user) diff --git a/src/pir_pipeline/dashboard/templates/base.html b/src/pir_pipeline/dashboard/templates/base.html index b53062a..49707ec 100644 --- a/src/pir_pipeline/dashboard/templates/base.html +++ b/src/pir_pipeline/dashboard/templates/base.html @@ -36,6 +36,9 @@
  • Finalize
    • +
  • + Metrics +
    • From 70e7423e49cc1445c732a1cf6fd00e40226879e3 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Mon, 23 Feb 2026 10:12:40 -0500 Subject: [PATCH 06/25] Metrics: Base metrics html and WIP metrics page --- .../dashboard/static/styles/metrics.css | 16 ++++++++++ .../dashboard/templates/metrics/base.html | 18 +++++++++++ .../dashboard/templates/metrics/metrics.html | 30 +++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 src/pir_pipeline/dashboard/static/styles/metrics.css create mode 100644 src/pir_pipeline/dashboard/templates/metrics/base.html create mode 100644 src/pir_pipeline/dashboard/templates/metrics/metrics.html diff --git a/src/pir_pipeline/dashboard/static/styles/metrics.css b/src/pir_pipeline/dashboard/static/styles/metrics.css new file mode 100644 index 0000000..0b6b69a --- /dev/null +++ b/src/pir_pipeline/dashboard/static/styles/metrics.css @@ -0,0 +1,16 @@ +section.content { + display: flex; +} + +section { + margin: 0; +} + +section.nav-panel { + align-self: start; + width: 10%; +} + +section.main-panel { + width: 100%; +} \ No newline at end of file diff --git a/src/pir_pipeline/dashboard/templates/metrics/base.html b/src/pir_pipeline/dashboard/templates/metrics/base.html new file mode 100644 index 0000000..3f04b03 --- /dev/null +++ b/src/pir_pipeline/dashboard/templates/metrics/base.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} + +{% block header %} + +{% endblock %} + +{% block content %} +
    + +
    +
    + {% block main_panel %}{% endblock %} +
    +{% endblock %} \ No newline at end of file diff --git a/src/pir_pipeline/dashboard/templates/metrics/metrics.html b/src/pir_pipeline/dashboard/templates/metrics/metrics.html new file mode 100644 index 0000000..03db4b3 --- /dev/null +++ b/src/pir_pipeline/dashboard/templates/metrics/metrics.html @@ -0,0 +1,30 @@ +{% extends "metrics/base.html" %} + +{% block header %} + +

    {% block title %}Metrics{% endblock %}

    +{% endblock %} + +{% block main_panel %} +

    Approval Rating By User

    +
    + + + + + + + + {% for row in approval_by_user.values() %} + + + + + {% endfor %} +
    UserPercent Approved
    {{ row["user"] }}{{ row["percent_approved"] }}
    +
    +{% endblock %} + +{% block scripts %} + +{% endblock %} \ No newline at end of file From a9f65c35e4040a58d00fcf6b66826b7d08d73a53 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Mon, 23 Feb 2026 10:24:47 -0500 Subject: [PATCH 07/25] Metrics: metrics.html remove div.card-body --- .../dashboard/templates/metrics/metrics.html | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/pir_pipeline/dashboard/templates/metrics/metrics.html b/src/pir_pipeline/dashboard/templates/metrics/metrics.html index 03db4b3..c195b4c 100644 --- a/src/pir_pipeline/dashboard/templates/metrics/metrics.html +++ b/src/pir_pipeline/dashboard/templates/metrics/metrics.html @@ -7,22 +7,20 @@

    {% block title %}Metrics{% endblock %}

    {% block main_panel %}

    Approval Rating By User

    -
    - - - - - - - - {% for row in approval_by_user.values() %} - - - - - {% endfor %} -
    UserPercent Approved
    {{ row["user"] }}{{ row["percent_approved"] }}
    -
    + + + + + + + + {% for row in approval_by_user.values() %} + + + + + {% endfor %} +
    UserPercent Approved
    {{ row["user"] }}{{ row["percent_approved"] }}
    {% endblock %} {% block scripts %} From 5471373395cd6b319a63b09df8017597511b6ae6 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Mon, 23 Feb 2026 10:30:05 -0500 Subject: [PATCH 08/25] Metrics: Progress page outline --- src/pir_pipeline/dashboard/metrics.py | 6 ++++++ src/pir_pipeline/dashboard/static/styles/metrics.css | 1 + src/pir_pipeline/dashboard/templates/metrics/base.html | 4 ++++ .../dashboard/templates/metrics/progress.html | 8 ++++++++ 4 files changed, 19 insertions(+) create mode 100644 src/pir_pipeline/dashboard/templates/metrics/progress.html diff --git a/src/pir_pipeline/dashboard/metrics.py b/src/pir_pipeline/dashboard/metrics.py index f015899..64fa47b 100644 --- a/src/pir_pipeline/dashboard/metrics.py +++ b/src/pir_pipeline/dashboard/metrics.py @@ -21,3 +21,9 @@ def index(): ) return render_template("metrics/metrics.html", approval_by_user=approval_by_user) + + +@bp.route("/progress") +@administrator +def progress(): + return render_template("metrics/progress.html") diff --git a/src/pir_pipeline/dashboard/static/styles/metrics.css b/src/pir_pipeline/dashboard/static/styles/metrics.css index 0b6b69a..d199aa1 100644 --- a/src/pir_pipeline/dashboard/static/styles/metrics.css +++ b/src/pir_pipeline/dashboard/static/styles/metrics.css @@ -9,6 +9,7 @@ section { section.nav-panel { align-self: start; width: 10%; + margin: 0 0 0 1em; } section.main-panel { diff --git a/src/pir_pipeline/dashboard/templates/metrics/base.html b/src/pir_pipeline/dashboard/templates/metrics/base.html index 3f04b03..acbe809 100644 --- a/src/pir_pipeline/dashboard/templates/metrics/base.html +++ b/src/pir_pipeline/dashboard/templates/metrics/base.html @@ -2,6 +2,7 @@ {% block header %} +

    {% block title %}Metrics{% endblock %}

    {% endblock %} {% block content %} @@ -10,6 +11,9 @@
  • Approval by User
    • +
  • + Progress +
    • diff --git a/src/pir_pipeline/dashboard/templates/metrics/progress.html b/src/pir_pipeline/dashboard/templates/metrics/progress.html new file mode 100644 index 0000000..d28568a --- /dev/null +++ b/src/pir_pipeline/dashboard/templates/metrics/progress.html @@ -0,0 +1,8 @@ +{% extends "metrics/base.html" %} + +{% block main_panel %} +{% endblock %} + +{% block scripts %} + +{% endblock %} \ No newline at end of file From 6ff529dec4dccba2308241a86198a655e44fb284 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Mon, 23 Feb 2026 11:32:51 -0500 Subject: [PATCH 09/25] Metrics: Beginning to fill progress page skeleton --- src/pir_pipeline/dashboard/metrics.py | 16 ++++++++- .../dashboard/static/styles/metrics.css | 2 +- .../dashboard/templates/metrics/base.html | 4 +-- .../dashboard/templates/metrics/metrics.html | 36 +++++++++---------- .../dashboard/templates/metrics/progress.html | 16 +++++++++ 5 files changed, 50 insertions(+), 24 deletions(-) diff --git a/src/pir_pipeline/dashboard/metrics.py b/src/pir_pipeline/dashboard/metrics.py index 64fa47b..a04ea13 100644 --- a/src/pir_pipeline/dashboard/metrics.py +++ b/src/pir_pipeline/dashboard/metrics.py @@ -13,6 +13,7 @@ def index(): db = get_db() df = db.get_records("SELECT * FROM link_history") + approval_by_user = ( df.groupby("user") .aggregate(percent_approved=("decision", "mean")) @@ -26,4 +27,17 @@ def index(): @bp.route("/progress") @administrator def progress(): - return render_template("metrics/progress.html") + db = get_db() + df = db.get_records("SELECT * FROM link_history") + + df["date"] = df["decision_timestamp"].map(lambda stamp: stamp.date()) + # TODO: Filter to only approved + approved = ( + df.groupby("date") + .size() + .reset_index() + .rename(columns={0: "approved"}) + .to_dict(orient="index") + ) + + return render_template("metrics/progress.html", approved=approved) diff --git a/src/pir_pipeline/dashboard/static/styles/metrics.css b/src/pir_pipeline/dashboard/static/styles/metrics.css index d199aa1..7e2c221 100644 --- a/src/pir_pipeline/dashboard/static/styles/metrics.css +++ b/src/pir_pipeline/dashboard/static/styles/metrics.css @@ -8,7 +8,7 @@ section { section.nav-panel { align-self: start; - width: 10%; + width: 15%; margin: 0 0 0 1em; } diff --git a/src/pir_pipeline/dashboard/templates/metrics/base.html b/src/pir_pipeline/dashboard/templates/metrics/base.html index acbe809..6c372d3 100644 --- a/src/pir_pipeline/dashboard/templates/metrics/base.html +++ b/src/pir_pipeline/dashboard/templates/metrics/base.html @@ -9,10 +9,10 @@

    {% block title %}Metrics{% endblock %}

    diff --git a/src/pir_pipeline/dashboard/templates/metrics/metrics.html b/src/pir_pipeline/dashboard/templates/metrics/metrics.html index c195b4c..be6ea73 100644 --- a/src/pir_pipeline/dashboard/templates/metrics/metrics.html +++ b/src/pir_pipeline/dashboard/templates/metrics/metrics.html @@ -1,26 +1,22 @@ {% extends "metrics/base.html" %} -{% block header %} - -

    {% block title %}Metrics{% endblock %}

    -{% endblock %} - {% block main_panel %} -

    Approval Rating By User

    - - - - - - - - {% for row in approval_by_user.values() %} - - - - - {% endfor %} -
    UserPercent Approved
    {{ row["user"] }}{{ row["percent_approved"] }}
    +
    + + + + + + + + {% for row in approval_by_user.values() %} + + + + + {% endfor %} +
    UserPercent Approved
    {{ row["user"] }}{{ row["percent_approved"] }}
    +
    {% endblock %} {% block scripts %} diff --git a/src/pir_pipeline/dashboard/templates/metrics/progress.html b/src/pir_pipeline/dashboard/templates/metrics/progress.html index d28568a..a4969b5 100644 --- a/src/pir_pipeline/dashboard/templates/metrics/progress.html +++ b/src/pir_pipeline/dashboard/templates/metrics/progress.html @@ -1,6 +1,22 @@ {% extends "metrics/base.html" %} {% block main_panel %} +
    + + + + + + + + {% for row in approved.values() %} + + + + + {% endfor %} +
    DateNumber Approved
    {{ row["date"] }}{{ row["approved"] }}
    +
    {% endblock %} {% block scripts %} From aa5fcec2d5a1507c11eeb06e08a623628bb24cec Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Tue, 24 Feb 2026 11:22:07 -0500 Subject: [PATCH 10/25] Dashboard: Daily workflow for number of records linked --- code/daily_confirmed_count.sh | 3 ++ .../dashboard/scripts/daily_confirmed.py | 45 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 code/daily_confirmed_count.sh create mode 100644 src/pir_pipeline/dashboard/scripts/daily_confirmed.py diff --git a/code/daily_confirmed_count.sh b/code/daily_confirmed_count.sh new file mode 100644 index 0000000..487040c --- /dev/null +++ b/code/daily_confirmed_count.sh @@ -0,0 +1,3 @@ +# Runs daily via crontab +# Command 00 08 * * * /bin/bash +python3.12 -m pir_pipeline.dashboard.scripts.daily_confirmed diff --git a/src/pir_pipeline/dashboard/scripts/daily_confirmed.py b/src/pir_pipeline/dashboard/scripts/daily_confirmed.py new file mode 100644 index 0000000..4dca716 --- /dev/null +++ b/src/pir_pipeline/dashboard/scripts/daily_confirmed.py @@ -0,0 +1,45 @@ +import json +import os +from datetime import datetime, timedelta + +from sqlalchemy import func, select + +from instance.config import DB_CONFIG, DB_NAME +from pir_pipeline.utils.SQLAlchemyUtils import SQLAlchemyUtils + +if __name__ == "__main__": + sql = SQLAlchemyUtils(**DB_CONFIG, database=DB_NAME) + path = os.path.join( + os.path.dirname(__file__), "..", "static", "data", "daily_confirmed_count.json" + ) + uqid_changelog = sql.tables["uqid_changelog"] + confirmed = sql.tables["confirmed"] + + # Confirmed uqids + confirmations = ( + select(uqid_changelog.c["original_uqid"]) + .where(uqid_changelog.c["timestamp"] > datetime.today() - timedelta(hours=24)) + .group_by(uqid_changelog.c["original_uqid"]) + .having(func.max(uqid_changelog.c["complete_series_flag"]) == 1) + .scalar_subquery() + ) + + # Number of confirmed questions + confirmed_count = select(func.count(confirmed.table_valued())).where( + confirmed.c.uqid.in_(confirmations) + ) + + try: + fp = open(path, "r+") + confirmed = json.load(fp) + except FileNotFoundError: + fp = open(path, "w+") + confirmed = {} + + fp.seek(0) + + confirmed[datetime.today().strftime("%Y-%m-%d")] = ( + sql.get_scalar(confirmed_count, {}) or 0 + ) + json.dump(confirmed, fp, indent=2) + fp.close() From d973dd4ad122352639b35910e9c7dcf80ed0b849 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Tue, 24 Feb 2026 11:22:40 -0500 Subject: [PATCH 11/25] Dashboard: Use daily_confirmed_count.json in metrics --- src/pir_pipeline/dashboard/metrics.py | 29 ++++++++++--------- .../dashboard/templates/metrics/progress.html | 6 ++-- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/pir_pipeline/dashboard/metrics.py b/src/pir_pipeline/dashboard/metrics.py index a04ea13..04419d3 100644 --- a/src/pir_pipeline/dashboard/metrics.py +++ b/src/pir_pipeline/dashboard/metrics.py @@ -1,5 +1,8 @@ """Routes and logic for the home page""" +import json +import os + from flask import Blueprint, render_template from pir_pipeline.dashboard.db import get_db @@ -27,17 +30,15 @@ def index(): @bp.route("/progress") @administrator def progress(): - db = get_db() - df = db.get_records("SELECT * FROM link_history") - - df["date"] = df["decision_timestamp"].map(lambda stamp: stamp.date()) - # TODO: Filter to only approved - approved = ( - df.groupby("date") - .size() - .reset_index() - .rename(columns={0: "approved"}) - .to_dict(orient="index") - ) - - return render_template("metrics/progress.html", approved=approved) + with open( + os.path.join( + os.path.dirname(__file__), "static/data/daily_confirmed_count.json" + ), + "r", + ) as f: + confirmed = json.load(f) + + confirmed = list(confirmed.items()) + confirmed.reverse() + + return render_template("metrics/progress.html", confirmed=confirmed) diff --git a/src/pir_pipeline/dashboard/templates/metrics/progress.html b/src/pir_pipeline/dashboard/templates/metrics/progress.html index a4969b5..eb48550 100644 --- a/src/pir_pipeline/dashboard/templates/metrics/progress.html +++ b/src/pir_pipeline/dashboard/templates/metrics/progress.html @@ -9,10 +9,10 @@ Number Approved - {% for row in approved.values() %} + {% for key, value in confirmed %} - {{ row["date"] }} - {{ row["approved"] }} + {{ key }} + {{ value }} {% endfor %} From bd41d659749a12f02cd5262eb8349ab376846949 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Tue, 24 Feb 2026 16:08:55 -0500 Subject: [PATCH 12/25] SQLAlchemyUtils: Correct type hint --- src/pir_pipeline/utils/SQLAlchemyUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pir_pipeline/utils/SQLAlchemyUtils.py b/src/pir_pipeline/utils/SQLAlchemyUtils.py index 6ced086..c458112 100644 --- a/src/pir_pipeline/utils/SQLAlchemyUtils.py +++ b/src/pir_pipeline/utils/SQLAlchemyUtils.py @@ -66,7 +66,7 @@ def __init__(self, user: str, password: str, host: str, port: int, database: str self.insert = insert self._dialect = self._engine.name - self._tables: dict[Table] = { + self._tables: dict[str, Table] = { "response": response, "question": question, "program": program, From 7072f14690de419e462b8342fdb1acf8bda0e766 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Tue, 24 Feb 2026 16:10:31 -0500 Subject: [PATCH 13/25] Dashboard: Rename progress.html and add projections page --- src/pir_pipeline/dashboard/templates/metrics/base.html | 5 ++++- .../templates/metrics/{progress.html => daily_links.html} | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) rename src/pir_pipeline/dashboard/templates/metrics/{progress.html => daily_links.html} (91%) diff --git a/src/pir_pipeline/dashboard/templates/metrics/base.html b/src/pir_pipeline/dashboard/templates/metrics/base.html index 6c372d3..58caa0d 100644 --- a/src/pir_pipeline/dashboard/templates/metrics/base.html +++ b/src/pir_pipeline/dashboard/templates/metrics/base.html @@ -12,7 +12,10 @@

    {% block title %}Metrics{% endblock %}

    Approval Rate by User
  • - Daily Links + Daily Links +
    • +
  • + Linking Projections
    • diff --git a/src/pir_pipeline/dashboard/templates/metrics/progress.html b/src/pir_pipeline/dashboard/templates/metrics/daily_links.html similarity index 91% rename from src/pir_pipeline/dashboard/templates/metrics/progress.html rename to src/pir_pipeline/dashboard/templates/metrics/daily_links.html index eb48550..f756243 100644 --- a/src/pir_pipeline/dashboard/templates/metrics/progress.html +++ b/src/pir_pipeline/dashboard/templates/metrics/daily_links.html @@ -9,7 +9,7 @@ Number Approved - {% for key, value in confirmed %} + {% for key, value in confirmed_dict.items() %} {{ key }} {{ value }} From 35274e2f051a41f8660ddc98b558b662c214b854 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Wed, 25 Feb 2026 09:06:14 -0500 Subject: [PATCH 14/25] Dashboard: Initial projections page --- .../templates/metrics/projections.html | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 src/pir_pipeline/dashboard/templates/metrics/projections.html diff --git a/src/pir_pipeline/dashboard/templates/metrics/projections.html b/src/pir_pipeline/dashboard/templates/metrics/projections.html new file mode 100644 index 0000000..c0ab5f4 --- /dev/null +++ b/src/pir_pipeline/dashboard/templates/metrics/projections.html @@ -0,0 +1,74 @@ +{% extends "metrics/base.html" %} + +{% block main_panel %} +
    + + + + + + + + + + + {% for key, value in overall_projection.items() %} + + + + + + + + {% endfor %} +
    RateAverageRemaining DaysProjected LinksShortfall
    {{ key }}{{ value.get("average") }}{{ value.get("remaining_days") }}{{ value.get("year_end") }}{{ value.get("shortfall") }}
    +
    +
    +
    +

    Weekly

    + + + + + + + + + + {% for row in weekly_by_year.values() %} + + + + + + + {% endfor %} +
    YearAverageProjected LinksShortfall
    {{ row['year'] }}{{ row['average'] }}{{ row['year_end'] }}{{ row['shortfall'] }}
    +
    +
    +

    Overall

    + + + + + + + + + + {% for row in overall_by_year.values() %} + + + + + + + {% endfor %} +
    YearAverageProjected LinksShortfall
    {{ row['year'] }}{{ row['average'] }}{{ row['year_end'] }}{{ row['shortfall'] }}
    +
    +
    +{% endblock %} + +{% block scripts %} + +{% endblock %} \ No newline at end of file From a851cc8c034ba7f3f099313c2fa9b4fd3c1083d8 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Wed, 25 Feb 2026 09:40:47 -0500 Subject: [PATCH 15/25] Dashboard: Metrics logic for daily links and linking projections --- src/pir_pipeline/dashboard/metrics.py | 177 ++++++++++++++++++++++++-- 1 file changed, 163 insertions(+), 14 deletions(-) diff --git a/src/pir_pipeline/dashboard/metrics.py b/src/pir_pipeline/dashboard/metrics.py index 04419d3..cd959e9 100644 --- a/src/pir_pipeline/dashboard/metrics.py +++ b/src/pir_pipeline/dashboard/metrics.py @@ -1,9 +1,10 @@ -"""Routes and logic for the home page""" +"""Routes and logic for the metrics page""" -import json -import os +from datetime import datetime, timedelta +import pandas as pd from flask import Blueprint, render_template +from sqlalchemy import func, select from pir_pipeline.dashboard.db import get_db from pir_pipeline.dashboard.utils import administrator @@ -11,6 +12,42 @@ bp = Blueprint("metrics", __name__, url_prefix="/metrics") +def get_confirmed_dict(by: list[str] = []): + db = get_db() + + uqid_changelog = db.tables["uqid_changelog"] + confirmed = db.tables["confirmed"] + + # Confirmed uqids + uqids = ( + select( + uqid_changelog.c["original_uqid"], + func.min(uqid_changelog.c["timestamp"]).label("timestamp"), + ) + .group_by(uqid_changelog.c["original_uqid"]) + .having(func.max(uqid_changelog.c["complete_series_flag"]) == 1) + .subquery() + ) + + # Number of confirmed questions + confirmed = select(confirmed, uqids.c["timestamp"]).join( + uqids, + onclause=uqids.c["original_uqid"] == confirmed.c["uqid"], + ) + confirmed_df = db.get_records(confirmed) + confirmed_df["date"] = confirmed_df["timestamp"].map( + lambda date: date.date().strftime("%Y-%m-%d") + ) + confirmed_dict = ( + confirmed_df.groupby(by=["date"] + by) + .size() + .sort_index(ascending=False) + .to_dict() + ) + + return confirmed_dict + + @bp.route("/") @administrator def index(): @@ -27,18 +64,130 @@ def index(): return render_template("metrics/metrics.html", approval_by_user=approval_by_user) -@bp.route("/progress") +@bp.route("/daily_links") +@administrator +def daily_links(): + confirmed_dict = get_confirmed_dict() + + return render_template("metrics/daily_links.html", confirmed_dict=confirmed_dict) + + +@bp.route("/projections") @administrator -def progress(): - with open( - os.path.join( - os.path.dirname(__file__), "static/data/daily_confirmed_count.json" +def projections(): + def get_projections( + average: int | pd.DataFrame, remaining_days: int, remaining: int | pd.DataFrame + ): + if isinstance(average, pd.DataFrame): + merged = average.join(remaining) + merged["year_end"] = merged["average"].map( + lambda x: round(x * remaining_days) + ) + merged["difference"] = merged["count"] - merged["year_end"] + merged["shortfall"] = merged["difference"].map(lambda x: x if x > 0 else 0) + merged["average"] = merged["average"].round(1) + out_dict = ( + merged.reset_index()[["year", "average", "year_end", "shortfall"]] + .sort_values("year", ascending=False) + .to_dict(orient="index") + ) + else: + projection = round(average * remaining_days) + difference = remaining - projection + shortfall = difference if difference > 0 else 0 + + out_dict = { + "average": round(average, 1), + "remaining_days": remaining_days, + "year_end": projection, + "shortfall": shortfall, + } + + return out_dict + + db = get_db() + confirmed_dict = get_confirmed_dict() + confirmed_df = pd.DataFrame.from_dict( + confirmed_dict, orient="index", columns=["count"] + ).reset_index(names=["date"]) + + confirmed_df["date"] = confirmed_df["date"].map(datetime.fromisoformat) + unconfirmed = db.tables["unconfirmed"] + + remaining_days = (datetime.fromisoformat("2026-10-01") - datetime.today()).days + remaining_questions = db.get_scalar( + select(func.count(unconfirmed.table_valued())), {} + ) + remaining_questions_by_year = db.get_records( + select( + func.count(unconfirmed.c["question_id"]), unconfirmed.c["year"] + ).group_by(unconfirmed.c["year"]) + ).set_index("year") + + # Full data set projections + weekly_average = ( + confirmed_df[ + confirmed_df["date"].map( + lambda date: date >= datetime.today() - timedelta(days=7) + ) + ]["count"].sum() + / 7 + ) + overall_average = ( + confirmed_df["count"].sum() + / (datetime.today() - confirmed_df["date"].min()).days + ) + + overall_projection = { + "Weekly": get_projections(weekly_average, remaining_days, remaining_questions), + "Overall": get_projections( + overall_average, remaining_days, remaining_questions ), - "r", - ) as f: - confirmed = json.load(f) + } - confirmed = list(confirmed.items()) - confirmed.reverse() + # Projections by year + confirmed_dict = get_confirmed_dict(["year"]) + confirmed_df = pd.DataFrame.from_dict( + confirmed_dict, orient="index", columns=["count"] + ).reset_index(names=["date", "year"]) - return render_template("metrics/progress.html", confirmed=confirmed) + confirmed_df["date"], confirmed_df["year"] = zip( + *confirmed_df["date"].map(lambda x: (x[0], x[1])) + ) + confirmed_df["date"] = confirmed_df["date"].map(datetime.fromisoformat) + + weekly_df = confirmed_df[ + confirmed_df["date"].map( + lambda date: date >= datetime.today() - timedelta(days=7) + ) + ] + weekly_average_df = ( + (weekly_df.groupby("year")["count"].sum() / 7) + .reset_index() + .rename(columns={"count": "average"}) + .set_index("year") + ) + + overall_average_df = ( + ( + confirmed_df.groupby("year")["count"].sum() + / (datetime.today() - confirmed_df["date"].min()).days + ) + .reset_index() + .rename(columns={"count": "average"}) + .set_index("year") + ) + + weekly_projections_by_year = get_projections( + weekly_average_df, remaining_days, remaining_questions_by_year + ) + overall_projections_by_year = get_projections( + overall_average_df, remaining_days, remaining_questions_by_year + ) + + return render_template( + "metrics/projections.html", + overall_projection=overall_projection, + weekly_by_year=weekly_projections_by_year, + overall_by_year=overall_projections_by_year, + ) From 1b598771b9824866b51350463731c1b0764e9323 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Wed, 25 Feb 2026 09:41:54 -0500 Subject: [PATCH 16/25] Remove daily counts. Logic is handled in metrics.py --- code/daily_confirmed_count.sh | 3 -- .../dashboard/scripts/daily_confirmed.py | 45 ------------------- 2 files changed, 48 deletions(-) delete mode 100644 code/daily_confirmed_count.sh delete mode 100644 src/pir_pipeline/dashboard/scripts/daily_confirmed.py diff --git a/code/daily_confirmed_count.sh b/code/daily_confirmed_count.sh deleted file mode 100644 index 487040c..0000000 --- a/code/daily_confirmed_count.sh +++ /dev/null @@ -1,3 +0,0 @@ -# Runs daily via crontab -# Command 00 08 * * * /bin/bash -python3.12 -m pir_pipeline.dashboard.scripts.daily_confirmed diff --git a/src/pir_pipeline/dashboard/scripts/daily_confirmed.py b/src/pir_pipeline/dashboard/scripts/daily_confirmed.py deleted file mode 100644 index 4dca716..0000000 --- a/src/pir_pipeline/dashboard/scripts/daily_confirmed.py +++ /dev/null @@ -1,45 +0,0 @@ -import json -import os -from datetime import datetime, timedelta - -from sqlalchemy import func, select - -from instance.config import DB_CONFIG, DB_NAME -from pir_pipeline.utils.SQLAlchemyUtils import SQLAlchemyUtils - -if __name__ == "__main__": - sql = SQLAlchemyUtils(**DB_CONFIG, database=DB_NAME) - path = os.path.join( - os.path.dirname(__file__), "..", "static", "data", "daily_confirmed_count.json" - ) - uqid_changelog = sql.tables["uqid_changelog"] - confirmed = sql.tables["confirmed"] - - # Confirmed uqids - confirmations = ( - select(uqid_changelog.c["original_uqid"]) - .where(uqid_changelog.c["timestamp"] > datetime.today() - timedelta(hours=24)) - .group_by(uqid_changelog.c["original_uqid"]) - .having(func.max(uqid_changelog.c["complete_series_flag"]) == 1) - .scalar_subquery() - ) - - # Number of confirmed questions - confirmed_count = select(func.count(confirmed.table_valued())).where( - confirmed.c.uqid.in_(confirmations) - ) - - try: - fp = open(path, "r+") - confirmed = json.load(fp) - except FileNotFoundError: - fp = open(path, "w+") - confirmed = {} - - fp.seek(0) - - confirmed[datetime.today().strftime("%Y-%m-%d")] = ( - sql.get_scalar(confirmed_count, {}) or 0 - ) - json.dump(confirmed, fp, indent=2) - fp.close() From b04fc297b5231096afdc76d7d6b3046878d1f7b8 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Wed, 25 Feb 2026 09:42:18 -0500 Subject: [PATCH 17/25] Navigation panel for metrics --- .../dashboard/static/metrics/metrics.js | 30 +++++ .../dashboard/static/styles/metrics.css | 120 ++++++++++++++++++ .../dashboard/templates/metrics/base.html | 31 +++-- .../templates/metrics/daily_links.html | 4 - .../dashboard/templates/metrics/metrics.html | 4 - .../templates/metrics/projections.html | 4 - 6 files changed, 170 insertions(+), 23 deletions(-) create mode 100644 src/pir_pipeline/dashboard/static/metrics/metrics.js diff --git a/src/pir_pipeline/dashboard/static/metrics/metrics.js b/src/pir_pipeline/dashboard/static/metrics/metrics.js new file mode 100644 index 0000000..55bc896 --- /dev/null +++ b/src/pir_pipeline/dashboard/static/metrics/metrics.js @@ -0,0 +1,30 @@ +// Smooth scroll +// Adapted from Claude +document.querySelectorAll('a[href^="#"]').forEach(anchor => { + anchor.addEventListener('click', function (e) { + e.preventDefault(); + const target = document.querySelector(this.getAttribute('href')); + if (target) { + target.scrollIntoView({ + behavior: 'smooth', + block: 'start' + }); + } + }); +}); + +// Change the active link +document.addEventListener("DOMContentLoaded", toggleActive()) + +function toggleActive() { + const links = document.querySelectorAll('.floating-nav-link'); + for (let i = 0; i < links.length; i++) { + console.log(links[i]); + if (document.URL.includes(links[i].getAttribute("href"))) { + links.forEach(link => { + link.classList.remove("active"); + }) + links[i].classList.add("active"); + } + } +} \ No newline at end of file diff --git a/src/pir_pipeline/dashboard/static/styles/metrics.css b/src/pir_pipeline/dashboard/static/styles/metrics.css index 7e2c221..0f27cea 100644 --- a/src/pir_pipeline/dashboard/static/styles/metrics.css +++ b/src/pir_pipeline/dashboard/static/styles/metrics.css @@ -14,4 +14,124 @@ section.nav-panel { section.main-panel { width: 100%; +} + +div#yearly-average-tables { + display: flex; +} + +div.projections { + flex-direction: column; +} + +/* Adapted from Claude */ +/* Floating Navigation Panel */ +.floating-nav { + background-color: var(--primary); + border-radius: 0.5em; + box-shadow: 0 4px 12px rgba(23, 44, 60, 0.2); + padding: 1em; + transition: all 0.3s ease; + margin-top: 1em; +} + +/* .floating-nav.collapsed { + right: -160px; +} */ + +.floating-nav-header { + display: flex; + justify-content: space-between; + align-items: center; + padding-bottom: 0.75rem; + margin-bottom: 0.75rem; + border-bottom: 2px solid var(--secondary); +} + +.floating-nav-title { + font-size: 1rem; + font-weight: 600; + color: var(--light); + margin: 0; +} + +.floating-nav-toggle { + background: none; + border: none; + color: var(--light); + cursor: pointer; + padding: 0.25rem; + display: flex; + align-items: center; + transition: color 0.2s ease; +} + +.floating-nav-toggle:hover { + color: var(--info); +} + +.floating-nav-list { + list-style: none; + padding: 0; + margin: 0; +} + +.floating-nav-item { + margin-bottom: 0.5rem; +} + +.floating-nav-link { + display: block; + color: var(--light); + text-decoration: none; + padding: 0.5rem 0.75rem; + border-radius: 0.25rem; + font-size: 0.875rem; + transition: all 0.2s ease; + border-left: 3px solid transparent; +} + +.floating-nav-link:hover { + background-color: rgba(255, 255, 255, 0.1); + color: var(--info); + border-left-color: var(--info); + padding-left: 1rem; +} + +.floating-nav-link.active { + background-color: var(--secondary); + color: var(--light); + border-left-color: var(--light); + font-weight: 600; +} + +.floating-nav-divider { + height: 1px; + background-color: var(--secondary); + margin: 0.75rem 0; + opacity: 0.5; +} + +/* Floating Nav Icon/Badge */ +.floating-nav-badge { + display: inline-block; + background-color: var(--accent); + color: var(--dark); + font-size: 0.75rem; + padding: 0.125rem 0.5rem; + border-radius: 0.25rem; + margin-left: 0.5rem; + font-weight: 600; +} + +/* Responsive adjustments */ +@media (max-width: 768px) { + .floating-nav { + right: 1em; + min-width: 180px; + } + + .floating-nav.left { + left: 1em; + } } \ No newline at end of file diff --git a/src/pir_pipeline/dashboard/templates/metrics/base.html b/src/pir_pipeline/dashboard/templates/metrics/base.html index 58caa0d..14576b6 100644 --- a/src/pir_pipeline/dashboard/templates/metrics/base.html +++ b/src/pir_pipeline/dashboard/templates/metrics/base.html @@ -7,19 +7,28 @@

    {% block title %}Metrics{% endblock %}

    {% block content %}
    - +
    {% block main_panel %}{% endblock %}
    +{% endblock %} + +{% block scripts %} + {% endblock %} \ No newline at end of file diff --git a/src/pir_pipeline/dashboard/templates/metrics/daily_links.html b/src/pir_pipeline/dashboard/templates/metrics/daily_links.html index f756243..e61a975 100644 --- a/src/pir_pipeline/dashboard/templates/metrics/daily_links.html +++ b/src/pir_pipeline/dashboard/templates/metrics/daily_links.html @@ -17,8 +17,4 @@ {% endfor %} -{% endblock %} - -{% block scripts %} - {% endblock %} \ No newline at end of file diff --git a/src/pir_pipeline/dashboard/templates/metrics/metrics.html b/src/pir_pipeline/dashboard/templates/metrics/metrics.html index be6ea73..a750d7e 100644 --- a/src/pir_pipeline/dashboard/templates/metrics/metrics.html +++ b/src/pir_pipeline/dashboard/templates/metrics/metrics.html @@ -17,8 +17,4 @@ {% endfor %} -{% endblock %} - -{% block scripts %} - {% endblock %} \ No newline at end of file diff --git a/src/pir_pipeline/dashboard/templates/metrics/projections.html b/src/pir_pipeline/dashboard/templates/metrics/projections.html index c0ab5f4..6a74d40 100644 --- a/src/pir_pipeline/dashboard/templates/metrics/projections.html +++ b/src/pir_pipeline/dashboard/templates/metrics/projections.html @@ -67,8 +67,4 @@

    Overall

    -{% endblock %} - -{% block scripts %} - {% endblock %} \ No newline at end of file From 221fbb4b3bb45340d3073e7356f0423b5cbe8daf Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Tue, 3 Mar 2026 13:55:43 -0500 Subject: [PATCH 18/25] Dashboard: Bash script to create users --- code/create_users.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 code/create_users.sh diff --git a/code/create_users.sh b/code/create_users.sh new file mode 100644 index 0000000..67607ac --- /dev/null +++ b/code/create_users.sh @@ -0,0 +1,10 @@ +set -a +source ./.env +set +a + +# https://stackoverflow.com/questions/78251247/how-to-use-systems-environnement-variables-in-sql-script +if [ $1 == "local" ]; then + psql -U $dbusername -p $dbport -h 127.0.0.1 -d pir -f <(envsubst < $2) +else + psql -U $DB_USER -p $DB_PORT -h $DB_HOST -d $DB_NAME -f <(envsubst < $2) +fi \ No newline at end of file From acea3db017b5d8b92d6ae7bad1dfa2ad912bb73b Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Tue, 3 Mar 2026 13:57:28 -0500 Subject: [PATCH 19/25] Dashboard: SQL scripts to create new local approver/reviewer users --- code/sql/user_local_approver.sql | 10 ++++++++++ code/sql/user_local_reviewer.sql | 9 +++++++++ 2 files changed, 19 insertions(+) create mode 100644 code/sql/user_local_approver.sql create mode 100644 code/sql/user_local_reviewer.sql diff --git a/code/sql/user_local_approver.sql b/code/sql/user_local_approver.sql new file mode 100644 index 0000000..cbc6840 --- /dev/null +++ b/code/sql/user_local_approver.sql @@ -0,0 +1,10 @@ +CREATE USER approver +WITH PASSWORD '$approver_user_password'; + +GRANT SELECT ON ALL TABLES IN SCHEMA public TO approver; + +GRANT ALL PRIVILEGES ON proposed_changes, uqid_changelog, uqid_changelog_id_seq, link_history TO approver; +GRANT INSERT, SELECT, UPDATE ON question TO approver; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT SELECT ON TABLES TO approver; diff --git a/code/sql/user_local_reviewer.sql b/code/sql/user_local_reviewer.sql new file mode 100644 index 0000000..4ae38a1 --- /dev/null +++ b/code/sql/user_local_reviewer.sql @@ -0,0 +1,9 @@ +CREATE USER reviewer +WITH PASSWORD '$reviewer_user_password'; + +GRANT SELECT ON ALL TABLES IN SCHEMA public TO reviewer; + +GRANT ALL PRIVILEGES ON proposed_changes, link_history TO reviewer; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT SELECT ON TABLES TO reviewer; From fffa9fcd925d0f5be82cf75e9aec9706b1564849 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Tue, 3 Mar 2026 14:12:51 -0500 Subject: [PATCH 20/25] Dashboard: Prod user templates --- code/sql/user_prod_approver.sql | 10 ++++++++++ code/sql/user_prod_reviewer.sql | 9 +++++++++ 2 files changed, 19 insertions(+) create mode 100644 code/sql/user_prod_approver.sql create mode 100644 code/sql/user_prod_reviewer.sql diff --git a/code/sql/user_prod_approver.sql b/code/sql/user_prod_approver.sql new file mode 100644 index 0000000..f902f28 --- /dev/null +++ b/code/sql/user_prod_approver.sql @@ -0,0 +1,10 @@ +CREATE USER approver +WITH PASSWORD '$USER_PASSWORD_APPROVER'; + +GRANT SELECT ON ALL TABLES IN SCHEMA public TO approver; + +GRANT ALL PRIVILEGES ON proposed_changes, uqid_changelog, uqid_changelog_id_seq, link_history TO approver; +GRANT INSERT, SELECT, UPDATE ON question TO approver; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT SELECT ON TABLES TO approver; diff --git a/code/sql/user_prod_reviewer.sql b/code/sql/user_prod_reviewer.sql new file mode 100644 index 0000000..1d0d352 --- /dev/null +++ b/code/sql/user_prod_reviewer.sql @@ -0,0 +1,9 @@ +CREATE USER reviewer +WITH PASSWORD '$USER_PASSWORD_REVIEWER'; + +GRANT SELECT ON ALL TABLES IN SCHEMA public TO reviewer; + +GRANT ALL PRIVILEGES ON proposed_changes, link_history TO reviewer; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT SELECT ON TABLES TO reviewer; From 5e743c157616fc664d5e1dad79081a7b1c85f27d Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Tue, 3 Mar 2026 16:38:45 -0500 Subject: [PATCH 21/25] Code: New script for serving the dashboard --- code/serve_qa_dashboard.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/code/serve_qa_dashboard.sh b/code/serve_qa_dashboard.sh index ea59f4b..1cc62d5 100755 --- a/code/serve_qa_dashboard.sh +++ b/code/serve_qa_dashboard.sh @@ -1,5 +1,14 @@ -cd ~/repos/ACF-pir-data/ -source .venv/bin/activate -export LOADING_DASHBOARD="True" -google-chrome http://localhost:8080 &>/dev/null & -waitress-serve --host 127.0.0.1 --call "pir_pipeline.dashboard:create_app" \ No newline at end of file +source ./application/.venv/bin/activate + +USER_NUMBER=$(id -u) +USER_NAME=$(id -un) +USER_PORT=$((5000 + $USER_NUMBER % 1000)) +export DB_ENV="production" + +if [ $USER_NAME == "reggie.gilliard" ]; then + export DB_PROFILE="approver" +else + export DB_PROFILE="reviewer" +fi + +waitress-serve --host 127.0.0.1 --port=${USER_PORT} --call "pir_pipeline.dashboard:create_app" \ No newline at end of file From 629b64af80761b72251a9f3078372b7dfdcaa53a Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Tue, 3 Mar 2026 18:25:49 -0500 Subject: [PATCH 22/25] Dashboard: Add approval by user page --- .../templates/metrics/approval_by_user.html | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 src/pir_pipeline/dashboard/templates/metrics/approval_by_user.html diff --git a/src/pir_pipeline/dashboard/templates/metrics/approval_by_user.html b/src/pir_pipeline/dashboard/templates/metrics/approval_by_user.html new file mode 100644 index 0000000..a750d7e --- /dev/null +++ b/src/pir_pipeline/dashboard/templates/metrics/approval_by_user.html @@ -0,0 +1,20 @@ +{% extends "metrics/base.html" %} + +{% block main_panel %} +
    + + + + + + + + {% for row in approval_by_user.values() %} + + + + + {% endfor %} +
    UserPercent Approved
    {{ row["user"] }}{{ row["percent_approved"] }}
    +
    +{% endblock %} \ No newline at end of file From 47e78fc032f1143cc898d0d4e7a7fcef6c4d108b Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Wed, 18 Mar 2026 16:18:28 -0400 Subject: [PATCH 23/25] Add Alecs as approver --- code/serve_qa_dashboard.sh | 5 +++++ src/pir_pipeline/dashboard/utils.py | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/code/serve_qa_dashboard.sh b/code/serve_qa_dashboard.sh index 1cc62d5..cedf0a7 100755 --- a/code/serve_qa_dashboard.sh +++ b/code/serve_qa_dashboard.sh @@ -3,10 +3,15 @@ source ./application/.venv/bin/activate USER_NUMBER=$(id -u) USER_NAME=$(id -un) USER_PORT=$((5000 + $USER_NUMBER % 1000)) + export DB_ENV="production" if [ $USER_NAME == "reggie.gilliard" ]; then export DB_PROFILE="approver" +elif [ $USER_NAME == "emily.kowall" ]; then + export DB_PROFILE="approver" +elif [ $USER_NAME == "alecsandra.velez" ]; then + export DB_PROFILE="approver" else export DB_PROFILE="reviewer" fi diff --git a/src/pir_pipeline/dashboard/utils.py b/src/pir_pipeline/dashboard/utils.py index b543bc4..42e213b 100644 --- a/src/pir_pipeline/dashboard/utils.py +++ b/src/pir_pipeline/dashboard/utils.py @@ -8,7 +8,12 @@ def administrator(view): @wraps(view) def wrapped_view(**kwargs): - if getuser() not in ["jesse.escobar", "emily.kowall", "reggie.gilliard"]: + if getuser() not in [ + "jesse.escobar", + "emily.kowall", + "reggie.gilliard", + "alecsandra.velez", + ]: flash("You are not authorized to access this page.") return redirect(url_for("index")) From eb63609a736020048a6a06b68cdad1266cb5f470 Mon Sep 17 00:00:00 2001 From: Gilliard Reggie Date: Thu, 26 Mar 2026 15:32:19 -0400 Subject: [PATCH 24/25] Dashboard: Ensure confirmations are passed through from search --- src/pir_pipeline/dashboard/review.py | 2 +- .../dashboard/static/search/search.js | 26 +++++++++++++++++-- .../dashboard/templates/search/search.html | 2 +- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/pir_pipeline/dashboard/review.py b/src/pir_pipeline/dashboard/review.py index 91d42c8..fdb18e3 100644 --- a/src/pir_pipeline/dashboard/review.py +++ b/src/pir_pipeline/dashboard/review.py @@ -157,7 +157,7 @@ def link(): session["link_dict"] = link_dict message = f"Data {data} queued for linking" elif action == "store": - link_dict = session["link_dict"] + link_dict = session.get("link_dict") ids = [ record.get("base_question_id", "") diff --git a/src/pir_pipeline/dashboard/static/search/search.js b/src/pir_pipeline/dashboard/static/search/search.js index 6dd0026..54bfb27 100644 --- a/src/pir_pipeline/dashboard/static/search/search.js +++ b/src/pir_pipeline/dashboard/static/search/search.js @@ -76,13 +76,35 @@ function getFlashcardData(e) { * * @param {*} e The event that triggered comitting the changes */ -function commitChanges(e) { +async function commitChanges(e) { e.preventDefault(); + let value = e.srcElement.getAttribute("value"); const questionTable = document.getElementById("flashcard-question-table"); + let baseRecord = rowToJSON(questionTable.getElementsByTagName("tr")[1]); + + const linkDetails = { + "link_type": value, + "base_question_id": baseRecord.question_id, + "match_question_id": null + } + + let payload = { + "action": "build", + "data": linkDetails + } + + // Add the confirm action to the dictionary of linking actions + await fetch("/review/link", { + "method": "POST", + "headers": { + "Content-type": "application/json" + }, + "body": JSON.stringify(payload) + }) // Commit changes to the database - const payload = { + payload = { "action": "store", "html": questionTable.outerHTML } diff --git a/src/pir_pipeline/dashboard/templates/search/search.html b/src/pir_pipeline/dashboard/templates/search/search.html index 0bb524a..7176cea 100644 --- a/src/pir_pipeline/dashboard/templates/search/search.html +++ b/src/pir_pipeline/dashboard/templates/search/search.html @@ -58,7 +58,7 @@

    Search the PIR Database

    Cancel - + Confirm Changes From fab2f44f27d7b54d8e1de9e7bf28ab33b86a6b77 Mon Sep 17 00:00:00 2001 From: RGilliard-ACF <238555752+RGilliard-ACF@users.noreply.github.com> Date: Fri, 10 Apr 2026 12:42:41 +0000 Subject: [PATCH 25/25] Code formatted with black --- src/pir_pipeline/utils/SQLAlchemyUtils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pir_pipeline/utils/SQLAlchemyUtils.py b/src/pir_pipeline/utils/SQLAlchemyUtils.py index c458112..ad83df6 100644 --- a/src/pir_pipeline/utils/SQLAlchemyUtils.py +++ b/src/pir_pipeline/utils/SQLAlchemyUtils.py @@ -171,13 +171,11 @@ def get_columns(self, table: str, where: str = "") -> list[str]: elif self._dialect == "postgresql": table_schema = "table_catalog" - query = text( - f""" + query = text(f""" SELECT column_name FROM information_schema.columns WHERE table_name = :table AND {table_schema} = :schema {where} - """ - ) + """) with self._engine.connect() as conn: result = conn.execute( query, {"table": table, "schema": self._database, "where": where}