diff --git a/alembic/versions/3f4dce7266b1_create_linker_table.py b/alembic/versions/3f4dce7266b1_create_linker_table.py new file mode 100644 index 00000000..55acbe04 --- /dev/null +++ b/alembic/versions/3f4dce7266b1_create_linker_table.py @@ -0,0 +1,38 @@ +"""create_linker_table + +Revision ID: 3f4dce7266b1 +Revises: +Create Date: 2026-02-20 09:23:02.062061 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql.json import JSONB + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "3f4dce7266b1" +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "link_history", + sa.Column("link_id", sa.String(255), nullable=False, primary_key=True), + sa.Column("user", sa.String(255), nullable=False), + sa.Column("link_dict", JSONB), + sa.Column("link_timestamp", sa.DateTime(timezone=True), default=sa.func.now()), + sa.Column("decision", sa.Boolean), + sa.Column("decision_timestamp", sa.DateTime(timezone=True)), + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_table("link_history") diff --git a/code/create_users.sh b/code/create_users.sh new file mode 100644 index 00000000..67607acc --- /dev/null +++ b/code/create_users.sh @@ -0,0 +1,10 @@ +set -a +source ./.env +set +a + +# https://stackoverflow.com/questions/78251247/how-to-use-systems-environnement-variables-in-sql-script +if [ $1 == "local" ]; then + psql -U $dbusername -p $dbport -h 127.0.0.1 -d pir -f <(envsubst < $2) +else + psql -U $DB_USER -p $DB_PORT -h $DB_HOST -d $DB_NAME -f <(envsubst < $2) +fi \ No newline at end of file diff --git a/code/serve_qa_dashboard.sh b/code/serve_qa_dashboard.sh index ea59f4b3..cedf0a7b 100755 --- a/code/serve_qa_dashboard.sh +++ b/code/serve_qa_dashboard.sh @@ -1,5 +1,19 @@ -cd ~/repos/ACF-pir-data/ -source .venv/bin/activate -export LOADING_DASHBOARD="True" -google-chrome http://localhost:8080 &>/dev/null & -waitress-serve --host 127.0.0.1 --call "pir_pipeline.dashboard:create_app" \ No newline at end of file +source ./application/.venv/bin/activate + +USER_NUMBER=$(id -u) +USER_NAME=$(id -un) +USER_PORT=$((5000 + $USER_NUMBER % 1000)) + +export DB_ENV="production" + +if [ $USER_NAME == "reggie.gilliard" ]; then + export DB_PROFILE="approver" +elif [ $USER_NAME == "emily.kowall" ]; then + export DB_PROFILE="approver" +elif [ $USER_NAME == "alecsandra.velez" ]; then + export DB_PROFILE="approver" +else + export DB_PROFILE="reviewer" +fi + +waitress-serve --host 127.0.0.1 --port=${USER_PORT} --call "pir_pipeline.dashboard:create_app" \ No newline at end of file diff --git a/code/sql/user_local_approver.sql b/code/sql/user_local_approver.sql new file mode 100644 index 00000000..cbc68403 --- /dev/null +++ b/code/sql/user_local_approver.sql @@ -0,0 +1,10 @@ +CREATE USER approver +WITH PASSWORD '$approver_user_password'; + +GRANT SELECT ON ALL TABLES IN SCHEMA public TO approver; + +GRANT ALL PRIVILEGES ON proposed_changes, uqid_changelog, uqid_changelog_id_seq, link_history TO approver; +GRANT INSERT, SELECT, UPDATE ON question TO approver; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT SELECT ON TABLES TO approver; diff --git a/code/sql/user_local_reviewer.sql b/code/sql/user_local_reviewer.sql new file mode 100644 index 00000000..4ae38a12 --- /dev/null +++ b/code/sql/user_local_reviewer.sql @@ -0,0 +1,9 @@ +CREATE USER reviewer +WITH PASSWORD '$reviewer_user_password'; + +GRANT SELECT ON ALL TABLES IN SCHEMA public TO reviewer; + +GRANT ALL PRIVILEGES ON proposed_changes, link_history TO reviewer; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT SELECT ON TABLES TO reviewer; diff --git a/code/sql/user_prod_approver.sql b/code/sql/user_prod_approver.sql new file mode 100644 index 00000000..f902f284 --- /dev/null +++ b/code/sql/user_prod_approver.sql @@ -0,0 +1,10 @@ +CREATE USER approver +WITH PASSWORD '$USER_PASSWORD_APPROVER'; + +GRANT SELECT ON ALL TABLES IN SCHEMA public TO approver; + +GRANT ALL PRIVILEGES ON proposed_changes, uqid_changelog, uqid_changelog_id_seq, link_history TO approver; +GRANT INSERT, SELECT, UPDATE ON question TO approver; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT SELECT ON TABLES TO approver; diff --git a/code/sql/user_prod_reviewer.sql b/code/sql/user_prod_reviewer.sql new file mode 100644 index 00000000..1d0d3528 --- /dev/null +++ b/code/sql/user_prod_reviewer.sql @@ -0,0 +1,9 @@ +CREATE USER reviewer +WITH PASSWORD '$USER_PASSWORD_REVIEWER'; + +GRANT SELECT ON ALL TABLES IN SCHEMA public TO reviewer; + +GRANT ALL PRIVILEGES ON proposed_changes, link_history TO reviewer; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public +GRANT SELECT ON TABLES TO reviewer; diff --git a/src/pir_pipeline/dashboard/__init__.py b/src/pir_pipeline/dashboard/__init__.py index 6e1996d1..b4b1b9ba 100644 --- a/src/pir_pipeline/dashboard/__init__.py +++ b/src/pir_pipeline/dashboard/__init__.py @@ -48,7 +48,7 @@ def create_app(test_config: dict = None, **kwargs) -> Flask: db.init_app(app) - from pir_pipeline.dashboard import finalize, index, review, search + from pir_pipeline.dashboard import finalize, index, metrics, review, search app.register_blueprint(index.bp) app.add_url_rule("/", endpoint="index") @@ -56,5 +56,6 @@ def create_app(test_config: dict = None, **kwargs) -> Flask: app.register_blueprint(search.bp) app.register_blueprint(review.bp) app.register_blueprint(finalize.bp) + app.register_blueprint(metrics.bp) return app diff --git a/src/pir_pipeline/dashboard/finalize.py b/src/pir_pipeline/dashboard/finalize.py index 2d203cbe..0b054838 100644 --- a/src/pir_pipeline/dashboard/finalize.py +++ b/src/pir_pipeline/dashboard/finalize.py @@ -7,6 +7,7 @@ from sqlalchemy import select, text from pir_pipeline.dashboard.db import get_db +from pir_pipeline.dashboard.utils import administrator bp = Blueprint("finalize", __name__, url_prefix="/finalize") @@ -80,6 +81,7 @@ def get_page(number_displayed: Optional[int] = None) -> WrappedList: @bp.route("/", methods=["GET"]) +@administrator def index(): session["finalize_page"] = 0 session["number_displayed"] = session.get("number_displayed", DEFAULT_DISPLAYED) diff --git a/src/pir_pipeline/dashboard/metrics.py b/src/pir_pipeline/dashboard/metrics.py new file mode 100644 index 00000000..cd959e9a --- /dev/null +++ b/src/pir_pipeline/dashboard/metrics.py @@ -0,0 +1,193 @@ +"""Routes and logic for the metrics page""" + +from datetime import datetime, timedelta + +import pandas as pd +from flask import Blueprint, render_template +from sqlalchemy import func, select + +from pir_pipeline.dashboard.db import get_db +from pir_pipeline.dashboard.utils import administrator + +bp = Blueprint("metrics", __name__, url_prefix="/metrics") + + +def get_confirmed_dict(by: list[str] = []): + db = get_db() + + uqid_changelog = db.tables["uqid_changelog"] + confirmed = db.tables["confirmed"] + + # Confirmed uqids + uqids = ( + select( + uqid_changelog.c["original_uqid"], + func.min(uqid_changelog.c["timestamp"]).label("timestamp"), + ) + .group_by(uqid_changelog.c["original_uqid"]) + .having(func.max(uqid_changelog.c["complete_series_flag"]) == 1) + .subquery() + ) + + # Number of confirmed questions + confirmed = select(confirmed, uqids.c["timestamp"]).join( + uqids, + onclause=uqids.c["original_uqid"] == confirmed.c["uqid"], + ) + confirmed_df = db.get_records(confirmed) + confirmed_df["date"] = confirmed_df["timestamp"].map( + lambda date: date.date().strftime("%Y-%m-%d") + ) + confirmed_dict = ( + confirmed_df.groupby(by=["date"] + by) + .size() + .sort_index(ascending=False) + .to_dict() + ) + + return confirmed_dict + + +@bp.route("/") +@administrator +def index(): + db = get_db() + df = db.get_records("SELECT * FROM link_history") + + approval_by_user = ( + df.groupby("user") + .aggregate(percent_approved=("decision", "mean")) + .reset_index() + .to_dict(orient="index") + ) + + return render_template("metrics/metrics.html", approval_by_user=approval_by_user) + + +@bp.route("/daily_links") +@administrator +def daily_links(): + confirmed_dict = get_confirmed_dict() + + return render_template("metrics/daily_links.html", confirmed_dict=confirmed_dict) + + +@bp.route("/projections") +@administrator +def projections(): + def get_projections( + average: int | pd.DataFrame, remaining_days: int, remaining: int | pd.DataFrame + ): + if isinstance(average, pd.DataFrame): + merged = average.join(remaining) + merged["year_end"] = merged["average"].map( + lambda x: round(x * remaining_days) + ) + merged["difference"] = merged["count"] - merged["year_end"] + merged["shortfall"] = merged["difference"].map(lambda x: x if x > 0 else 0) + merged["average"] = merged["average"].round(1) + out_dict = ( + merged.reset_index()[["year", "average", "year_end", "shortfall"]] + .sort_values("year", ascending=False) + .to_dict(orient="index") + ) + else: + projection = round(average * remaining_days) + difference = remaining - projection + shortfall = difference if difference > 0 else 0 + + out_dict = { + "average": round(average, 1), + "remaining_days": remaining_days, + "year_end": projection, + "shortfall": shortfall, + } + + return out_dict + + db = get_db() + confirmed_dict = get_confirmed_dict() + confirmed_df = pd.DataFrame.from_dict( + confirmed_dict, orient="index", columns=["count"] + ).reset_index(names=["date"]) + + confirmed_df["date"] = confirmed_df["date"].map(datetime.fromisoformat) + unconfirmed = db.tables["unconfirmed"] + + remaining_days = (datetime.fromisoformat("2026-10-01") - datetime.today()).days + remaining_questions = db.get_scalar( + select(func.count(unconfirmed.table_valued())), {} + ) + remaining_questions_by_year = db.get_records( + select( + func.count(unconfirmed.c["question_id"]), unconfirmed.c["year"] + ).group_by(unconfirmed.c["year"]) + ).set_index("year") + + # Full data set projections + weekly_average = ( + confirmed_df[ + confirmed_df["date"].map( + lambda date: date >= datetime.today() - timedelta(days=7) + ) + ]["count"].sum() + / 7 + ) + overall_average = ( + confirmed_df["count"].sum() + / (datetime.today() - confirmed_df["date"].min()).days + ) + + overall_projection = { + "Weekly": get_projections(weekly_average, remaining_days, remaining_questions), + "Overall": get_projections( + overall_average, remaining_days, remaining_questions + ), + } + + # Projections by year + confirmed_dict = get_confirmed_dict(["year"]) + confirmed_df = pd.DataFrame.from_dict( + confirmed_dict, orient="index", columns=["count"] + ).reset_index(names=["date", "year"]) + + confirmed_df["date"], confirmed_df["year"] = zip( + *confirmed_df["date"].map(lambda x: (x[0], x[1])) + ) + confirmed_df["date"] = confirmed_df["date"].map(datetime.fromisoformat) + + weekly_df = confirmed_df[ + confirmed_df["date"].map( + lambda date: date >= datetime.today() - timedelta(days=7) + ) + ] + weekly_average_df = ( + (weekly_df.groupby("year")["count"].sum() / 7) + .reset_index() + .rename(columns={"count": "average"}) + .set_index("year") + ) + + overall_average_df = ( + ( + confirmed_df.groupby("year")["count"].sum() + / (datetime.today() - confirmed_df["date"].min()).days + ) + .reset_index() + .rename(columns={"count": "average"}) + .set_index("year") + ) + + weekly_projections_by_year = get_projections( + weekly_average_df, remaining_days, remaining_questions_by_year + ) + overall_projections_by_year = get_projections( + overall_average_df, remaining_days, remaining_questions_by_year + ) + + return render_template( + "metrics/projections.html", + overall_projection=overall_projection, + weekly_by_year=weekly_projections_by_year, + overall_by_year=overall_projections_by_year, + ) diff --git a/src/pir_pipeline/dashboard/review.py b/src/pir_pipeline/dashboard/review.py index e439b6ee..fdb18e37 100644 --- a/src/pir_pipeline/dashboard/review.py +++ b/src/pir_pipeline/dashboard/review.py @@ -2,12 +2,15 @@ import json from collections import OrderedDict +from datetime import datetime +from getpass import getuser from hashlib import sha1 from flask import Blueprint, render_template, request, session from sqlalchemy import bindparam, delete, func, select from pir_pipeline.dashboard.db import get_db +from pir_pipeline.models.pir_sql_models import link_history from pir_pipeline.utils.dashboard_utils import ( QuestionLinker, get_matches, @@ -154,7 +157,7 @@ def link(): session["link_dict"] = link_dict message = f"Data {data} queued for linking" elif action == "store": - link_dict = session["link_dict"] + link_dict = session.get("link_dict") ids = [ record.get("base_question_id", "") @@ -175,32 +178,76 @@ def link(): ], "proposed_changes", ) + db.insert_records( + [ + { + "link_id": proposed_id, + "user": getuser(), + "link_dict": list(link_dict.values()), + } + ], + "link_history", + ) del session["link_dict"] message = f"Record {link_dict} written to proposed changes." # Execute all linking actions elif action == "confirm": + link_id = payload["id"] proposed_changes = db.tables["proposed_changes"] link_dict_query = select(proposed_changes.c["link_dict"]).where( proposed_changes.c["id"] == bindparam("id") ) - link_dict = db.get_scalar(link_dict_query, {"id": payload["id"]}) + link_dict = db.get_scalar(link_dict_query, {"id": link_id}) QuestionLinker(link_dict, db).update_links() delete_query = delete(proposed_changes).where( - proposed_changes.c["id"] == payload["id"] + proposed_changes.c["id"] == link_id ) with db.engine.begin() as conn: conn.execute(delete_query) + db.update_records( + link_history, + { + "decision": bindparam("decision"), + "decision_timestamp": bindparam("decision_timestamp"), + }, + link_history.c["link_id"] == bindparam("b_link_id"), + [ + { + "b_link_id": link_id, + "decision": True, + "decision_timestamp": datetime.now(), + } + ], + ) + message = "Links Updated!" elif action == "deny": + link_id = payload["id"] proposed_changes = db.tables["proposed_changes"] delete_query = delete(proposed_changes).where( - proposed_changes.c["id"] == payload["id"] + proposed_changes.c["id"] == link_id ) with db.engine.begin() as conn: conn.execute(delete_query) + db.update_records( + link_history, + { + "decision": bindparam("decision"), + "decision_timestamp": bindparam("decision_timestamp"), + }, + link_history.c["link_id"] == bindparam("b_link_id"), + [ + { + "b_link_id": link_id, + "decision": False, + "decision_timestamp": datetime.now(), + } + ], + ) + message = f"Removed link associated with id: {payload["id"]}" return {"message": message} diff --git a/src/pir_pipeline/dashboard/static/metrics/metrics.js b/src/pir_pipeline/dashboard/static/metrics/metrics.js new file mode 100644 index 00000000..55bc896a --- /dev/null +++ b/src/pir_pipeline/dashboard/static/metrics/metrics.js @@ -0,0 +1,30 @@ +// Smooth scroll +// Adapted from Claude +document.querySelectorAll('a[href^="#"]').forEach(anchor => { + anchor.addEventListener('click', function (e) { + e.preventDefault(); + const target = document.querySelector(this.getAttribute('href')); + if (target) { + target.scrollIntoView({ + behavior: 'smooth', + block: 'start' + }); + } + }); +}); + +// Change the active link +document.addEventListener("DOMContentLoaded", toggleActive()) + +function toggleActive() { + const links = document.querySelectorAll('.floating-nav-link'); + for (let i = 0; i < links.length; i++) { + console.log(links[i]); + if (document.URL.includes(links[i].getAttribute("href"))) { + links.forEach(link => { + link.classList.remove("active"); + }) + links[i].classList.add("active"); + } + } +} \ No newline at end of file diff --git a/src/pir_pipeline/dashboard/static/search/search.js b/src/pir_pipeline/dashboard/static/search/search.js index 6dd0026d..54bfb27e 100644 --- a/src/pir_pipeline/dashboard/static/search/search.js +++ b/src/pir_pipeline/dashboard/static/search/search.js @@ -76,13 +76,35 @@ function getFlashcardData(e) { * * @param {*} e The event that triggered comitting the changes */ -function commitChanges(e) { +async function commitChanges(e) { e.preventDefault(); + let value = e.srcElement.getAttribute("value"); const questionTable = document.getElementById("flashcard-question-table"); + let baseRecord = rowToJSON(questionTable.getElementsByTagName("tr")[1]); + + const linkDetails = { + "link_type": value, + "base_question_id": baseRecord.question_id, + "match_question_id": null + } + + let payload = { + "action": "build", + "data": linkDetails + } + + // Add the confirm action to the dictionary of linking actions + await fetch("/review/link", { + "method": "POST", + "headers": { + "Content-type": "application/json" + }, + "body": JSON.stringify(payload) + }) // Commit changes to the database - const payload = { + payload = { "action": "store", "html": questionTable.outerHTML } diff --git a/src/pir_pipeline/dashboard/static/styles/metrics.css b/src/pir_pipeline/dashboard/static/styles/metrics.css new file mode 100644 index 00000000..0f27cea4 --- /dev/null +++ b/src/pir_pipeline/dashboard/static/styles/metrics.css @@ -0,0 +1,137 @@ +section.content { + display: flex; +} + +section { + margin: 0; +} + +section.nav-panel { + align-self: start; + width: 15%; + margin: 0 0 0 1em; +} + +section.main-panel { + width: 100%; +} + +div#yearly-average-tables { + display: flex; +} + +div.projections { + flex-direction: column; +} + +/* Adapted from Claude */ +/* Floating Navigation Panel */ +.floating-nav { + background-color: var(--primary); + border-radius: 0.5em; + box-shadow: 0 4px 12px rgba(23, 44, 60, 0.2); + padding: 1em; + transition: all 0.3s ease; + margin-top: 1em; +} + +/* .floating-nav.collapsed { + right: -160px; +} */ + +.floating-nav-header { + display: flex; + justify-content: space-between; + align-items: center; + padding-bottom: 0.75rem; + margin-bottom: 0.75rem; + border-bottom: 2px solid var(--secondary); +} + +.floating-nav-title { + font-size: 1rem; + font-weight: 600; + color: var(--light); + margin: 0; +} + +.floating-nav-toggle { + background: none; + border: none; + color: var(--light); + cursor: pointer; + padding: 0.25rem; + display: flex; + align-items: center; + transition: color 0.2s ease; +} + +.floating-nav-toggle:hover { + color: var(--info); +} + +.floating-nav-list { + list-style: none; + padding: 0; + margin: 0; +} + +.floating-nav-item { + margin-bottom: 0.5rem; +} + +.floating-nav-link { + display: block; + color: var(--light); + text-decoration: none; + padding: 0.5rem 0.75rem; + border-radius: 0.25rem; + font-size: 0.875rem; + transition: all 0.2s ease; + border-left: 3px solid transparent; +} + +.floating-nav-link:hover { + background-color: rgba(255, 255, 255, 0.1); + color: var(--info); + border-left-color: var(--info); + padding-left: 1rem; +} + +.floating-nav-link.active { + background-color: var(--secondary); + color: var(--light); + border-left-color: var(--light); + font-weight: 600; +} + +.floating-nav-divider { + height: 1px; + background-color: var(--secondary); + margin: 0.75rem 0; + opacity: 0.5; +} + +/* Floating Nav Icon/Badge */ +.floating-nav-badge { + display: inline-block; + background-color: var(--accent); + color: var(--dark); + font-size: 0.75rem; + padding: 0.125rem 0.5rem; + border-radius: 0.25rem; + margin-left: 0.5rem; + font-weight: 600; +} + +/* Responsive adjustments */ +@media (max-width: 768px) { + .floating-nav { + right: 1em; + min-width: 180px; + } + + .floating-nav.left { + left: 1em; + } +} \ No newline at end of file diff --git a/src/pir_pipeline/dashboard/templates/base.html b/src/pir_pipeline/dashboard/templates/base.html index b53062a6..49707ecb 100644 --- a/src/pir_pipeline/dashboard/templates/base.html +++ b/src/pir_pipeline/dashboard/templates/base.html @@ -36,6 +36,9 @@
  • Finalize
    • +
  • + Metrics +
    • diff --git a/src/pir_pipeline/dashboard/templates/metrics/approval_by_user.html b/src/pir_pipeline/dashboard/templates/metrics/approval_by_user.html new file mode 100644 index 00000000..a750d7e8 --- /dev/null +++ b/src/pir_pipeline/dashboard/templates/metrics/approval_by_user.html @@ -0,0 +1,20 @@ +{% extends "metrics/base.html" %} + +{% block main_panel %} +
    + + + + + + + + {% for row in approval_by_user.values() %} + + + + + {% endfor %} +
    UserPercent Approved
    {{ row["user"] }}{{ row["percent_approved"] }}
    +
    +{% endblock %} \ No newline at end of file diff --git a/src/pir_pipeline/dashboard/templates/metrics/base.html b/src/pir_pipeline/dashboard/templates/metrics/base.html new file mode 100644 index 00000000..14576b6c --- /dev/null +++ b/src/pir_pipeline/dashboard/templates/metrics/base.html @@ -0,0 +1,34 @@ +{% extends "base.html" %} + +{% block header %} + +

    {% block title %}Metrics{% endblock %}

    +{% endblock %} + +{% block content %} +
    + +
    +
    + {% block main_panel %}{% endblock %} +
    +{% endblock %} + +{% block scripts %} + +{% endblock %} \ No newline at end of file diff --git a/src/pir_pipeline/dashboard/templates/metrics/daily_links.html b/src/pir_pipeline/dashboard/templates/metrics/daily_links.html new file mode 100644 index 00000000..e61a9756 --- /dev/null +++ b/src/pir_pipeline/dashboard/templates/metrics/daily_links.html @@ -0,0 +1,20 @@ +{% extends "metrics/base.html" %} + +{% block main_panel %} +
    + + + + + + + + {% for key, value in confirmed_dict.items() %} + + + + + {% endfor %} +
    DateNumber Approved
    {{ key }}{{ value }}
    +
    +{% endblock %} \ No newline at end of file diff --git a/src/pir_pipeline/dashboard/templates/metrics/metrics.html b/src/pir_pipeline/dashboard/templates/metrics/metrics.html new file mode 100644 index 00000000..a750d7e8 --- /dev/null +++ b/src/pir_pipeline/dashboard/templates/metrics/metrics.html @@ -0,0 +1,20 @@ +{% extends "metrics/base.html" %} + +{% block main_panel %} +
    + + + + + + + + {% for row in approval_by_user.values() %} + + + + + {% endfor %} +
    UserPercent Approved
    {{ row["user"] }}{{ row["percent_approved"] }}
    +
    +{% endblock %} \ No newline at end of file diff --git a/src/pir_pipeline/dashboard/templates/metrics/projections.html b/src/pir_pipeline/dashboard/templates/metrics/projections.html new file mode 100644 index 00000000..6a74d40b --- /dev/null +++ b/src/pir_pipeline/dashboard/templates/metrics/projections.html @@ -0,0 +1,70 @@ +{% extends "metrics/base.html" %} + +{% block main_panel %} +
    + + + + + + + + + + + {% for key, value in overall_projection.items() %} + + + + + + + + {% endfor %} +
    RateAverageRemaining DaysProjected LinksShortfall
    {{ key }}{{ value.get("average") }}{{ value.get("remaining_days") }}{{ value.get("year_end") }}{{ value.get("shortfall") }}
    +
    +
    +
    +

    Weekly

    + + + + + + + + + + {% for row in weekly_by_year.values() %} + + + + + + + {% endfor %} +
    YearAverageProjected LinksShortfall
    {{ row['year'] }}{{ row['average'] }}{{ row['year_end'] }}{{ row['shortfall'] }}
    +
    +
    +

    Overall

    + + + + + + + + + + {% for row in overall_by_year.values() %} + + + + + + + {% endfor %} +
    YearAverageProjected LinksShortfall
    {{ row['year'] }}{{ row['average'] }}{{ row['year_end'] }}{{ row['shortfall'] }}
    +
    +
    +{% endblock %} \ No newline at end of file diff --git a/src/pir_pipeline/dashboard/templates/search/search.html b/src/pir_pipeline/dashboard/templates/search/search.html index 0bb524a6..7176cea9 100644 --- a/src/pir_pipeline/dashboard/templates/search/search.html +++ b/src/pir_pipeline/dashboard/templates/search/search.html @@ -58,7 +58,7 @@

    Search the PIR Database

    Cancel - + Confirm Changes diff --git a/src/pir_pipeline/dashboard/utils.py b/src/pir_pipeline/dashboard/utils.py new file mode 100644 index 00000000..42e213b3 --- /dev/null +++ b/src/pir_pipeline/dashboard/utils.py @@ -0,0 +1,22 @@ +from functools import wraps +from getpass import getuser + +from flask import flash, redirect, url_for + + +# https://flask.palletsprojects.com/en/stable/tutorial/views/ +def administrator(view): + @wraps(view) + def wrapped_view(**kwargs): + if getuser() not in [ + "jesse.escobar", + "emily.kowall", + "reggie.gilliard", + "alecsandra.velez", + ]: + flash("You are not authorized to access this page.") + return redirect(url_for("index")) + + return view(**kwargs) + + return wrapped_view diff --git a/src/pir_pipeline/models/pir_sql_models.py b/src/pir_pipeline/models/pir_sql_models.py index 18f34072..65869aa5 100644 --- a/src/pir_pipeline/models/pir_sql_models.py +++ b/src/pir_pipeline/models/pir_sql_models.py @@ -1,6 +1,7 @@ """SQLAlchemy models for creating and interacting with the PIR database""" from sqlalchemy import ( + Boolean, Column, DateTime, Float, @@ -115,6 +116,17 @@ Column("html", Text), ) +link_history = Table( + "link_history", + sql_metadata, + Column("link_id", String(255), nullable=False, primary_key=True), + Column("user", String(255), nullable=False), + Column("link_dict", JSONB), + Column("link_timestamp", DateTime(timezone=True), default=func.now()), + Column("decision", Boolean), + Column("decision_timestamp", DateTime(timezone=True)), +) + # Here to proposed_ids definition written with GPT dictionaries = ( func.jsonb_array_elements(proposed_changes.c.link_dict) @@ -183,7 +195,6 @@ select(question.c.uqid) .where(and_(question.c.question_id.in_(proposed_ids), question.c.uqid != null())) .distinct() - .subquery() ) query = ( diff --git a/src/pir_pipeline/utils/SQLAlchemyUtils.py b/src/pir_pipeline/utils/SQLAlchemyUtils.py index 4bf972a9..ad83df6a 100644 --- a/src/pir_pipeline/utils/SQLAlchemyUtils.py +++ b/src/pir_pipeline/utils/SQLAlchemyUtils.py @@ -13,6 +13,7 @@ from pir_pipeline.models.pir_sql_models import ( confirmed, flashcard, + link_history, linked, program, proposed_changes, @@ -65,12 +66,13 @@ def __init__(self, user: str, password: str, host: str, port: int, database: str self.insert = insert self._dialect = self._engine.name - self._tables: dict[Table] = { + self._tables: dict[str, Table] = { "response": response, "question": question, "program": program, "proposed_changes": proposed_changes, "linked": linked, + "link_history": link_history, "unlinked": unlinked, "uqid_changelog": uqid_changelog, "confirmed": confirmed, diff --git a/tests/dashboard/test_review.py b/tests/dashboard/test_review.py index 89ffb3c9..5c056ad7 100644 --- a/tests/dashboard/test_review.py +++ b/tests/dashboard/test_review.py @@ -136,21 +136,40 @@ def test_post_link(self, client, sql_utils, error_message_constructor): result = conn.execute(select(sql_utils.tables["proposed_changes"])) records = result.all() + result = conn.execute( + select(sql_utils.tables["link_history"]).where( + sql_utils.tables["link_history"].c["link_id"] == deny_id + ) + ) + decision = result.one()[-2] + expected = 1 got = len(records) assert expected == got, error_message_constructor( "Incorrect number of records", expected, got ) + assert decision is False, error_message_constructor( + "Decision should be false", expected, got + ) + confirm_id = records[0][0] client.post("/review/link", json={"action": "confirm", "id": confirm_id}) with sql_utils.engine.connect() as conn: result = conn.execute(select(sql_utils.tables["proposed_changes"])) proposed_records = result.all() + result = conn.execute(select(sql_utils.tables["uqid_changelog"])) confirmed_records = result.all() + result = conn.execute( + select(sql_utils.tables["link_history"]).where( + sql_utils.tables["link_history"].c["link_id"] == confirm_id + ) + ) + decision = result.one()[-2] + expected = 0 got = len(proposed_records) assert expected == got, error_message_constructor( @@ -169,6 +188,10 @@ def test_post_link(self, client, sql_utils, error_message_constructor): "Incorrect record in db", expected, got ) + assert decision is True, error_message_constructor( + "Decision should be true", expected, got + ) + if __name__ == "__main__": pytest.main([__file__, "-sk", "test_post_link"])