From 7b827ca7dbcaea572e778399fc02c340a30bd607 Mon Sep 17 00:00:00 2001 From: Ester Beltrami Date: Sun, 18 Jan 2026 17:28:45 +0000 Subject: [PATCH 1/4] Add standard deviation to grants review recap for vote divergence Add `std_dev` annotation to the grants review recap view to help identify controversial grants that need discussion during review calls. Standard deviation measures how much reviewers disagree: - High std dev = genuine disagreement across reviewers - Low std dev = consensus, grant can be quickly accepted/rejected This helps prioritize which grants to discuss in review meetings by surfacing cases where reviewers have diverging opinions. --- backend/reviews/admin.py | 5 + backend/reviews/templates/grants-recap.html | 7 ++ backend/reviews/tests/test_admin.py | 118 ++++++++++++++++++++ 3 files changed, 130 insertions(+) diff --git a/backend/reviews/admin.py b/backend/reviews/admin.py index e388423bdd..219d3f1a2b 100644 --- a/backend/reviews/admin.py +++ b/backend/reviews/admin.py @@ -13,6 +13,7 @@ OuterRef, Prefetch, Q, + StdDev, Subquery, Sum, ) @@ -426,6 +427,10 @@ def _review_grants_recap_view(self, request, review_session): F("total_score") / F("vote_count"), output_field=FloatField(), ), + std_dev=StdDev( + "userreview__score__numeric_value", + filter=Q(userreview__review_session_id=review_session_id), + ), has_sent_a_proposal=Exists( Submission.objects.non_cancelled().filter( speaker_id=OuterRef("user_id"), diff --git a/backend/reviews/templates/grants-recap.html b/backend/reviews/templates/grants-recap.html index 9177c9d416..4734cdd669 100644 --- a/backend/reviews/templates/grants-recap.html +++ b/backend/reviews/templates/grants-recap.html @@ -432,6 +432,12 @@

+ +
+ Std Dev +
+
+
Votes @@ -573,6 +579,7 @@

{{ item.score }} + {{ item.std_dev|floatformat:2 }}
    {% for reviewer in item.userreview_set.all %} diff --git a/backend/reviews/tests/test_admin.py b/backend/reviews/tests/test_admin.py index 3b151a5fc2..1c7265aac8 100644 --- a/backend/reviews/tests/test_admin.py +++ b/backend/reviews/tests/test_admin.py @@ -191,6 +191,124 @@ def test_grants_review_scores(rf, scores, avg): assert grant_to_check.score == avg +@pytest.mark.parametrize( + "scores, expected_std_dev", + [ + # Multiple different scores: mean=0.6, std_dev ≈ 1.744 + ( + [ + {"user": 0, "score": 2}, + {"user": 1, "score": 2}, + {"user": 2, "score": 2}, + {"user": 3, "score": -1}, + {"user": 4, "score": -2}, + ], + 1.744, + ), + # All same scores: std_dev = 0 + ( + [ + {"user": 0, "score": -2}, + {"user": 1, "score": -2}, + {"user": 2, "score": -2}, + {"user": 3, "score": -2}, + {"user": 4, "score": -2}, + ], + 0.0, + ), + # Single score: std_dev = 0 + ( + [ + {"user": 0, "score": 1}, + ], + 0.0, + ), + # No scores: std_dev = None + ([], None), + # Two different scores (1 and -1): mean=0, std_dev = sqrt(((1-0)^2 + (-1-0)^2) / 2) = 1.0 + ( + [ + {"user": 0, "score": 1}, + {"user": 1, "score": -1}, + ], + 1.0, + ), + # Three scores with same value: std_dev = 0 + ( + [ + {"user": 0, "score": 1}, + {"user": 1, "score": 1}, + {"user": 2, "score": 1}, + ], + 0.0, + ), + # Mixed scores showing consensus with outlier: 3x score=1, 1x score=-2 + # mean = (1+1+1-2)/4 = 0.25 + # std_dev = sqrt(((1-0.25)^2 + (1-0.25)^2 + (1-0.25)^2 + (-2-0.25)^2) / 4) + # = sqrt((0.5625 + 0.5625 + 0.5625 + 5.0625) / 4) = sqrt(1.6875) ≈ 1.299 + ( + [ + {"user": 0, "score": 1}, + {"user": 1, "score": 1}, + {"user": 2, "score": 1}, + {"user": 3, "score": -2}, + ], + 1.299, + ), + ], +) +def test_grants_review_std_dev(rf, scores, expected_std_dev): + conference = ConferenceFactory() + review_session = ReviewSessionFactory( + conference=conference, + session_type=ReviewSession.SessionType.GRANTS, + status=ReviewSession.Status.COMPLETED, + ) + + users = UserFactory.create_batch(10, is_staff=True, is_superuser=True) + all_scores = { + -2: AvailableScoreOptionFactory( + review_session=review_session, numeric_value=-2, label="Rejected" + ), + -1: AvailableScoreOptionFactory( + review_session=review_session, numeric_value=-1, label="Not convinced" + ), + 0: AvailableScoreOptionFactory( + review_session=review_session, numeric_value=0, label="Maybe" + ), + 1: AvailableScoreOptionFactory( + review_session=review_session, numeric_value=1, label="Yes" + ), + 2: AvailableScoreOptionFactory( + review_session=review_session, numeric_value=2, label="Absolutely" + ), + } + + grant = GrantFactory(conference=conference) + for score in scores: + UserReviewFactory( + review_session=review_session, + grant=grant, + user=users[score["user"]], + score=all_scores[score["score"]], + ) + + request = rf.get("/") + request.user = users[5] + + admin = ReviewSessionAdmin(ReviewSession, AdminSite()) + response = admin._review_grants_recap_view(request, review_session) + context_data = response.context_data + items = context_data["items"] + grant_to_check = next(item for item in items if item.id == grant.id) + + assert grant_to_check.id == grant.id + if expected_std_dev is None: + assert grant_to_check.std_dev is None + else: + assert grant_to_check.std_dev == pytest.approx(expected_std_dev, abs=0.01) + + def test_review_start_view_when_no_items_are_left(rf, mocker): mock_messages = mocker.patch("reviews.admin.messages") From 3aa8dafa793ed555d6bf03a1f05716f3cf6cb53d Mon Sep 17 00:00:00 2001 From: Ester Beltrami Date: Sun, 18 Jan 2026 20:46:38 +0000 Subject: [PATCH 2/4] Add tooltip to Std Dev column in grants recap --- backend/reviews/templates/grants-recap.html | 47 ++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/backend/reviews/templates/grants-recap.html b/backend/reviews/templates/grants-recap.html index 4734cdd669..9aab460b5d 100644 --- a/backend/reviews/templates/grants-recap.html +++ b/backend/reviews/templates/grants-recap.html @@ -181,6 +181,49 @@ tr:nth-of-type(odd) { background-color: var(--body-bg); } + + /* Tooltip */ + .tooltip { + position: relative; + display: inline-block; + } + + .tooltip .tooltiptext { + visibility: hidden; + opacity: 0; + width: 220px; + background-color: #333; + color: #fff; + text-align: left; + text-transform: none; + padding: 8px 10px; + border-radius: 6px; + position: absolute; + z-index: 1; + top: 125%; + left: 50%; + margin-left: -110px; + font-size: 12px; + font-weight: normal; + line-height: 1.4; + transition: opacity 0.2s ease-in-out 0.5s, visibility 0.2s ease-in-out 0.5s; + } + + .tooltip .tooltiptext::after { + content: ""; + position: absolute; + bottom: 100%; + left: 50%; + margin-left: -5px; + border-width: 5px; + border-style: solid; + border-color: transparent transparent #333 transparent; + } + + .tooltip:hover .tooltiptext { + visibility: visible; + opacity: 1; + }
    • @@ -471,13 +559,13 @@

      - Score + Score ▼
      - Std Dev + Std Dev Standard Deviation: measures reviewer disagreement. High value = controversial (needs discussion), Low value = consensus.
      @@ -532,6 +620,8 @@

      id="grant-{{ item.id }}" data-type="{{ item.type }}" data-num-of-votes="{{ item.userreview_set.count }}" + data-score="{{ item.score|default_if_none:'-999' }}" + data-std-dev="{{ item.std_dev|default_if_none:'-1' }}" > {{ forloop.counter }} From 98aea1cbe46171cd23739d63a6fdb4a163029fb4 Mon Sep 17 00:00:00 2001 From: Ester Beltrami Date: Sun, 18 Jan 2026 21:01:22 +0000 Subject: [PATCH 4/4] Center-align columns in grants recap --- backend/reviews/templates/grants-recap.html | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/backend/reviews/templates/grants-recap.html b/backend/reviews/templates/grants-recap.html index 7fb2ff051f..f710df78ca 100644 --- a/backend/reviews/templates/grants-recap.html +++ b/backend/reviews/templates/grants-recap.html @@ -234,6 +234,21 @@ .sortable:hover { text-decoration: underline; } + + /* Center-align columns */ + .results-table th:nth-child(1), /* number column */ + .results-table th:nth-child(3), /* score column */ + .results-table th:nth-child(4), /* std dev column */ + .results-table th:nth-child(6), /* current status column */ + .results-table th:nth-child(7), /* pending status column */ + .results-table td:nth-child(1), /* number column */ + .results-table td:nth-child(3), /* score column */ + .results-table td:nth-child(4), /* std dev column */ + .results-table td:nth-child(6), /* current status column */ + .results-table td:nth-child(7) /* pending status column */ + { + text-align: center; + }