From 569e61449e38b20507155ab678d3e4fac7665089 Mon Sep 17 00:00:00 2001
From: Muhamed Fazal PS <muhamedfazalps@gmail.com>
Date: Wed, 15 Jul 2026 16:52:10 +0530
Subject: [PATCH] =?UTF-8?q?fix:=20comparison=20report=20winner=20logic=20?=
 =?UTF-8?q?=E2=80=94=20use=20common=20metrics=20only?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The winner determination and overall scores now only consider metrics
that exist in BOTH baseline and experiment results, preventing unfair
comparison when metric sets differ.

Previously, each side averaged all its own metrics, so adding extra
metrics to one side could inflate or deflate its overall score
independent of actual per-metric performance. The per-metric delta
table already defaulted missing metrics to 0.0, but the overall
score and winner used different denominator sets, creating
inconsistencies.

Closes #120
---
 openagent_eval/reports/comparison.py | 52 ++++++++++++++++++----------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/openagent_eval/reports/comparison.py b/openagent_eval/reports/comparison.py
index 2fb0324..6ddfeea 100644
--- a/openagent_eval/reports/comparison.py
+++ b/openagent_eval/reports/comparison.py
@@ -99,28 +99,42 @@ def generate(self, report: Any) -> str:
             lines.append(f"  Metrics unchanged: {len(all_metrics) - improved - regressed}")
             lines.append("")
 
-            # Overall scores
-            if baseline_metrics:
-                base_overall = sum(baseline_metrics.values()) / len(baseline_metrics)
-                lines.append(f"  Baseline overall:    {base_overall:.4f}")
-            if experiment_metrics:
-                exp_overall = sum(experiment_metrics.values()) / len(experiment_metrics)
-                lines.append(f"  Experiment overall:  {exp_overall:.4f}")
-                if baseline_metrics:
+            # Overall scores (common metrics only for fair comparison)
+            if baseline_metrics and experiment_metrics:
+                common_metrics = sorted(
+                    set(baseline_metrics) & set(experiment_metrics)
+                )
+                if common_metrics:
+                    base_overall = (
+                        sum(baseline_metrics[m] for m in common_metrics)
+                        / len(common_metrics)
+                    )
+                    exp_overall = (
+                        sum(experiment_metrics[m] for m in common_metrics)
+                        / len(common_metrics)
+                    )
+                    lines.append(f"  Baseline overall:    {base_overall:.4f}")
+                    lines.append(f"  Experiment overall:  {exp_overall:.4f}")
                     overall_delta = exp_overall - base_overall
                     lines.append(f"  Overall delta:       {overall_delta:+.4f}")
-            lines.append("")
-
-            # Winner
-            if baseline_metrics and experiment_metrics:
-                base_overall = sum(baseline_metrics.values()) / len(baseline_metrics)
-                exp_overall = sum(experiment_metrics.values()) / len(experiment_metrics)
-                if exp_overall > base_overall:
-                    lines.append(f"  >> WINNER: {report.experiment_name}")
-                elif exp_overall < base_overall:
-                    lines.append(f"  >> WINNER: {report.baseline_name}")
+                    lines.append("")
+
+                    # Winner
+                    if exp_overall > base_overall:
+                        lines.append(f"  >> WINNER: {report.experiment_name}")
+                    elif exp_overall < base_overall:
+                        lines.append(f"  >> WINNER: {report.baseline_name}")
+                    else:
+                        lines.append("  >> TIE")
                 else:
-                    lines.append("  >> TIE")
+                    lines.append("  No common metrics to compare overall scores.")
+                    lines.append("")
+            elif baseline_metrics:
+                lines.append(f"  Baseline overall:    {sum(baseline_metrics.values()) / len(baseline_metrics):.4f}")
+                lines.append("")
+            elif experiment_metrics:
+                lines.append(f"  Experiment overall:  {sum(experiment_metrics.values()) / len(experiment_metrics):.4f}")
+                lines.append("")
         else:
             lines.append("  No metrics available for comparison.")
             lines.append("")