diff --git a/openagent_eval/reports/comparison.py b/openagent_eval/reports/comparison.py index 2fb0324..6ddfeea 100644 --- a/openagent_eval/reports/comparison.py +++ b/openagent_eval/reports/comparison.py @@ -99,28 +99,42 @@ def generate(self, report: Any) -> str: lines.append(f" Metrics unchanged: {len(all_metrics) - improved - regressed}") lines.append("") - # Overall scores - if baseline_metrics: - base_overall = sum(baseline_metrics.values()) / len(baseline_metrics) - lines.append(f" Baseline overall: {base_overall:.4f}") - if experiment_metrics: - exp_overall = sum(experiment_metrics.values()) / len(experiment_metrics) - lines.append(f" Experiment overall: {exp_overall:.4f}") - if baseline_metrics: + # Overall scores (common metrics only for fair comparison) + if baseline_metrics and experiment_metrics: + common_metrics = sorted( + set(baseline_metrics) & set(experiment_metrics) + ) + if common_metrics: + base_overall = ( + sum(baseline_metrics[m] for m in common_metrics) + / len(common_metrics) + ) + exp_overall = ( + sum(experiment_metrics[m] for m in common_metrics) + / len(common_metrics) + ) + lines.append(f" Baseline overall: {base_overall:.4f}") + lines.append(f" Experiment overall: {exp_overall:.4f}") overall_delta = exp_overall - base_overall lines.append(f" Overall delta: {overall_delta:+.4f}") - lines.append("") - - # Winner - if baseline_metrics and experiment_metrics: - base_overall = sum(baseline_metrics.values()) / len(baseline_metrics) - exp_overall = sum(experiment_metrics.values()) / len(experiment_metrics) - if exp_overall > base_overall: - lines.append(f" >> WINNER: {report.experiment_name}") - elif exp_overall < base_overall: - lines.append(f" >> WINNER: {report.baseline_name}") + lines.append("") + + # Winner + if exp_overall > base_overall: + lines.append(f" >> WINNER: {report.experiment_name}") + elif exp_overall < base_overall: + lines.append(f" >> WINNER: {report.baseline_name}") + else: + lines.append(" >> TIE") else: - lines.append(" >> TIE") + lines.append(" No common metrics to compare overall scores.") + lines.append("") + elif baseline_metrics: + lines.append(f" Baseline overall: {sum(baseline_metrics.values()) / len(baseline_metrics):.4f}") + lines.append("") + elif experiment_metrics: + lines.append(f" Experiment overall: {sum(experiment_metrics.values()) / len(experiment_metrics):.4f}") + lines.append("") else: lines.append(" No metrics available for comparison.") lines.append("")