main.py

"""Repeatable experiment pipeline for the FaceGuard-Risk prototype."""

from __future__ import annotations

import csv
from pathlib import Path
from typing import Any

import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split

from continuous.sequence_generator import (
    generate_all_session_types,
    session_to_feature_array,
    get_session_labels,
)
from continuous.session_risk_model import SessionRiskModel
from continuous.session_policy import (
    compute_session_states,
    summarize_session,
)
from continuous.session_visualization import plot_session_risk
from evaluation.continuous_metrics import (
    build_continuous_summary_rows,
    save_continuous_summary_csv,
    save_continuous_session_markdown,
    print_continuous_summary,
)
from evaluation.metrics import evaluate_scores, get_roc_points
from evaluation.policy_evaluation import (
    evaluate_policy_scenarios,
    format_policy_summary,
    flatten_policy_seed_results,
    aggregate_policy_results,
    save_policy_by_seed_csv,
    save_policy_aggregated_csv,
    save_policy_comparison_markdown,
    print_policy_terminal_summary,
)
from evaluation.statistics import compute_confidence_interval, paired_significance_test
from evaluation.threshold_analysis import run_threshold_sensitivity_analysis
from explainability.risk_explainer import build_example_explanations
from challenge.challenge_policy import (
    build_adaptive_challenge_examples,
    evaluate_challenge_policies,
    print_challenge_examples,
    save_adaptive_challenge_csv,
    save_adaptive_challenge_markdown,
)
from reporting.generate_research_report import generate_research_report, save_report
from models.risk_model import (
    RiskAwareFaceModel,
    baseline_face_only_score,
    rule_based_score,
)
from simulation.generate_data import generate_synthetic_access_data


FEATURE_NAMES = ["face_score", "location_score", "time_score", "device_score"]
METRIC_NAMES = ["auc", "accuracy", "far", "frr"]
METHOD_ORDER = [
    "Face-only baseline",
    "Rule-based baseline",
    "Full risk-aware model",
    "Without location_score",
    "Without time_score",
    "Without device_score",
]
EXPERIMENT_SEEDS = [7, 21, 42, 84, 126]
REPRESENTATIVE_SEED = 42
SIGNIFICANCE_COMPARISONS = [
    ("Full risk-aware model", "Face-only baseline"),
    ("Full risk-aware model", "Rule-based baseline"),
    ("Full risk-aware model", "Without location_score"),
    ("Full risk-aware model", "Without time_score"),
    ("Full risk-aware model", "Without device_score"),
]


def evaluate_logistic_subset(
    X_train: np.ndarray,
    X_test: np.ndarray,
    y_train: np.ndarray,
    y_test: np.ndarray,
    feature_indices: list[int],
    threshold: float,
    random_state: int,
) -> tuple[RiskAwareFaceModel, np.ndarray, dict[str, float]]:
    """Fit and evaluate a logistic model on a selected feature subset."""
    model = RiskAwareFaceModel(random_state=random_state)
    model.fit(X_train[:, feature_indices], y_train)
    scores = model.predict_proba(X_test[:, feature_indices])
    metrics = evaluate_scores(y_test, scores, threshold=threshold)
    return model, scores, metrics


def run_single_seed(seed: int) -> dict[str, Any]:
    """Run one full train/test experiment for a single random seed."""
    X, y = generate_synthetic_access_data(n_samples=3000, random_state=seed)

    X_train, X_test, y_train, y_test = train_test_split(
        X,
        y,
        test_size=0.30,
        random_state=seed,
        stratify=y,
    )

    face_scores = baseline_face_only_score(X_test)
    rule_scores = rule_based_score(X_test)

    face_metrics = evaluate_scores(y_test, face_scores, threshold=0.68)
    rule_metrics = evaluate_scores(y_test, rule_scores, threshold=0.58)

    full_model, full_scores, full_metrics = evaluate_logistic_subset(
        X_train=X_train,
        X_test=X_test,
        y_train=y_train,
        y_test=y_test,
        feature_indices=[0, 1, 2, 3],
        threshold=0.50,
        random_state=seed,
    )

    ablations = [
        ("Without location_score", [0, 2, 3]),
        ("Without time_score", [0, 1, 3]),
        ("Without device_score", [0, 1, 2]),
    ]

    method_results: dict[str, dict[str, float]] = {
        "Face-only baseline": face_metrics,
        "Rule-based baseline": rule_metrics,
        "Full risk-aware model": full_metrics,
    }

    for method_name, feature_indices in ablations:
        _, _, metrics = evaluate_logistic_subset(
            X_train=X_train,
            X_test=X_test,
            y_train=y_train,
            y_test=y_test,
            feature_indices=feature_indices,
            threshold=0.50,
            random_state=seed,
        )
        method_results[method_name] = metrics

    return {
        "seed": seed,
        "train_size": len(X_train),
        "test_size": len(X_test),
        "positive_rate": float(np.mean(y)),
        "method_results": method_results,
        "representative": {
            "y_test": y_test,
            "face_scores": face_scores,
            "rule_scores": rule_scores,
            "full_scores": full_scores,
            "full_model": full_model,
        },
    }


def flatten_seed_results(seed_runs: list[dict[str, Any]]) -> list[dict[str, float | int | str]]:
    """Turn nested per-seed experiment outputs into CSV-ready rows."""
    rows: list[dict[str, float | int | str]] = []
    for run in seed_runs:
        seed = int(run["seed"])
        for method_name in METHOD_ORDER:
            metrics = run["method_results"][method_name]
            rows.append(
                {
                    "seed": seed,
                    "method": method_name,
                    "auc": metrics["auc"],
                    "accuracy": metrics["accuracy"],
                    "far": metrics["far"],
                    "frr": metrics["frr"],
                }
            )
    return rows


def aggregate_results(seed_rows: list[dict[str, float | int | str]]) -> list[dict[str, float | str]]:
    """Aggregate metric rows into mean, std, and 95% confidence intervals."""
    aggregated: list[dict[str, float | str]] = []
    for method_name in METHOD_ORDER:
        method_rows = [row for row in seed_rows if row["method"] == method_name]
        result_row: dict[str, float | str] = {"method": method_name}

        for metric_name in METRIC_NAMES:
            values = [float(row[metric_name]) for row in method_rows]
            stats_row = compute_confidence_interval(
                values,
                confidence=0.95,
                lower_bound=0.0,
                upper_bound=1.0,
            )
            result_row[f"{metric_name}_mean"] = stats_row["mean"]
            result_row[f"{metric_name}_std"] = stats_row["std"]
            result_row[f"{metric_name}_ci_lower"] = stats_row["ci_lower"]
            result_row[f"{metric_name}_ci_upper"] = stats_row["ci_upper"]

        aggregated.append(result_row)
    return aggregated


def build_significance_rows(
    seed_rows: list[dict[str, float | int | str]],
    alpha: float = 0.05,
) -> list[dict[str, float | str | bool]]:
    """Compute paired significance results for selected comparisons."""
    rows: list[dict[str, float | str | bool]] = []

    for full_method, comparison_method in SIGNIFICANCE_COMPARISONS:
        full_rows = [row for row in seed_rows if row["method"] == full_method]
        comparison_rows = [row for row in seed_rows if row["method"] == comparison_method]

        for metric_name in METRIC_NAMES:
            full_values = [float(row[metric_name]) for row in full_rows]
            comparison_values = [float(row[metric_name]) for row in comparison_rows]
            result = paired_significance_test(full_values, comparison_values, alpha=alpha)
            rows.append(
                {
                    "comparison": f"{full_method} vs {comparison_method}",
                    "metric": metric_name,
                    "full_mean": float(np.mean(full_values)),
                    "comparison_mean": float(np.mean(comparison_values)),
                    "mean_difference": result["mean_difference"],
                    "ttest_p_value": result["ttest_p_value"],
                    "wilcoxon_p_value": result["wilcoxon_p_value"],
                    "significant": result["significant"],
                }
            )
    return rows


def save_csv(path: Path, fieldnames: list[str], rows: list[dict[str, Any]]) -> None:
    """Save experiment rows to a CSV file."""
    with path.open("w", newline="", encoding="utf-8") as csv_file:
        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(rows)


def save_results_markdown_table(path: Path, aggregated_rows: list[dict[str, float | str]]) -> None:
    """Create a paper-style markdown table with mean, std, and confidence intervals."""
    lines = [
        "# FaceGuard-Risk Results Table",
        "",
        "Mean +/- standard deviation and 95% confidence intervals across repeated random seeds.",
        "",
        "| Method | AUC | Accuracy | FAR | FRR |",
        "|---|---:|---:|---:|---:|",
    ]

    for row in aggregated_rows:
        lines.append(
            "| "
            f"{row['method']} | "
            f"{float(row['auc_mean']):.4f} +/- {float(row['auc_std']):.4f} "
            f"[{float(row['auc_ci_lower']):.4f}, {float(row['auc_ci_upper']):.4f}] | "
            f"{float(row['accuracy_mean']):.4f} +/- {float(row['accuracy_std']):.4f} "
            f"[{float(row['accuracy_ci_lower']):.4f}, {float(row['accuracy_ci_upper']):.4f}] | "
            f"{float(row['far_mean']):.4f} +/- {float(row['far_std']):.4f} "
            f"[{float(row['far_ci_lower']):.4f}, {float(row['far_ci_upper']):.4f}] | "
            f"{float(row['frr_mean']):.4f} +/- {float(row['frr_std']):.4f} "
            f"[{float(row['frr_ci_lower']):.4f}, {float(row['frr_ci_upper']):.4f}] |"
        )

    path.write_text("\n".join(lines) + "\n", encoding="utf-8")


def save_significance_markdown_table(
    path: Path,
    significance_rows: list[dict[str, float | str | bool]],
) -> None:
    """Create a paper-style markdown table for significance testing."""
    lines = [
        "# FaceGuard-Risk Significance Table",
        "",
        "Paired significance tests across repeated random seeds with alpha = 0.05.",
        "",
    ]

    for metric_name in METRIC_NAMES:
        lines.extend(
            [
                f"## Metric: {metric_name.upper()}",
                "",
                "| Comparison | Full Mean | Comparison Mean | Mean Difference | paired t-test p | Wilcoxon p | Significant |",
                "|---|---:|---:|---:|---:|---:|:---:|",
            ]
        )

        metric_rows = [row for row in significance_rows if row["metric"] == metric_name]
        for row in metric_rows:
            lines.append(
                "| "
                f"{row['comparison']} | "
                f"{float(row['full_mean']):.4f} | "
                f"{float(row['comparison_mean']):.4f} | "
                f"{float(row['mean_difference']):.4f} | "
                f"{float(row['ttest_p_value']):.4f} | "
                f"{float(row['wilcoxon_p_value']):.4f} | "
                f"{'Yes' if bool(row['significant']) else 'No'} |"
            )
        lines.append("")

    path.write_text("\n".join(lines), encoding="utf-8")


def save_risk_explanation_examples_csv(path: Path, rows: list[dict[str, Any]]) -> None:
    """Save representative interpretable risk explanations to CSV."""
    fieldnames = [
        "case",
        "face_score",
        "location_score",
        "time_score",
        "device_score",
        "model_score",
        "decision",
        "risk_level",
        "main_risk_reason",
        "secondary_risk_reasons",
        "suggested_action",
    ]
    csv_rows: list[dict[str, Any]] = []

    for row in rows:
        csv_rows.append(
            {
                **row,
                "secondary_risk_reasons": "; ".join(row["secondary_risk_reasons"]),
            }
        )

    save_csv(path=path, fieldnames=fieldnames, rows=csv_rows)


def save_risk_explanation_examples_markdown(path: Path, rows: list[dict[str, Any]]) -> None:
    """Save representative interpretable risk explanations to a markdown table."""
    lines = [
        "# Interpretable Risk Explanation Examples",
        "",
        "Representative examples showing how FaceGuard-Risk explains access decisions.",
        "",
        "| Case | Scores | Decision | Risk Level | Main Reason | Secondary Reasons | Suggested Action |",
        "|---|---|---:|---:|---|---|---|",
    ]

    for row in rows:
        score_text = (
            f"face={float(row['face_score']):.2f}, "
            f"location={float(row['location_score']):.2f}, "
            f"time={float(row['time_score']):.2f}, "
            f"device={float(row['device_score']):.2f}, "
            f"model={float(row['model_score']):.2f}"
        )
        secondary_reasons = "; ".join(row["secondary_risk_reasons"]) or "none"
        lines.append(
            "| "
            f"{row['case']} | "
            f"{score_text} | "
            f"{row['decision']} | "
            f"{row['risk_level']} | "
            f"{row['main_risk_reason']} | "
            f"{secondary_reasons} | "
            f"{row['suggested_action']} |"
        )

    path.write_text("\n".join(lines) + "\n", encoding="utf-8")


def print_risk_explanation_summary(rows: list[dict[str, Any]]) -> None:
    """Print a compact summary of generated explanation examples."""
    print("\nInterpretable Risk Explanation Examples")
    print("=======================================")
    for row in rows:
        print(
            f"{row['case']:<46} "
            f"{row['decision']:<10} "
            f"risk={row['risk_level']:<6} "
            f"reason={row['main_risk_reason']}"
        )


def print_threshold_sensitivity_summary(rows: list[dict[str, Any]]) -> None:
    """Print the best threshold pair for each sensitivity scenario."""
    print("\nThreshold Sensitivity Analysis")
    print("==============================")
    print(
        f"{'Scenario':<20} {'Accept':>8} {'Reject':>8} "
        f"{'Risk':>10} {'FAR':>8} {'FRR':>8} {'Challenge':>10}"
    )
    for row in rows:
        scenario = str(row["scenario"]).replace("_", " ").title()
        print(
            f"{scenario:<20} "
            f"{float(row['accept_threshold']):>8.2f} "
            f"{float(row['reject_threshold']):>8.2f} "
            f"{float(row['expected_risk']):>10.4f} "
            f"{float(row['far']):>8.4f} "
            f"{float(row['frr']):>8.4f} "
            f"{float(row['challenge_rate']):>10.4f}"
        )


def print_seed_overview(seed_runs: list[dict[str, Any]]) -> None:
    """Print high-level experiment configuration."""
    representative = next(run for run in seed_runs if run["seed"] == REPRESENTATIVE_SEED)
    print("\nFaceGuard-Risk Repeatable Experiment")
    print("====================================")
    print(f"Seeds           : {', '.join(str(seed) for seed in EXPERIMENT_SEEDS)}")
    print(f"Representative  : {REPRESENTATIVE_SEED}")
    print(f"Train samples   : {representative['train_size']}")
    print(f"Test samples    : {representative['test_size']}")
    print(f"Positive rate   : {representative['positive_rate']:.4f}")
    print(f"Feature space   : {', '.join(FEATURE_NAMES)}")


def print_aggregated_summary(aggregated_rows: list[dict[str, float | str]]) -> None:
    """Print a clean terminal summary table with confidence intervals."""
    print("\nAggregated Results (mean +/- std, 95% CI)")
    print("=========================================")
    print(f"{'Method':<26} {'AUC':>39} {'Accuracy':>39}")
    for row in aggregated_rows:
        auc_text = (
            f"{float(row['auc_mean']):.4f} +/- {float(row['auc_std']):.4f} "
            f"[{float(row['auc_ci_lower']):.4f}, {float(row['auc_ci_upper']):.4f}]"
        )
        accuracy_text = (
            f"{float(row['accuracy_mean']):.4f} +/- {float(row['accuracy_std']):.4f} "
            f"[{float(row['accuracy_ci_lower']):.4f}, {float(row['accuracy_ci_upper']):.4f}]"
        )
        print(f"{str(row['method']):<26} {auc_text:>39} {accuracy_text:>39}")


def print_significance_summary(significance_rows: list[dict[str, float | str | bool]]) -> None:
    """Print a concise paper-style statistical summary in terminal."""
    print("\nMain Statistical Results (AUC)")
    print("==============================")
    print(
        f"{'Comparison':<58} {'Diff':>10} {'t-test p':>12} {'Wilcoxon p':>12} {'Sig':>8}"
    )
    auc_rows = [row for row in significance_rows if row["metric"] == "auc"]
    for row in auc_rows:
        print(
            f"{str(row['comparison']):<58} "
            f"{float(row['mean_difference']):>10.4f} "
            f"{float(row['ttest_p_value']):>12.4f} "
            f"{float(row['wilcoxon_p_value']):>12.4f} "
            f"{'Yes' if bool(row['significant']) else 'No':>8}"
        )

    print("\nInterpretation")
    print("--------------")
    print("The full risk-aware model remains strongest in mean AUC across all reported comparisons.")
    print("Paired t-tests suggest consistent gains, but under the stricter dual-test criterion")
    print("the five-seed setting is too small for Wilcoxon to confirm most differences at alpha = 0.05.")
    print("This should be interpreted as encouraging evidence rather than definitive statistical confirmation.")


def print_representative_coefficients(seed_runs: list[dict[str, Any]]) -> None:
    """Print coefficients for one representative full-model run."""
    representative = next(run for run in seed_runs if run["seed"] == REPRESENTATIVE_SEED)
    full_model: RiskAwareFaceModel = representative["representative"]["full_model"]
    print("\nRepresentative full-model coefficients")
    print("======================================")
    for feature_name, coef in zip(FEATURE_NAMES, full_model.coefficients()):
        print(f"{feature_name:<15} {coef:.4f}")
    print(f"{'intercept':<15} {full_model.intercept():.4f}")


def save_representative_roc(seed_runs: list[dict[str, Any]], results_dir: Path) -> Path:
    """Save ROC figure for one representative run."""
    representative = next(run for run in seed_runs if run["seed"] == REPRESENTATIVE_SEED)
    data = representative["representative"]
    y_test = data["y_test"]

    face_fpr, face_tpr = get_roc_points(y_test, data["face_scores"])
    rule_fpr, rule_tpr = get_roc_points(y_test, data["rule_scores"])
    full_fpr, full_tpr = get_roc_points(y_test, data["full_scores"])

    face_metrics = representative["method_results"]["Face-only baseline"]
    rule_metrics = representative["method_results"]["Rule-based baseline"]
    full_metrics = representative["method_results"]["Full risk-aware model"]

    output_path = results_dir / "roc_curve.png"
    plt.figure(figsize=(8, 6))
    plt.plot(face_fpr, face_tpr, label=f"Face-only (AUC={face_metrics['auc']:.3f})", linewidth=2)
    plt.plot(rule_fpr, rule_tpr, label=f"Rule-based (AUC={rule_metrics['auc']:.3f})", linewidth=2)
    plt.plot(full_fpr, full_tpr, label=f"Risk-aware logistic (AUC={full_metrics['auc']:.3f})", linewidth=2)
    plt.plot([0, 1], [0, 1], linestyle="--", color="gray", linewidth=1, label="Random guess")
    plt.title(f"FaceGuard-Risk ROC Comparison (seed={REPRESENTATIVE_SEED})")
    plt.xlabel("False Positive Rate")
    plt.ylabel("True Positive Rate")
    plt.legend(loc="lower right")
    plt.grid(True, linestyle="--", alpha=0.4)
    plt.tight_layout()
    plt.savefig(output_path, dpi=150)
    plt.close()
    return output_path


def save_policy_summary_csv(path: Path, policy_results: list[dict[str, Any]]) -> None:
    """Save policy evaluation results to CSV."""
    if not policy_results:
        return

    fieldnames = [
        "scenario",
        "accept_threshold",
        "reject_threshold",
        "cost_false_accept",
        "cost_false_reject",
        "cost_challenge",
        "expected_risk",
        "far",
        "frr",
        "accept_rate",
        "reject_rate",
        "challenge_rate",
    ]

    with path.open("w", newline="", encoding="utf-8") as csv_file:
        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(policy_results)


def save_policy_summary_markdown(path: Path, policy_results: list[dict[str, Any]]) -> None:
    """Save policy evaluation results to markdown table."""
    if not policy_results:
        return

    lines = [
        "# Risk-Aware Multi-Stage Decision Policy Summary",
        "",
        "Optimal threshold pairs and expected risk under different operational scenarios.",
        "",
        "| Scenario | Accept Thr. | Reject Thr. | Expected Risk | FAR | FRR | Challenge Rate |",
        "|---|---:|---:|---:|---:|---:|---:|",
    ]

    for result in policy_results:
        lines.append(
            "| "
            f"{result['scenario'].replace('_', ' ').title()} | "
            f"{float(result['accept_threshold']):.3f} | "
            f"{float(result['reject_threshold']):.3f} | "
            f"{float(result['expected_risk']):.4f} | "
            f"{float(result['far']):.4f} | "
            f"{float(result['frr']):.4f} | "
            f"{float(result['challenge_rate']):.4f} |"
        )

    path.write_text("\n".join(lines) + "\n", encoding="utf-8")


def print_policy_summary(policy_results: list[dict[str, Any]]) -> None:
    """Print a clean policy summary in terminal."""
    if not policy_results:
        print("\n(Policy evaluation not available)")
        return

    print("\n" + "=" * 80)
    print("RISK-AWARE MULTI-STAGE ACCESS DECISION POLICY")
    print("=" * 80)

    for result in policy_results:
        scenario_name = result["scenario"].replace("_", " ").upper()
        print(f"\n{scenario_name}")
        print("-" * 40)
        print(f"  Accept Threshold:   {float(result['accept_threshold']):.3f}")
        print(f"  Reject Threshold:   {float(result['reject_threshold']):.3f}")
        print(f"  Expected Risk:      {float(result['expected_risk']):.4f}")
        print(f"  False Accept Rate:  {float(result['far']):.4f}")
        print(f"  False Reject Rate:  {float(result['frr']):.4f}")
        print(f"  Challenge Rate:     {float(result['challenge_rate']):.4f}")
        print(f"  Decision Breakdown:")
        print(f"    ACCEPT:   {float(result['accept_rate'])*100:5.2f}%")
        print(f"    CHALLENGE: {float(result['challenge_rate'])*100:5.2f}%")
        print(f"    REJECT:   {float(result['reject_rate'])*100:5.2f}%")

    print("=" * 80)


def run_continuous_sessions(
    seed_runs: list[dict[str, Any]],
    results_dir: Path,
) -> tuple[list[dict[str, Any]], dict[str, np.ndarray], list[Path]]:
    """Run continuous session-level risk monitoring on representative model.

    Uses the full risk-aware model from the representative seed to evaluate
    all session types and produce summary CSVs, markdown, and risk plots.

    Args:
        seed_runs: Per-seed experiment results.
        results_dir: Directory for output files.

    Returns:
        Tuple of (summary_rows, labels_dict, plot_paths).
    """
    representative = next(run for run in seed_runs if run["seed"] == REPRESENTATIVE_SEED)
    full_model = representative["representative"]["full_model"]

    # Train the session risk model using the pre-trained full model
    session_model = SessionRiskModel(risk_model=full_model, alpha=0.30)

    # Generate all session types using the same random state for reproducibility
    sessions = generate_all_session_types(random_state=42)

    summaries: list[Any] = []
    labels_dict: dict[str, np.ndarray] = {}
    plot_paths: list[Path] = []

    for session_type, session_events in sessions.items():
        features = session_to_feature_array(session_events)
        labels = get_session_labels(session_events)
        labels_dict[session_type] = labels

        # Compute event and session risk
        event_risk, session_risk = session_model.compute_session_risk(features)

        # Classify session states and record transitions
        states, transitions = compute_session_states(session_risk, event_risk)

        # Build summary
        summary = summarize_session(
            session_type=session_type,
            states=states,
            session_risk=session_risk,
            event_risk=event_risk,
            transitions=transitions,
        )
        summaries.append(summary)

        # Generate and save risk plot
        plot_path = plot_session_risk(
            session_type=session_type,
            event_risk=event_risk,
            session_risk=session_risk,
            states=states,
            transitions=transitions,
            output_path=results_dir / f"session_risk_curve_{session_type}.png",
        )
        plot_paths.append(plot_path)

    # Build CSV-ready rows and save
    summary_rows = build_continuous_summary_rows(summaries, labels_dict)
    save_continuous_summary_csv(
        results_dir / "continuous_session_summary.csv", summary_rows
    )
    save_continuous_session_markdown(
        results_dir / "continuous_session_examples.md", summary_rows
    )
    print_continuous_summary(summary_rows)

    return summary_rows, labels_dict, plot_paths


def main() -> None:
    """Run the repeatable multi-seed experiment pipeline."""
    seed_runs = [run_single_seed(seed) for seed in EXPERIMENT_SEEDS]
    seed_rows = flatten_seed_results(seed_runs)
    aggregated_rows = aggregate_results(seed_rows)
    significance_rows = build_significance_rows(seed_rows, alpha=0.05)

    # Policy evaluation across all seeds
    policy_seed_rows = flatten_policy_seed_results(seed_runs)
    policy_aggregated = aggregate_policy_results(policy_seed_rows)

    # Get representative run for legacy policy evaluation (single seed)
    representative = next(run for run in seed_runs if run["seed"] == REPRESENTATIVE_SEED)
    y_test = representative["representative"]["y_test"]
    full_scores = representative["representative"]["full_scores"]

    # Evaluate policy under different scenarios (legacy single-seed evaluation)
    policy_results_dict = evaluate_policy_scenarios(y_test, full_scores)
    policy_results = [policy_results_dict[scenario] for scenario in ["security_first", "convenience_first", "balanced"]]

    results_dir = Path("results")
    results_dir.mkdir(exist_ok=True)

    by_seed_path = results_dir / "metrics_by_seed.csv"
    aggregated_path = results_dir / "metrics_aggregated.csv"
    results_markdown_path = results_dir / "results_table.md"
    significance_csv_path = results_dir / "significance_summary.csv"
    significance_markdown_path = results_dir / "significance_table.md"
    policy_csv_path = results_dir / "policy_summary.csv"
    policy_markdown_path = results_dir / "policy_table.md"
    policy_by_seed_path = results_dir / "policy_by_seed.csv"
    policy_aggregated_path = results_dir / "policy_aggregated.csv"
    policy_comparison_path = results_dir / "policy_comparison_table.md"
    explanation_csv_path = results_dir / "risk_explanation_examples.csv"
    explanation_markdown_path = results_dir / "risk_explanation_examples.md"
    explanation_rows = build_example_explanations()
    threshold_sensitivity_rows = run_threshold_sensitivity_analysis(
        y_true=y_test,
        y_score=full_scores,
        results_dir=results_dir,
    )

    save_csv(
        path=by_seed_path,
        fieldnames=["seed", "method", "auc", "accuracy", "far", "frr"],
        rows=seed_rows,
    )
    save_csv(
        path=aggregated_path,
        fieldnames=[
            "method",
            "auc_mean",
            "auc_std",
            "auc_ci_lower",
            "auc_ci_upper",
            "accuracy_mean",
            "accuracy_std",
            "accuracy_ci_lower",
            "accuracy_ci_upper",
            "far_mean",
            "far_std",
            "far_ci_lower",
            "far_ci_upper",
            "frr_mean",
            "frr_std",
            "frr_ci_lower",
            "frr_ci_upper",
        ],
        rows=aggregated_rows,
    )
    save_csv(
        path=significance_csv_path,
        fieldnames=[
            "comparison",
            "metric",
            "full_mean",
            "comparison_mean",
            "mean_difference",
            "ttest_p_value",
            "wilcoxon_p_value",
            "significant",
        ],
        rows=significance_rows,
    )
    save_results_markdown_table(results_markdown_path, aggregated_rows)
    save_significance_markdown_table(significance_markdown_path, significance_rows)
    save_policy_summary_csv(policy_csv_path, policy_results)
    save_policy_summary_markdown(policy_markdown_path, policy_results)
    save_policy_by_seed_csv(policy_by_seed_path, policy_seed_rows)
    save_policy_aggregated_csv(policy_aggregated_path, policy_aggregated)
    save_policy_comparison_markdown(policy_comparison_path, policy_aggregated)
    save_risk_explanation_examples_csv(explanation_csv_path, explanation_rows)
    save_risk_explanation_examples_markdown(explanation_markdown_path, explanation_rows)
    roc_path = save_representative_roc(seed_runs, results_dir)

    # Continuous risk-based authentication session evaluation
    continuous_rows, _, continuous_plot_paths = run_continuous_sessions(
        seed_runs, results_dir
    )

    # Adaptive challenge policy evaluation (v1.3)

    from decision.policy import make_decisions, ThresholdPair
    accept_thr = policy_results_dict["balanced"]["accept_threshold"]
    reject_thr = policy_results_dict["balanced"]["reject_threshold"]

    thr_pair = ThresholdPair(accept_threshold=accept_thr, reject_threshold=reject_thr)
    raw_decisions = make_decisions(full_scores, thr_pair)
    decisions_list = [str(d.value) for d in raw_decisions]
    n_test = len(y_test)

    X_full, y_full = generate_synthetic_access_data(n_samples=3000, random_state=REPRESENTATIVE_SEED)
    _, X_test_rep, _, _ = train_test_split(
        X_full, y_full, test_size=0.30, random_state=REPRESENTATIVE_SEED, stratify=y_full
    )

    feature_dicts = [
        {"face_score": float(X_test_rep[i, 0]),
         "location_score": float(X_test_rep[i, 1]),
         "time_score": float(X_test_rep[i, 2]),
         "device_score": float(X_test_rep[i, 3])}
        for i in range(n_test)
    ]

    challenge_comparison = evaluate_challenge_policies(
        features_list=feature_dicts,
        scores=full_scores,
        decisions=decisions_list,
    )

    adaptive_examples = build_adaptive_challenge_examples()
    save_adaptive_challenge_csv(
        str(results_dir / "adaptive_challenge_summary.csv"),
        adaptive_examples,
    )
    save_adaptive_challenge_markdown(
        str(results_dir / "adaptive_challenge_examples.md"),
        adaptive_examples,
    )

    print_seed_overview(seed_runs)
    print_aggregated_summary(aggregated_rows)
    print_significance_summary(significance_rows)
    print_representative_coefficients(seed_runs)
    print_policy_summary(policy_results)
    print_policy_terminal_summary(policy_aggregated)
    print_threshold_sensitivity_summary(threshold_sensitivity_rows)
    print_risk_explanation_summary(explanation_rows)
    print_challenge_examples(adaptive_examples)
    print("\nAdaptive Challenge Policy Comparison")
    print("===================================")
    print(f"  Challenge events evaluated: {challenge_comparison['challenge_count']}")
    print(f"  Avg generic challenge cost:  {challenge_comparison['avg_generic_challenge_cost']:.4f}")
    print(f"  Avg adaptive challenge cost: {challenge_comparison['avg_adaptive_challenge_cost']:.4f}")
    print(f"  Cost saving (adaptive):      {challenge_comparison['cost_saving']:.4f}")
    print(f"  Avg generic residual risk:   {challenge_comparison['avg_generic_residual_risk']:.4f}")
    print(f"  Avg adaptive residual risk:  {challenge_comparison['avg_adaptive_residual_risk']:.4f}")
    print(f"  Risk improvement (adaptive):  {challenge_comparison['risk_improvement']:.4f}")
    print(f"  Avg user friction (adaptive):{challenge_comparison['avg_user_friction']:.4f}")
    print(f"  Avg security gain (adaptive):{challenge_comparison['avg_security_gain']:.4f}")

    print(f"\nPer-seed metrics saved to                    : {by_seed_path}")
    print(f"Aggregated metrics saved to                 : {aggregated_path}")
    print(f"Results table saved to                      : {results_markdown_path}")
    print(f"Significance CSV saved to                   : {significance_csv_path}")
    print(f"Significance table saved to                 : {significance_markdown_path}")
    print(f"Policy summary CSV saved to                 : {policy_csv_path}")
    print(f"Policy table (single seed) saved to         : {policy_markdown_path}")
    print(f"Policy by seed CSV saved to                 : {policy_by_seed_path}")
    print(f"Policy aggregated CSV saved to              : {policy_aggregated_path}")
    print(f"Policy comparison table saved to            : {policy_comparison_path}")
    print(f"Risk explanation CSV saved to               : {explanation_csv_path}")
    print(f"Risk explanation table saved to             : {explanation_markdown_path}")
    for row in threshold_sensitivity_rows:
        scenario_label = str(row["scenario"]).replace("_", " ").title()
        print(f"Threshold CSV ({scenario_label}) saved to          : {row['csv_path']}")
        print(f"Threshold heatmap ({scenario_label}) saved to      : {row['heatmap_path']}")
    print(f"Representative ROC saved to                 : {roc_path}")
    print(f"Continuous session summary CSV saved to      : {results_dir / 'continuous_session_summary.csv'}")
    print(f"Continuous session examples markdown saved to: {results_dir / 'continuous_session_examples.md'}")
    print(f"Adaptive challenge summary CSV saved to      : {results_dir / 'adaptive_challenge_summary.csv'}")
    print(f"Adaptive challenge examples saved to         : {results_dir / 'adaptive_challenge_examples.md'}")
    for plot_path in continuous_plot_paths:
        print(f"Session risk plot saved to                  : {plot_path}")
    
    # Generate research summary report
    print("\n" + "="*70)
    print("Generating Research Summary Report")
    print("="*70)
    report_content = generate_research_report(str(results_dir))
    report_path = save_report(report_content, str(results_dir / "research_summary_report.md"))
    print(f"\nResearch summary report saved to             : {report_path}")
    print("\nReport provides a human-readable explanation of:")
    print("  - Project overview and core research questions")
    print("  - Classification results and statistical evidence")
    print("  - Policy optimization across security/usability tradeoffs")
    print("  - Continuous authentication capabilities")
    print("  - Adaptive challenge recommendations")
    print("  - Project value, limitations, and next steps")


if __name__ == "__main__":
    main()