app.py

import os
import numpy as np
from flask import Flask, render_template, request, jsonify, session, send_from_directory
from flask_cors import CORS
from werkzeug.utils import secure_filename
from services.parser_service import parse_shortlist_request
from services.shortlist_service import run_shortlist
from utils.file_loader import load_and_clean_csv

app = Flask(__name__)
CORS(app)
app.secret_key = "fairness-audit-key-2024"
app.config["UPLOAD_FOLDER"] = os.path.join(os.path.dirname(__file__), "uploads")
app.config["MAX_CONTENT_LENGTH"] = 50 * 1024 * 1024  # 50MB

os.makedirs(app.config["UPLOAD_FOLDER"], exist_ok=True)

# ───────────── ROUTES ─────────────


@app.route("/")
def index():
    return render_template("index.html")


@app.route("/sample/<filename>")
def sample_file(filename):
    return send_from_directory(os.path.dirname(__file__), filename)


@app.route("/api/upload", methods=["POST"])
def upload():
    if "file" not in request.files:
        return jsonify({"error": "No file uploaded"}), 400
    f = request.files["file"]
    if f.filename == "":
        return jsonify({"error": "Empty filename"}), 400

    filename = secure_filename(f.filename)
    filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename)
    f.save(filepath)

    df = load_and_clean_csv(filepath)
    columns = df.columns.tolist()
    dtypes = {c: str(df[c].dtype) for c in columns}
    preview = df.head(10).fillna("").to_dict(orient="records")

    # Auto-detect dataset
    dataset_name = "Unknown"
    suggestions = {}
    if "education.num" in columns:
        dataset_name = "UCI Adult Income"
        suggestions = {
            "gender_col": "sex",
            "target_col": "income",
            "positive_val": ">50K",
            "occupation_col": "occupation",
        }
    elif "Attrition" in columns:
        dataset_name = "IBM HR Analytics"
        suggestions = {
            "gender_col": "Gender",
            "target_col": "HighIncome",
            "positive_val": "Yes",
            "occupation_col": "JobRole",
        }

    return jsonify(
        {
            "filename": filename,
            "filepath": filepath,
            "columns": columns,
            "dtypes": dtypes,
            "rows": len(df),
            "preview": preview,
            "dataset_name": dataset_name,
            "suggestions": suggestions,
        }
    )


@app.route("/api/unique_values", methods=["POST"])
def unique_values():
    data = request.json
    df = load_and_clean_csv(data["filepath"])
    col = data["column"]
    values = df[col].dropna().unique().tolist()
    return jsonify({"values": values})


# ───────────── BIAS ANALYSIS (solution.py) ─────────────


@app.route("/api/bias_analysis", methods=["POST"])
def bias_analysis():
    data = request.json
    df = load_and_clean_csv(data["filepath"])
    gender_col = data["gender_col"]
    target_col = data["target_col"]
    occupation_col = data["occupation_col"]
    positive_outcome = data["positive_val"]

    audit_df = df[[gender_col, target_col, occupation_col]].copy()
    audit_df[occupation_col] = audit_df[occupation_col].replace(
        "?", "Private/Undisclosed"
    )

    # Overall bias
    groups = audit_df[gender_col].unique().tolist()
    group_rates = {}
    for group in groups:
        subset = audit_df[audit_df[gender_col] == group]
        rate = float((subset[target_col] == positive_outcome).mean())
        group_rates[group] = round(rate, 4)

    rates = list(group_rates.values())
    di_score = round(min(rates) / max(rates), 4) if max(rates) > 0 else 0
    parity = round(abs(rates[0] - rates[1]), 4) if len(rates) >= 2 else 0

    if di_score >= 0.8:
        di_verdict = "PASSES"
    elif di_score >= 0.6:
        di_verdict = "WARNING"
    else:
        di_verdict = "FAILS"

    # Within-occupation bias
    occupations = audit_df[occupation_col].unique().tolist()
    biased_jobs = []
    fair_jobs = []
    skipped_jobs = []

    for occ in occupations:
        occ_subset = audit_df[audit_df[occupation_col] == occ]
        occ_rates = {}
        valid = True
        for group in groups:
            grp_subset = occ_subset[occ_subset[gender_col] == group]
            if len(grp_subset) < 5:
                valid = False
                break
            rate = float((grp_subset[target_col] == positive_outcome).mean())
            occ_rates[group] = round(rate, 4)

        if not valid:
            skipped_jobs.append(occ)
            continue

        occ_rate_values = list(occ_rates.values())
        occ_di = (
            round(min(occ_rate_values) / max(occ_rate_values), 4)
            if max(occ_rate_values) > 0
            else 0
        )

        if occ_di < 0.8:
            biased_jobs.append({"name": occ, "di_score": occ_di, "rates": occ_rates})
        else:
            fair_jobs.append({"name": occ, "di_score": occ_di, "rates": occ_rates})

    # Conclusion
    if di_score < 0.8 and len(biased_jobs) > 0:
        conclusion = "SYSTEMIC GENDER BIAS DETECTED"
        conclusion_detail = "Bias exists both overall AND within specific job roles. This is pure gender discrimination, not just occupational difference."
    elif di_score < 0.8 and len(biased_jobs) == 0:
        conclusion = "OCCUPATIONAL SEGREGATION"
        conclusion_detail = "Overall bias exists BUT jobs individually are fair. Women may be concentrated in lower-paying job categories."
    elif di_score >= 0.8 and len(biased_jobs) > 0:
        conclusion = "HIDDEN BIAS IN SPECIFIC ROLES"
        conclusion_detail = "Overall numbers look fair BUT specific jobs show bias. Needs targeted investigation."
    else:
        conclusion = "NO SIGNIFICANT BIAS DETECTED"
        conclusion_detail = "System appears fair both overall and within occupations."

    return jsonify(
        {
            "groups": groups,
            "group_rates": group_rates,
            "di_score": di_score,
            "di_verdict": di_verdict,
            "parity": parity,
            "parity_pass": abs(parity) < 0.1,
            "biased_jobs": sorted(biased_jobs, key=lambda x: x["di_score"]),
            "fair_jobs": sorted(fair_jobs, key=lambda x: x["di_score"]),
            "skipped_jobs": skipped_jobs,
            "conclusion": conclusion,
            "conclusion_detail": conclusion_detail,
            "total_occupations": len(occupations),
        }
    )


# ───────────── REWEIGHTING (reweight.py) ─────────────


@app.route("/api/reweight", methods=["POST"])
def reweight():
    data = request.json
    df = load_and_clean_csv(data["filepath"])
    gender_col = data["gender_col"]
    income_col = data["target_col"]
    positive_val = data["positive_val"]
    scoring_weights = data.get("scoring_weights", {})

    n = len(df)
    p_gender = df[gender_col].value_counts() / n
    p_income = df[income_col].value_counts() / n
    p_joint = df.groupby([gender_col, income_col]).size() / n

    def compute_fairness_weight(row):
        g = row[gender_col]
        y = row[income_col]
        joint = p_joint.get((g, y), 1e-9)
        return (p_gender[g] * p_income[y]) / joint

    df["fairness_weight"] = df.apply(compute_fairness_weight, axis=1)

    weight_stats = {
        "mean": round(float(df["fairness_weight"].mean()), 4),
        "std": round(float(df["fairness_weight"].std()), 4),
        "min": round(float(df["fairness_weight"].min()), 4),
        "max": round(float(df["fairness_weight"].max()), 4),
    }

    weight_by_group = {}
    for g, grp in df.groupby(gender_col):
        weight_by_group[g] = round(float(grp["fairness_weight"].mean()), 4)

    weight_by_combo = {}
    for (g, y), grp in df.groupby([gender_col, income_col]):
        weight_by_combo[f"{g} + {y}"] = round(float(grp["fairness_weight"].mean()), 4)

    # Merit scoring
    numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
    numeric_cols = [c for c in numeric_cols if c not in ["fairness_weight"]]

    if not scoring_weights:
        if "education.num" in numeric_cols:
            scoring_weights = {
                "education.num": 40,
                "hours.per.week": 35,
                "age": 25,
            }
        else:
            scoring_weights = {numeric_cols[0]: 100} if numeric_cols else {}

    total_w = sum(scoring_weights.values())
    if total_w > 0:
        scoring_weights = {k: v / total_w for k, v in scoring_weights.items()}

    for feat in scoring_weights:
        if feat in df.columns:
            min_v = df[feat].min()
            max_v = df[feat].max()
            df[f"_norm_{feat}"] = (df[feat] - min_v) / (max_v - min_v + 1e-9)

    df["merit_score"] = sum(
        df[f"_norm_{feat}"] * w
        for feat, w in scoring_weights.items()
        if f"_norm_{feat}" in df.columns
    )
    df["final_score"] = df["merit_score"] * df["fairness_weight"]

    drop_cols = [c for c in df.columns if c.startswith("_norm_")]
    df_out = df.drop(columns=drop_cols)

    out_path = os.path.join(
        app.config["UPLOAD_FOLDER"], "reweighted_" + os.path.basename(data["filepath"])
    )
    df_out.to_csv(out_path, index=False)

    return jsonify(
        {
            "weight_stats": weight_stats,
            "weight_by_group": weight_by_group,
            "weight_by_combo": weight_by_combo,
            "numeric_cols": numeric_cols,
            "scoring_weights": {
                k: round(v * 100, 1) for k, v in scoring_weights.items()
            },
            "merit_range": [
                round(float(df["merit_score"].min()), 4),
                round(float(df["merit_score"].max()), 4),
            ],
            "final_range": [
                round(float(df["final_score"].min()), 4),
                round(float(df["final_score"].max()), 4),
            ],
            "output_path": out_path,
            "rows": len(df_out),
        }
    )


# ───────────── SHORTLIST (shortlist.py — Random Forest) ─────────────


@app.route("/api/shortlist", methods=["POST"])
def shortlist():
    params = parse_shortlist_request(request.json)

    if not params["filepath"]:
        return jsonify({"success": False, "error": "Missing filepath"}), 400
    if not os.path.exists(params["filepath"]):
        return jsonify({"success": False, "error": "File not found"}), 400

    try:
        result = run_shortlist(**params)
        return jsonify(result)
    except ValueError as exc:
        return jsonify({"success": False, "error": str(exc)}), 400
    except Exception as exc:
        return jsonify({"success": False, "error": str(exc)}), 500


# ───────────── PROXY DETECTION ─────────────


@app.route("/api/detect_proxies", methods=["POST"])
def detect_proxies():
    from sklearn.preprocessing import LabelEncoder

    data = request.json
    df = load_and_clean_csv(data["filepath"])
    gender_col = data["gender_col"]
    threshold = data.get("threshold", 0.3)

    df_encoded = df.copy()
    le = LabelEncoder()
    for col in df_encoded.columns:
        if df_encoded[col].dtype == "object":
            df_encoded[col] = le.fit_transform(df_encoded[col].astype(str))

    gender_encoded = df_encoded[gender_col]
    correlations = {}
    for col in df_encoded.columns:
        if col == gender_col:
            continue
        corr = abs(float(df_encoded[col].corr(gender_encoded)))
        correlations[col] = round(corr, 4)

    correlations = dict(
        sorted(correlations.items(), key=lambda x: x[1], reverse=True)
    )
    flagged = [col for col, corr in correlations.items() if corr >= threshold]

    return jsonify(
        {
            "correlations": correlations,
            "flagged": flagged,
            "threshold": threshold,
        }
    )


if __name__ == "__main__":
    app.run(host="0.0.0.0", port=int(os.environ.get("PORT", 5000)))