SmartDataCleaner.py

"""
SmartDataCleaner v2.0.0
Professional Smart Data Cleanup Tool
Heuristic Scoring | Type Normalization | Column Renaming Suggestions
"""

import os, sys, threading, time, json
import tkinter as tk
from tkinter import filedialog, messagebox
import ttkbootstrap as tb
from ttkbootstrap.constants import *
from datetime import datetime
import pandas as pd
import numpy as np
import re

# =================== GLOBALS ===================
stop_event = threading.Event()
cleanup_results = {}
log_file = os.path.join(os.getcwd(), "datacleaner.log")

# =================== UTIL ===================
def resource_path(file_name):
    base_path = getattr(sys, "_MEIPASS", os.path.dirname(os.path.abspath(__file__)))
    return os.path.join(base_path, file_name)

def log_error(msg):
    with open(log_file, "a", encoding="utf-8") as f:
        f.write(f"[{datetime.now().isoformat()}] {msg}\n")

def clean_column_name(name):
    # Convert to snake_case and remove special chars
    name = name.strip().lower()
    name = re.sub(r"[^\w\s]", "", name)
    name = re.sub(r"\s+", "_", name)
    return name

def convert_numpy(obj):
    if isinstance(obj, np.integer):
        return int(obj)
    if isinstance(obj, np.floating):
        return float(obj)
    if isinstance(obj, np.ndarray):
        return obj.tolist()
    raise TypeError

# ---------------------- Helper for consistent messages ----------------------
def _show_message(title, text, msg_type="info", val_geometry="420x180"):
    """Custom popup window for messages (info, success, error)."""
    colors = {"info": "#2563eb", "success": "#16a34a", "error": "#dc2626"}  # blue, green, red
    win = tb.Toplevel(app)
    win.title(title)
    win.geometry(val_geometry)
    win.resizable(False, False)
    win.grab_set()
    win.attributes("-toolwindow", True)

    app.update_idletasks()
    x = app.winfo_x() + (app.winfo_width() // 2) - 210
    y = app.winfo_y() + (app.winfo_height() // 2) - 90
    win.geometry(f"+{x}+{y}")

    frame = tb.Frame(win, padding=15)
    frame.pack(fill="both", expand=True)

    tb.Label(frame, text=title, font=("Segoe UI", 14, "bold"), foreground=colors.get(msg_type, "#000")).pack(pady=(0, 10))
    tb.Label(frame, text=text, font=("Segoe UI", 11), wraplength=380, justify="left").pack(pady=(0, 15))
    tb.Button(frame, text="Close", bootstyle="success-outline", width=12, command=win.destroy).pack(pady=5)

# =================== ROOT ===================
app = tb.Window(themename="darkly")
app.title("SmartDataCleaner v2.0.0")
app.geometry("1100x650")

try:
    app.iconbitmap(resource_path("logo.ico"))
except Exception:
    pass

# =================== TITLE ===================
tb.Label(app, text="SmartDataCleaner", font=("Segoe UI", 22, "bold")).pack(pady=(10, 2))
tb.Label(
    app,
    text="Professional Smart Data Cleanup & Preprocessing Tool",
    font=("Segoe UI", 10, "italic"),
    foreground="#9ca3af"
).pack(pady=(0, 8))

# =================== TARGET FILE ===================
row1 = tb.Labelframe(app, text="Select CSV File", padding=10)
row1.pack(fill="x", padx=10, pady=6)

file_path = tk.StringVar()
tb.Label(row1, text="File:", width=10).pack(side="left")
tb.Entry(row1, textvariable=file_path, width=60).pack(side="left", padx=6)
tb.Button(row1, text="📄 CSV File", bootstyle="secondary", 
          command=lambda: file_path.set(filedialog.askopenfilename(filetypes=[("CSV Files", "*.csv")]))).pack(side="left", padx=4)

# =================== CONTROLS ===================
row2 = tb.Labelframe(app, text="Cleanup Controls", padding=10)
row2.pack(fill="x", padx=10, pady=6)

start_btn = tb.Button(row2, text="🧹 CLEAN DATA", bootstyle="success")
stop_btn = tb.Button(row2, text="🛑 STOP", bootstyle="danger-outline", state="disabled")

start_btn.pack(side="left", padx=6)
stop_btn.pack(side="left", padx=6)

tb.Button(row2, text="ℹ About", bootstyle="info-outline", command=lambda: show_about()).pack(side="right", padx=4)
tb.Button(row2, text="📄 PDF", bootstyle="primary-outline", command=lambda: export_pdf()).pack(side="right", padx=4)
tb.Button(row2, text="📄 JSON", bootstyle="secondary-outline", command=lambda: export_json()).pack(side="right", padx=4)
tb.Button(row2, text="📃 TXT", bootstyle="secondary-outline", command=lambda: export_txt()).pack(side="right", padx=4)

# =================== RESULTS ===================
row3 = tb.Labelframe(app, text="Cleanup Results & Heuristic Suggestions", padding=10)
row3.pack(fill="both", expand=True, padx=10, pady=6)

cols = ("column", "original_type", "suggested_type", "cleaned_type", "missing_values", "duplicates_removed", "heuristic_score", "rename_suggestion")
tree = tb.Treeview(row3, columns=cols, show="headings")

for col in cols:
    tree.heading(col, text=col.upper())
    tree.column(col, width=130, anchor="w")

tree.pack(fill="both", expand=True)

# =================== HEURISTIC CLEANUP ENGINE ===================
def heuristic_score(missing, duplicates, type_issue):
    """
    Compute a heuristic score for column health.
    0-30: LOW risk
    31-70: MEDIUM
    71-100: HIGH
    """
    score = 0
    score += min(30, missing * 2)          # penalize missing values
    score += min(30, duplicates * 2)       # penalize duplicates
    score += 40 if type_issue else 0       # type mismatch penalty
    return min(score, 100)

def assess_and_clean(df: pd.DataFrame):
    results = []

    for col in df.columns:
        # ----- STOP SUPPORT (thread-safe) -----
        if stop_event.is_set():
            return df, results

        series = df[col]
        orig_type = series.dtype

        missing = series.isna().sum()
        duplicates = series.duplicated().sum()

        # ----- TYPE ANALYSIS (non-destructive) -----
        type_issue = False

        if pd.api.types.is_numeric_dtype(series):
            suggested_type = "float"
            coerced = pd.to_numeric(series, errors="coerce")
            type_issue = coerced.isna().sum() > missing
            cleaned_series = coerced
        else:
            suggested_type = "string"
            cleaned_series = series.astype("string")
            type_issue = False

        # ----- MISSING VALUE HANDLING -----
        if pd.api.types.is_numeric_dtype(cleaned_series):
            fill_value = cleaned_series.mean()
            cleaned_series = cleaned_series.fillna(fill_value)
            imputation = "mean"
        else:
            mode = cleaned_series.mode()
            cleaned_series = cleaned_series.fillna(mode[0] if not mode.empty else "")
            imputation = "mode"

        # Apply cleaned column safely
        df[col] = cleaned_series

        # ----- COLUMN RENAMING SUGGESTION -----
        cleaned_name = clean_column_name(col)
        rename_suggestion = cleaned_name if cleaned_name != col else ""

        # ----- HEURISTIC SCORE -----
        score = heuristic_score(missing, duplicates, type_issue)

        results.append({
            "column": col,
            "original_type": str(orig_type),
            "suggested_type": suggested_type,
            "cleaned_type": str(df[col].dtype),
            "missing_values": int(missing),
            "duplicates_detected": int(duplicates),
            "heuristic_score": int(score),
            "rename_suggestion": rename_suggestion,
            "imputation_method": imputation
        })

    return df, results

# =================== THREAD FUNCTIONS ===================
def stop_cleanup():
    stop_event.set()
    stop_btn.config(state="disabled")

def run_cleanup():
    path = file_path.get()
    if not path:
        _show_message("Warning ⚠️", "Please select a CSV file to clean.", "info", "420x180")
        return

    stop_event.clear()
    start_btn.config(state="disabled")
    stop_btn.config(state="normal")
    tree.delete(*tree.get_children())
    cleanup_results.clear()

    # ----- LOAD CSV -----
    try:
        df = pd.read_csv(path)
    except Exception as e:
        _show_message("Load Error ❌", f"Failed to read CSV:\n{e}", "error", "420x180")
        start_btn.config(state="normal")
        stop_btn.config(state="disabled")
        return

    # ----- RUN CLEANUP (background-safe) -----
    cleaned_df, results = assess_and_clean(df)

    # ----- HANDLE STOP -----
    if stop_event.is_set():
        start_btn.config(state="normal")
        stop_btn.config(state="disabled")
        _show_message("Stopped ⛔", "Cleanup was cancelled by user.", "info", "420x180")
        return

    # ----- DISPLAY RESULTS -----
    for r in results:
        tree.insert("", "end", values=(
            r["column"],
            r["original_type"],
            r["suggested_type"],
            r["cleaned_type"],
            r["missing_values"],
            r.get("duplicates_detected", r.get("duplicates_removed", 0)),
            r["heuristic_score"],
            r["rename_suggestion"]
        ))

    cleanup_results["path"] = path
    cleanup_results["results"] = results
    cleanup_results["timestamp"] = datetime.utcnow().isoformat()
    cleanup_results["rows_after_cleanup"] = len(cleaned_df)

    start_btn.config(state="normal")
    stop_btn.config(state="disabled")

    _show_message(
        "Cleanup Complete ✅",
        f"Data cleanup finished successfully.\nRows after cleanup: {len(cleaned_df)}",
        "success",
        "420x180"
    )

start_btn.config(command=lambda: threading.Thread(target=run_cleanup, daemon=True).start())
stop_btn.config(command=stop_cleanup)

# =================== EXPORT JSON ===================
def export_json():
    if not cleanup_results:
        _show_message("No Data ⚠️", "No cleanup results to export!", "info", "420x180")
        return
    path = filedialog.asksaveasfilename(defaultextension=".json", filetypes=[("JSON Files", "*.json")])
    if not path:
        return
    try:
        with open(path, "w", encoding="utf-8") as f:
            json.dump(cleanup_results, f, indent=2, default=convert_numpy)
        _show_message("Export Success ✅", f"JSON saved successfully at:\n{path}", "success", "420x220")
    except Exception as e:
        _show_message("Export Error ❌", f"Failed to save JSON:\n{e}", "error", "420x180")

# =================== EXPORT TXT ===================
def export_txt():
    if not cleanup_results:
        _show_message("No Data ⚠️", "No cleanup results to export!", "info", "420x180")
        return
    path = filedialog.asksaveasfilename(defaultextension=".txt", filetypes=[("Text Files", "*.txt")])
    if not path:
        return
    try:
        with open(path, "w", encoding="utf-8") as f:
            for r in cleanup_results.get("results", []):
                f.write(f"{r}\n")
        _show_message("Export Success ✅", f"TXT saved successfully at:\n{path}", "success", "420x220")
    except Exception as e:
        _show_message("Export Error ❌", f"Failed to save TXT:\n{e}", "error", "420x180")

# =================== EXPORT PDF ===================
def export_pdf():
    if not cleanup_results:
        _show_message("No Data ⚠️", "No cleanup results to export!", "info", "420x180")
        return

    path = filedialog.asksaveasfilename(defaultextension=".pdf", filetypes=[("PDF Files", "*.pdf")])
    if not path:
        return

    try:
        from reportlab.pdfgen import canvas
        from reportlab.lib.pagesizes import A4
        from reportlab.lib.units import mm
        from reportlab.lib.colors import red, orange, green, black

        c = canvas.Canvas(path, pagesize=A4)
        w, h = A4
        margin = 20 * mm
        max_width = w - 2 * margin
        y = h - margin
        page_number = 1

        # ===== Color helper =====
        def score_color(score):
            if score >= 71:
                return red
            elif score >= 31:
                return orange
            else:
                return green

        # ===== Utility to wrap text =====
        def draw_wrapped_text(text, x, y, font_name="Helvetica", font_size=9, leading=12, color=black):
            c.setFont(font_name, font_size)
            c.setFillColor(color)
            lines = []
            for part in text.split(" "):
                if not lines:
                    lines.append(part)
                else:
                    if c.stringWidth(" ".join(lines[-1:] + [part]), font_name, font_size) < max_width - x:
                        lines[-1] += " " + part
                    else:
                        lines.append(part)
            for line in lines:
                c.drawString(x, y, line)
                y -= leading
            return y

        # ===== Page number =====
        def draw_page_number():
            c.setFont("Helvetica", 8)
            c.setFillColor(black)
            c.drawRightString(w - margin, margin / 2, f"Page {page_number}")

        # ===== Title =====
        c.setFont("Helvetica-Bold", 16)
        y = draw_wrapped_text("SmartDataCleaner – Data Cleanup Report", margin, y)
        y -= 10

        # ===== Summary =====
        c.setFont("Helvetica-Bold", 14)
        y = draw_wrapped_text("Summary", margin, y)
        y -= 6
        total_cols = len(cleanup_results.get("results", []))
        y = draw_wrapped_text(f"Total columns cleaned: {total_cols}", margin + 10, y)
        y -= 10

        # ===== Details =====
        for r in cleanup_results.get("results", []):
            if y < margin + 60:  # create new page if needed
                draw_page_number()
                c.showPage()
                page_number += 1
                y = h - margin

            # Determine color based on heuristic_score
            color = score_color(r["heuristic_score"])

            y = draw_wrapped_text(f"Column: {r['column']} (Suggested Rename: {r['rename_suggestion']})", margin, y)

            dup_count = r.get("duplicates_detected", r.get("duplicates_removed", 0))

            y = draw_wrapped_text(
                f"Type: {r['original_type']} → {r['cleaned_type']} | "
                f"Missing: {r['missing_values']} | "
                f"Duplicates Detected: {dup_count}",
                margin + 10,
                y
            )

            y = draw_wrapped_text(f"Heuristic Score: {r['heuristic_score']}", margin + 10, y, color=color)
            y -= 6

        # Draw page number for the last page
        draw_page_number()
        c.save()

        _show_message("Export Success ✅", f"PDF saved successfully at:\n{path}", "success", "420x220")

    except Exception as e:
        _show_message("Export Error ❌", f"Failed to save PDF:\n{e}", "error", "420x180")

# ================= HELP ===================
def show_about():
    win = tb.Toplevel(app)
    win.title("🧹 SmartDataCleaner v2.0 – About & Guide")
    win.resizable(False, False)
    win.grab_set()
    win.attributes("-toolwindow", True)

    # ===== Center window relative to root =====
    app.update_idletasks()
    win_w, win_h = 500, 400
    root_x = app.winfo_x()
    root_y = app.winfo_y()
    root_w = app.winfo_width()
    root_h = app.winfo_height()

    pos_x = root_x + (root_w // 2) - (win_w // 2)
    pos_y = root_y + (root_h // 2) - (win_h // 2)

    win.geometry(f"{win_w}x{win_h}+{pos_x}+{pos_y}")

    # ===== Content =====
    frame = tb.Frame(win, padding=15)
    frame.pack(fill="both", expand=True)

    tb.Label(frame, text="About SmartDataCleaner v2.0",
             font=("Segoe UI", 12, "bold")).pack(anchor="w", pady=(10, 0))

    tb.Label(
        frame,
        text=(
            "SmartDataCleaner automatically detects and fixes data quality issues. "
            "It performs type normalization, fills missing values, removes duplicates, "
            "and suggests cleaned column names."
        ),
        font=("Segoe UI", 10),
        wraplength=460,
        justify="left"
    ).pack(anchor="w", pady=(4, 8))

    tb.Label(frame, text="Heuristic Suggestions",
             font=("Segoe UI", 12, "bold")).pack(anchor="w", pady=(10, 0))

    tb.Label(
        frame,
        text=(
            "• Columns with HIGH heuristic score need immediate attention\n"
            "• Rename columns based on suggestions for consistency\n"
            "• Review missing values and duplicates before export"
        ),
        font=("Segoe UI", 10),
        wraplength=460,
        justify="left"
    ).pack(anchor="w", pady=(4, 8))

    tb.Label(frame, text="Developer",
             font=("Segoe UI", 12, "bold")).pack(anchor="w", pady=(10, 0))

    tb.Label(
        frame,
        text=(
            "SmartDataCleaner v2.0\n"
            "Developed by Mate Technologies\n"
            "https://matetools.gumroad.com"
        ),
        font=("Segoe UI", 10),
        wraplength=460,
        justify="left"
    ).pack(anchor="w", pady=(4, 8))

    tb.Button(
        frame,
        text="Close",
        bootstyle="danger-outline",
        width=15,
        command=win.destroy
    ).pack(pady=15)

# =================== START ===================
app.mainloop()