'
+ f'{label[:8]}' if pct > 10 else
+ f'
'
+ )
+
+ dist_label = "标签分布" if lang == "zh" else "Label Dist."
+ return (
+ f'
'
+ f'
'
+ f'{dist_label}
'
+ f'
'
+ f'{"".join(segments)}
'
+ )
diff --git a/socialscikit/core/icr.py b/socialscikit/core/icr.py
new file mode 100644
index 0000000..cfa99cd
--- /dev/null
+++ b/socialscikit/core/icr.py
@@ -0,0 +1,630 @@
+"""Inter-Coder Reliability (ICR) — Cohen's Kappa, Krippendorff's Alpha, Jaccard agreement.
+
+Compute agreement metrics between human-human or human-LLM coders.
+Supports both single-label (QuantiKit) and multi-label (QualiKit) coding.
+
+References:
+- Cohen, J. (1960). A coefficient of agreement for nominal scales.
+ Educational and Psychological Measurement, 20(1), 37–46.
+- Krippendorff, K. (2011). Computing Krippendorff's Alpha-Reliability.
+ https://repository.upenn.edu/asc_papers/43
+- Landis, J. R., & Koch, G. G. (1977). The measurement of observer
+ agreement for categorical data. Biometrics, 33(1), 159–174.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+from dataclasses import dataclass, field
+from itertools import combinations
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class ICRResult:
+ """Result for a single metric computation."""
+
+ metric_name: str # "cohens_kappa" | "krippendorffs_alpha" | "jaccard_agreement"
+ value: float
+ interpretation: str # e.g. "Moderate agreement"
+ n_coders: int = 2
+ n_items: int = 0
+ n_categories: int = 0
+
+
+@dataclass
+class PerCategoryAgreement:
+ """Agreement breakdown for a single category/theme."""
+
+ category: str
+ observed_agreement: float
+ expected_agreement: float
+ specific_agreement: float # category-specific agreement
+
+
+@dataclass
+class ICRReport:
+ """Full inter-coder reliability report."""
+
+ results: list[ICRResult] = field(default_factory=list)
+ per_category: list[PerCategoryAgreement] = field(default_factory=list)
+ pairwise_matrix: list[list[float]] | None = None
+ coder_labels: list[str] = field(default_factory=list)
+ summary_text: str = ""
+
+
+# ---------------------------------------------------------------------------
+# ICR Calculator
+# ---------------------------------------------------------------------------
+
+
+class ICRCalculator:
+ """Compute inter-coder reliability metrics.
+
+ Supports two input modes:
+ 1. Two label lists (simple 2-coder case, single-label)
+ 2. Multi-label theme sets (QualiKit qualitative coding)
+
+ Also supports Krippendorff's Alpha with a reliability matrix for 2+ coders.
+
+ Usage::
+
+ calc = ICRCalculator()
+
+ # Single-label (QuantiKit)
+ report = calc.compute_all(
+ coder1_labels=["pos", "neg", "pos"],
+ coder2_labels=["pos", "pos", "pos"],
+ )
+
+ # Multi-label (QualiKit)
+ report = calc.compute_all_multilabel(
+ coder1_themes=[{"economy", "policy"}, {"health"}],
+ coder2_themes=[{"economy"}, {"health", "education"}],
+ )
+ """
+
+ # ------------------------------------------------------------------
+ # Cohen's Kappa (2 coders, single-label)
+ # ------------------------------------------------------------------
+
+ def compute_cohens_kappa(
+ self,
+ coder1_labels: list[str],
+ coder2_labels: list[str],
+ labels: list[str] | None = None,
+ ) -> ICRResult:
+ """Cohen's Kappa for two coders with single-label classification.
+
+ Parameters
+ ----------
+ coder1_labels, coder2_labels : list[str]
+ Labels assigned by each coder (same length).
+ labels : list[str] or None
+ Explicit label set. If None, derived from the union.
+
+ Returns
+ -------
+ ICRResult
+ """
+ if len(coder1_labels) != len(coder2_labels):
+ raise ValueError(
+ f"Length mismatch: {len(coder1_labels)} vs {len(coder2_labels)}."
+ )
+
+ n = len(coder1_labels)
+ if n == 0:
+ return ICRResult(
+ metric_name="cohens_kappa", value=0.0,
+ interpretation="No data", n_items=0,
+ )
+
+ c1 = [str(x).strip() for x in coder1_labels]
+ c2 = [str(x).strip() for x in coder2_labels]
+
+ if labels is None:
+ labels = sorted(set(c1) | set(c2))
+ label_idx = {l: i for i, l in enumerate(labels)}
+ k = len(labels)
+
+ # Build confusion matrix
+ cm = [[0] * k for _ in range(k)]
+ for a, b in zip(c1, c2):
+ ai = label_idx.get(a)
+ bi = label_idx.get(b)
+ if ai is not None and bi is not None:
+ cm[ai][bi] += 1
+
+ # Observed agreement
+ p_o = sum(cm[i][i] for i in range(k)) / n
+
+ # Expected agreement by chance
+ p_e = 0.0
+ for i in range(k):
+ row_sum = sum(cm[i][j] for j in range(k))
+ col_sum = sum(cm[j][i] for j in range(k))
+ p_e += (row_sum / n) * (col_sum / n)
+
+ if p_e >= 1.0:
+ kappa = 0.0
+ else:
+ kappa = (p_o - p_e) / (1.0 - p_e)
+
+ return ICRResult(
+ metric_name="cohens_kappa",
+ value=round(kappa, 4),
+ interpretation=self.interpret_kappa(kappa),
+ n_coders=2,
+ n_items=n,
+ n_categories=k,
+ )
+
+ # ------------------------------------------------------------------
+ # Krippendorff's Alpha (2+ coders, handles missing data)
+ # ------------------------------------------------------------------
+
+ def compute_krippendorffs_alpha(
+ self,
+ reliability_matrix: list[list[str | None]],
+ data_type: str = "nominal",
+ ) -> ICRResult:
+ """Krippendorff's Alpha for 2+ coders.
+
+ Parameters
+ ----------
+ reliability_matrix : list[list[str | None]]
+ Shape: (n_items, n_coders). Cell = label or None for missing.
+ data_type : str
+ "nominal" (default). Others reserved for future.
+
+ Returns
+ -------
+ ICRResult
+ """
+ if not reliability_matrix:
+ return ICRResult(
+ metric_name="krippendorffs_alpha", value=0.0,
+ interpretation="No data", n_items=0,
+ )
+
+ n_items = len(reliability_matrix)
+ n_coders = len(reliability_matrix[0]) if reliability_matrix else 0
+
+ if data_type != "nominal":
+ raise ValueError(
+ f"Currently only 'nominal' data type is supported, got '{data_type}'."
+ )
+
+ # Collect all categories
+ all_values: set[str] = set()
+ for row in reliability_matrix:
+ for v in row:
+ if v is not None:
+ all_values.add(str(v))
+
+ if len(all_values) <= 1:
+ # All same value or empty — perfect agreement (by convention)
+ return ICRResult(
+ metric_name="krippendorffs_alpha", value=1.0,
+ interpretation=self.interpret_alpha(1.0),
+ n_coders=n_coders, n_items=n_items,
+ n_categories=len(all_values),
+ )
+
+ # For each item, count the number of non-missing values
+ # and the value frequencies
+ item_counts: list[dict[str, int]] = []
+ item_mu: list[int] = [] # number of coders per item (non-missing)
+
+ for row in reliability_matrix:
+ counts: dict[str, int] = Counter()
+ mu = 0
+ for v in row:
+ if v is not None:
+ counts[str(v)] += 1
+ mu += 1
+ item_counts.append(counts)
+ item_mu.append(mu)
+
+ # Only keep items where at least 2 coders provided values
+ valid_items = [i for i in range(n_items) if item_mu[i] >= 2]
+
+ if not valid_items:
+ return ICRResult(
+ metric_name="krippendorffs_alpha", value=0.0,
+ interpretation="Insufficient data (need ≥2 coders per item)",
+ n_coders=n_coders, n_items=0,
+ )
+
+ # Observed disagreement
+ # D_o = (1 / sum_of_pairable_values) * sum_over_items(
+ # 1/(m_u - 1) * sum_c_k( n_uc * n_uk * delta(c,k) )
+ # )
+ total_pairable = sum(item_mu[i] * (item_mu[i] - 1) for i in valid_items)
+
+ if total_pairable == 0:
+ return ICRResult(
+ metric_name="krippendorffs_alpha", value=0.0,
+ interpretation="Insufficient data",
+ n_coders=n_coders, n_items=0,
+ )
+
+ d_observed = 0.0
+ for i in valid_items:
+ mu = item_mu[i]
+ counts = item_counts[i]
+ # Sum over all pairs of categories (c, k) where c != k
+ for c, n_c in counts.items():
+ for k, n_k in counts.items():
+ if c != k:
+ # For nominal: delta(c, k) = 1 when c != k
+ d_observed += n_c * n_k
+
+ d_observed /= total_pairable
+
+ # Expected disagreement
+ # Marginal frequencies across all items
+ n_total_values = sum(item_mu[i] for i in valid_items)
+ marginal: Counter = Counter()
+ for i in valid_items:
+ for v, cnt in item_counts[i].items():
+ marginal[v] += cnt
+
+ d_expected = 0.0
+ for c in marginal:
+ for k in marginal:
+ if c != k:
+ d_expected += marginal[c] * marginal[k]
+
+ d_expected /= (n_total_values * (n_total_values - 1))
+
+ if d_expected == 0:
+ alpha = 1.0
+ else:
+ alpha = 1.0 - d_observed / d_expected
+
+ return ICRResult(
+ metric_name="krippendorffs_alpha",
+ value=round(alpha, 4),
+ interpretation=self.interpret_alpha(alpha),
+ n_coders=n_coders,
+ n_items=len(valid_items),
+ n_categories=len(all_values),
+ )
+
+ # ------------------------------------------------------------------
+ # Multi-label Jaccard agreement (QualiKit)
+ # ------------------------------------------------------------------
+
+ def compute_multilabel_agreement(
+ self,
+ coder1_themes: list[set[str]],
+ coder2_themes: list[set[str]],
+ ) -> ICRResult:
+ """Jaccard-based agreement for multi-label coding.
+
+ For each segment, agreement = |intersection| / |union|.
+ If both coders assign no themes, agreement = 1.0 (both agree: no themes).
+ Returns the average across all segments.
+
+ Parameters
+ ----------
+ coder1_themes, coder2_themes : list[set[str]]
+ Theme sets per segment (same length).
+
+ Returns
+ -------
+ ICRResult
+ """
+ if len(coder1_themes) != len(coder2_themes):
+ raise ValueError(
+ f"Length mismatch: {len(coder1_themes)} vs {len(coder2_themes)}."
+ )
+
+ n = len(coder1_themes)
+ if n == 0:
+ return ICRResult(
+ metric_name="jaccard_agreement", value=0.0,
+ interpretation="No data", n_items=0,
+ )
+
+ total_jaccard = 0.0
+ for s1, s2 in zip(coder1_themes, coder2_themes):
+ if not s1 and not s2:
+ total_jaccard += 1.0
+ elif not s1 or not s2:
+ total_jaccard += 0.0
+ else:
+ intersection = len(s1 & s2)
+ union = len(s1 | s2)
+ total_jaccard += intersection / union if union > 0 else 0.0
+
+ avg_jaccard = total_jaccard / n
+ all_themes = set()
+ for s in coder1_themes + coder2_themes:
+ all_themes |= s
+
+ return ICRResult(
+ metric_name="jaccard_agreement",
+ value=round(avg_jaccard, 4),
+ interpretation=self._interpret_jaccard(avg_jaccard),
+ n_coders=2,
+ n_items=n,
+ n_categories=len(all_themes),
+ )
+
+ # ------------------------------------------------------------------
+ # Per-category agreement (single-label)
+ # ------------------------------------------------------------------
+
+ def _compute_per_category(
+ self,
+ coder1_labels: list[str],
+ coder2_labels: list[str],
+ labels: list[str],
+ ) -> list[PerCategoryAgreement]:
+ """Compute category-specific agreement for each label."""
+ n = len(coder1_labels)
+ if n == 0:
+ return []
+
+ result = []
+ for cat in labels:
+ # Binary: does coder assign this category or not?
+ c1_binary = [1 if l == cat else 0 for l in coder1_labels]
+ c2_binary = [1 if l == cat else 0 for l in coder2_labels]
+
+ # Observed agreement for this category
+ agree = sum(1 for a, b in zip(c1_binary, c2_binary) if a == b)
+ p_o = agree / n
+
+ # Expected agreement
+ p1 = sum(c1_binary) / n
+ p2 = sum(c2_binary) / n
+ p_e = p1 * p2 + (1 - p1) * (1 - p2)
+
+ # Specific agreement: proportion of cases where both say "yes"
+ both_yes = sum(1 for a, b in zip(c1_binary, c2_binary) if a == 1 and b == 1)
+ either_yes = sum(1 for a, b in zip(c1_binary, c2_binary) if a == 1 or b == 1)
+ specific = both_yes / either_yes if either_yes > 0 else 0.0
+
+ result.append(PerCategoryAgreement(
+ category=cat,
+ observed_agreement=round(p_o, 4),
+ expected_agreement=round(p_e, 4),
+ specific_agreement=round(specific, 4),
+ ))
+
+ return result
+
+ # ------------------------------------------------------------------
+ # Comprehensive reports
+ # ------------------------------------------------------------------
+
+ def compute_all(
+ self,
+ coder1_labels: list[str],
+ coder2_labels: list[str],
+ labels: list[str] | None = None,
+ ) -> ICRReport:
+ """Compute Cohen's Kappa, Krippendorff's Alpha, and per-category.
+
+ Parameters
+ ----------
+ coder1_labels, coder2_labels : list[str]
+ labels : list[str] or None
+
+ Returns
+ -------
+ ICRReport
+ """
+ if labels is None:
+ labels = sorted(
+ set(str(x).strip() for x in coder1_labels)
+ | set(str(x).strip() for x in coder2_labels)
+ )
+
+ kappa_result = self.compute_cohens_kappa(coder1_labels, coder2_labels, labels)
+
+ # Build reliability matrix for Krippendorff's Alpha
+ c1 = [str(x).strip() for x in coder1_labels]
+ c2 = [str(x).strip() for x in coder2_labels]
+ reliability_matrix = [[a, b] for a, b in zip(c1, c2)]
+ alpha_result = self.compute_krippendorffs_alpha(reliability_matrix, "nominal")
+
+ per_category = self._compute_per_category(c1, c2, labels)
+
+ report = ICRReport(
+ results=[kappa_result, alpha_result],
+ per_category=per_category,
+ coder_labels=["Coder 1", "Coder 2"],
+ )
+ report.summary_text = self.format_report(report)
+ return report
+
+ def compute_all_multilabel(
+ self,
+ coder1_themes: list[set[str]],
+ coder2_themes: list[set[str]],
+ all_themes: list[str] | None = None,
+ ) -> ICRReport:
+ """Full report for multi-label coding comparison.
+
+ Computes Jaccard agreement and per-theme binary Kappa.
+
+ Parameters
+ ----------
+ coder1_themes, coder2_themes : list[set[str]]
+ all_themes : list[str] or None
+
+ Returns
+ -------
+ ICRReport
+ """
+ jaccard_result = self.compute_multilabel_agreement(coder1_themes, coder2_themes)
+
+ if all_themes is None:
+ all_t: set[str] = set()
+ for s in coder1_themes + coder2_themes:
+ all_t |= s
+ all_themes = sorted(all_t)
+
+ # Per-theme binary Kappa
+ per_category = []
+ per_theme_kappas = []
+ for theme in all_themes:
+ c1_binary = ["yes" if theme in s else "no" for s in coder1_themes]
+ c2_binary = ["yes" if theme in s else "no" for s in coder2_themes]
+ kappa_r = self.compute_cohens_kappa(c1_binary, c2_binary, ["yes", "no"])
+ per_theme_kappas.append(kappa_r)
+
+ # Specific agreement for this theme
+ n = len(coder1_themes)
+ both_yes = sum(
+ 1 for s1, s2 in zip(coder1_themes, coder2_themes)
+ if theme in s1 and theme in s2
+ )
+ either_yes = sum(
+ 1 for s1, s2 in zip(coder1_themes, coder2_themes)
+ if theme in s1 or theme in s2
+ )
+ specific = both_yes / either_yes if either_yes > 0 else 0.0
+
+ per_category.append(PerCategoryAgreement(
+ category=theme,
+ observed_agreement=round(
+ sum(1 for a, b in zip(c1_binary, c2_binary) if a == b) / n, 4
+ ) if n > 0 else 0.0,
+ expected_agreement=0.0, # not as meaningful for multi-label
+ specific_agreement=round(specific, 4),
+ ))
+
+ # Average per-theme Kappa as a summary metric
+ valid_kappas = [r.value for r in per_theme_kappas if r.n_items > 0]
+ avg_kappa = sum(valid_kappas) / len(valid_kappas) if valid_kappas else 0.0
+ avg_kappa_result = ICRResult(
+ metric_name="avg_per_theme_kappa",
+ value=round(avg_kappa, 4),
+ interpretation=self.interpret_kappa(avg_kappa),
+ n_coders=2,
+ n_items=len(coder1_themes),
+ n_categories=len(all_themes),
+ )
+
+ report = ICRReport(
+ results=[jaccard_result, avg_kappa_result],
+ per_category=per_category,
+ coder_labels=["Coder 1", "Coder 2"],
+ )
+ report.summary_text = self.format_report(report, multilabel=True)
+ return report
+
+ # ------------------------------------------------------------------
+ # Interpretation scales
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def interpret_kappa(value: float) -> str:
+ """Landis & Koch (1977) interpretation scale for Kappa."""
+ if value < 0:
+ return "Poor agreement"
+ elif value < 0.21:
+ return "Slight agreement"
+ elif value < 0.41:
+ return "Fair agreement"
+ elif value < 0.61:
+ return "Moderate agreement"
+ elif value < 0.81:
+ return "Substantial agreement"
+ else:
+ return "Almost perfect agreement"
+
+ @staticmethod
+ def interpret_alpha(value: float) -> str:
+ """Krippendorff interpretation scale for Alpha."""
+ if value < 0.667:
+ return "Unreliable — discard or recode"
+ elif value < 0.8:
+ return "Tentatively reliable"
+ else:
+ return "Reliable"
+
+ @staticmethod
+ def _interpret_jaccard(value: float) -> str:
+ """Interpret average Jaccard agreement."""
+ if value < 0.2:
+ return "Poor agreement"
+ elif value < 0.4:
+ return "Fair agreement"
+ elif value < 0.6:
+ return "Moderate agreement"
+ elif value < 0.8:
+ return "Substantial agreement"
+ else:
+ return "Almost perfect agreement"
+
+ # ------------------------------------------------------------------
+ # Formatting
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def format_report(
+ report: ICRReport,
+ lang: str = "zh",
+ multilabel: bool = False,
+ ) -> str:
+ """Format an ICRReport as a human-readable string."""
+ lines: list[str] = []
+
+ if lang == "zh":
+ lines.append("═══ 编码者间信度报告 ═══")
+ else:
+ lines.append("═══ Inter-Coder Reliability Report ═══")
+ lines.append("")
+
+ for r in report.results:
+ label_map = {
+ "cohens_kappa": "Cohen's Kappa",
+ "krippendorffs_alpha": "Krippendorff's Alpha",
+ "jaccard_agreement": "Jaccard Agreement (avg)" if lang == "en"
+ else "Jaccard 一致性（均值）",
+ "avg_per_theme_kappa": "Avg Per-Theme Kappa" if lang == "en"
+ else "各主题 Kappa 均值",
+ }
+ name = label_map.get(r.metric_name, r.metric_name)
+ lines.append(f"{name}: {r.value:.4f}")
+ if lang == "zh":
+ lines.append(f" 解释: {r.interpretation}")
+ else:
+ lines.append(f" Interpretation: {r.interpretation}")
+ if r.n_items > 0:
+ items_label = "项目数" if lang == "zh" else "Items"
+ cats_label = "类别数" if lang == "zh" else "Categories"
+ lines.append(f" {items_label}: {r.n_items} | {cats_label}: {r.n_categories}")
+ lines.append("")
+
+ if report.per_category:
+ if multilabel:
+ header_label = "各主题一致性" if lang == "zh" else "Per-Theme Agreement"
+ else:
+ header_label = "各类别一致性" if lang == "zh" else "Per-Category Agreement"
+ lines.append(f"── {header_label} ──")
+
+ cat_label = "类别" if lang == "zh" else "Category"
+ obs_label = "观测一致" if lang == "zh" else "Observed"
+ spec_label = "特定一致" if lang == "zh" else "Specific"
+ lines.append(f"{cat_label:<24} {obs_label:>10} {spec_label:>10}")
+ lines.append("─" * 46)
+ for pc in report.per_category:
+ lines.append(
+ f"{pc.category:<24} {pc.observed_agreement:>10.4f} "
+ f"{pc.specific_agreement:>10.4f}"
+ )
+ lines.append("")
+
+ return "\n".join(lines)
diff --git a/socialscikit/core/methods_writer.py b/socialscikit/core/methods_writer.py
new file mode 100644
index 0000000..820cf72
--- /dev/null
+++ b/socialscikit/core/methods_writer.py
@@ -0,0 +1,448 @@
+"""Methods Section Auto-generation — template-based methods paragraph for papers.
+
+Generates a Methods paragraph draft from pipeline metadata. Template-based
+(no LLM calls) so every number and method name is deterministic and verifiable.
+
+Outputs both English and Chinese versions.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+# ---------------------------------------------------------------------------
+# Pipeline metadata
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class QuantiKitPipelineMetadata:
+ """Metadata collected from a QuantiKit analysis session."""
+
+ dataset_name: str = ""
+ n_samples: int = 0
+ n_classes: int = 0
+ class_labels: list[str] = field(default_factory=list)
+ classification_method: str = "" # "zero-shot" | "few-shot" | "fine-tune-local" | "fine-tune-api"
+ model_name: str = ""
+ model_backend: str = "" # "openai" | "anthropic" | "ollama"
+ n_annotations: int = 0
+ prompt_optimization_used: bool = False
+ n_prompt_variants: int = 0
+ # Evaluation metrics
+ accuracy: float = 0.0
+ macro_f1: float = 0.0
+ weighted_f1: float = 0.0
+ cohens_kappa: float = 0.0
+ # ICR metrics (from ICR module, if run)
+ icr_kappa: float = 0.0
+ icr_alpha: float = 0.0
+
+
+@dataclass
+class QualiKitPipelineMetadata:
+ """Metadata collected from a QualiKit analysis session."""
+
+ dataset_name: str = ""
+ n_segments: int = 0
+ deidentification_performed: bool = False
+ n_pii_detected: int = 0
+ n_themes: int = 0
+ theme_names: list[str] = field(default_factory=list)
+ coding_model_name: str = ""
+ coding_model_backend: str = ""
+ # Consensus
+ consensus_coding_used: bool = False
+ n_consensus_models: int = 0
+ consensus_model_names: list[str] = field(default_factory=list)
+ consensus_agreement: float = 0.0
+ # Confidence tiers
+ n_high_confidence: int = 0
+ n_medium_confidence: int = 0
+ n_low_confidence: int = 0
+ # Review
+ n_accepted: int = 0
+ n_rejected: int = 0
+ n_edited: int = 0
+ # ICR
+ icr_jaccard: float = 0.0
+ icr_per_theme_kappa: float = 0.0
+
+
+@dataclass
+class MethodsSection:
+ """Generated methods section output."""
+
+ text_en: str = ""
+ text_zh: str = ""
+ metadata_used: dict = field(default_factory=dict)
+
+
+# ---------------------------------------------------------------------------
+# Methods Writer
+# ---------------------------------------------------------------------------
+
+
+class MethodsWriter:
+ """Generate a Methods section paragraph for academic papers.
+
+ Uses template-based generation with slot-filling from pipeline metadata.
+ No LLM calls — purely deterministic to ensure accuracy.
+
+ Usage::
+
+ writer = MethodsWriter()
+ meta = QuantiKitPipelineMetadata(
+ n_samples=5000, n_classes=3, classification_method="few-shot",
+ model_name="gpt-4o", accuracy=0.87, macro_f1=0.85,
+ )
+ section = writer.generate_quantikit_methods(meta)
+ print(section.text_en)
+ """
+
+ # ------------------------------------------------------------------
+ # Public API
+ # ------------------------------------------------------------------
+
+ def generate_quantikit_methods(
+ self, metadata: QuantiKitPipelineMetadata,
+ ) -> MethodsSection:
+ """Generate methods section for a QuantiKit text classification analysis."""
+ return MethodsSection(
+ text_en=self._build_quantikit_en(metadata),
+ text_zh=self._build_quantikit_zh(metadata),
+ metadata_used=self._meta_to_dict(metadata),
+ )
+
+ def generate_qualikit_methods(
+ self, metadata: QualiKitPipelineMetadata,
+ ) -> MethodsSection:
+ """Generate methods section for a QualiKit qualitative coding analysis."""
+ return MethodsSection(
+ text_en=self._build_qualikit_en(metadata),
+ text_zh=self._build_qualikit_zh(metadata),
+ metadata_used=self._meta_to_dict(metadata),
+ )
+
+ # ------------------------------------------------------------------
+ # QuantiKit — English
+ # ------------------------------------------------------------------
+
+ def _build_quantikit_en(self, m: QuantiKitPipelineMetadata) -> str:
+ parts: list[str] = []
+
+ # Opening
+ n_str = f"{m.n_samples:,}" if m.n_samples else "N"
+ classes_str = f"{m.n_classes}" if m.n_classes else "multiple"
+ labels_str = (
+ f" ({', '.join(m.class_labels)})" if m.class_labels else ""
+ )
+ parts.append(
+ f"We classified {n_str} text samples into {classes_str} "
+ f"categories{labels_str} using SocialSciKit (Sun et al., 2026)."
+ )
+
+ # Classification method
+ method_desc = self._method_desc_en(m.classification_method, m.model_name, m.model_backend)
+ if method_desc:
+ parts.append(method_desc)
+
+ # Annotations
+ if m.n_annotations > 0:
+ parts.append(
+ f"A total of {m.n_annotations} samples were manually annotated "
+ f"to serve as the training/evaluation set."
+ )
+
+ # Prompt optimization
+ if m.prompt_optimization_used:
+ opt_str = (
+ f" across {m.n_prompt_variants} prompt variants"
+ if m.n_prompt_variants > 1 else ""
+ )
+ parts.append(
+ f"Automated Prompt Engineering (APE) was applied to optimize "
+ f"classification prompts{opt_str}."
+ )
+
+ # Evaluation
+ eval_parts = []
+ if m.accuracy > 0:
+ eval_parts.append(f"accuracy of {m.accuracy:.2%}")
+ if m.macro_f1 > 0:
+ eval_parts.append(f"macro-F1 of {m.macro_f1:.4f}")
+ if m.weighted_f1 > 0:
+ eval_parts.append(f"weighted-F1 of {m.weighted_f1:.4f}")
+ if m.cohens_kappa > 0:
+ eval_parts.append(f"Cohen's Kappa of {m.cohens_kappa:.4f}")
+ if eval_parts:
+ parts.append(
+ f"The model achieved {', '.join(eval_parts)} on the held-out "
+ f"evaluation set."
+ )
+
+ # ICR
+ icr_parts = []
+ if m.icr_kappa > 0:
+ icr_parts.append(f"Cohen's Kappa = {m.icr_kappa:.4f}")
+ if m.icr_alpha > 0:
+ icr_parts.append(f"Krippendorff's Alpha = {m.icr_alpha:.4f}")
+ if icr_parts:
+ parts.append(
+ f"Inter-coder reliability was assessed ({', '.join(icr_parts)})."
+ )
+
+ return " ".join(parts)
+
+ # ------------------------------------------------------------------
+ # QuantiKit — Chinese
+ # ------------------------------------------------------------------
+
+ def _build_quantikit_zh(self, m: QuantiKitPipelineMetadata) -> str:
+ parts: list[str] = []
+
+ n_str = f"{m.n_samples:,}" if m.n_samples else "N"
+ classes_str = f"{m.n_classes}" if m.n_classes else "多个"
+ labels_str = (
+ f"（{', '.join(m.class_labels)}）" if m.class_labels else ""
+ )
+ parts.append(
+ f"本研究使用 SocialSciKit（Sun et al., 2026）对 {n_str} 条文本进行"
+ f" {classes_str} 分类{labels_str}。"
+ )
+
+ method_desc = self._method_desc_zh(m.classification_method, m.model_name, m.model_backend)
+ if method_desc:
+ parts.append(method_desc)
+
+ if m.n_annotations > 0:
+ parts.append(f"共人工标注 {m.n_annotations} 条样本作为训练/评估数据。")
+
+ if m.prompt_optimization_used:
+ opt_str = f"，共测试 {m.n_prompt_variants} 种 Prompt 变体" if m.n_prompt_variants > 1 else ""
+ parts.append(f"使用自动提示工程（APE）优化分类 Prompt{opt_str}。")
+
+ eval_parts = []
+ if m.accuracy > 0:
+ eval_parts.append(f"准确率 {m.accuracy:.2%}")
+ if m.macro_f1 > 0:
+ eval_parts.append(f"宏平均 F1 = {m.macro_f1:.4f}")
+ if m.weighted_f1 > 0:
+ eval_parts.append(f"加权 F1 = {m.weighted_f1:.4f}")
+ if m.cohens_kappa > 0:
+ eval_parts.append(f"Cohen's Kappa = {m.cohens_kappa:.4f}")
+ if eval_parts:
+ parts.append(f"模型在评估集上的表现为：{', '.join(eval_parts)}。")
+
+ icr_parts = []
+ if m.icr_kappa > 0:
+ icr_parts.append(f"Cohen's Kappa = {m.icr_kappa:.4f}")
+ if m.icr_alpha > 0:
+ icr_parts.append(f"Krippendorff's Alpha = {m.icr_alpha:.4f}")
+ if icr_parts:
+ parts.append(f"编码者间信度检验：{', '.join(icr_parts)}。")
+
+ return "".join(parts)
+
+ # ------------------------------------------------------------------
+ # QualiKit — English
+ # ------------------------------------------------------------------
+
+ def _build_qualikit_en(self, m: QualiKitPipelineMetadata) -> str:
+ parts: list[str] = []
+
+ n_str = f"{m.n_segments:,}" if m.n_segments else "N"
+ parts.append(
+ f"Qualitative coding was performed on {n_str} text segments "
+ f"using SocialSciKit (Sun et al., 2026)."
+ )
+
+ # De-identification
+ if m.deidentification_performed:
+ pii_str = (
+ f", detecting and masking {m.n_pii_detected} personally "
+ f"identifiable items" if m.n_pii_detected > 0 else ""
+ )
+ parts.append(
+ f"Prior to coding, all texts were de-identified{pii_str}."
+ )
+
+ # Research framework
+ if m.n_themes > 0:
+ themes_str = (
+ f" ({', '.join(m.theme_names[:5])}{'...' if len(m.theme_names) > 5 else ''})"
+ if m.theme_names else ""
+ )
+ parts.append(
+ f"A coding framework with {m.n_themes} themes was defined{themes_str}."
+ )
+
+ # Coding method
+ if m.consensus_coding_used and m.n_consensus_models >= 2:
+ models_str = ", ".join(m.consensus_model_names) if m.consensus_model_names else f"{m.n_consensus_models} models"
+ parts.append(
+ f"Multi-LLM consensus coding was employed: {models_str} independently "
+ f"coded each segment, and themes were retained only when a majority "
+ f"of coders agreed (overall agreement: {m.consensus_agreement:.2%})."
+ )
+ elif m.coding_model_name:
+ backend_str = f" ({m.coding_model_backend})" if m.coding_model_backend else ""
+ parts.append(
+ f"LLM-assisted coding was performed using {m.coding_model_name}"
+ f"{backend_str}."
+ )
+
+ # Confidence tiers
+ total_conf = m.n_high_confidence + m.n_medium_confidence + m.n_low_confidence
+ if total_conf > 0:
+ parts.append(
+ f"Coding confidence was categorized into three tiers: "
+ f"high ({m.n_high_confidence}), medium ({m.n_medium_confidence}), "
+ f"and low ({m.n_low_confidence})."
+ )
+
+ # Human review
+ total_reviewed = m.n_accepted + m.n_rejected + m.n_edited
+ if total_reviewed > 0:
+ parts.append(
+ f"Human review was conducted on all coded segments: "
+ f"{m.n_accepted} accepted, {m.n_rejected} rejected, "
+ f"and {m.n_edited} manually edited."
+ )
+
+ # ICR
+ icr_parts = []
+ if m.icr_jaccard > 0:
+ icr_parts.append(f"Jaccard agreement = {m.icr_jaccard:.4f}")
+ if m.icr_per_theme_kappa > 0:
+ icr_parts.append(f"average per-theme Kappa = {m.icr_per_theme_kappa:.4f}")
+ if icr_parts:
+ parts.append(
+ f"Inter-coder reliability between human review and LLM coding "
+ f"was assessed ({', '.join(icr_parts)})."
+ )
+
+ return " ".join(parts)
+
+ # ------------------------------------------------------------------
+ # QualiKit — Chinese
+ # ------------------------------------------------------------------
+
+ def _build_qualikit_zh(self, m: QualiKitPipelineMetadata) -> str:
+ parts: list[str] = []
+
+ n_str = f"{m.n_segments:,}" if m.n_segments else "N"
+ parts.append(
+ f"本研究使用 SocialSciKit（Sun et al., 2026）对 {n_str} 条文本段落"
+ f"进行质性编码。"
+ )
+
+ if m.deidentification_performed:
+ pii_str = f"，共检测并脱敏 {m.n_pii_detected} 项个人信息" if m.n_pii_detected > 0 else ""
+ parts.append(f"编码前对所有文本进行了脱敏处理{pii_str}。")
+
+ if m.n_themes > 0:
+ themes_str = (
+ f"（{', '.join(m.theme_names[:5])}{'...' if len(m.theme_names) > 5 else ''}）"
+ if m.theme_names else ""
+ )
+ parts.append(f"定义了包含 {m.n_themes} 个主题的编码框架{themes_str}。")
+
+ if m.consensus_coding_used and m.n_consensus_models >= 2:
+ models_str = "、".join(m.consensus_model_names) if m.consensus_model_names else f"{m.n_consensus_models} 个模型"
+ parts.append(
+ f"采用多模型共识编码策略：{models_str}分别独立编码，"
+ f"仅保留多数模型一致的主题标签（总体一致性：{m.consensus_agreement:.2%}）。"
+ )
+ elif m.coding_model_name:
+ backend_str = f"（{m.coding_model_backend}）" if m.coding_model_backend else ""
+ parts.append(f"使用 {m.coding_model_name}{backend_str} 进行 LLM 辅助编码。")
+
+ total_conf = m.n_high_confidence + m.n_medium_confidence + m.n_low_confidence
+ if total_conf > 0:
+ parts.append(
+ f"编码置信度分为三档：高（{m.n_high_confidence} 条）、"
+ f"中（{m.n_medium_confidence} 条）、低（{m.n_low_confidence} 条）。"
+ )
+
+ total_reviewed = m.n_accepted + m.n_rejected + m.n_edited
+ if total_reviewed > 0:
+ parts.append(
+ f"对全部编码进行人工审核：接受 {m.n_accepted} 条、"
+ f"拒绝 {m.n_rejected} 条、手动编辑 {m.n_edited} 条。"
+ )
+
+ icr_parts = []
+ if m.icr_jaccard > 0:
+ icr_parts.append(f"Jaccard 一致性 = {m.icr_jaccard:.4f}")
+ if m.icr_per_theme_kappa > 0:
+ icr_parts.append(f"各主题平均 Kappa = {m.icr_per_theme_kappa:.4f}")
+ if icr_parts:
+ parts.append(f"人工审核与 LLM 编码的编码者间信度：{', '.join(icr_parts)}。")
+
+ return "".join(parts)
+
+ # ------------------------------------------------------------------
+ # Helpers
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def _method_desc_en(method: str, model: str, backend: str) -> str:
+ """Build the classification method sentence (English)."""
+ model_str = model or "the language model"
+ backend_str = f" ({backend})" if backend else ""
+
+ method_map = {
+ "zero-shot": (
+ f"A zero-shot prompting approach was used with "
+ f"{model_str}{backend_str}, where no labeled examples were "
+ f"provided in the prompt."
+ ),
+ "few-shot": (
+ f"A few-shot prompting approach was used with "
+ f"{model_str}{backend_str}, where labeled examples were "
+ f"included in the prompt for in-context learning."
+ ),
+ "fine-tune-local": (
+ f"The model {model_str} was fine-tuned locally on the "
+ f"annotated training data."
+ ),
+ "fine-tune-api": (
+ f"The model {model_str}{backend_str} was fine-tuned via "
+ f"the provider's API on the annotated training data."
+ ),
+ }
+ return method_map.get(method, "")
+
+ @staticmethod
+ def _method_desc_zh(method: str, model: str, backend: str) -> str:
+ """Build the classification method sentence (Chinese)."""
+ model_str = model or "语言模型"
+ backend_str = f"（{backend}）" if backend else ""
+
+ method_map = {
+ "zero-shot": (
+ f"采用零样本提示（zero-shot prompting）方法，"
+ f"使用 {model_str}{backend_str}，Prompt 中不包含标注示例。"
+ ),
+ "few-shot": (
+ f"采用少样本提示（few-shot prompting）方法，"
+ f"使用 {model_str}{backend_str}，Prompt 中包含标注示例进行上下文学习。"
+ ),
+ "fine-tune-local": (
+ f"在标注训练数据上对 {model_str} 进行本地微调。"
+ ),
+ "fine-tune-api": (
+ f"通过 API 在标注训练数据上对 {model_str}{backend_str} 进行微调。"
+ ),
+ }
+ return method_map.get(method, "")
+
+ @staticmethod
+ def _meta_to_dict(metadata: object) -> dict:
+ """Convert a metadata dataclass to a plain dict."""
+ if hasattr(metadata, "__dataclass_fields__"):
+ from dataclasses import asdict
+ return asdict(metadata)
+ return {}
diff --git a/socialscikit/core/project_io.py b/socialscikit/core/project_io.py
new file mode 100644
index 0000000..a7797a3
--- /dev/null
+++ b/socialscikit/core/project_io.py
@@ -0,0 +1,394 @@
+"""Project save & restore — serialize / deserialize all session state to JSON.
+
+Allows users to save their progress and resume later without losing any
+annotation, coding, or review state. All complex types are converted to
+plain dicts with a ``__type__`` discriminator for safe, human-readable
+round-tripping.
+
+Usage::
+
+ from socialscikit.core.project_io import save_project, load_project
+
+ json_str = save_project({"qt_df": df, "ql_segments": segments, ...})
+ states = load_project(json_str)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from typing import Any
+
+import pandas as pd
+
+from socialscikit.quantikit.annotator import (
+ Annotation,
+ AnnotationSession,
+ AnnotationStatus,
+)
+from socialscikit.qualikit.extraction_reviewer import (
+ ExtractionReviewSession,
+ ReviewAction,
+ ReviewedExtraction,
+)
+from socialscikit.qualikit.segment_extractor import (
+ ExtractionResult,
+ ResearchQuestion,
+)
+from socialscikit.qualikit.segmenter import TextPosition, TextSegment
+
+logger = logging.getLogger(__name__)
+
+PROJECT_VERSION = "1.0"
+
+
+# ======================================================================
+# Serializers (Python object → JSON-safe dict)
+# ======================================================================
+
+
+def _ser_dataframe(df: pd.DataFrame) -> dict:
+ return {
+ "__type__": "DataFrame",
+ "records": json.loads(df.to_json(orient="records", force_ascii=False)),
+ "columns": list(df.columns),
+ }
+
+
+def _ser_text_position(pos: TextPosition) -> dict:
+ return {
+ "__type__": "TextPosition",
+ "line_start": pos.line_start,
+ "line_end": pos.line_end,
+ "char_start": pos.char_start,
+ "char_end": pos.char_end,
+ "paragraph_index": pos.paragraph_index,
+ }
+
+
+def _ser_text_segment(seg: TextSegment) -> dict:
+ return {
+ "__type__": "TextSegment",
+ "segment_id": seg.segment_id,
+ "text": seg.text,
+ "position": _ser_text_position(seg.position),
+ "core_sentence": seg.core_sentence,
+ "core_char_start": seg.core_char_start,
+ "core_char_end": seg.core_char_end,
+ }
+
+
+def _ser_research_question(rq: ResearchQuestion) -> dict:
+ return {
+ "__type__": "ResearchQuestion",
+ "rq_id": rq.rq_id,
+ "description": rq.description,
+ "sub_themes": list(rq.sub_themes),
+ }
+
+
+def _ser_extraction_result(r: ExtractionResult) -> dict:
+ return {
+ "__type__": "ExtractionResult",
+ "segment_id": r.segment_id,
+ "text": r.text,
+ "rq_label": r.rq_label,
+ "sub_theme": r.sub_theme,
+ "confidence": r.confidence,
+ "reasoning": r.reasoning,
+ "evidence_span": getattr(r, "evidence_span", ""),
+ "position": _ser_text_position(r.position) if r.position else None,
+ }
+
+
+def _ser_reviewed_extraction(item: ReviewedExtraction) -> dict:
+ return {
+ "__type__": "ReviewedExtraction",
+ "result": _ser_extraction_result(item.result),
+ "action": item.action.value,
+ "edited_rq_label": item.edited_rq_label,
+ "edited_sub_theme": item.edited_sub_theme,
+ }
+
+
+def _ser_extraction_review_session(sess: ExtractionReviewSession) -> dict:
+ return {
+ "__type__": "ExtractionReviewSession",
+ "items": [_ser_reviewed_extraction(i) for i in sess.items],
+ "original_text": sess.original_text,
+ "segments": [_ser_text_segment(s) for s in sess.segments],
+ "research_questions": [_ser_research_question(rq) for rq in sess.research_questions],
+ }
+
+
+def _ser_annotation(a: Annotation) -> dict:
+ return {
+ "__type__": "Annotation",
+ "idx": a.idx,
+ "text": a.text,
+ "label": a.label,
+ "status": a.status.value,
+ "timestamp": a.timestamp,
+ "annotator_note": a.annotator_note,
+ }
+
+
+def _ser_annotation_session(sess: AnnotationSession) -> dict:
+ elapsed = time.monotonic() - sess._start_time
+ return {
+ "__type__": "AnnotationSession",
+ "labels": list(sess.labels),
+ "items": [_ser_annotation(a) for a in sess._items],
+ "cursor": sess._cursor,
+ "history": list(sess._history),
+ "elapsed_seconds": round(elapsed, 1),
+ }
+
+
+# ======================================================================
+# Deserializers (JSON-safe dict → Python object)
+# ======================================================================
+
+
+def _de_dataframe(d: dict) -> pd.DataFrame:
+ df = pd.DataFrame(d["records"])
+ # Restore column order
+ if d.get("columns"):
+ cols = [c for c in d["columns"] if c in df.columns]
+ df = df[cols]
+ return df
+
+
+def _de_text_position(d: dict) -> TextPosition:
+ return TextPosition(
+ line_start=d["line_start"],
+ line_end=d["line_end"],
+ char_start=d["char_start"],
+ char_end=d["char_end"],
+ paragraph_index=d.get("paragraph_index", 0),
+ )
+
+
+def _de_text_segment(d: dict) -> TextSegment:
+ return TextSegment(
+ segment_id=d["segment_id"],
+ text=d["text"],
+ position=_de_text_position(d["position"]),
+ core_sentence=d.get("core_sentence"),
+ core_char_start=d.get("core_char_start"),
+ core_char_end=d.get("core_char_end"),
+ )
+
+
+def _de_research_question(d: dict) -> ResearchQuestion:
+ return ResearchQuestion(
+ rq_id=d["rq_id"],
+ description=d["description"],
+ sub_themes=d.get("sub_themes", []),
+ )
+
+
+def _de_extraction_result(d: dict) -> ExtractionResult:
+ pos_data = d.get("position")
+ return ExtractionResult(
+ segment_id=d["segment_id"],
+ text=d["text"],
+ rq_label=d["rq_label"],
+ sub_theme=d["sub_theme"],
+ confidence=d["confidence"],
+ reasoning=d.get("reasoning", ""),
+ evidence_span=d.get("evidence_span", ""),
+ position=_de_text_position(pos_data) if pos_data else None,
+ )
+
+
+def _de_reviewed_extraction(d: dict) -> ReviewedExtraction:
+ return ReviewedExtraction(
+ result=_de_extraction_result(d["result"]),
+ action=ReviewAction(d["action"]),
+ edited_rq_label=d.get("edited_rq_label"),
+ edited_sub_theme=d.get("edited_sub_theme"),
+ )
+
+
+def _de_extraction_review_session(d: dict) -> ExtractionReviewSession:
+ return ExtractionReviewSession(
+ items=[_de_reviewed_extraction(i) for i in d.get("items", [])],
+ original_text=d.get("original_text", ""),
+ segments=[_de_text_segment(s) for s in d.get("segments", [])],
+ research_questions=[_de_research_question(rq) for rq in d.get("research_questions", [])],
+ )
+
+
+def _de_annotation(d: dict) -> Annotation:
+ return Annotation(
+ idx=d["idx"],
+ text=d["text"],
+ label=d.get("label"),
+ status=AnnotationStatus(d.get("status", "pending")),
+ timestamp=d.get("timestamp"),
+ annotator_note=d.get("annotator_note", ""),
+ )
+
+
+def _de_annotation_session(d: dict) -> AnnotationSession:
+ items = [_de_annotation(a) for a in d.get("items", [])]
+ labels = d.get("labels", [])
+ sess = AnnotationSession(items=items, labels=labels, shuffle=False)
+ sess._cursor = d.get("cursor", 0)
+ sess._history = d.get("history", [])
+ elapsed = d.get("elapsed_seconds", 0.0)
+ sess._start_time = time.monotonic() - elapsed
+ return sess
+
+
+# ======================================================================
+# Top-level dispatch
+# ======================================================================
+
+_TYPE_SERIALIZERS = {
+ "DataFrame": _ser_dataframe,
+ "TextPosition": _ser_text_position,
+ "TextSegment": _ser_text_segment,
+ "ResearchQuestion": _ser_research_question,
+ "ExtractionResult": _ser_extraction_result,
+ "ReviewedExtraction": _ser_reviewed_extraction,
+ "ExtractionReviewSession": _ser_extraction_review_session,
+ "Annotation": _ser_annotation,
+ "AnnotationSession": _ser_annotation_session,
+}
+
+_TYPE_DESERIALIZERS = {
+ "DataFrame": _de_dataframe,
+ "TextPosition": _de_text_position,
+ "TextSegment": _de_text_segment,
+ "ResearchQuestion": _de_research_question,
+ "ExtractionResult": _de_extraction_result,
+ "ReviewedExtraction": _de_reviewed_extraction,
+ "ExtractionReviewSession": _de_extraction_review_session,
+ "Annotation": _de_annotation,
+ "AnnotationSession": _de_annotation_session,
+}
+
+_TYPE_MAP = {
+ pd.DataFrame: "DataFrame",
+ TextPosition: "TextPosition",
+ TextSegment: "TextSegment",
+ ResearchQuestion: "ResearchQuestion",
+ ExtractionResult: "ExtractionResult",
+ ReviewedExtraction: "ReviewedExtraction",
+ ExtractionReviewSession: "ExtractionReviewSession",
+ Annotation: "Annotation",
+ AnnotationSession: "AnnotationSession",
+}
+
+
+def _serialize_value(val: Any) -> Any:
+ """Recursively serialize a value to JSON-safe form."""
+ if val is None:
+ return None
+ # Check known types
+ type_name = _TYPE_MAP.get(type(val))
+ if type_name:
+ return _TYPE_SERIALIZERS[type_name](val)
+ # Lists
+ if isinstance(val, list):
+ return [_serialize_value(v) for v in val]
+ # Dicts
+ if isinstance(val, dict):
+ return {str(k): _serialize_value(v) for k, v in val.items()}
+ # Primitives
+ if isinstance(val, (str, int, float, bool)):
+ return val
+ # Fallback: try str
+ logger.warning("Cannot serialize type %s, converting to str", type(val).__name__)
+ return str(val)
+
+
+def _deserialize_value(val: Any) -> Any:
+ """Recursively deserialize a JSON-safe value back to Python objects."""
+ if val is None:
+ return None
+ if isinstance(val, dict):
+ type_tag = val.get("__type__")
+ if type_tag and type_tag in _TYPE_DESERIALIZERS:
+ return _TYPE_DESERIALIZERS[type_tag](val)
+ # Regular dict
+ return {k: _deserialize_value(v) for k, v in val.items()}
+ if isinstance(val, list):
+ return [_deserialize_value(v) for v in val]
+ return val
+
+
+# ======================================================================
+# Public API
+# ======================================================================
+
+
+def save_project(states: dict[str, Any]) -> str:
+ """Serialize all session states to a JSON string.
+
+ Parameters
+ ----------
+ states : dict
+ Mapping of state names to their Python values. Keys include
+ ``qt_df``, ``qt_result_df``, ``qt_ann_session``, ``ql_raw_text``,
+ ``ql_segments``, ``ql_rqs``, ``ql_ext_session``, ``ql_lang``.
+
+ Returns
+ -------
+ str
+ JSON string that can be written to a file.
+ """
+ payload: dict[str, Any] = {
+ "__project_version__": PROJECT_VERSION,
+ "__toolkit__": "SocialSciKit",
+ }
+ for key, val in states.items():
+ try:
+ payload[key] = _serialize_value(val)
+ except Exception as e:
+ logger.warning("Failed to serialize '%s': %s", key, e)
+ payload[key] = None
+ return json.dumps(payload, ensure_ascii=False, indent=2)
+
+
+def load_project(json_str: str) -> dict[str, Any]:
+ """Deserialize a JSON project file back into session states.
+
+ Parameters
+ ----------
+ json_str : str
+ The JSON string from a previously saved project file.
+
+ Returns
+ -------
+ dict
+ Mapping of state names to reconstructed Python objects.
+
+ Raises
+ ------
+ ValueError
+ If the JSON cannot be parsed or is not a valid project file.
+ """
+ try:
+ raw = json.loads(json_str)
+ except json.JSONDecodeError as e:
+ raise ValueError(f"Invalid project file: {e}") from e
+
+ if not isinstance(raw, dict):
+ raise ValueError("Project file must be a JSON object.")
+
+ version = raw.pop("__project_version__", "unknown")
+ raw.pop("__toolkit__", None)
+ logger.info("Loading project file (version %s)", version)
+
+ states: dict[str, Any] = {}
+ for key, val in raw.items():
+ try:
+ states[key] = _deserialize_value(val)
+ except Exception as e:
+ logger.warning("Failed to deserialize '%s': %s", key, e)
+ states[key] = None
+ return states
diff --git a/socialscikit/qualikit/__init__.py b/socialscikit/qualikit/__init__.py
index 57687e6..871f230 100644
--- a/socialscikit/qualikit/__init__.py
+++ b/socialscikit/qualikit/__init__.py
@@ -13,9 +13,15 @@
ReviewAction,
ReviewedExtraction,
)
+from socialscikit.qualikit.consensus import (
+ ConsensusCoder,
+ ConsensusReport,
+ SegmentConsensus,
+)
__all__ = [
"Segmenter", "TextPosition", "TextSegment",
"ExtractionReport", "ExtractionResult", "ResearchQuestion", "SegmentExtractor",
"ExtractionReviewer", "ExtractionReviewSession", "ReviewAction", "ReviewedExtraction",
+ "ConsensusCoder", "ConsensusReport", "SegmentConsensus",
]
diff --git a/socialscikit/qualikit/coder.py b/socialscikit/qualikit/coder.py
index fa366ea..88749d7 100644
--- a/socialscikit/qualikit/coder.py
+++ b/socialscikit/qualikit/coder.py
@@ -44,6 +44,7 @@ class CodingResult:
themes: list[str] = field(default_factory=list)
confidences: dict[str, float] = field(default_factory=dict)
trigger_words: dict[str, list[str]] = field(default_factory=dict)
+ evidence_spans: dict[str, str] = field(default_factory=dict)
reasoning: str = ""
@property
@@ -103,12 +104,13 @@ def low_confidence_count(self) -> int:
Assign relevant themes to this text. For each theme:
1. Confidence (0.0-1.0): how certain you are this theme applies
2. Trigger words: specific words/phrases from the text that support this coding
-3. Brief reasoning
+3. Evidence span: copy the exact phrase or sentence from the text that most directly supports this coding (verbatim, do not paraphrase)
+4. Brief reasoning
If NO themes apply, return an empty "themes" array.
Return ONLY valid JSON (no markdown fencing):
-{{"themes": [{{"name": "theme_name", "confidence": 0.85, "trigger_words": ["word1", "word2"], "reasoning": "brief reason"}}]}}"""
+{{"themes": [{{"name": "theme_name", "confidence": 0.85, "trigger_words": ["word1", "word2"], "evidence_span": "exact quote from text", "reasoning": "brief reason"}}]}}"""
# ---------------------------------------------------------------------------
@@ -253,6 +255,7 @@ def _parse_response(
assigned_themes = []
confidences = {}
trigger_words = {}
+ evidence_spans = {}
reasoning_parts = []
raw_themes = parsed.get("themes", [])
@@ -271,9 +274,13 @@ def _parse_response(
triggers = item.get("trigger_words", [])
reason = item.get("reasoning", "")
+ evidence = str(item.get("evidence_span", "")).strip()
+
assigned_themes.append(matched)
confidences[matched] = round(conf, 3)
trigger_words[matched] = triggers if isinstance(triggers, list) else []
+ if evidence:
+ evidence_spans[matched] = evidence
if reason:
reasoning_parts.append(f"{matched}: {reason}")
@@ -283,6 +290,7 @@ def _parse_response(
themes=assigned_themes,
confidences=confidences,
trigger_words=trigger_words,
+ evidence_spans=evidence_spans,
reasoning="; ".join(reasoning_parts),
)
diff --git a/socialscikit/qualikit/consensus.py b/socialscikit/qualikit/consensus.py
new file mode 100644
index 0000000..9533682
--- /dev/null
+++ b/socialscikit/qualikit/consensus.py
@@ -0,0 +1,369 @@
+"""Multi-LLM Consensus Coding — run multiple LLMs, majority vote, agreement report.
+
+Run 2–3 different LLMs independently on the same text segments. For each
+segment the final theme set is determined by majority vote: a theme is
+included only if ≥ ceil(n_coders / 2) coders assigned it.
+
+The merged results are standard ``CodingResult`` objects so they flow
+through the existing ``ConfidenceRanker → CodingReviewer → Exporter``
+pipeline unchanged.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import math
+from collections import Counter
+from dataclasses import dataclass, field
+
+from socialscikit.core.llm_client import LLMClient
+from socialscikit.qualikit.coder import Coder, CodingReport, CodingResult
+from socialscikit.qualikit.theme_definer import Theme
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class SegmentConsensus:
+ """Consensus result for a single text segment."""
+
+ text_id: int
+ text: str
+ consensus_themes: list[str] = field(default_factory=list)
+ consensus_confidences: dict[str, float] = field(default_factory=dict)
+ agreement_rate: float = 0.0
+ individual_results: list[CodingResult] = field(default_factory=list)
+ vote_counts: dict[str, int] = field(default_factory=dict)
+
+ def to_coding_result(self) -> CodingResult:
+ """Convert to a standard CodingResult for downstream compatibility."""
+ # Merge trigger words from all coders who assigned consensus themes
+ merged_triggers: dict[str, list[str]] = {}
+ merged_reasoning_parts: list[str] = []
+
+ for theme in self.consensus_themes:
+ triggers_set: set[str] = set()
+ for r in self.individual_results:
+ if theme in r.themes:
+ triggers_set.update(r.trigger_words.get(theme, []))
+ merged_triggers[theme] = list(triggers_set)
+
+ # Merge reasoning
+ for r in self.individual_results:
+ if r.reasoning:
+ merged_reasoning_parts.append(r.reasoning)
+
+ return CodingResult(
+ text_id=self.text_id,
+ text=self.text,
+ themes=self.consensus_themes,
+ confidences=self.consensus_confidences,
+ trigger_words=merged_triggers,
+ reasoning=" | ".join(merged_reasoning_parts) if merged_reasoning_parts else "",
+ )
+
+
+@dataclass
+class ConsensusReport:
+ """Full report from a consensus coding run."""
+
+ segments: list[SegmentConsensus] = field(default_factory=list)
+ n_coders: int = 0
+ coder_models: list[str] = field(default_factory=list)
+ n_total: int = 0
+ n_coded: int = 0
+ n_failed: int = 0
+ overall_agreement: float = 0.0
+ theme_distribution: dict[str, int] = field(default_factory=dict)
+ per_coder_cost: list[float] = field(default_factory=list)
+ total_cost: float = 0.0
+
+ def to_coding_report(self) -> CodingReport:
+ """Convert to a standard CodingReport for downstream compatibility."""
+ results = [seg.to_coding_result() for seg in self.segments]
+ return CodingReport(
+ results=results,
+ n_total=self.n_total,
+ n_coded=self.n_coded,
+ n_failed=self.n_failed,
+ theme_distribution=dict(self.theme_distribution),
+ )
+
+
+# ---------------------------------------------------------------------------
+# Consensus Coder
+# ---------------------------------------------------------------------------
+
+
+class ConsensusCoder:
+ """Multi-LLM consensus coding engine.
+
+ Instantiates a ``Coder`` per ``LLMClient``, runs all coders on the same
+ segments, then merges results via majority vote.
+
+ Usage::
+
+ clients = [
+ LLMClient(backend="openai", model="gpt-4o"),
+ LLMClient(backend="anthropic", model="claude-sonnet-4-20250514"),
+ ]
+ consensus = ConsensusCoder(clients)
+ report = consensus.code(texts, themes)
+ """
+
+ def __init__(
+ self,
+ llm_clients: list[LLMClient],
+ majority_threshold: int | None = None,
+ ):
+ """
+ Parameters
+ ----------
+ llm_clients : list[LLMClient]
+ At least 2 LLM clients for consensus coding.
+ majority_threshold : int or None
+ Minimum number of coders that must agree for a theme to be included.
+ Defaults to ``ceil(len(llm_clients) / 2)``.
+ """
+ if len(llm_clients) < 2:
+ raise ValueError("Consensus coding requires at least 2 LLM clients.")
+
+ self.llm_clients = llm_clients
+ self.coders = [Coder(client) for client in llm_clients]
+ self.n_coders = len(llm_clients)
+ self.majority_threshold = majority_threshold or math.ceil(self.n_coders / 2)
+
+ def code(
+ self,
+ texts: list[str],
+ themes: list[Theme],
+ ) -> ConsensusReport:
+ """Synchronous consensus coding — runs coders sequentially.
+
+ Parameters
+ ----------
+ texts : list[str]
+ Text segments to code.
+ themes : list[Theme]
+ Theme definitions.
+
+ Returns
+ -------
+ ConsensusReport
+ """
+ all_reports: list[CodingReport] = []
+ for coder in self.coders:
+ report = coder.code(texts, themes)
+ all_reports.append(report)
+
+ return self._merge_results(all_reports, texts)
+
+ async def code_async(
+ self,
+ texts: list[str],
+ themes: list[Theme],
+ batch_size: int = 50,
+ ) -> ConsensusReport:
+ """Async consensus coding — runs coders in parallel.
+
+ Parameters
+ ----------
+ texts : list[str]
+ themes : list[Theme]
+ batch_size : int
+ Batch size for each coder's async coding.
+
+ Returns
+ -------
+ ConsensusReport
+ """
+ tasks = [
+ coder.code_async(texts, themes, batch_size)
+ for coder in self.coders
+ ]
+ all_reports = await asyncio.gather(*tasks, return_exceptions=True)
+
+ # Filter out exceptions
+ valid_reports: list[CodingReport] = []
+ for r in all_reports:
+ if isinstance(r, CodingReport):
+ valid_reports.append(r)
+
+ if len(valid_reports) < 2:
+ raise RuntimeError(
+ f"Only {len(valid_reports)}/{self.n_coders} coders succeeded. "
+ "Need at least 2 for consensus."
+ )
+
+ return self._merge_results(valid_reports, texts)
+
+ # ------------------------------------------------------------------
+ # Internal: merge results
+ # ------------------------------------------------------------------
+
+ def _merge_results(
+ self,
+ all_reports: list[CodingReport],
+ texts: list[str],
+ ) -> ConsensusReport:
+ """Apply majority vote to merge results from multiple coders."""
+ n_coders = len(all_reports)
+ threshold = self.majority_threshold
+ n_total = len(texts)
+
+ segments: list[SegmentConsensus] = []
+ theme_dist: dict[str, int] = Counter()
+ n_failed = 0
+ agreement_sum = 0.0
+
+ for idx in range(n_total):
+ # Gather individual results for this segment
+ individual: list[CodingResult] = []
+ for report in all_reports:
+ if idx < len(report.results):
+ individual.append(report.results[idx])
+ else:
+ individual.append(CodingResult(text_id=idx, text=texts[idx]))
+
+ # Majority vote
+ consensus_themes, avg_confs, votes = self._majority_vote(
+ individual, threshold,
+ )
+
+ # Agreement rate: for each theme mentioned by any coder,
+ # agreement = votes / n_coders. Average across all mentioned themes.
+ if votes:
+ all_mentioned_themes = set(votes.keys())
+ rates = [votes[t] / n_coders for t in all_mentioned_themes]
+ seg_agreement = sum(rates) / len(rates)
+ else:
+ # All coders agree: no themes
+ seg_agreement = 1.0
+
+ agreement_sum += seg_agreement
+
+ for t in consensus_themes:
+ theme_dist[t] = theme_dist.get(t, 0) + 1
+
+ if not any(r.themes for r in individual):
+ n_failed += 1
+
+ segments.append(SegmentConsensus(
+ text_id=idx,
+ text=texts[idx],
+ consensus_themes=consensus_themes,
+ consensus_confidences=avg_confs,
+ agreement_rate=round(seg_agreement, 4),
+ individual_results=individual,
+ vote_counts=votes,
+ ))
+
+ overall_agreement = agreement_sum / n_total if n_total > 0 else 0.0
+
+ # Cost tracking
+ coder_models = []
+ per_coder_cost = []
+ for i, client in enumerate(self.llm_clients[:n_coders]):
+ coder_models.append(f"{client.backend}:{client.model}")
+ # Sum cost from call log
+ cost = sum(log.cost_usd for log in client.call_log) if client.call_log else 0.0
+ per_coder_cost.append(cost)
+
+ return ConsensusReport(
+ segments=segments,
+ n_coders=n_coders,
+ coder_models=coder_models,
+ n_total=n_total,
+ n_coded=n_total - n_failed,
+ n_failed=n_failed,
+ overall_agreement=round(overall_agreement, 4),
+ theme_distribution=dict(theme_dist),
+ per_coder_cost=per_coder_cost,
+ total_cost=sum(per_coder_cost),
+ )
+
+ @staticmethod
+ def _majority_vote(
+ individual_results: list[CodingResult],
+ threshold: int,
+ ) -> tuple[list[str], dict[str, float], dict[str, int]]:
+ """Compute majority vote for a single segment.
+
+ Returns
+ -------
+ consensus_themes : list[str]
+ Themes meeting the majority threshold.
+ avg_confidences : dict[str, float]
+ Average confidence among coders who assigned each consensus theme.
+ vote_counts : dict[str, int]
+ Theme -> number of coders who assigned it (for ALL mentioned themes).
+ """
+ # Count votes for each theme
+ vote_counts: dict[str, int] = Counter()
+ confidence_sums: dict[str, float] = {}
+ confidence_counts: dict[str, int] = {}
+
+ for result in individual_results:
+ for theme in result.themes:
+ vote_counts[theme] += 1
+ confidence_sums[theme] = confidence_sums.get(theme, 0.0) + result.confidences.get(theme, 0.5)
+ confidence_counts[theme] = confidence_counts.get(theme, 0) + 1
+
+ # Filter by threshold
+ consensus_themes = [
+ t for t, count in vote_counts.items() if count >= threshold
+ ]
+ consensus_themes.sort()
+
+ # Average confidence for consensus themes
+ avg_confidences = {}
+ for t in consensus_themes:
+ avg_confidences[t] = round(
+ confidence_sums[t] / confidence_counts[t], 3
+ )
+
+ return consensus_themes, avg_confidences, dict(vote_counts)
+
+ # ------------------------------------------------------------------
+ # Formatting
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def format_report(report: ConsensusReport, lang: str = "zh") -> str:
+ """Format a ConsensusReport as a human-readable string."""
+ lines: list[str] = []
+
+ if lang == "zh":
+ lines.append("═══ 多模型共识编码报告 ═══")
+ lines.append("")
+ lines.append(f"编码模型数量：{report.n_coders}")
+ lines.append(f"模型列表：{', '.join(report.coder_models)}")
+ lines.append(f"总文本数：{report.n_total}")
+ lines.append(f"成功编码：{report.n_coded}")
+ lines.append(f"总体一致性：{report.overall_agreement:.2%}")
+ lines.append(f"总费用：${report.total_cost:.4f}")
+ else:
+ lines.append("═══ Multi-LLM Consensus Coding Report ═══")
+ lines.append("")
+ lines.append(f"Number of coders: {report.n_coders}")
+ lines.append(f"Models: {', '.join(report.coder_models)}")
+ lines.append(f"Total segments: {report.n_total}")
+ lines.append(f"Successfully coded: {report.n_coded}")
+ lines.append(f"Overall agreement: {report.overall_agreement:.2%}")
+ lines.append(f"Total cost: ${report.total_cost:.4f}")
+
+ lines.append("")
+
+ if report.theme_distribution:
+ dist_label = "主题分布" if lang == "zh" else "Theme Distribution"
+ lines.append(f"── {dist_label} ──")
+ for theme, count in sorted(
+ report.theme_distribution.items(), key=lambda x: -x[1],
+ ):
+ lines.append(f" {theme}: {count}")
+ lines.append("")
+
+ return "\n".join(lines)
diff --git a/socialscikit/qualikit/extraction_reviewer.py b/socialscikit/qualikit/extraction_reviewer.py
index 70a1ba9..cb6e785 100644
--- a/socialscikit/qualikit/extraction_reviewer.py
+++ b/socialscikit/qualikit/extraction_reviewer.py
@@ -216,5 +216,6 @@ def export_to_dataframe(self, session: ExtractionReviewSession) -> pd.DataFrame:
"paragraph_index": pos.paragraph_index if pos else 0,
"review_status": item.action.value,
"reasoning": item.result.reasoning,
+ "evidence_span": getattr(item.result, "evidence_span", ""),
})
return pd.DataFrame(rows) if rows else pd.DataFrame()
diff --git a/socialscikit/qualikit/segment_extractor.py b/socialscikit/qualikit/segment_extractor.py
index 22985b0..4e4b38d 100644
--- a/socialscikit/qualikit/segment_extractor.py
+++ b/socialscikit/qualikit/segment_extractor.py
@@ -46,6 +46,7 @@ class ExtractionResult:
sub_theme: str # sub-theme label
confidence: float # 0.0 – 1.0
reasoning: str = ""
+ evidence_span: str = "" # verbatim quote from text supporting the coding
position: TextPosition | None = None
@@ -87,9 +88,10 @@ class ExtractionReport:
2. 只在文本明确涉及时才标记，避免过度匹配
3. 不相关的段落不要包含在结果中
4. 如果研究问题指定了子主题列表，sub_theme 必须从中选择；如果标注了「自动生成」，请自行生成 2-8 字的子主题标签
+5. evidence_span: 从原文中逐字复制最能支持该编码判断的关键短语或句子（原文原样，不要改写）
仅返回 JSON，不要 markdown 代码块：
-{{"matches": [{{"segment_id": 1, "rq_id": "RQ1", "sub_theme": "子主题名", "confidence": 0.85, "reasoning": "简要依据"}}]}}"""
+{{"matches": [{{"segment_id": 1, "rq_id": "RQ1", "sub_theme": "子主题名", "confidence": 0.85, "reasoning": "简要依据", "evidence_span": "原文中的关键句"}}]}}"""
# ---------------------------------------------------------------------------
@@ -274,6 +276,7 @@ def _parse_response(
sub_theme=sub_theme or "未分类",
confidence=round(confidence, 3),
reasoning=str(item.get("reasoning", "")).strip(),
+ evidence_span=str(item.get("evidence_span", "")).strip(),
))
return results
diff --git a/socialscikit/ui/i18n.py b/socialscikit/ui/i18n.py
index d05d390..5d8a71f 100644
--- a/socialscikit/ui/i18n.py
+++ b/socialscikit/ui/i18n.py
@@ -23,8 +23,8 @@ def t(key: str, lang: str = "en") -> str:
"zh": "# SocialSciKit",
},
"landing.subtitle": {
- "en": "Zero-code text analysis toolkit for social science researchers",
- "zh": "面向社会科学研究者的零代码文本分析工具",
+ "en": "Zero-code text analysis & research methods toolkit for social science researchers",
+ "zh": "面向社会科学研究者的零代码文本分析与研究方法工具包",
},
"landing.quantikit_card": {
"en": (
@@ -70,18 +70,36 @@ def t(key: str, lang: str = "en") -> str:
"- 摘录表 + 共现矩阵 + 分析备忘录\n"
),
},
+ "landing.toolbox_card": {
+ "en": (
+ "### Toolbox — Research Methods Tools\n\n"
+ "**Standalone tools** that work independently or together with QuantiKit / QualiKit:\n\n"
+ "- **ICR Calculator** — Inter-coder reliability (Cohen's Kappa, Krippendorff's Alpha, multi-label Jaccard); supports 2+ coders with auto metric selection\n"
+ "- **Consensus Coding** — Multi-LLM majority-vote coding with 2\u20135 configurable LLM backends\n"
+ "- **Methods Generator** — Auto-generate methods section paragraphs (EN/ZH) from pipeline logs or manual input\n"
+ ),
+ "zh": (
+ "### 工具箱 — 研究方法工具\n\n"
+ "**独立工具**，可单独使用或与 QuantiKit / QualiKit 搭配：\n\n"
+ "- **ICR 计算器** — 编码者间信度（Cohen's Kappa、Krippendorff's Alpha、多标签 Jaccard）；支持 2+ 编码者自动选择指标\n"
+ "- **共识编码** — 多 LLM 多数投票编码，支持 2\u20135 个可配置的 LLM 后端\n"
+ "- **方法论生成器** — 从流水线日志或手动输入自动生成方法部分段落（中英双语）\n"
+ ),
+ },
"landing.quickstart": {
"en": (
"### Quick Start\n\n"
- "1. Click the **QuantiKit** or **QualiKit** tab above to enter the corresponding module\n"
+ "1. Click the **QuantiKit**, **QualiKit**, or **Toolbox** tab above to enter the corresponding module\n"
"2. Follow the numbered steps in order \u2014 results at each step can be reviewed and edited before proceeding\n"
- "3. When LLM features are needed, provide an API Key (OpenAI / Anthropic) or use Ollama for local inference"
+ "3. When LLM features are needed, provide an API Key (OpenAI / Anthropic) or use Ollama for local inference\n"
+ "4. Use the **Toolbox** for standalone tools: ICR calculation, multi-LLM consensus coding, or auto-generating methods sections"
),
"zh": (
"### 快速开始\n\n"
- "1. 点击上方 **QuantiKit** 或 **QualiKit** 标签页进入对应模块\n"
+ "1. 点击上方 **QuantiKit**、**QualiKit** 或 **工具箱** 标签页进入对应模块\n"
"2. 按步骤编号依次操作 \u2014 每步结果可审核编辑，确认后再进入下一步\n"
- "3. 需要 LLM 功能时提供 API Key（OpenAI / Anthropic），或使用 Ollama 本地推理"
+ "3. 需要 LLM 功能时提供 API Key（OpenAI / Anthropic），或使用 Ollama 本地推理\n"
+ "4. 使用**工具箱**可独立进行：ICR 信度计算、多 LLM 共识编码、自动生成方法论段落"
),
},
"landing.examples": {
@@ -90,14 +108,20 @@ def t(key: str, lang: str = "en") -> str:
"`examples/sentiment_example.csv` (sentiment classification \u00b7 QuantiKit) \u00b7 "
"`examples/policy_example.csv` (policy instrument classification \u00b7 QuantiKit) \u00b7 "
"`examples/interview_example.txt` (single interview \u00b7 QualiKit) \u00b7 "
- "`examples/interview_focus_group.txt` (focus group \u00b7 QualiKit)"
+ "`examples/interview_focus_group.txt` (focus group \u00b7 QualiKit) \u00b7 "
+ "`examples/icr_example.csv` (inter-coder reliability \u00b7 Toolbox) \u00b7 "
+ "`examples/consensus_example.csv` (consensus coding \u00b7 Toolbox) \u00b7 "
+ "`examples/methods_log_quantikit.json` / `methods_log_qualikit.json` (methods generator \u00b7 Toolbox)"
),
"zh": (
"**示例数据：**\n"
"`examples/sentiment_example.csv`（情感分类 \u00b7 QuantiKit）\u00b7 "
"`examples/policy_example.csv`（政策工具分类 \u00b7 QuantiKit）\u00b7 "
"`examples/interview_example.txt`（单人访谈 \u00b7 QualiKit）\u00b7 "
- "`examples/interview_focus_group.txt`（焦点小组 \u00b7 QualiKit）"
+ "`examples/interview_focus_group.txt`（焦点小组 \u00b7 QualiKit）\u00b7 "
+ "`examples/icr_example.csv`（编码者间信度 \u00b7 工具箱）\u00b7 "
+ "`examples/consensus_example.csv`（共识编码 \u00b7 工具箱）\u00b7 "
+ "`examples/methods_log_quantikit.json` / `methods_log_qualikit.json`（方法论生成 \u00b7 工具箱）"
),
},
"landing.references": {
@@ -536,6 +560,22 @@ def t(key: str, lang: str = "en") -> str:
"en": "Add",
"zh": "添加",
},
+ "ql.s5.charts_title": {
+ "en": "Visualization Dashboard",
+ "zh": "可视化仪表盘",
+ },
+ "ql.s5.review_progress": {
+ "en": "Review Progress",
+ "zh": "审查进度",
+ },
+ "ql.s5.confidence": {
+ "en": "Confidence Distribution",
+ "zh": "置信度分布",
+ },
+ "ql.s5.themes": {
+ "en": "Theme Distribution",
+ "zh": "主题分布",
+ },
# ======================================================================
# QualiKit Step 6 - Export
@@ -1360,6 +1400,18 @@ def t(key: str, lang: str = "en") -> str:
"en": "Evaluation report",
"zh": "评估报告",
},
+ "qt.s5.confusion_matrix": {
+ "en": "Confusion Matrix",
+ "zh": "混淆矩阵",
+ },
+ "qt.s5.per_class": {
+ "en": "Per-Class Metrics",
+ "zh": "各类别指标",
+ },
+ "qt.s5.text_report": {
+ "en": "Full Text Report",
+ "zh": "完整文字报告",
+ },
# ======================================================================
# QuantiKit Step 6 - Export
@@ -1431,4 +1483,279 @@ def t(key: str, lang: str = "en") -> str:
"en": "API Key",
"zh": "API Key",
},
+
+ # ======================================================================
+ # Inter-Coder Reliability (ICR)
+ # ======================================================================
+
+ "icr.title": {
+ "en": "Inter-Coder Reliability",
+ "zh": "编码者间信度",
+ },
+ "icr.description": {
+ "en": "Compute agreement metrics between two sets of labels or coders.",
+ "zh": "计算两组标签或编码者之间的一致性指标。",
+ },
+ "icr.upload_second_labels": {
+ "en": "Upload second coder's labels (CSV)",
+ "zh": "上传第二编码者标签（CSV）",
+ },
+ "icr.second_label_col": {
+ "en": "Second coder label column",
+ "zh": "第二编码者标签列名",
+ },
+ "icr.compute_btn": {
+ "en": "Compute ICR",
+ "zh": "计算编码者间信度",
+ },
+ "icr.report": {
+ "en": "ICR Report",
+ "zh": "信度报告",
+ },
+ "icr.human_vs_llm": {
+ "en": "Human vs LLM Agreement",
+ "zh": "人工 vs LLM 一致性",
+ },
+ "icr.human_vs_llm_desc": {
+ "en": "Compare human-reviewed themes against original LLM coding.",
+ "zh": "对比人工审核主题与原始 LLM 编码结果。",
+ },
+ "icr.compute_human_llm_btn": {
+ "en": "Compute Human vs LLM ICR",
+ "zh": "计算人工 vs LLM 信度",
+ },
+ "msg.run_eval_first": {
+ "en": "Please run evaluation first.",
+ "zh": "请先运行评估。",
+ },
+ "msg.no_review_data": {
+ "en": "No review data available. Please complete coding and review first.",
+ "zh": "暂无审核数据，请先完成编码和审核。",
+ },
+
+ # ======================================================================
+ # Multi-LLM Consensus Coding
+ # ======================================================================
+
+ "consensus.title": {
+ "en": "Consensus Coding (Multi-LLM)",
+ "zh": "共识编码（多模型）",
+ },
+ "consensus.description": {
+ "en": (
+ "Run 2–3 LLMs independently on the same segments. "
+ "Themes are retained only when a majority of models agree."
+ ),
+ "zh": (
+ "使用 2–3 个 LLM 分别独立编码同一批文本，"
+ "仅保留多数模型一致同意的主题。"
+ ),
+ },
+ "consensus.backend_n": {
+ "en": "LLM {} Backend",
+ "zh": "LLM {} 后端",
+ },
+ "consensus.model_n": {
+ "en": "Model {}",
+ "zh": "模型 {}",
+ },
+ "consensus.api_key_n": {
+ "en": "API Key {}",
+ "zh": "API Key {}",
+ },
+ "consensus.run_btn": {
+ "en": "Run Consensus Coding",
+ "zh": "运行共识编码",
+ },
+ "consensus.summary": {
+ "en": "Consensus Summary",
+ "zh": "共识摘要",
+ },
+ "consensus.results": {
+ "en": "Consensus Results",
+ "zh": "共识结果",
+ },
+ "consensus.agreement": {
+ "en": "Agreement Report",
+ "zh": "一致性报告",
+ },
+ "msg.at_least_two_llms": {
+ "en": "Please configure at least 2 LLMs for consensus coding.",
+ "zh": "共识编码需要至少配置 2 个 LLM。",
+ },
+ "msg.consensus_done": {
+ "en": "Consensus coding complete. {} segments coded with {} models.",
+ "zh": "共识编码完成。{} 条文本使用 {} 个模型编码。",
+ },
+ "msg.lock_themes_first": {
+ "en": "Please define and lock themes first.",
+ "zh": "请先定义并锁定主题框架。",
+ },
+
+ # ======================================================================
+ # Methods Section Auto-generation
+ # ======================================================================
+
+ "methods.title": {
+ "en": "Methods Section Generator",
+ "zh": "方法论段落生成",
+ },
+ "methods.description": {
+ "en": "Generate a Methods paragraph draft for your paper based on the analysis pipeline.",
+ "zh": "根据分析流程自动生成论文方法论段落草稿。",
+ },
+ "methods.generate_btn": {
+ "en": "Generate Methods Section",
+ "zh": "生成方法论段落",
+ },
+ "methods.text_en": {
+ "en": "Methods (English)",
+ "zh": "方法论（英文）",
+ },
+ "methods.text_zh": {
+ "en": "Methods (Chinese)",
+ "zh": "方法论（中文）",
+ },
+ "methods.copy_hint": {
+ "en": "Auto-generated draft. Copy, edit, and cite appropriately before publication.",
+ "zh": "自动生成草稿，请复制后编辑，并在发表前适当引用。",
+ },
+ "methods.no_data": {
+ "en": "Please complete the analysis pipeline before generating.",
+ "zh": "请先完成分析流程再生成。",
+ },
+
+ # ======================================================================
+ # Toolbox
+ # ======================================================================
+
+ "toolbox.title": {
+ "en": "Toolbox",
+ "zh": "工具箱",
+ },
+ "toolbox.description": {
+ "en": "Standalone research tools — ICR Calculator, Multi-LLM Consensus Coding, and Methods Section Generator. These tools work independently from QuantiKit and QualiKit.",
+ "zh": "独立研究工具 — 编码者间信度计算、多模型共识编码、方法论段落生成。这些工具独立于 QuantiKit 和 QualiKit 使用。",
+ },
+ "toolbox.icr_tab": {
+ "en": "ICR Calculator",
+ "zh": "编码者间信度",
+ },
+ "toolbox.consensus_tab": {
+ "en": "Consensus Coding",
+ "zh": "共识编码",
+ },
+ "toolbox.methods_tab": {
+ "en": "Methods Generator",
+ "zh": "方法论生成",
+ },
+ "toolbox.import_log": {
+ "en": "Import Pipeline Log",
+ "zh": "导入流水线日志",
+ },
+ "toolbox.export_log": {
+ "en": "Export Pipeline Log",
+ "zh": "导出流水线日志",
+ },
+ "toolbox.manual_input": {
+ "en": "Manual Input (without log file)",
+ "zh": "手动输入（无日志文件时使用）",
+ },
+ "toolbox.pipeline_type": {
+ "en": "Pipeline Type",
+ "zh": "流水线类型",
+ },
+ "toolbox.icr_upload": {
+ "en": "Upload Labels CSV (each column = one coder)",
+ "zh": "上传标签 CSV（每列代表一个编码者）",
+ },
+ "toolbox.icr_file_info": {
+ "en": "File info",
+ "zh": "文件信息",
+ },
+ "toolbox.icr_select_cols": {
+ "en": "Select coder columns",
+ "zh": "选择编码者列",
+ },
+ "toolbox.icr_select_cols_info": {
+ "en": "Pick 2+ columns. 2 coders → Cohen's Kappa; 3+ coders → Krippendorff's Alpha.",
+ "zh": "选择 2 列以上。2 人 → Cohen's Kappa；3 人以上 → Krippendorff's Alpha。",
+ },
+ "toolbox.icr_mode": {
+ "en": "Label Mode",
+ "zh": "标签模式",
+ },
+ "toolbox.icr_mode_info": {
+ "en": "single-label: one label per cell. multi-label: comma-separated values per cell.",
+ "zh": "单标签：每格一个标签。多标签：每格用逗号分隔多个标签。",
+ },
+ "toolbox.add_llm": {
+ "en": "+ Add LLM",
+ "zh": "+ 添加模型",
+ },
+ "toolbox.remove_llm": {
+ "en": "- Remove LLM",
+ "zh": "- 移除模型",
+ },
+ "toolbox.data_file": {
+ "en": "Data File (CSV)",
+ "zh": "数据文件（CSV）",
+ },
+ "toolbox.text_col": {
+ "en": "Text Column",
+ "zh": "文本列名",
+ },
+ "toolbox.themes_input": {
+ "en": "Themes (one per line, format: name: description)",
+ "zh": "主题（每行一个，格式：名称: 描述）",
+ },
+ "toolbox.download_example": {
+ "en": "Download Example",
+ "zh": "下载示例",
+ },
+ "toolbox.example_file": {
+ "en": "Example File",
+ "zh": "示例文件",
+ },
+ "toolbox.example_qt_log": {
+ "en": "QuantiKit Log Example",
+ "zh": "QuantiKit 日志示例",
+ },
+ "toolbox.example_ql_log": {
+ "en": "QualiKit Log Example",
+ "zh": "QualiKit 日志示例",
+ },
+
+ # ======================================================================
+ # Project Save / Load
+ # ======================================================================
+
+ "project.save_title": {
+ "en": "Save Project",
+ "zh": "保存项目",
+ },
+ "project.save_btn": {
+ "en": "Save Project File",
+ "zh": "保存项目文件",
+ },
+ "project.load_title": {
+ "en": "Load Saved Project",
+ "zh": "加载已保存的项目",
+ },
+ "project.load_desc": {
+ "en": "Upload a previously saved `.json` project file to restore your progress.",
+ "zh": "上传之前保存的 `.json` 项目文件以恢复进度。",
+ },
+ "project.load_btn": {
+ "en": "Load Project",
+ "zh": "加载项目",
+ },
+ "project.file": {
+ "en": "Project File",
+ "zh": "项目文件",
+ },
+ "project.status": {
+ "en": "Status",
+ "zh": "状态",
+ },
}
diff --git a/socialscikit/ui/main_app.py b/socialscikit/ui/main_app.py
index e871d36..8891e33 100644
--- a/socialscikit/ui/main_app.py
+++ b/socialscikit/ui/main_app.py
@@ -14,10 +14,12 @@
import pandas as pd
from socialscikit.core.data_loader import get_template_path
+from socialscikit.core.project_io import save_project, load_project
from socialscikit.quantikit.feature_extractor import TASK_TYPES
import socialscikit.ui.quantikit_app as qn
import socialscikit.ui.qualikit_app as ql
+import socialscikit.ui.toolbox_app as tb
from socialscikit.ui.i18n import t, LANGUAGES
# ---------------------------------------------------------------------------
@@ -116,11 +118,15 @@
/* Initial fix for file-drop text (browser locale → Chinese) */
fixDropText();
- new MutationObserver(fixDropText).observe(document.body, {
- childList: true, subtree: true
- });
+ /* Body-wide observer: re-apply drop-text AND tab labels on any DOM change.
+ Needed because Gradio re-renders tabs individually when selected, which
+ reverts our translations. Cheap text-only updates — no layout work. */
+ new MutationObserver(() => {
+ fixDropText();
+ switchTabs();
+ }).observe(document.body, { childList: true, subtree: true });
/* Fallback for stubborn Gradio re-renders */
- setInterval(fixDropText, 3000);
+ setInterval(() => { fixDropText(); switchTabs(); }, 2000);
}
"""
@@ -408,16 +414,35 @@
.hero-title h1 { font-size: 2rem !important; color: #222 !important; margin-bottom: 0.25rem !important; }
.hero-sub { text-align: center; color: #888 !important; font-size: 1rem !important; margin-bottom: 2rem; }
-/* ---- Module cards — just a subtle bg, no border ---- */
+/* ---- Module cards — subtle bg with colored left border ---- */
.module-card {
border: none !important;
border-radius: 8px;
padding: 1.5rem;
background: #f7f7f8;
+ border-left: 4px solid #4A90D9;
+ transition: transform 0.15s ease, box-shadow 0.15s ease;
+}
+.module-card:nth-child(2) { border-left-color: #5BA88D; }
+.module-card:nth-child(3) { border-left-color: #E8734A; }
+.module-card:hover {
+ transform: translateY(-2px);
+ box-shadow: 0 4px 12px rgba(0,0,0,0.06) !important;
}
.module-card h3 { margin-top: 0 !important; color: #222 !important; }
.module-card strong { color: #222 !important; }
.module-card p, .module-card li { color: #444 !important; }
+
+/* ---- Plot containers — clean borders ---- */
+.plot-container {
+ border: 1px solid #f0f0f0 !important;
+ border-radius: 8px !important;
+ overflow: hidden;
+}
+/* Ensure plots render on white background */
+.plot-container img, .plot-container canvas {
+ background: #ffffff !important;
+}
"""
# ---------------------------------------------------------------------------
@@ -534,12 +559,15 @@ def _build_landing(lang: str = "en") -> str:
'
\n\n'
f'{t("landing.subtitle", lang)}\n\n'
'
\n\n---\n\n'
- '
\n'
+ '
\n'
'
\n\n'
f'{t("landing.quantikit_card", lang)}\n\n'
'
\n'
'
\n\n'
f'{t("landing.qualikit_card", lang)}\n\n'
+ '
\n'
+ '
\n\n'
+ f'{t("landing.toolbox_card", lang)}\n\n'
'
\n
\n\n'
f'{t("landing.quickstart", lang)}\n\n'
f'{t("landing.examples", lang)}\n\n'
@@ -647,6 +675,23 @@ def create_app() -> gr.Blocks:
with gr.Tab("Home"):
landing_md = gr.Markdown(value=_build_landing("en"))
+ with gr.Accordion(t("project.load_title", "en"), open=False):
+ gr.Markdown(t("project.load_desc", "en"))
+ with gr.Row():
+ proj_load_file = gr.File(
+ label=t("project.file", "en"),
+ file_types=[".json"],
+ )
+ with gr.Column(scale=0, min_width=160):
+ proj_load_btn = gr.Button(
+ t("project.load_btn", "en"),
+ variant="primary", size="sm",
+ )
+ proj_load_msg = gr.Textbox(
+ label=t("project.status", "en"),
+ interactive=False, lines=1,
+ )
+
# ==================================================================
# QuantiKit
# ==================================================================
@@ -715,7 +760,9 @@ def create_app() -> gr.Blocks:
qa_labels = gr.Textbox(label=t("qt.s3.labels", "en"), value="positive, negative, neutral", placeholder=t("qt.s3.labels_placeholder", "en"))
qa_shuf = gr.Checkbox(label=t("qt.s3.shuffle", "en"), value=False)
qa_create = gr.Button(t("qt.s3.create_session", "en"), variant="primary")
- qa_stats = gr.Textbox(label=t("qt.s3.progress", "en"), interactive=False)
+ with gr.Row():
+ qa_stats = gr.Textbox(label=t("qt.s3.progress", "en"), interactive=False, scale=3)
+ qa_ann_plot = gr.Plot(label="", scale=1, min_width=180)
qa_idx = gr.Textbox(label=t("qt.s3.current_pos", "en"), interactive=False)
qa_text = gr.Textbox(label=t("qt.s3.text_to_annotate", "en"), lines=4, interactive=False)
with gr.Row():
@@ -741,12 +788,18 @@ def create_app() -> gr.Blocks:
qa_mmsg = gr.Textbox(label="", interactive=False, show_label=False)
_so = [qt_ann_session, qa_stats, qa_text, qa_idx, qa_msg]
+ _ann_chart_then = dict(fn=qn._make_annotation_chart, inputs=[qt_ann_session], outputs=[qa_ann_plot])
qa_create.click(fn=qn._create_annotation_session,
- inputs=[qt_df, qt_tcol, qt_lcol, qa_labels, qa_shuf], outputs=_so)
- qa_sub.click(fn=qn._annotate_item, inputs=[qt_ann_session, qa_input], outputs=_so)
- qa_skip.click(fn=qn._skip_item, inputs=[qt_ann_session], outputs=_so)
- qa_flag.click(fn=qn._flag_item, inputs=[qt_ann_session, qa_fnote], outputs=_so)
- qa_undo.click(fn=qn._undo_annotation, inputs=[qt_ann_session], outputs=_so)
+ inputs=[qt_df, qt_tcol, qt_lcol, qa_labels, qa_shuf], outputs=_so
+ ).then(**_ann_chart_then)
+ qa_sub.click(fn=qn._annotate_item, inputs=[qt_ann_session, qa_input], outputs=_so
+ ).then(**_ann_chart_then)
+ qa_skip.click(fn=qn._skip_item, inputs=[qt_ann_session], outputs=_so
+ ).then(**_ann_chart_then)
+ qa_flag.click(fn=qn._flag_item, inputs=[qt_ann_session, qa_fnote], outputs=_so
+ ).then(**_ann_chart_then)
+ qa_undo.click(fn=qn._undo_annotation, inputs=[qt_ann_session], outputs=_so
+ ).then(**_ann_chart_then)
qa_exp.click(fn=qn._export_annotations, inputs=[qt_ann_session, qa_all], outputs=[qa_tbl, qa_msg])
qa_dl.click(fn=qn._download_annotations_csv, inputs=[qt_ann_session, qa_all], outputs=[qa_dlf])
qa_merge.click(fn=qn._update_main_df_from_annotations,
@@ -908,8 +961,15 @@ def _ft_wrap(df, tc, lc, m, bs, ep, lr):
with gr.Tab("Step 5 · Evaluation"):
qt_s5_md = gr.Markdown(t("qt.s5.title", "en"))
qt_ebtn = gr.Button(t("qt.s5.run_btn", "en"), variant="primary")
- qt_eout = gr.Textbox(label=t("qt.s5.report", "en"), lines=18, interactive=False)
- qt_ebtn.click(fn=qn._evaluate_results, inputs=[qt_result_df, qt_df, qt_lcol], outputs=[qt_eout])
+ qt_metrics_html = gr.HTML()
+ with gr.Row():
+ qt_cm_plot = gr.Plot(label=t("qt.s5.confusion_matrix", "en"))
+ qt_pc_plot = gr.Plot(label=t("qt.s5.per_class", "en"))
+ qt_s5_text_acc = gr.Accordion(t("qt.s5.text_report", "en"), open=False)
+ with qt_s5_text_acc:
+ qt_eout = gr.Textbox(label=t("qt.s5.report", "en"), lines=18, interactive=False)
+ qt_ebtn.click(fn=qn._evaluate_results, inputs=[qt_result_df, qt_df, qt_lcol],
+ outputs=[qt_eout, qt_metrics_html, qt_cm_plot, qt_pc_plot])
# -- 6. Export ------------------------------------------------
with gr.Tab("Step 6 · Export"):
@@ -918,6 +978,18 @@ def _ft_wrap(df, tc, lc, m, bs, ep, lr):
qt_xfile = gr.File(label=t("qt.s6.file", "en"))
qt_xbtn.click(fn=qn._export_results, inputs=[qt_result_df], outputs=[qt_xfile])
+ qt_log_btn = gr.Button(t("toolbox.export_log", "en"), variant="secondary")
+ qt_log_file = gr.File(label="Pipeline Log")
+ qt_log_btn.click(fn=qn._export_pipeline_log,
+ inputs=[qt_result_df, qt_df, qt_lcol],
+ outputs=[qt_log_file])
+
+ with gr.Accordion(t("project.save_title", "en"), open=False):
+ proj_qt_save_btn = gr.Button(
+ t("project.save_btn", "en"), variant="secondary",
+ )
+ proj_qt_save_file = gr.File(label=t("project.file", "en"))
+
# ==================================================================
# QualiKit
# ==================================================================
@@ -1143,6 +1215,13 @@ def _ft_wrap(df, tc, lc, m, bs, ep, lr):
ql_rev_stats = gr.Textbox(label=t("ql.s5.stats", "en"), interactive=False)
ql_rev_tbl = gr.Dataframe(label=t("ql.s5.table", "en"), interactive=False)
+ with gr.Accordion(t("ql.s5.charts_title", "en"), open=True):
+ ql_rev_cards = gr.HTML()
+ with gr.Row():
+ ql_review_plot = gr.Plot(label=t("ql.s5.review_progress", "en"))
+ ql_conf_plot = gr.Plot(label=t("ql.s5.confidence", "en"))
+ ql_theme_plot = gr.Plot(label=t("ql.s5.themes", "en"))
+
ql_s5_detail_md = gr.Markdown("---\n" + t("ql.s5.detail_title", "en"))
with gr.Row():
ql_rev_idx = gr.Number(label=t("ql.s5.index", "en"), precision=0, value=0, minimum=0)
@@ -1186,6 +1265,20 @@ def _ft_wrap(df, tc, lc, m, bs, ep, lr):
outputs=[ql_xl_file, ql_xl_msg],
)
+ ql_log_btn = gr.Button(t("toolbox.export_log", "en"), variant="secondary")
+ ql_log_file = gr.File(label="Pipeline Log")
+ ql_log_btn.click(
+ fn=ql._export_pipeline_log,
+ inputs=[ql_ext_session, ql_rqs, ql_ext_session],
+ outputs=[ql_log_file],
+ )
+
+ with gr.Accordion(t("project.save_title", "en"), open=False):
+ proj_ql_save_btn = gr.Button(
+ t("project.save_btn", "en"), variant="secondary",
+ )
+ proj_ql_save_file = gr.File(label=t("project.file", "en"))
+
# ==========================================================
# Cross-tab event wiring
# (placed here so all components are defined)
@@ -1199,12 +1292,18 @@ def _ft_wrap(df, tc, lc, m, bs, ep, lr):
)
# Step 4 → extraction outputs to Step 5 review table
+ _ql_chart_out = [ql_rev_cards, ql_review_plot, ql_conf_plot, ql_theme_plot]
+
ql_ext_btn.click(
fn=ql._run_extraction_v2,
inputs=[ql_raw_text, ql_segments, ql_rqs,
ql_ext_be, ql_ext_mod, ql_ext_key],
outputs=[ql_ext_session, ql_ext_msg, ql_ext_tbl,
ql_rev_tbl, ql_rev_stats],
+ ).then(
+ fn=ql._make_ql_charts,
+ inputs=[ql_ext_session],
+ outputs=_ql_chart_out,
)
# Step 5 — review events
@@ -1222,6 +1321,10 @@ def _ft_wrap(df, tc, lc, m, bs, ep, lr):
fn=ql._ext_show_context,
inputs=[ql_ext_session, ql_rev_idx, ql_raw_text],
outputs=[ql_rev_ctx],
+ ).then(
+ fn=ql._make_ql_charts,
+ inputs=[ql_ext_session],
+ outputs=_ql_chart_out,
)
ql_rev_rej.click(
fn=ql._ext_reject, inputs=[ql_ext_session, ql_rev_idx],
@@ -1230,6 +1333,10 @@ def _ft_wrap(df, tc, lc, m, bs, ep, lr):
fn=ql._ext_show_context,
inputs=[ql_ext_session, ql_rev_idx, ql_raw_text],
outputs=[ql_rev_ctx],
+ ).then(
+ fn=ql._make_ql_charts,
+ inputs=[ql_ext_session],
+ outputs=_ql_chart_out,
)
ql_rev_edit_btn.click(
fn=ql._ext_edit,
@@ -1239,11 +1346,19 @@ def _ft_wrap(df, tc, lc, m, bs, ep, lr):
fn=ql._ext_show_context,
inputs=[ql_ext_session, ql_rev_idx, ql_raw_text],
outputs=[ql_rev_ctx],
+ ).then(
+ fn=ql._make_ql_charts,
+ inputs=[ql_ext_session],
+ outputs=_ql_chart_out,
)
ql_rev_bulk.click(
fn=ql._ext_accept_all_high,
inputs=[ql_ext_session, ql_rev_thr],
outputs=_rev_out,
+ ).then(
+ fn=ql._make_ql_charts,
+ inputs=[ql_ext_session],
+ outputs=_ql_chart_out,
)
# Manual add — segment preview on ID change
@@ -1264,6 +1379,296 @@ def _ft_wrap(df, tc, lc, m, bs, ep, lr):
outputs=_rev_out,
)
+ # ==================================================================
+ # Project Save / Load wiring
+ # ==================================================================
+
+ _project_state_inputs = [
+ qt_df, qt_result_df, qt_ann_session,
+ ql_raw_text, ql_segments, ql_rqs, ql_ext_session,
+ ql_lang,
+ ]
+ _project_state_keys = [
+ "qt_df", "qt_result_df", "qt_ann_session",
+ "ql_raw_text", "ql_segments", "ql_rqs", "ql_ext_session",
+ "ql_lang",
+ ]
+
+ def _save_project_fn(*state_values):
+ states = dict(zip(_project_state_keys, state_values))
+ json_str = save_project(states)
+ path = os.path.join(tempfile.gettempdir(), "socialscikit_project.json")
+ with open(path, "w", encoding="utf-8") as f:
+ f.write(json_str)
+ return path
+
+ # Load returns all state objects + a status message
+ _load_outputs = _project_state_inputs + [proj_load_msg]
+
+ def _load_project_fn(file):
+ n_states = len(_project_state_keys)
+ if file is None:
+ return [gr.update()] * n_states + ["Please upload a project file."]
+ try:
+ with open(file, "r", encoding="utf-8") as f:
+ json_str = f.read()
+ states = load_project(json_str)
+ results = []
+ for key in _project_state_keys:
+ results.append(states.get(key))
+ results.append("Project loaded successfully!")
+ return results
+ except Exception as e:
+ return [gr.update()] * n_states + [f"Load failed: {e}"]
+
+ proj_qt_save_btn.click(
+ fn=_save_project_fn,
+ inputs=_project_state_inputs,
+ outputs=[proj_qt_save_file],
+ )
+ proj_ql_save_btn.click(
+ fn=_save_project_fn,
+ inputs=_project_state_inputs,
+ outputs=[proj_ql_save_file],
+ )
+ proj_load_btn.click(
+ fn=_load_project_fn,
+ inputs=[proj_load_file],
+ outputs=_load_outputs,
+ )
+
+ # ==================================================================
+ # Toolbox
+ # ==================================================================
+ with gr.Tab(t("toolbox.title", "en")) as toolbox_tab:
+
+ tb_intro_md = gr.Markdown(t("toolbox.description", "en"))
+
+ # ---------- ICR Calculator ----------
+ with gr.Tab(t("toolbox.icr_tab", "en")) as tb_icr_tab:
+ tb_icr_md = gr.Markdown(t("icr.description", "en"))
+ with gr.Row():
+ tb_icr_file = gr.File(label=t("toolbox.icr_upload", "en"), file_types=[".csv"])
+ with gr.Column(scale=0, min_width=160):
+ tb_icr_ex_btn = gr.Button(t("toolbox.download_example", "en"), variant="secondary", size="sm")
+ tb_icr_ex_file = gr.File(label=t("toolbox.example_file", "en"), visible=False)
+ tb_icr_ex_btn.click(fn=tb._download_icr_example, outputs=[tb_icr_ex_file])
+ tb_icr_info = gr.Textbox(label=t("toolbox.icr_file_info", "en"), interactive=False, lines=1)
+ tb_icr_cols = gr.CheckboxGroup(
+ choices=[], label=t("toolbox.icr_select_cols", "en"),
+ info=t("toolbox.icr_select_cols_info", "en"),
+ )
+ tb_icr_mode = gr.Radio(
+ choices=["single-label", "multi-label"],
+ value="single-label",
+ label=t("toolbox.icr_mode", "en"),
+ info=t("toolbox.icr_mode_info", "en"),
+ )
+ tb_icr_btn = gr.Button(t("icr.compute_btn", "en"), variant="primary")
+ with gr.Row():
+ tb_icr_out = gr.Textbox(label=t("icr.report", "en"), lines=18, interactive=False, scale=2)
+ tb_icr_plot = gr.Plot(label="Agreement Chart", scale=1)
+
+ tb_icr_file.change(
+ fn=tb._icr_on_upload,
+ inputs=[tb_icr_file],
+ outputs=[tb_icr_cols, tb_icr_info],
+ )
+ tb_icr_btn.click(
+ fn=tb._compute_icr,
+ inputs=[tb_icr_file, tb_icr_cols, tb_icr_mode],
+ outputs=[tb_icr_out],
+ ).then(
+ fn=tb._make_icr_chart,
+ inputs=[tb_icr_file, tb_icr_cols, tb_icr_mode],
+ outputs=[tb_icr_plot],
+ )
+
+ # ---------- Consensus Coding ----------
+ with gr.Tab(t("toolbox.consensus_tab", "en")) as tb_con_tab:
+ tb_con_md = gr.Markdown(t("consensus.description", "en"))
+ with gr.Row():
+ tb_con_file = gr.File(label=t("toolbox.data_file", "en"), file_types=[".csv"])
+ tb_con_tcol = gr.Textbox(label=t("toolbox.text_col", "en"), value="text")
+ with gr.Column(scale=0, min_width=160):
+ tb_con_ex_btn = gr.Button(t("toolbox.download_example", "en"), variant="secondary", size="sm")
+ tb_con_ex_file = gr.File(label=t("toolbox.example_file", "en"), visible=False)
+ tb_con_ex_btn.click(fn=tb._download_consensus_example, outputs=[tb_con_ex_file])
+ tb_con_themes = gr.Textbox(
+ label=t("toolbox.themes_input", "en"),
+ placeholder="theme1: description\ntheme2: description\n...",
+ lines=4,
+ )
+
+ # Dynamic LLM slots (up to 5, first 2 visible)
+ tb_con_llm_rows = [] # list of (gr.Row, backend, model, key)
+ tb_con_backends = []
+ tb_con_models = []
+ tb_con_keys = []
+ _defaults = [
+ ("openai", "gpt-4o-mini"),
+ ("anthropic", "claude-sonnet-4-20250514"),
+ ("openai", "gpt-4o"),
+ ("ollama", ""),
+ ("anthropic", ""),
+ ]
+ for idx in range(tb.MAX_LLM_SLOTS):
+ visible = idx < 2
+ be_default, mod_default = _defaults[idx]
+ with gr.Row(visible=visible) as llm_row:
+ be = gr.Dropdown(
+ choices=["openai", "anthropic", "ollama"],
+ value=be_default,
+ label=f"Backend {idx + 1}",
+ )
+ md = gr.Textbox(label=f"Model {idx + 1}", value=mod_default if visible else "")
+ ky = gr.Textbox(label=f"API Key {idx + 1}", type="password")
+ tb_con_llm_rows.append(llm_row)
+ tb_con_backends.append(be)
+ tb_con_models.append(md)
+ tb_con_keys.append(ky)
+
+ tb_con_n_llms = gr.State(2) # tracks how many slots are visible
+
+ with gr.Row():
+ tb_con_add = gr.Button(t("toolbox.add_llm", "en"), variant="secondary", size="sm")
+ tb_con_rm = gr.Button(t("toolbox.remove_llm", "en"), variant="secondary", size="sm")
+
+ def _add_llm_slot(n):
+ n = min(n + 1, tb.MAX_LLM_SLOTS)
+ updates = [gr.update(visible=(i < n)) for i in range(tb.MAX_LLM_SLOTS)]
+ return [n] + updates
+
+ def _remove_llm_slot(n):
+ n = max(n - 1, 2)
+ updates = [gr.update(visible=(i < n)) for i in range(tb.MAX_LLM_SLOTS)]
+ return [n] + updates
+
+ tb_con_add.click(
+ fn=_add_llm_slot,
+ inputs=[tb_con_n_llms],
+ outputs=[tb_con_n_llms] + tb_con_llm_rows,
+ )
+ tb_con_rm.click(
+ fn=_remove_llm_slot,
+ inputs=[tb_con_n_llms],
+ outputs=[tb_con_n_llms] + tb_con_llm_rows,
+ )
+
+ tb_con_btn = gr.Button(t("consensus.run_btn", "en"), variant="primary")
+ tb_con_summary = gr.Textbox(label=t("consensus.summary", "en"), lines=10, interactive=False)
+ tb_con_results = gr.Dataframe(label=t("consensus.results", "en"), interactive=False)
+ tb_con_agreement = gr.Textbox(label=t("consensus.agreement", "en"), lines=4, interactive=False)
+
+ # Collect all backend/model/key inputs in order
+ _con_inputs = [tb_con_file, tb_con_tcol, tb_con_themes]
+ for i in range(tb.MAX_LLM_SLOTS):
+ _con_inputs.extend([tb_con_backends[i], tb_con_models[i], tb_con_keys[i]])
+
+ tb_con_btn.click(
+ fn=tb._run_standalone_consensus,
+ inputs=_con_inputs,
+ outputs=[tb_con_summary, tb_con_results, tb_con_agreement],
+ )
+
+ # ---------- Methods Generator ----------
+ with gr.Tab(t("toolbox.methods_tab", "en")) as tb_meth_tab:
+ tb_meth_md = gr.Markdown(t("methods.description", "en"))
+
+ # Primary: import log
+ gr.Markdown(f"### {t('toolbox.import_log', 'en')}")
+ with gr.Row():
+ tb_meth_log = gr.File(
+ label=t("toolbox.import_log", "en"),
+ file_types=[".json"],
+ )
+ with gr.Column(scale=0, min_width=200):
+ tb_meth_ex_qt_btn = gr.Button(
+ t("toolbox.example_qt_log", "en"),
+ variant="secondary", size="sm",
+ )
+ tb_meth_ex_qt_file = gr.File(
+ label=t("toolbox.example_file", "en"), visible=False,
+ )
+ tb_meth_ex_qt_btn.click(
+ fn=tb._download_methods_example_qt,
+ outputs=[tb_meth_ex_qt_file],
+ )
+ tb_meth_ex_ql_btn = gr.Button(
+ t("toolbox.example_ql_log", "en"),
+ variant="secondary", size="sm",
+ )
+ tb_meth_ex_ql_file = gr.File(
+ label=t("toolbox.example_file", "en"), visible=False,
+ )
+ tb_meth_ex_ql_btn.click(
+ fn=tb._download_methods_example_ql,
+ outputs=[tb_meth_ex_ql_file],
+ )
+ tb_meth_log_btn = gr.Button(t("methods.generate_btn", "en"), variant="primary")
+
+ tb_meth_en = gr.Textbox(label=t("methods.text_en", "en"), lines=8, interactive=True)
+ tb_meth_zh = gr.Textbox(label=t("methods.text_zh", "en"), lines=8, interactive=True)
+ gr.Markdown(t("methods.copy_hint", "en"))
+
+ tb_meth_log_btn.click(
+ fn=tb._generate_methods_from_log,
+ inputs=[tb_meth_log],
+ outputs=[tb_meth_en, tb_meth_zh],
+ )
+
+ # Fallback: manual form
+ with gr.Accordion(t("toolbox.manual_input", "en"), open=False):
+ tb_meth_type = gr.Radio(
+ choices=["QuantiKit", "QualiKit"],
+ value="QuantiKit",
+ label=t("toolbox.pipeline_type", "en"),
+ )
+ # QuantiKit fields
+ with gr.Group(visible=True) as tb_qt_group:
+ with gr.Row():
+ tb_qt_ns = gr.Number(label="N samples", value=0, precision=0)
+ tb_qt_nc = gr.Number(label="N classes", value=0, precision=0)
+ tb_qt_labels = gr.Textbox(label="Class labels (comma-separated)", value="")
+ tb_qt_model = gr.Textbox(label="Model name", value="")
+ with gr.Row():
+ tb_qt_acc = gr.Number(label="Accuracy", value=0, precision=3)
+ tb_qt_f1 = gr.Number(label="Macro F1", value=0, precision=3)
+ tb_qt_kappa = gr.Number(label="Cohen's Kappa", value=0, precision=3)
+
+ # QualiKit fields
+ with gr.Group(visible=False) as tb_ql_group:
+ with gr.Row():
+ tb_ql_nseg = gr.Number(label="N segments", value=0, precision=0)
+ tb_ql_nth = gr.Number(label="N themes", value=0, precision=0)
+ tb_ql_themes = gr.Textbox(label="Theme names (comma-separated)", value="")
+ tb_ql_model = gr.Textbox(label="Model name", value="")
+ tb_ql_consensus = gr.Checkbox(label="Consensus coding used", value=False)
+ tb_ql_ncon = gr.Number(label="N consensus models", value=0, precision=0)
+
+ def _toggle_pipeline_fields(choice):
+ return (
+ gr.update(visible=choice == "QuantiKit"),
+ gr.update(visible=choice == "QualiKit"),
+ )
+
+ tb_meth_type.change(
+ fn=_toggle_pipeline_fields,
+ inputs=[tb_meth_type],
+ outputs=[tb_qt_group, tb_ql_group],
+ )
+
+ tb_meth_form_btn = gr.Button(t("methods.generate_btn", "en"), variant="secondary")
+ tb_meth_form_btn.click(
+ fn=tb._generate_methods_from_form,
+ inputs=[tb_meth_type,
+ tb_qt_ns, tb_qt_nc, tb_qt_labels, tb_qt_model,
+ tb_qt_acc, tb_qt_f1, tb_qt_kappa,
+ tb_ql_nseg, tb_ql_nth, tb_ql_themes, tb_ql_model,
+ tb_ql_consensus, tb_ql_ncon],
+ outputs=[tb_meth_en, tb_meth_zh],
+ )
+
# ==================================================================
# Language switching
# ==================================================================
@@ -1276,6 +1681,12 @@ def _switch_language(choice):
lang,
# Landing page
_build_landing(lang),
+ # Project Load UI (on Home tab)
+ gr.update(value=t("project.load_btn", lang)), # proj_load_btn
+ gr.update(label=t("project.status", lang)), # proj_load_msg
+ # Project Save buttons
+ gr.update(value=t("project.save_btn", lang)), # proj_qt_save_btn
+ gr.update(value=t("project.save_btn", lang)), # proj_ql_save_btn
# -- QualiKit Step 1 --
t("ql.s1.title", lang), # ql_s1_md
gr.update(label=t("ql.s1.upload", lang)), # ql_file
@@ -1344,6 +1755,9 @@ def _switch_language(choice):
t("ql.s5.title", lang), # ql_s5_md
gr.update(label=t("ql.s5.stats", lang)), # ql_rev_stats
gr.update(label=t("ql.s5.table", lang)), # ql_rev_tbl
+ gr.update(label=t("ql.s5.review_progress", lang)), # ql_review_plot
+ gr.update(label=t("ql.s5.confidence", lang)), # ql_conf_plot
+ gr.update(label=t("ql.s5.themes", lang)), # ql_theme_plot
"---\n" + t("ql.s5.detail_title", lang), # ql_s5_detail_md
gr.update(label=t("ql.s5.index", lang)), # ql_rev_idx
gr.update(value=t("ql.s5.accept", lang)), # ql_rev_acc
@@ -1487,11 +1901,41 @@ def _switch_language(choice):
# -- QuantiKit Step 5 --
t("qt.s5.title", lang), # qt_s5_md
gr.update(value=t("qt.s5.run_btn", lang)), # qt_ebtn
+ gr.update(label=t("qt.s5.confusion_matrix", lang)), # qt_cm_plot
+ gr.update(label=t("qt.s5.per_class", lang)), # qt_pc_plot
+ gr.update(label=t("qt.s5.text_report", lang)), # qt_s5_text_acc
gr.update(label=t("qt.s5.report", lang)), # qt_eout
# -- QuantiKit Step 6 --
t("qt.s6.title", lang), # qt_s6_md
gr.update(value=t("qt.s6.export_btn", lang)), # qt_xbtn
gr.update(label=t("qt.s6.file", lang)), # qt_xfile
+ # -- Toolbox --
+ t("toolbox.description", lang), # tb_intro_md
+ t("icr.description", lang), # tb_icr_md
+ gr.update(label=t("toolbox.icr_upload", lang)), # tb_icr_file
+ gr.update(label=t("toolbox.icr_select_cols", lang)), # tb_icr_cols
+ gr.update(label=t("toolbox.icr_mode", lang)), # tb_icr_mode
+ gr.update(value=t("icr.compute_btn", lang)), # tb_icr_btn
+ gr.update(label=t("icr.report", lang)), # tb_icr_out
+ gr.update(value=t("toolbox.download_example", lang)), # tb_icr_ex_btn
+ t("consensus.description", lang), # tb_con_md
+ gr.update(label=t("toolbox.data_file", lang)), # tb_con_file
+ gr.update(label=t("toolbox.text_col", lang)), # tb_con_tcol
+ gr.update(label=t("toolbox.themes_input", lang)), # tb_con_themes
+ gr.update(value=t("toolbox.add_llm", lang)), # tb_con_add
+ gr.update(value=t("toolbox.remove_llm", lang)), # tb_con_rm
+ gr.update(value=t("consensus.run_btn", lang)), # tb_con_btn
+ gr.update(label=t("consensus.summary", lang)), # tb_con_summary
+ gr.update(label=t("consensus.results", lang)), # tb_con_results
+ gr.update(label=t("consensus.agreement", lang)), # tb_con_agreement
+ gr.update(value=t("toolbox.download_example", lang)), # tb_con_ex_btn
+ t("methods.description", lang), # tb_meth_md
+ gr.update(label=t("toolbox.import_log", lang)), # tb_meth_log
+ gr.update(value=t("methods.generate_btn", lang)), # tb_meth_log_btn
+ gr.update(label=t("methods.text_en", lang)), # tb_meth_en
+ gr.update(label=t("methods.text_zh", lang)), # tb_meth_zh
+ gr.update(value=t("toolbox.example_qt_log", lang)), # tb_meth_ex_qt_btn
+ gr.update(value=t("toolbox.example_ql_log", lang)), # tb_meth_ex_ql_btn
]
_lang_outputs = [
@@ -1499,6 +1943,9 @@ def _switch_language(choice):
ql_lang,
# Landing
landing_md,
+ # Project UI
+ proj_load_btn, proj_load_msg,
+ proj_qt_save_btn, proj_ql_save_btn,
# Step 1
ql_s1_md, ql_file, ql_tpl_btn, ql_tpl_file,
ql_text_preview, ql_s1_seg_md, ql_seg_mode, ql_seg_cw,
@@ -1517,7 +1964,9 @@ def _switch_language(choice):
ql_s4_md, ql_ext_be, ql_ext_mod, ql_ext_key,
ql_ext_btn, ql_ext_msg, ql_ext_tbl,
# Step 5
- ql_s5_md, ql_rev_stats, ql_rev_tbl, ql_s5_detail_md,
+ ql_s5_md, ql_rev_stats, ql_rev_tbl,
+ ql_review_plot, ql_conf_plot, ql_theme_plot,
+ ql_s5_detail_md,
ql_rev_idx, ql_rev_acc, ql_rev_rej, ql_rev_ctx,
ql_rev_edit_rq, ql_rev_edit_sub, ql_rev_edit_btn,
ql_s5_bulk_md, ql_rev_thr, ql_rev_bulk,
@@ -1553,9 +2002,20 @@ def _switch_language(choice):
qt_s4_aft_md, qt_aft_key, qt_aft_mod, qt_aft_ep, qt_aft_sfx,
qt_aft_btn, qt_aft_stat, qt_aft_chk, qt_aft_cnl, qt_aft_res,
# ---- QuantiKit Step 5 ----
- qt_s5_md, qt_ebtn, qt_eout,
+ qt_s5_md, qt_ebtn, qt_cm_plot, qt_pc_plot, qt_s5_text_acc, qt_eout,
# ---- QuantiKit Step 6 ----
qt_s6_md, qt_xbtn, qt_xfile,
+ # ---- Toolbox ----
+ tb_intro_md,
+ tb_icr_md, tb_icr_file, tb_icr_cols,
+ tb_icr_mode, tb_icr_btn, tb_icr_out,
+ tb_icr_ex_btn,
+ tb_con_md, tb_con_file, tb_con_tcol, tb_con_themes,
+ tb_con_add, tb_con_rm,
+ tb_con_btn, tb_con_summary, tb_con_results, tb_con_agreement,
+ tb_con_ex_btn,
+ tb_meth_md, tb_meth_log, tb_meth_log_btn, tb_meth_en, tb_meth_zh,
+ tb_meth_ex_qt_btn, tb_meth_ex_ql_btn,
]
lang_selector.change(
diff --git a/socialscikit/ui/qualikit_app.py b/socialscikit/ui/qualikit_app.py
index 99e8e8e..3726f1f 100644
--- a/socialscikit/ui/qualikit_app.py
+++ b/socialscikit/ui/qualikit_app.py
@@ -37,6 +37,7 @@
from socialscikit.qualikit.segmenter import Segmenter
from socialscikit.qualikit.segment_extractor import ResearchQuestion, SegmentExtractor
from socialscikit.qualikit.theme_reviewer import ThemeReviewer
+from socialscikit.core import charts
# ---------------------------------------------------------------------------
# Shared instances
@@ -450,6 +451,67 @@ def _accept_all_high_coding(review_session_state):
return review_session_state, msg
+# ---------------------------------------------------------------------------
+# Step 4b: Export Pipeline Log
+# ---------------------------------------------------------------------------
+
+
+def _export_pipeline_log(ext_session_state, rqs_state, review_session_state):
+ """Export QualiKit pipeline metadata as JSON for the Toolbox Methods Generator."""
+ import json, tempfile
+
+ if ext_session_state is None:
+ return None
+
+ log = {"pipeline": "qualikit"}
+
+ # Segment & theme info from extraction session
+ if hasattr(ext_session_state, "results") and ext_session_state.results:
+ results = ext_session_state.results
+ log["n_segments"] = len(results)
+
+ # Collect all themes from extraction results
+ all_themes = set()
+ for r in results:
+ if hasattr(r, "themes"):
+ all_themes.update(r.themes)
+ elif hasattr(r, "research_question"):
+ all_themes.add(r.research_question)
+ if all_themes:
+ log["theme_names"] = sorted(all_themes)
+ log["n_themes"] = len(all_themes)
+
+ # Research questions
+ if rqs_state:
+ rq_names = []
+ for rq in rqs_state:
+ if hasattr(rq, "name"):
+ rq_names.append(rq.name)
+ elif hasattr(rq, "question"):
+ rq_names.append(rq.question)
+ if rq_names:
+ log["research_questions"] = rq_names
+
+ # Review stats
+ if review_session_state is not None:
+ stats = {"accepted": 0, "rejected": 0, "edited": 0, "pending": 0}
+ for tier in [review_session_state.high, review_session_state.medium, review_session_state.low]:
+ for item in tier:
+ action = item.action.value if hasattr(item.action, "value") else str(item.action)
+ if action in stats:
+ stats[action] += 1
+ else:
+ stats["pending"] += 1
+ log["review_stats"] = stats
+
+ tmp = tempfile.NamedTemporaryFile(
+ mode="w", suffix="_qualikit_log.json", delete=False, encoding="utf-8",
+ )
+ json.dump(log, tmp, ensure_ascii=False, indent=2)
+ tmp.close()
+ return tmp.name
+
+
# ---------------------------------------------------------------------------
# Step 5: Export
# ---------------------------------------------------------------------------
@@ -622,6 +684,55 @@ def _ext_stats_text(session):
)
+def _make_ql_charts(session):
+ """Generate QualiKit visualization charts for the review dashboard.
+
+ Returns
+ -------
+ tuple
+ (review_cards_html, review_progress_fig, confidence_fig, theme_fig)
+ """
+ _empty = ("", None, None, None)
+ if session is None or not session.items:
+ return _empty
+
+ try:
+ stats = _extraction_reviewer.stats(session)
+
+ # Metric cards HTML
+ html = charts.format_review_stats_html(
+ stats["total"], stats["accepted"], stats["edited"],
+ stats["rejected"], stats["pending"],
+ )
+
+ # Review progress donut
+ review_fig = charts.plot_review_progress(
+ stats["accepted"], stats["edited"],
+ stats["rejected"], stats["pending"],
+ )
+
+ # Confidence histogram
+ confidences = [
+ item.result.confidence
+ for item in session.items
+ if item.result.confidence is not None
+ ]
+ conf_fig = charts.plot_confidence_histogram(confidences)
+
+ # Theme / RQ distribution
+ themes: dict[str, int] = {}
+ for item in session.items:
+ rq = item.final_rq_label
+ themes[rq] = themes.get(rq, 0) + 1
+ theme_fig = charts.plot_theme_distribution(themes)
+
+ return html, review_fig, conf_fig, theme_fig
+ except Exception as e:
+ import logging
+ logging.getLogger(__name__).warning("Chart generation failed: %s", e)
+ return _empty
+
+
def _run_extraction(raw_text, segments, rqs_text, backend, model, api_key):
"""LLM extraction — returns (session, summary, ext_tbl, rev_tbl, stats)."""
if not segments:
@@ -734,7 +845,41 @@ def _ext_show_context(session, index, raw_text):
}
status = status_map.get(item.action.value, item.action.value)
- # --- Part 1: full segment text ---
+ # --- Part 1: full segment text (with evidence highlighting if available) ---
+ seg_text = item.result.text
+ evidence = getattr(item.result, "evidence_span", "")
+
+ # Build segment body — highlight evidence span inline if found
+ if evidence:
+ ev_lower = evidence.lower()
+ seg_lower = seg_text.lower()
+ ev_idx = seg_lower.find(ev_lower)
+ if ev_idx >= 0:
+ before = esc(seg_text[:ev_idx])
+ match_text = esc(seg_text[ev_idx:ev_idx + len(evidence)])
+ after = esc(seg_text[ev_idx + len(evidence):])
+ seg_body = (
+ f'{before}'
+ '
'
+ f'{match_text}'
+ f'{after}'
+ )
+ else:
+ # Evidence not found verbatim — show segment + evidence block
+ seg_body = esc(seg_text)
+ else:
+ seg_body = esc(seg_text)
+
+ # Evidence block shown below segment text when present
+ evidence_html = ""
+ if evidence:
+ evidence_html = (
+ '
'
+ f'Evidence: \u201c{esc(evidence)}\u201d
'
+ )
+
full_text_html = (
'
'
@@ -747,7 +892,8 @@ def _ext_show_context(session, index, raw_text):
'
'
f'
'
- f'{esc(item.result.text)}
'
+ f'{seg_body}
'
+ f'{evidence_html}'
f'
'
f'判断依据：{esc(item.result.reasoning)}
'
'
- 
+ 
@@ -23,10 +23,18 @@
SocialSciKit is an open-source Python toolkit that enables social science researchers to perform text analysis **without writing a single line of code**. It provides a Gradio-based web interface with full bilingual support (English / Chinese).
-Two core modules:
+Three core modules:
- **QuantiKit** — End-to-end text classification pipeline (method recommendation → annotation → prompt/fine-tuning classification → evaluation → export)
-- **QualiKit** — End-to-end qualitative coding pipeline (upload → de-identification → research framework → LLM coding → human review → export)
+- **QualiKit** — End-to-end qualitative coding pipeline (upload → de-identification → research framework → LLM coding with evidence grounding → human review → export)
+- **Toolbox** — Standalone research methods tools: Inter-Coder Reliability (ICR) calculator, Multi-LLM Consensus Coding, and Methods Section Generator
+
+### Highlights
+
+- **Visualization Dashboard** — academic-style matplotlib charts (confusion matrix heatmaps, per-class P/R/F1 bars, confidence histograms, progress donuts, theme distribution) embedded throughout both pipelines
+- **Evidence Highlighting** — LLM codings include a verbatim `evidence_span` from the source text; the review UI highlights the supporting quote inline in the original document
+- **Project Save & Restore** — serialize the entire research project state (data, annotations, sessions, coding results) to a single JSON file; resume work later from the Home tab
+- **Zero-code web UI** — Gradio 4.44+ with full EN/ZH language switching at runtime
---
@@ -36,6 +44,8 @@ Two core modules:
- [Quick Start](#quick-start)
- [QuantiKit: Text Classification](#quantikit-text-classification)
- [QualiKit: Qualitative Coding](#qualikit-qualitative-coding)
+- [Toolbox: Research Methods Tools](#toolbox-research-methods-tools)
+- [Project Save & Restore](#project-save--restore)
- [Supported LLM Backends](#supported-llm-backends)
- [Example Datasets](#example-datasets)
- [Project Structure](#project-structure)
@@ -161,8 +171,8 @@ QuantiKit guides you through the full text classification workflow in 6 steps.
- Built-in annotation UI — no need for external tools
- Label each text sample, with **skip**, **undo**, **flag for review** support
+- **Real-time progress donut chart** — visual progress tracker updates after every action
- Export annotated data as CSV, merge with original dataset
-- Progress tracker shows completion percentage
### Step 4 · Classification
@@ -181,13 +191,18 @@ Three sub-approaches available in parallel tabs:
### Step 5 · Evaluation
-- Metrics: Accuracy, Macro-F1, Cohen's Kappa, per-class Precision / Recall / F1
-- Confusion matrix visualization
-- Detailed classification report
+Full visualization dashboard:
+
+- **Metric summary cards** (HTML) — Accuracy, Macro-F1, Weighted-F1, Cohen's Kappa, total/correct counts
+- **Confusion matrix heatmap** — row-normalized, annotated with counts + percentages
+- **Per-class metrics bar chart** — Precision / Recall / F1 grouped bars per class
+- Collapsible full text report below charts
### Step 6 · Export
- Download classification results as CSV (original text + predicted labels + confidence)
+- **Pipeline log export** — JSON metadata usable by the Toolbox Methods Generator
+- **Save project** — persist all research state (data, predictions, annotation session) to a single JSON file
---
@@ -221,12 +236,18 @@ QualiKit supports the full qualitative coding workflow for interview transcripts
### Step 4 · LLM Coding
- Batch coding: LLM reads each segment and assigns RQ + sub-theme labels with confidence scores
+- **Evidence grounding**: the LLM prompt requires a verbatim `evidence_span` — the exact phrase or sentence from the source text that supports the coding decision
- Supports OpenAI, Anthropic, and Ollama backends
-- Results displayed with segment text, assigned codes, and confidence levels
+- Results displayed with segment text, assigned codes, confidence levels, and evidence spans
### Step 5 · Review
- Review coding results in a table sorted by confidence
+- **Evidence highlighting**: when you select an item, the original text is shown with the LLM's `evidence_span` highlighted in green, so you can verify the coding decision at a glance; if the exact quote isn't found, a fallback "Evidence" block displays the cited text
+- Visualization dashboard (collapsible accordion):
+ - **Review progress donut** — accepted / edited / rejected / pending counts
+ - **Confidence histogram** — low/medium/high tier shading + median marker
+ - **Theme distribution** — horizontal bar chart of RQ frequencies
- Per-item actions: accept, reject, or edit (reassign RQ/sub-theme)
- Bulk accept by confidence threshold
- **Manual coding**: select a segment, preview its content, and manually assign RQ + sub-theme labels
@@ -235,6 +256,54 @@ QualiKit supports the full qualitative coding workflow for interview transcripts
### Step 6 · Export
- Export reviewed coding results as structured Excel file
+- **Pipeline log export** — JSON metadata usable by the Toolbox Methods Generator
+- **Save project** — persist the entire coding session (segments, RQs, review state, evidence spans) to a single JSON file
+
+---
+
+## Toolbox: Research Methods Tools
+
+The Toolbox provides standalone research utilities that work independently or in combination with QuantiKit / QualiKit.
+
+### ICR Calculator
+
+Compute inter-coder reliability for 2 or more coders with automatic metric selection:
+
+| Scenario | Metric |
+|----------|--------|
+| 2 coders, single-label | Cohen's Kappa + Krippendorff's Alpha + per-category agreement |
+| 3+ coders, single-label | Krippendorff's Alpha + pairwise Cohen's Kappa |
+| 2 coders, multi-label | Jaccard index (pairwise) |
+| 3+ coders, multi-label | Average pairwise Jaccard |
+
+- Upload a CSV with coder columns, select which columns to compare
+- Interpretation follows the Landis & Koch (1977) scale
+
+### Consensus Coding
+
+Multi-LLM majority-vote coding for qualitative data:
+
+- Configure 2–5 LLM backends (OpenAI, Anthropic, Ollama) with independent models
+- Each LLM codes every text segment; final label is determined by majority vote
+- Agreement statistics across LLMs are reported automatically
+
+### Methods Section Generator
+
+Auto-generate a methods section paragraph (English + Chinese) for your paper:
+
+- **From pipeline log**: QuantiKit and QualiKit can export a pipeline log (JSON) capturing all metadata (sample size, model, metrics, themes, etc.). Import the log and generate a ready-to-use methods paragraph.
+- **Manual input**: Fill in metadata fields manually if you prefer not to use the pipeline log.
+
+---
+
+## Project Save & Restore
+
+Long research projects rarely finish in one session. SocialSciKit serialises the full state of your work — loaded DataFrames, annotation sessions (including cursor and history), extraction review sessions, research questions, de-identification results — into a single JSON file:
+
+- **Save**: at the end of any pipeline, click "Save Project" in Step 6 to download a `.json` archive
+- **Restore**: return to the **Home** tab, expand "Load Saved Project", upload the JSON file, and all state is restored across both pipelines
+- **Tagged-union serialisation**: complex types (`pd.DataFrame`, `AnnotationSession`, `ExtractionReviewSession`, `ResearchQuestion`, `ExtractionResult`, enums) round-trip losslessly; elapsed-time counters are preserved via monotonic time offsets
+- **Version-aware**: project files include a `__project_version__` field so future readers can migrate old archives
---
@@ -258,12 +327,16 @@ ollama pull llama3
The `examples/` directory contains ready-to-use sample data:
-| File | Type | Description |
-|------|------|-------------|
+| File | Module | Description |
+|------|--------|-------------|
| `sentiment_example.csv` | QuantiKit | 50 Chinese product/service reviews with 3 sentiment labels |
| `policy_example.csv` | QuantiKit | 40 Chinese policy text excerpts with 8 policy-instrument labels |
| `interview_example.txt` | QualiKit | Single-person community healthcare interview transcript |
| `interview_focus_group.txt` | QualiKit | 4-person focus group on elderly digital service experiences |
+| `icr_example.csv` | Toolbox | 20 policy texts coded by 3 coders (A/B/C) for ICR calculation |
+| `consensus_example.csv` | Toolbox | 15 interview segments for multi-LLM consensus coding |
+| `methods_log_quantikit.json` | Toolbox | Sample QuantiKit pipeline log for methods generation |
+| `methods_log_qualikit.json` | Toolbox | Sample QualiKit pipeline log for methods generation |
### Cookbook: Sentiment Classification (QuantiKit)
@@ -298,6 +371,10 @@ socialscikit/
│ ├── data_validator.py # Schema validation + auto-fix
│ ├── data_diagnostics.py # Data quality diagnostics report
│ ├── llm_client.py # Unified LLM client (OpenAI/Anthropic/Ollama)
+│ ├── icr.py # Inter-coder reliability (Kappa/Alpha/Jaccard)
+│ ├── methods_writer.py # Methods section generator (EN/ZH templates)
+│ ├── charts.py # Academic-style matplotlib charts (viz dashboard)
+│ ├── project_io.py # Project state serialization (save/restore)
│ └── templates/ # Template files for download
│
├── quantikit/ # Text classification module
@@ -322,18 +399,21 @@ socialscikit/
│ ├── confidence_ranker.py # Confidence scoring & ranking
│ ├── coding_reviewer.py # Human-in-the-loop coding review
│ ├── extraction_reviewer.py # Extraction result review
+│ ├── consensus.py # Multi-LLM consensus coding (majority vote)
│ └── exporter.py # Excel / Markdown export
│
├── ui/ # Gradio web interface
-│ ├── main_app.py # Unified app (Home + QuantiKit + QualiKit)
+│ ├── main_app.py # Unified app (Home + QuantiKit + QualiKit + Toolbox)
│ ├── quantikit_app.py # QuantiKit UI callbacks
│ ├── qualikit_app.py # QualiKit UI callbacks
+│ ├── toolbox_app.py # Toolbox UI callbacks (ICR/Consensus/Methods)
│ └── i18n.py # Internationalization (EN / ZH)
│
├── cli.py # Command-line entry point
│
examples/ # Sample datasets
-tests/ # Test suite (562 tests)
+tests/ # Test suite (676 tests)
+promo/ # Promotional posters + HTML sources
pyproject.toml # Package metadata & dependencies
CITATION.cff # Citation metadata
```
diff --git a/README_zh.md b/README_zh.md
index f52fad1..610ae73 100644
--- a/README_zh.md
+++ b/README_zh.md
@@ -7,7 +7,7 @@