diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..fbd323e --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +OPENAI_API_KEY= +DEEPSEEK_API_KEY= diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a4bfa48 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.env +__pycache__/ +*.pyc +results/ +batch_results/ +*.csv +_sample_rows.json diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..613f112 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 MAKIEval contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 230e3f9..c19061d 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,13 @@ We have released the MAKIEval dataset on Hugging Face: The released dataset currently contains: -* 🤖 13 LLMs +* 🤖 **7 LLMs** (see paper Table 1) * 🌐 13 Languages -* 🗺️ Multiple Countries and Regions +* 🗺️ 19 Countries and Regions * 🎭 6 Cultural Domains +> **Note on scale:** The Hugging Face release contains approximately **5.25M rows**. The paper reports 85.8 million generated texts; at the response level the expected order of magnitude is ≈6M (1,716 prompts × 500 responses × 7 models). The difference may reflect aggregation, filtering, or release packaging — we report HF counts here rather than restating the paper figure. + ### Cultural Domains * 🍽️ Food @@ -55,17 +57,134 @@ entities ```text code/ - analysis_*.py + analysis_*.py # topic-specific Wikidata linking + data_loading.py # Hugging Face dataset loader entity_extraction.py + lm_utils.py + metrics.py # granularity, diversity, specificity, consensus prompt_construct.py run_experiment.py + run_metrics.py # CLI for metric computation + validate_fidelity.py # paper-facing fidelity checks meta_info/ country.json name.json prompt.json + +tests/ + test_data_loading.py + test_metrics.py + +docs/ + DEVIATIONS.md + FIDELITY_REPORT.md +``` + +--- + +## 🔁 Reproduce (metrics) + +> ⚠️ **Use `--full` for paper-comparable numbers. The default 5,000-row limit is only a quick smoke test; Diversity and Consensus in this mode are not comparable to the paper values.** + +### 1. Install + +```bash +python -m venv .venv +source .venv/bin/activate # Windows: .venv\Scripts\activate +pip install -r requirements.txt +``` + +### 2. Compute metrics from the public dataset + +No API key is required for `run_metrics.py`; API-backed generation and entity +extraction still require `OPENAI_API_KEY` / `DEEPSEEK_API_KEY` in the +environment. + +```bash +python code/run_metrics.py --model Qwen2.5-7B-Instruct --topic beverage --language ar --seed 42 --limit 5000 +``` + +Outputs are written to `results/metrics/`: + +* `group_metrics.csv` — granularity, diversity, culture specificity per (model, topic, language, country) +* `culture_consensus.csv` — pairwise Jaccard scores across languages +* `parse_report.json` — entity JSON parse statistics + +`run_metrics.py` defaults to a 5,000-row limit for quick checks. Add `--full` to process all rows matching the selected filters. + +### 3. Run tests + +```bash +pytest -q tests/ +``` + +--- + +## Published-data Quality Audit + +`quality_report.py` audits the published Hugging Face rows without generation, +GPU, or API keys. The default command reservoir-samples 200 rows per +`model x language x topic` slice and writes `docs/DATA_QUALITY_REPORT.md`. +See docs/RESULTS.md for an audit summary with figures. + +```bash +python code/quality_report.py --sample 200 --seed 42 +``` + +Use `--full` to scan every row for the quality checks. The report includes +degenerate text checks, language leakage estimates, missing-QID rates, +surface-form to QID inconsistency, suspicious extraction review candidates, +and observed group coverage. + +## Smoke Reproduction With Together + +Small-scale generation can be run through Together AI to avoid local GPU/model +downloads. Set `TOGETHER_API_KEY` in the environment; do not commit keys. + +```bash +python code/smoke_generation.py \ + --model Qwen2.5-7B-Instruct \ + --language en \ + --topic books \ + --country "United States" \ + --num-responses 3 \ + --faithful \ + --seed 42 ``` +Outputs are written to `results/smoke/`. Compare a smoke run with the matching +published Hugging Face slice: + +```bash +python code/compare_to_published.py \ + --model Qwen2.5-7B-Instruct \ + --language en \ + --topic books \ + --country "United States" \ + --published-limit 3 +``` + +`--faithful` locks local-model decoding values to the paper setup: +`temperature=0.7`, `top_p=0.9`, `top_k=10`, and `max_tokens=100`. By default, +the CLI reuses the exact prompt string from the selected Hugging Face slice. + +### Model Access Matrix + +| Paper model | Default Together route | Task B feasibility | +|---|---|---| +| Qwen2.5-7B-Instruct | `Qwen/Qwen2.5-7B-Instruct-Turbo` | Recommended smoke default; provider Turbo endpoint. | +| Llama-3.1-8B-Instruct | `meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo` | Depends on Together availability. | +| Mistral-7B-Instruct-v0.1 | `mistralai/Mistral-7B-Instruct-v0.1` | Depends on Together availability. | +| aya-expanse-8b | `CohereLabs/aya-expanse-8b` | Depends on Together availability. | +| Llama-3.3-70B-Instruct | `meta-llama/Llama-3.3-70B-Instruct-Turbo` | API feasible; local full-scale requires large GPUs. | +| ChatGPT-4o-mini | `openai/gpt-oss-120b` | **PROXY MODEL - NOT IN PAPER SET** for generation/extraction when OpenAI API is unavailable. | +| DeepSeek-V3 | `deepseek-ai/DeepSeek-V3` | Depends on Together availability; native DeepSeek API remains separate. | + +Entity extraction in the paper uses GPT-4o-mini. This branch can substitute a +Together model when `OPENAI_API_KEY` is intentionally unavailable; such runs are +reported as proxy extraction and are not exact paper-faithful extraction. + --- ## ⚙️ Pipeline diff --git a/code/analysis.py b/code/analysis.py new file mode 100644 index 0000000..d22014b --- /dev/null +++ b/code/analysis.py @@ -0,0 +1,96 @@ +# RECONSTRUCTED from paper Section 3.3 & 4.3; original analysis.py was not released in upstream repo. + +import re +import time +from functools import lru_cache + +import pandas as pd +from SPARQLWrapper import SPARQLWrapper, JSON + +sparql = SPARQLWrapper("https://query.wikidata.org/sparql") +sparql.addCustomHttpHeader("User-Agent", "MAKIEval/1.0 (cultural-awareness-evaluation)") + +COUNTRY_SOURCE_COLUMNS = ( + "Author Countries", + "Performer Countries", + "Origin Countries", +) + + +@lru_cache(maxsize=4096) +def _query_wikidata_origin_country(qid: str) -> str: + """Resolve origin country for a QID (P495 preferred, then P17).""" + if not qid or qid == "NA": + return "NA" + + query = f""" + SELECT ?countryLabel WHERE {{ + OPTIONAL {{ wd:{qid} wdt:P495 ?country. }} + OPTIONAL {{ wd:{qid} wdt:P17 ?country. }} + ?country rdfs:label ?countryLabel. + FILTER(LANG(?countryLabel) = "en") + }} + LIMIT 1 + """ + for attempt in range(3): + try: + sparql.setQuery(query) + sparql.setReturnFormat(JSON) + results = sparql.query().convert() + bindings = results["results"]["bindings"] + if bindings and "countryLabel" in bindings[0]: + return bindings[0]["countryLabel"]["value"] + return "NA" + except Exception: + time.sleep(1 + attempt) + return "NA" + + +def _parse_countries_from_information(information: str) -> str: + """Best-effort parse of country names embedded in the Information field.""" + if not information or information == "NA": + return "NA" + + # Wikidata info strings often end with comma-separated English country labels. + parts = [part.strip() for part in information.split(",") if part.strip()] + if not parts: + return "NA" + + # Heuristic: trailing tokens that look like country names (Title Case words). + country_like = [] + for part in reversed(parts): + if re.match(r"^[A-Z][a-zA-Z\s\-']+$", part): + country_like.insert(0, part) + else: + break + + return ", ".join(country_like) if country_like else "NA" + + +def _row_origin_country(row: pd.Series) -> str: + for column in COUNTRY_SOURCE_COLUMNS: + if column in row.index and row[column] not in (None, "", "NA"): + return str(row[column]) + + information = row.get("Information", "NA") + parsed = _parse_countries_from_information(str(information)) + if parsed != "NA": + return parsed + + qid = row.get("Q_ID", "NA") + return _query_wikidata_origin_country(str(qid)) + + +def process_wikidata_country_info(df: pd.DataFrame) -> pd.DataFrame: + """ + Enrich Wikidata entity rows with origin-country metadata for culture specificity. + + Expected input columns include: + Q_ID, Original Label, Label, Information, is_category, Alternative QIDs + """ + if df.empty: + return df + + result = df.copy() + result["Origin Country"] = result.apply(_row_origin_country, axis=1) + return result diff --git a/code/compare_to_published.py b/code/compare_to_published.py new file mode 100644 index 0000000..4a9d094 --- /dev/null +++ b/code/compare_to_published.py @@ -0,0 +1,232 @@ +"""Compare a small smoke-generation result with the published HF slice.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from collections import Counter +from pathlib import Path +from typing import Any + +import pandas as pd + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from data_loading import filter_entities_for_metrics, load_makieval_dataset +from metrics import collect_qids, culture_consensus, culture_specificity, diversity, granularity +from quality_report import detect_language, has_repeated_ngram_block, has_repeated_sentence, language_matches +from smoke_generation import aggregate_metrics, canonical_model_name, normalize_topic, safe_slug + + +def load_smoke(path: str | None, model: str, language: str, topic: str, country: str) -> dict[str, Any]: + if path: + return json.loads(Path(path).read_text(encoding="utf-8")) + candidate = Path("results/smoke") / ( + f"{safe_slug(canonical_model_name(model))}_{safe_slug(language)}_" + f"{safe_slug(normalize_topic(topic))}_{safe_slug(country)}.json" + ) + if not candidate.exists(): + raise FileNotFoundError(f"Smoke result not found: {candidate}") + return json.loads(candidate.read_text(encoding="utf-8")) + + +def flatten_entities_from_smoke(smoke: dict[str, Any]) -> list[dict[str, Any]]: + entities = [] + for row in smoke.get("rows", []): + filtered, _excluded = filter_entities_for_metrics(row.get("entities", [])) + entities.extend(filtered) + return entities + + +def flatten_entities_from_df(df: pd.DataFrame) -> list[dict[str, Any]]: + entities = [] + for row_entities in df.get("entities_parsed", []): + entities.extend(row_entities) + return entities + + +def dominant_qid(entities: list[dict[str, Any]]) -> tuple[str, int]: + qids = [str(entity.get("qid")) for entity in entities if entity.get("qid")] + if not qids: + return "NA", 0 + qid, count = Counter(qids).most_common(1)[0] + return qid, count + + +def quality_rates(rows: list[dict[str, Any]], language: str) -> dict[str, float]: + if not rows: + return {"degenerate": 0.0, "language_leakage": 0.0} + degenerate = 0 + leakage_checked = 0 + leakage = 0 + for row in rows: + text = str(row.get("generated_text") or "") + if has_repeated_ngram_block(text) or has_repeated_sentence(text): + degenerate += 1 + detected = detect_language(text) + if detected not in {"und", "unknown"}: + leakage_checked += 1 + leakage += int(not language_matches(language, detected)) + return { + "degenerate": degenerate / len(rows), + "language_leakage": leakage / leakage_checked if leakage_checked else 0.0, + } + + +def published_rows(df: pd.DataFrame) -> list[dict[str, Any]]: + return df.to_dict(orient="records") + + +def label(value: float, close_threshold: float = 0.15) -> str: + if value == 0: + return "MATCH" + if abs(value) <= close_threshold: + return "CLOSE" + return "DIVERGE" + + +def load_country_origin_map(path: str | None) -> dict[str, str]: + if not path: + return {} + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def build_report( + smoke: dict[str, Any], + published_df: pd.DataFrame, + published_limit: int | None, + country_origin_map: dict[str, str] | None = None, +) -> str: + meta = smoke["metadata"] + smoke_rows = smoke.get("rows", []) + smoke_entities = flatten_entities_from_smoke(smoke) + published_entities = flatten_entities_from_df(published_df) + country = meta["country"] + published_lookup = country_origin_map or {} + + smoke_metrics = smoke.get("metrics") or aggregate_metrics( + smoke_rows, + country, + smoke.get("country_lookup", {}), + ) + published_metrics = { + "granularity": granularity(published_entities), + "diversity": diversity(published_entities), + "culture_specificity": culture_specificity( + published_entities, country, published_lookup + ), + "entity_count": len(published_entities), + "unique_qids": len(collect_qids(published_entities)), + } + + smoke_qids = collect_qids(smoke_entities) + published_qids = collect_qids(published_entities) + qid_jaccard = culture_consensus(smoke_qids, published_qids) + smoke_dom = dominant_qid(smoke_entities) + published_dom = dominant_qid(published_entities) + smoke_quality = quality_rates(smoke_rows, meta["language"]) + published_quality = quality_rates(published_rows(published_df), meta["language"]) + + lines = [ + "# MAKIEval Smoke Reproduction Comparison", + "", + "Scope: small-sample, not paper-scale. Published data may be an older snapshot per the author.", + "", + f"- Model: `{meta['model']}`", + f"- Together endpoint: `{meta.get('together_model')}`", + f"- Slice: `{meta['language']} / {meta['topic']} / {meta['country']}`", + f"- Our N: `{len(smoke_rows)}`", + f"- Published N used: `{len(published_df)}`" + + (f" (limit={published_limit})" if published_limit else ""), + f"- Extraction status: `{meta.get('extraction_status')}`", + "", + "## Deviations", + "", + ] + deviations = meta.get("deviations") or [] + if deviations: + lines.extend(f"- {item}" for item in deviations) + else: + lines.append("- None recorded.") + + lines.extend( + [ + "", + "## Metric Comparison", + "", + "| Metric | Ours | Published | Status | Comment |", + "|---|---:|---:|---|---|", + ] + ) + for metric in ("granularity", "diversity", "culture_specificity", "entity_count", "unique_qids"): + ours = smoke_metrics.get(metric, 0) + published = published_metrics.get(metric, 0) + delta = float(ours) - float(published) + if metric == "culture_specificity" and not published_lookup: + status = "N/A (origin lookup not supplied)" + comment = "Per-QID origin metadata required for specificity comparison." + else: + status = label(delta) + comment = "Small N; exact match is not expected." + lines.append( + f"| {metric} | {ours:.4f} | {published:.4f} | {status} | {comment} |" + ) + + lines.extend( + [ + f"| qid_set_jaccard | {qid_jaccard:.4f} | n/a | {'CLOSE' if qid_jaccard > 0 else 'DIVERGE'} | Entity-set overlap between our slice and published slice. |", + f"| dominant_qid | {smoke_dom[0]} ({smoke_dom[1]}) | {published_dom[0]} ({published_dom[1]}) | {'MATCH' if smoke_dom[0] == published_dom[0] else 'DIVERGE'} | Mode QID comparison. |", + f"| degenerate_rate | {smoke_quality['degenerate']:.4f} | {published_quality['degenerate']:.4f} | {label(smoke_quality['degenerate'] - published_quality['degenerate'])} | Rule-based repeated text check. |", + f"| language_leakage_rate | {smoke_quality['language_leakage']:.4f} | {published_quality['language_leakage']:.4f} | {label(smoke_quality['language_leakage'] - published_quality['language_leakage'])} | langid/script detector. |", + "", + "## Notes", + "", + "- Diversity and consensus are N-sensitive; this comparison is directional only.", + "- Published culture specificity is marked N/A unless `--country-origin-map` supplies per-QID origin metadata.", + "- GPT-4o-mini extraction was not used because OPENAI_API_KEY is intentionally unavailable.", + ] + ) + return "\n".join(lines) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Compare smoke output to HF published data.") + parser.add_argument("--smoke-result", default=None) + parser.add_argument("--model", default="Qwen2.5-7B-Instruct") + parser.add_argument("--language", required=True) + parser.add_argument("--topic", required=True) + parser.add_argument("--country", required=True) + parser.add_argument("--published-limit", type=int, default=None) + parser.add_argument("--output", default="docs/REPRODUCTION_COMPARISON.md") + parser.add_argument( + "--country-origin-map", + default=None, + help="JSON map of QID -> origin country for published culture specificity.", + ) + args = parser.parse_args() + + smoke = load_smoke(args.smoke_result, args.model, args.language, args.topic, args.country) + country_origin_map = load_country_origin_map(args.country_origin_map) + limit = args.published_limit or int(smoke["metadata"].get("num_responses", 0)) or None + filters = { + "model": canonical_model_name(args.model), + "topic": normalize_topic(args.topic), + "language": args.language, + "country_region": args.country, + } + published_df, _report = load_makieval_dataset(filters=filters, limit=limit) + if published_df.empty: + raise ValueError(f"No published rows matched filters: {filters}") + + report = build_report(smoke, published_df, limit, country_origin_map) + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(report, encoding="utf-8") + print(f"Wrote comparison report to {output}") + + +if __name__ == "__main__": + main() diff --git a/code/data_loading.py b/code/data_loading.py new file mode 100644 index 0000000..27a0c5a --- /dev/null +++ b/code/data_loading.py @@ -0,0 +1,328 @@ +"""Load and clean the Raoyuan/MAKIEval Hugging Face dataset.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from functools import lru_cache +from itertools import product +from typing import Any + +import pandas as pd +import requests + +EXCLUDED_ENTITY_TYPES = frozenset( + {"place", "person_name", "listener_name", "reader_name"} +) + +DEFAULT_COLUMNS = [ + "model", + "topic", + "language", + "country_region", + "prompt", + "generated_text", + "entities", +] + +GROUP_COLUMNS = ["model", "topic", "language", "country_region"] + +COUNTRY_ALIASES = { + "us": "united states", + "usa": "united states", + "u.s.": "united states", + "united states": "united states", + "uk": "united kingdom", + "u.k.": "united kingdom", + "united kingdom": "united kingdom", + "uae": "united arab emirates", + "united arab emirates": "united arab emirates", +} + +TOPIC_ALIASES = { + "books": "book", +} + + +@dataclass +class ParseReport: + total_rows: int + parse_errors: int + empty_entity_rows: int + excluded_entity_count: int + + +def parse_entities_field(raw_entities: Any) -> tuple[list[dict[str, Any]], bool]: + """ + Parse the entities column into a list of dicts. + + Returns (entities, ok) where ok is False when the raw value could not be parsed. + """ + if raw_entities is None: + return [], False + + if isinstance(raw_entities, list): + return raw_entities, True + + if isinstance(raw_entities, str): + text = raw_entities.strip() + if not text: + return [], True + try: + parsed = json.loads(text) + except json.JSONDecodeError: + return [], False + if isinstance(parsed, list): + return parsed, True + return [], False + + return [], False + + +def filter_entities_for_metrics( + entities: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], int]: + """Drop entity types excluded from metric computation.""" + kept: list[dict[str, Any]] = [] + excluded = 0 + for entity in entities: + entity_type = str(entity.get("entity_type", "")).lower() + if entity_type in EXCLUDED_ENTITY_TYPES: + excluded += 1 + continue + kept.append(entity) + return kept, excluded + + +def clean_entities_column(df: pd.DataFrame) -> tuple[pd.DataFrame, ParseReport]: + """Parse entities JSON and attach metric-eligible entity lists.""" + parsed_entities: list[list[dict[str, Any]]] = [] + parse_errors = 0 + empty_rows = 0 + excluded_total = 0 + + for raw in df["entities"]: + entities, ok = parse_entities_field(raw) + if not ok: + parse_errors += 1 + parsed_entities.append([]) + continue + if not entities: + empty_rows += 1 + filtered, excluded = filter_entities_for_metrics(entities) + excluded_total += excluded + parsed_entities.append(filtered) + + result = df.copy() + result["entities_parsed"] = parsed_entities + report = ParseReport( + total_rows=len(df), + parse_errors=parse_errors, + empty_entity_rows=empty_rows, + excluded_entity_count=excluded_total, + ) + return result, report + + +def _filter_tuples(filters: dict[str, str] | None) -> list[tuple[str, str, str]] | None: + if not filters: + return None + return [(column, "==", value) for column, value in filters.items()] + + +def normalize_filters(filters: dict[str, str] | None) -> dict[str, str] | None: + """Normalize user-facing filter aliases to Hugging Face column values.""" + if not filters: + return None + + normalized = dict(filters) + if "topic" in normalized: + topic = normalized["topic"].strip().lower() + normalized["topic"] = TOPIC_ALIASES.get(topic, topic) + if "language" in normalized: + normalized["language"] = normalized["language"].strip().lower() + if "country_region" in normalized: + country = " ".join(normalized["country_region"].strip().lower().split()) + normalized["country_region"] = COUNTRY_ALIASES.get(country, country) + return normalized + + +def _hf_parquet_urls(split: str) -> list[str]: + response = requests.get( + "https://datasets-server.huggingface.co/parquet", + params={"dataset": "Raoyuan/MAKIEval"}, + timeout=30, + ) + response.raise_for_status() + payload = response.json() + return [ + item["url"] + for item in payload.get("parquet_files", []) + if item.get("config") == "default" and item.get("split") == split + ] + + +@lru_cache(maxsize=8) +def _hf_group_column_values(split: str) -> dict[str, list[str]]: + response = requests.get( + "https://datasets-server.huggingface.co/statistics", + params={ + "dataset": "Raoyuan/MAKIEval", + "config": "default", + "split": split, + }, + timeout=30, + ) + response.raise_for_status() + payload = response.json() + values: dict[str, list[str]] = {} + for item in payload.get("statistics", []): + column_name = item.get("column_name") + if column_name not in GROUP_COLUMNS: + continue + frequencies = item.get("column_statistics", {}).get("frequencies", {}) + values[column_name] = sorted(frequencies) + return values + + +def _load_from_parquet( + split: str, + filters: dict[str, str] | None, + limit: int | None, + columns: list[str] | None = None, +) -> pd.DataFrame: + if columns is None: + columns = DEFAULT_COLUMNS + frames: list[pd.DataFrame] = [] + + for shard in iter_makieval_parquet_frames( + split=split, + filters=filters, + columns=columns, + ): + if shard.empty: + continue + frames.append(shard) + if limit is not None and sum(len(frame) for frame in frames) >= limit: + break + + if not frames: + return pd.DataFrame(columns=columns) + + df = pd.concat(frames, ignore_index=True) + if limit is not None: + df = df.head(limit) + return df + + +def iter_makieval_parquet_frames( + split: str = "train", + filters: dict[str, str] | None = None, + columns: list[str] | None = None, +): + """Yield filtered MAKIEval parquet shards as pandas DataFrames.""" + if columns is None: + columns = DEFAULT_COLUMNS + normalized_filters = normalize_filters(filters) + parquet_filters = _filter_tuples(normalized_filters) + + for url in _hf_parquet_urls(split): + yield pd.read_parquet( + url, + engine="pyarrow", + columns=columns, + filters=parquet_filters, + ) + + +def load_expected_group_keys( + split: str = "train", + filters: dict[str, str] | None = None, + exact: bool = False, +) -> list[tuple[str, str, str, str]]: + """Return expected group keys for the given filters. + + By default this uses HF Dataset Viewer statistics to build a fast group + cross-product. Set exact=True for a slower filtered parquet scan of present keys. + """ + normalized_filters = normalize_filters(filters) + if not exact: + column_values = _hf_group_column_values(split) + values_by_column = [] + for column in GROUP_COLUMNS: + if normalized_filters and normalized_filters.get(column): + values_by_column.append([normalized_filters[column]]) + else: + values_by_column.append(column_values.get(column, [])) + return [tuple(group) for group in product(*values_by_column)] + + groups: set[tuple[str, str, str, str]] = set() + for shard in iter_makieval_parquet_frames( + split=split, + filters=normalized_filters, + columns=GROUP_COLUMNS, + ): + if shard.empty: + continue + unique = shard.drop_duplicates(GROUP_COLUMNS).sort_values(GROUP_COLUMNS) + groups.update( + tuple(row) + for row in unique[GROUP_COLUMNS].itertuples(index=False, name=None) + ) + return sorted(groups) + + +def load_makieval_dataset( + split: str = "train", + streaming: bool | None = None, + filters: dict[str, str] | None = None, + limit: int | None = None, +) -> tuple[pd.DataFrame, ParseReport]: + """ + Load Raoyuan/MAKIEval from Hugging Face and return a cleaned DataFrame. + + filters may include model, topic, language, country_region keys matching dataset columns. + Uses filtered parquet reads when filters are provided, and streaming otherwise. + """ + normalized_filters = normalize_filters(filters) + if filters: + try: + df = _load_from_parquet(split=split, filters=normalized_filters, limit=limit) + if df.empty: + report = ParseReport(0, 0, 0, 0) + return df, report + return clean_entities_column(df) + except Exception: + # Fall back to datasets streaming if Dataset Viewer/parquet is unavailable. + pass + + from datasets import load_dataset + + if streaming is None: + streaming = True + + dataset = load_dataset("Raoyuan/MAKIEval", split=split, streaming=streaming) + + rows: list[dict[str, Any]] = [] + for idx, row in enumerate(dataset): + if normalized_filters: + if normalized_filters.get("model") and row["model"] != normalized_filters["model"]: + continue + if normalized_filters.get("topic") and row["topic"] != normalized_filters["topic"]: + continue + if normalized_filters.get("language") and row["language"] != normalized_filters["language"]: + continue + if ( + normalized_filters.get("country_region") + and row["country_region"] != normalized_filters["country_region"] + ): + continue + rows.append(dict(row)) + if limit is not None and len(rows) >= limit: + break + + df = pd.DataFrame(rows) + if df.empty: + report = ParseReport(0, 0, 0, 0) + return df, report + return clean_entities_column(df) diff --git a/code/entity_extraction.py b/code/entity_extraction.py index a79d3a5..e3343d8 100644 --- a/code/entity_extraction.py +++ b/code/entity_extraction.py @@ -132,7 +132,12 @@ def parse_batch_results(filename): # **Complete get_key_words function** def get_key_words(response_text_dict, category): - client = OpenAI(api_key )= # Replace with actual API Key) + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + raise ValueError( + "OPENAI_API_KEY environment variable is required for entity extraction." + ) + client = OpenAI(api_key=api_key) file_id = prepare_batch_requests(response_text_dict, category, client) # Generate JSONL file batch_id = submit_batch(client, file_id) # Submit batch request diff --git a/code/lm_utils.py b/code/lm_utils.py index d787efc..2f80776 100644 --- a/code/lm_utils.py +++ b/code/lm_utils.py @@ -10,6 +10,22 @@ import wikipedia as wp from transformers import AutoModelForCausalLM, AutoTokenizer + +def _require_env(name): + value = os.environ.get(name) + if not value: + raise ValueError( + f"{name} environment variable is required. Set it in your environment." + ) + return value + + +def _resolve_deepseek_api_key(deep_seek_api=None): + if deep_seek_api: + return deep_seek_api + return _require_env("DEEPSEEK_API_KEY") + + def llm_init(model_name): global device global model @@ -53,13 +69,16 @@ def llm_init(model_name): model = AutoModelForCausalLM.from_pretrained("CohereLabs/aya-expanse-8b", device_map="auto", torch_dtype=torch.bfloat16) tokenizer = AutoTokenizer.from_pretrained("CohereLabs/aya-expanse-8b") if model_name == "deepseek": - client = OpenAI(api_key="API Keys", base_url="https://api.deepseek.com") + client = OpenAI( + api_key=_require_env("DEEPSEEK_API_KEY"), + base_url="https://api.deepseek.com", + ) if model_name == "chatgpt": - client = OpenAI(api_key="API Keys") # Fill in your API key + client = OpenAI(api_key=_require_env("OPENAI_API_KEY")) -def llm_response(prompt, model_name, num_responses, language=None, save_path = None, deep_seek_api = "sk-fe66b828e09e45a3ba34ec9b3caf05ee",temperature = 0.7, max_new_tokens = 100): +def llm_response(prompt, model_name, num_responses, language=None, save_path = None, deep_seek_api = None,temperature = 0.7, max_new_tokens = 100): language_instruction_map = { "en": "Please answer in English.", "zh": "请用中文作答。", @@ -129,7 +148,7 @@ def llm_response(prompt, model_name, num_responses, language=None, save_path = N if model_name == "chatgpt": responses = [] - client = OpenAI(api_key="API Keys") # Fill in your API key + client = OpenAI(api_key=_require_env("OPENAI_API_KEY")) for i in range(num_responses): response = client.chat.completions.create( @@ -145,7 +164,10 @@ def llm_response(prompt, model_name, num_responses, language=None, save_path = N if model_name == "deepseek": responses = [] # for backward compatibility, you can still use `https://api.deepseek.com/v1` as `base_url`. - client = OpenAI(api_key=deep_seek_api, base_url="https://api.deepseek.com") + client = OpenAI( + api_key=_resolve_deepseek_api_key(deep_seek_api), + base_url="https://api.deepseek.com", + ) for i in range(num_responses): diff --git a/code/metrics.py b/code/metrics.py new file mode 100644 index 0000000..7fea354 --- /dev/null +++ b/code/metrics.py @@ -0,0 +1,166 @@ +"""MAKIEval cultural-awareness metrics (paper Section 4).""" + +from __future__ import annotations + +from typing import Any, Iterable + +# Granularity: 1 = specific entity, 0 = category-level entity (Appendix B.2 / Section 4.1). +CATEGORY_ENTITY_TYPES = frozenset( + { + "dish_category", + "ingredient_category", + "beverage_category", + "book_genre", + "music_genre", + "fashion_style", + "vehicle_type", + } +) + +SPECIFIC_ENTITY_TYPES = frozenset( + { + "dish_name", + "dish", + "specific_ingredient", + "drink_name", + "book_title", + "song_name", + "clothing_type", + "mode_of_transport", + "artist_name", + "author_name", + } +) + +# Country aliases for matching target regions in culture specificity. +COUNTRY_ALIASES: dict[str, set[str]] = { + "united states": {"united states", "us", "usa", "u.s.", "america"}, + "united kingdom": {"united kingdom", "uk", "u.k.", "britain", "great britain"}, + "south korea": {"south korea", "korea", "republic of korea"}, + "united arab emirates": {"united arab emirates", "uae"}, + "taiwan": {"taiwan", "republic of china"}, + "china": {"china", "people's republic of china", "prc"}, + "mexico": {"mexico"}, + "spain": {"spain"}, + "argentina": {"argentina"}, + "germany": {"germany"}, + "iran": {"iran"}, + "india": {"india"}, + "italy": {"italy"}, + "japan": {"japan"}, + "thailand": {"thailand"}, + "turkey": {"turkey", "türkiye"}, + "canada": {"canada"}, + "australia": {"australia"}, + "nigeria": {"nigeria"}, +} + + +def _normalize_country(name: str) -> str: + return " ".join(name.strip().lower().replace("_", " ").split()) + + +def _entity_type(entity: dict[str, Any]) -> str: + return str(entity.get("entity_type", "")).lower() + + +def _entity_qid(entity: dict[str, Any]) -> str | None: + qid = entity.get("qid") + if qid is None or qid == "" or str(qid).upper() == "NA": + return None + return str(qid) + + +def _granularity_score(entity: dict[str, Any]) -> float | None: + entity_type = _entity_type(entity) + if entity_type in CATEGORY_ENTITY_TYPES: + return 0.0 + if entity_type in SPECIFIC_ENTITY_TYPES: + return 1.0 + # Unknown types are excluded from the average (paper uses typed cultural entities). + return None + + +def granularity(entities: Iterable[dict[str, Any]]) -> float: + """ + Eq. (1): Granularity = (1/|E|) * sum(Gran(e)). + + Specific cultural entities score 1; category-level entities score 0. + Entities with unknown types are excluded from |E|. + """ + scores = [s for s in (_granularity_score(e) for e in entities) if s is not None] + if not scores: + return 0.0 + return sum(scores) / len(scores) + + +def diversity(entities: Iterable[dict[str, Any]]) -> int: + """ + Section 4.2: count of unique Wikidata QIDs in entity set E. + + Rows with null/NA QIDs are excluded from diversity (QID-required metric). + """ + qids = {_entity_qid(e) for e in entities} + qids.discard(None) + return len(qids) + + +def _countries_match(target_country: str, origin_country: str) -> bool: + target_norm = _normalize_country(target_country) + origin_norm = _normalize_country(origin_country) + + if target_norm == origin_norm: + return True + + for canonical, aliases in COUNTRY_ALIASES.items(): + if target_norm in aliases or target_norm == canonical: + if origin_norm in aliases or origin_norm == canonical: + return True + return False + + +def culture_specificity( + entities: Iterable[dict[str, Any]], + target_country: str, + country_lookup: dict[str, str], +) -> float: + """ + Section 4.3: proportion of entities whose origin country matches target_country. + + country_lookup maps QID -> English country label from Wikidata (P495/P17). + Entities without a QID or without lookup metadata are excluded from the denominator. + """ + eligible = [] + matches = 0 + for entity in entities: + qid = _entity_qid(entity) + if not qid: + continue + origin = country_lookup.get(qid) + if not origin or origin == "NA": + continue + eligible.append(entity) + if _countries_match(target_country, origin): + matches += 1 + + if not eligible: + return 0.0 + return matches / len(eligible) + + +def culture_consensus(qids_lang_a: Iterable[str], qids_lang_b: Iterable[str]) -> float: + """ + Eq. (2) in Section 4.4: Jaccard similarity |Q_A ∩ Q_B| / |Q_A ∪ Q_B|. + """ + set_a = {qid for qid in qids_lang_a if qid} + set_b = {qid for qid in qids_lang_b if qid} + union = set_a | set_b + if not union: + return 0.0 + return len(set_a & set_b) / len(union) + + +def collect_qids(entities: Iterable[dict[str, Any]]) -> set[str]: + qids = {_entity_qid(e) for e in entities} + qids.discard(None) + return qids diff --git a/code/quality_report.py b/code/quality_report.py new file mode 100644 index 0000000..8252712 --- /dev/null +++ b/code/quality_report.py @@ -0,0 +1,539 @@ +"""Quality checks for the published Raoyuan/MAKIEval Hugging Face dataset.""" + +from __future__ import annotations + +import argparse +import json +import os +import random +import re +import sys +import unicodedata +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from tqdm import tqdm + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from data_loading import ( + DEFAULT_COLUMNS, + filter_entities_for_metrics, + iter_makieval_parquet_frames, + parse_entities_field, +) + +try: + import langid +except Exception: # pragma: no cover - exercised only when optional dep is absent. + langid = None + + +REPORT_NOTE = ( + "Scope: published HF dataset, version may differ from paper (per author)." +) + +LANG_ALIASES = { + "zh-tw": {"zh", "zh-tw"}, + "zh": {"zh", "zh-tw"}, +} + +SUSPICIOUS_LABELS = { + "cocaine", + "coca\u00edna", + "kokain", + "caffeine", + "kafein", + "curiosity", + "merak", +} + +SUSPICIOUS_QIDS = { + "Q422", + "Q60235", +} + +SUSPICIOUS_TYPES = { + "dish_name", + "dish", + "specific_ingredient", + "drink_name", + "beverage_category", +} + + +def normalize_label(label: str) -> str: + """Normalize entity surface forms for rough cross-row comparisons.""" + text = unicodedata.normalize("NFKD", str(label).strip().lower()) + text = "".join(ch for ch in text if not unicodedata.combining(ch)) + text = re.sub(r"^[\"'`]+|[\"'`]+$", "", text) + text = re.sub(r"^(the|a|an)\s+", "", text) + text = re.sub(r"^\u0627\u0644", "", text) + text = re.sub(r"\s+", " ", text) + return text + + +def entity_label(entity: dict[str, Any]) -> str: + for key in ("entity", "label", "text", "name"): + value = entity.get(key) + if value: + return str(value) + return "" + + +def entity_qid(entity: dict[str, Any]) -> str | None: + qid = entity.get("qid") + if qid is None or qid == "" or str(qid).upper() == "NA": + return None + return str(qid) + + +def is_empty_text(text: Any) -> bool: + return not str(text or "").strip() + + +def has_repeated_ngram_block(text: str, threshold: int = 10) -> bool: + if re.search(r"(.)\1{30,}", text): + return True + tokens = re.findall(r"\w+|[^\w\s]", text.lower(), flags=re.UNICODE) + if len(tokens) < threshold * 2: + return False + for n in (1, 2, 3): + previous = None + run = 0 + for idx in range(0, len(tokens) - n + 1, n): + chunk = tuple(tokens[idx : idx + n]) + if chunk == previous: + run += 1 + else: + previous = chunk + run = 1 + if run >= threshold: + return True + return False + + +def has_repeated_sentence(text: str, threshold: int = 3) -> bool: + sentences = [ + re.sub(r"\s+", " ", part.strip().lower()) + for part in re.split(r"[.!?\n\u3002\uff01\uff1f]+", text) + if part.strip() + ] + if not sentences: + return False + counts = Counter(sentences) + return any(count >= threshold for count in counts.values()) + + +def detect_language(text: str) -> str: + if is_empty_text(text): + return "und" + if langid is not None: + try: + return langid.classify(text)[0].lower() + except Exception: + return "und" + + # Fallback keeps the report runnable when langid is not installed. + if re.search(r"[\u0600-\u06ff]", text): + return "ar" + if re.search(r"[\u3040-\u30ff]", text): + return "ja" + if re.search(r"[\uac00-\ud7af]", text): + return "ko" + if re.search(r"[\u4e00-\u9fff]", text): + return "zh" + if re.search(r"[\u0e00-\u0e7f]", text): + return "th" + if re.search(r"[\u0900-\u097f]", text): + return "hi" + return "unknown" + + +def language_matches(expected: str, detected: str) -> bool: + expected_norm = str(expected).lower() + detected_norm = str(detected).lower() + if expected_norm == detected_norm: + return True + return detected_norm in LANG_ALIASES.get(expected_norm, set()) + + +def short_text(text: Any, limit: int = 180) -> str: + value = re.sub(r"\s+", " ", str(text or "").strip()) + if len(value) <= limit: + return value + return value[: limit - 3] + "..." + + +@dataclass +class BucketStats: + rows: int = 0 + repeated_ngram_rows: int = 0 + repeated_sentence_rows: int = 0 + empty_entity_rows: int = 0 + language_checked_rows: int = 0 + language_mismatch_rows: int = 0 + entity_count_all_types: int = 0 + missing_qid_all_types: int = 0 + entity_count_cultural_only: int = 0 + missing_qid_cultural_only: int = 0 + suspicious_count: int = 0 + + +@dataclass +class QualityState: + overall: BucketStats = field(default_factory=BucketStats) + by_slice: dict[tuple[str, str, str], BucketStats] = field( + default_factory=lambda: defaultdict(BucketStats) + ) + by_topic: dict[str, BucketStats] = field(default_factory=lambda: defaultdict(BucketStats)) + examples: dict[str, list[dict[str, Any]]] = field( + default_factory=lambda: defaultdict(list) + ) + label_qids: dict[str, Counter] = field(default_factory=lambda: defaultdict(Counter)) + observed_groups: set[tuple[str, str, str, str]] = field(default_factory=set) + + +def add_example(state: QualityState, kind: str, row: dict[str, Any], extra: str = "") -> None: + if len(state.examples[kind]) >= 3: + return + state.examples[kind].append( + { + "model": row.get("model"), + "topic": row.get("topic"), + "language": row.get("language"), + "country_region": row.get("country_region"), + "detail": extra, + "generated_text": short_text(row.get("generated_text")), + } + ) + + +def update_bucket( + bucket: BucketStats, + row_flags: dict[str, bool], + entities: list[dict[str, Any]], + cultural_entities: list[dict[str, Any]], +) -> None: + bucket.rows += 1 + bucket.repeated_ngram_rows += int(row_flags["repeated_ngram"]) + bucket.repeated_sentence_rows += int(row_flags["repeated_sentence"]) + bucket.empty_entity_rows += int(row_flags["empty_entities"]) + bucket.language_checked_rows += int(row_flags["language_checked"]) + bucket.language_mismatch_rows += int(row_flags["language_mismatch"]) + bucket.suspicious_count += int(row_flags["suspicious"]) + bucket.entity_count_all_types += len(entities) + bucket.missing_qid_all_types += sum(1 for entity in entities if not entity_qid(entity)) + bucket.entity_count_cultural_only += len(cultural_entities) + bucket.missing_qid_cultural_only += sum( + 1 for entity in cultural_entities if not entity_qid(entity) + ) + + +def process_row(state: QualityState, row: dict[str, Any]) -> None: + key = ( + str(row.get("model")), + str(row.get("topic")), + str(row.get("language")), + str(row.get("country_region")), + ) + state.observed_groups.add(key) + + entities, ok = parse_entities_field(row.get("entities")) + if not ok: + entities = [] + cultural_entities, _excluded = filter_entities_for_metrics(entities) + + text = str(row.get("generated_text") or "") + detected_language = detect_language(text) + language_checked = detected_language not in {"und", "unknown"} + language_mismatch = language_checked and not language_matches( + str(row.get("language")), detected_language + ) + + suspicious_entities = [] + for entity in entities: + label = normalize_label(entity_label(entity)) + qid = entity_qid(entity) + entity_type = str(entity.get("entity_type", "")).lower() + if label and qid: + state.label_qids[label][qid] += 1 + if entity_type in SUSPICIOUS_TYPES and ( + label in SUSPICIOUS_LABELS or qid in SUSPICIOUS_QIDS + ): + suspicious_entities.append(entity) + + row_flags = { + "repeated_ngram": has_repeated_ngram_block(text), + "repeated_sentence": has_repeated_sentence(text), + "empty_entities": len(entities) == 0, + "language_checked": language_checked, + "language_mismatch": language_mismatch, + "suspicious": bool(suspicious_entities), + } + + update_bucket(state.overall, row_flags, entities, cultural_entities) + update_bucket( + state.by_slice[(str(row.get("model")), str(row.get("language")), str(row.get("topic")))], + row_flags, + entities, + cultural_entities, + ) + update_bucket(state.by_topic[str(row.get("topic"))], row_flags, entities, cultural_entities) + + if row_flags["repeated_ngram"]: + add_example(state, "repeated_ngram", row) + if row_flags["repeated_sentence"]: + add_example(state, "repeated_sentence", row) + if row_flags["empty_entities"]: + add_example(state, "empty_entities", row) + if row_flags["language_mismatch"]: + add_example(state, "language_mismatch", row, f"detected={detected_language}") + if suspicious_entities: + detail = ", ".join( + f"{entity_label(entity)} ({entity.get('entity_type')}, {entity_qid(entity) or 'NA'})" + for entity in suspicious_entities[:3] + ) + add_example(state, "suspicious_entities", row, detail) + + +def scan_rows(sample: int | None, seed: int, full: bool) -> QualityState: + rng = random.Random(seed) + state = QualityState() + reservoirs: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list) + seen_by_sample_key: Counter = Counter() + columns = DEFAULT_COLUMNS + + for shard in tqdm( + iter_makieval_parquet_frames(columns=columns), + desc="Reading MAKIEval parquet shards", + ): + if shard.empty: + continue + for record in shard.to_dict(orient="records"): + group_key = ( + str(record.get("model")), + str(record.get("topic")), + str(record.get("language")), + str(record.get("country_region")), + ) + state.observed_groups.add(group_key) + + if full or sample is None: + process_row(state, record) + continue + + sample_key = ( + str(record.get("model")), + str(record.get("language")), + str(record.get("topic")), + ) + seen_by_sample_key[sample_key] += 1 + seen = seen_by_sample_key[sample_key] + bucket = reservoirs[sample_key] + if len(bucket) < sample: + bucket.append(record) + else: + replace_at = rng.randrange(seen) + if replace_at < sample: + bucket[replace_at] = record + + if not full and sample is not None: + for bucket in reservoirs.values(): + for record in bucket: + process_row(state, record) + + return state + + +def pct(numerator: int, denominator: int) -> str: + if denominator == 0: + return "0.00%" + return f"{100 * numerator / denominator:.2f}%" + + +def bucket_row(label: str, stats: BucketStats) -> str: + cultural_qid_rate = pct( + stats.missing_qid_cultural_only, + stats.entity_count_cultural_only, + ) + all_qid_rate = pct(stats.missing_qid_all_types, stats.entity_count_all_types) + lang_rate = pct(stats.language_mismatch_rows, stats.language_checked_rows) + return ( + f"| {label} | {stats.rows} | {pct(stats.repeated_ngram_rows, stats.rows)} | " + f"{pct(stats.repeated_sentence_rows, stats.rows)} | " + f"{pct(stats.empty_entity_rows, stats.rows)} | {lang_rate} | " + f"{cultural_qid_rate} | {all_qid_rate} | {stats.suspicious_count} |" + ) + + +def top_surface_inconsistencies(state: QualityState, limit: int = 20) -> list[tuple[str, Counter]]: + rows = [ + (label, counts) + for label, counts in state.label_qids.items() + if len(counts) > 1 + ] + rows.sort(key=lambda item: (len(item[1]), sum(item[1].values())), reverse=True) + return rows[:limit] + + +def examples_markdown(state: QualityState, kind: str) -> list[str]: + rows = state.examples.get(kind, []) + if not rows: + return ["No examples captured in this run."] + lines = [] + for row in rows: + context = ( + f"{row['model']} / {row['topic']} / {row['language']} / " + f"{row['country_region']}" + ) + detail = f" ({row['detail']})" if row.get("detail") else "" + lines.append(f"- {context}{detail}: {row['generated_text']}") + return lines + + +def build_report(state: QualityState, mode_label: str) -> str: + lines = [ + "# MAKIEval Published Data Quality Report", + "", + REPORT_NOTE, + "", + f"- Mode: `{mode_label}`", + f"- Rows analyzed for quality checks: `{state.overall.rows}`", + f"- Observed `(model, topic, language, country_region)` cells: `{len(state.observed_groups)}`", + "", + "## Summary", + "", + "| Scope | Rows | Repeated n-gram | Repeated sentence | Empty entities | Language mismatch | missing_qid_cultural_only | missing_qid_all_types | Suspicious rows |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|", + bucket_row("Overall", state.overall), + "", + "## Slice Breakdown", + "", + "| Model / Language / Topic | Rows | Repeated n-gram | Repeated sentence | Empty entities | Language mismatch | missing_qid_cultural_only | missing_qid_all_types | Suspicious rows |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + + slice_rows = sorted( + state.by_slice.items(), + key=lambda item: ( + item[1].language_mismatch_rows / max(1, item[1].language_checked_rows), + item[1].empty_entity_rows / max(1, item[1].rows), + ), + reverse=True, + ) + for (model, language, topic), stats in slice_rows[:30]: + lines.append(bucket_row(f"{model} / {language} / {topic}", stats)) + + lines.extend( + [ + "", + "## Missing QID By Topic", + "", + "| Topic | cultural_entities | missing_qid_cultural_only | missing_qid_cultural_only_rate | all_entities | missing_qid_all_types | missing_qid_all_types_rate |", + "|---|---:|---:|---:|---:|---:|---:|", + ] + ) + for topic, stats in sorted(state.by_topic.items()): + lines.append( + f"| {topic} | {stats.entity_count_cultural_only} | " + f"{stats.missing_qid_cultural_only} | " + f"{pct(stats.missing_qid_cultural_only, stats.entity_count_cultural_only)} | " + f"{stats.entity_count_all_types} | {stats.missing_qid_all_types} | " + f"{pct(stats.missing_qid_all_types, stats.entity_count_all_types)} |" + ) + + lines.extend( + [ + "", + "## Surface Form To QID Inconsistency", + "", + "| Normalized label | Distinct QIDs | Top QIDs |", + "|---|---:|---|", + ] + ) + inconsistencies = top_surface_inconsistencies(state) + if not inconsistencies: + lines.append("| None captured | 0 | |") + for label, counts in inconsistencies: + top_qids = ", ".join(f"{qid} ({count})" for qid, count in counts.most_common(5)) + lines.append(f"| {label} | {len(counts)} | {top_qids} |") + + example_sections = [ + ("Repeated n-gram examples", "repeated_ngram"), + ("Repeated sentence examples", "repeated_sentence"), + ("Empty entity examples", "empty_entities"), + ("Language mismatch examples", "language_mismatch"), + ("Suspicious extraction review candidates", "suspicious_entities"), + ] + for title, key in example_sections: + lines.extend(["", f"## {title}", ""]) + lines.extend(examples_markdown(state, key)) + + lines.extend( + [ + "", + "## Notes", + "", + "- Language detection uses `langid` when installed, with a script-based fallback.", + "- Suspicious extraction rows are rule-based review candidates, not confirmed false positives.", + "- `missing_qid_cultural_only` excludes `place`, `person_name`, `listener_name`, and `reader_name` via `data_loading.filter_entities_for_metrics`; this is the Table 10-comparable missing-QID rate.", + "- `missing_qid_all_types` keeps every extracted entity type for broader extraction/linking completeness diagnostics.", + "- Missing-QID rates are compared descriptively against the paper's 26-35% range; released data may use a different snapshot.", + ] + ) + return "\n".join(lines) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Audit published MAKIEval data quality.") + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--sample", type=int, default=200, help="Reservoir sample rows per model-language-topic slice.") + mode.add_argument("--full", action="store_true", help="Analyze every row in the published dataset.") + parser.add_argument("--seed", type=int, default=42, help="Sampling seed") + parser.add_argument( + "--output", + default="docs/DATA_QUALITY_REPORT.md", + help="Markdown report path", + ) + args = parser.parse_args() + + sample = None if args.full else args.sample + state = scan_rows(sample=sample, seed=args.seed, full=args.full) + mode_label = "full" if args.full else f"sample={sample} per model-language-topic" + report = build_report(state, mode_label) + + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(report, encoding="utf-8") + + print(f"Wrote quality report to {output}") + print( + json.dumps( + { + "mode": mode_label, + "rows_analyzed": state.overall.rows, + "observed_cells": len(state.observed_groups), + "empty_entity_rate": pct(state.overall.empty_entity_rows, state.overall.rows), + "language_mismatch_rate": pct( + state.overall.language_mismatch_rows, + state.overall.language_checked_rows, + ), + "missing_qid_cultural_only_rate": pct( + state.overall.missing_qid_cultural_only, + state.overall.entity_count_cultural_only, + ), + "missing_qid_all_types_rate": pct( + state.overall.missing_qid_all_types, + state.overall.entity_count_all_types, + ), + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/code/run_experiment.py b/code/run_experiment.py index 2908ae0..edbbcfa 100644 --- a/code/run_experiment.py +++ b/code/run_experiment.py @@ -3,7 +3,7 @@ import argparse import os import logging -from entity_extraction_nobatch import get_key_words +from entity_extraction import get_key_words from analysis_food import fetch_wikidata_food_info from analysis_beverage import fetch_wikidata_beverage_info from analysis_clothing import fetch_wikidata_clothing_info @@ -17,7 +17,7 @@ import json import shlex from collections import defaultdict, Counter -sys.path.append(PATH) +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import prompt_construct import lm_utils import re @@ -415,4 +415,4 @@ def filter_nan_qid(group): } with open(runtime_log_path, "w", encoding="utf-8") as f: json.dump(runtime_data, f, indent=4, ensure_ascii=False) - logging.info(f"Execution time log saved to {runtime_log_path}") \ No newline at end of file + logging.info(f"Execution time log saved to {runtime_log_path}") diff --git a/code/run_metrics.py b/code/run_metrics.py new file mode 100644 index 0000000..2687406 --- /dev/null +++ b/code/run_metrics.py @@ -0,0 +1,236 @@ +"""CLI to compute MAKIEval metrics from the Hugging Face dataset.""" + +from __future__ import annotations + +import argparse +import json +import os +import random +import sys +from collections import defaultdict +from pathlib import Path + +import numpy as np +import pandas as pd + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from data_loading import GROUP_COLUMNS, load_expected_group_keys, load_makieval_dataset +from metrics import ( + collect_qids, + culture_consensus, + culture_specificity, + diversity, + granularity, +) + + +def _set_seed(seed: int) -> None: + random.seed(seed) + np.random.seed(seed) + + +def _aggregate_group_metrics( + group_df: pd.DataFrame, + country_lookup: dict[str, str], +) -> dict: + all_entities: list[dict] = [] + for entities in group_df["entities_parsed"]: + all_entities.extend(entities) + + target_country = str(group_df["country_region"].iloc[0]) + return { + "granularity": granularity(all_entities), + "diversity": diversity(all_entities), + "culture_specificity": culture_specificity( + all_entities, target_country, country_lookup + ), + "entity_count": len(all_entities), + "unique_qids": len(collect_qids(all_entities)), + } + + +def _compute_consensus_rows(df: pd.DataFrame) -> list[dict]: + """Pairwise culture consensus across languages for each (model, topic, country).""" + rows = [] + group_cols = ["model", "topic", "country_region"] + for (model, topic, country), group in df.groupby(group_cols): + lang_qids: dict[str, set[str]] = {} + for language, lang_df in group.groupby("language"): + qids: set[str] = set() + for entities in lang_df["entities_parsed"]: + qids.update(collect_qids(entities)) + lang_qids[language] = qids + + languages = sorted(lang_qids) + for i, lang_a in enumerate(languages): + for lang_b in languages[i + 1 :]: + rows.append( + { + "model": model, + "topic": topic, + "country_region": country, + "language_a": lang_a, + "language_b": lang_b, + "culture_consensus": culture_consensus( + lang_qids[lang_a], lang_qids[lang_b] + ), + } + ) + return rows + + +def _add_run_metadata( + df: pd.DataFrame, + is_partial_sample: bool, + n_rows_used: int, + warning: str, +) -> pd.DataFrame: + result = df.copy() + result["is_partial_sample"] = is_partial_sample + result["n_rows_used"] = n_rows_used + result["warning"] = warning + return result + + +def _computed_group_keys(metrics_df: pd.DataFrame) -> set[tuple[str, str, str, str]]: + if metrics_df.empty: + return set() + return { + tuple(row) + for row in metrics_df[GROUP_COLUMNS].itertuples(index=False, name=None) + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Compute MAKIEval metrics from HF data.") + parser.add_argument("--model", default=None, help="Filter by model name") + parser.add_argument("--topic", default=None, help="Filter by topic") + parser.add_argument("--language", default=None, help="Filter by language") + parser.add_argument("--country", default=None, help="Filter by country_region") + parser.add_argument( + "--limit", + type=int, + default=5000, + help="Optional row limit for quick reproducibility checks (default: 5000)", + ) + parser.add_argument( + "--full", + action="store_true", + help="Process all rows matching the filters instead of the default quick limit", + ) + parser.add_argument("--seed", type=int, default=42, help="Random seed") + parser.add_argument( + "--output-dir", + default="results/metrics", + help="Directory for CSV/JSON outputs", + ) + parser.add_argument( + "--country-lookup", + default=None, + help="Optional JSON file mapping QID -> origin country label", + ) + args = parser.parse_args() + _set_seed(args.seed) + if args.full: + args.limit = None + + filters = { + k: v + for k, v in { + "model": args.model, + "topic": args.topic, + "language": args.language, + "country_region": args.country, + }.items() + if v + } + + df, report = load_makieval_dataset(filters=filters or None, limit=args.limit) + if df.empty: + print("No rows matched the provided filters.") + return + + country_lookup: dict[str, str] = {} + if args.country_lookup and os.path.exists(args.country_lookup): + with open(args.country_lookup, encoding="utf-8") as f: + country_lookup = json.load(f) + + metric_rows = [] + group_cols = GROUP_COLUMNS + for keys, group in df.groupby(group_cols): + if not isinstance(keys, tuple): + keys = (keys,) + metrics = _aggregate_group_metrics(group, country_lookup) + row = dict(zip(group_cols, keys)) + row.update(metrics) + metric_rows.append(row) + + metrics_df = pd.DataFrame(metric_rows) + consensus_df = pd.DataFrame(_compute_consensus_rows(df)) + is_partial_sample = args.limit is not None + warning = "" + if is_partial_sample: + warning = ( + f"WARNING: partial sample ({len(df)} rows); diversity/consensus NOT " + "comparable to paper. Use --full." + ) + print(warning, file=sys.stderr) + + expected_groups = load_expected_group_keys(filters=filters or None, exact=True) + computed_groups = _computed_group_keys(metrics_df) + missing_groups = sorted(set(expected_groups) - computed_groups) + completeness_warning = "" + if len(expected_groups) != len(computed_groups): + completeness_warning = ( + f"WARNING: expected {len(expected_groups)} groups, computed " + f"{len(computed_groups)}; missing: {missing_groups}" + ) + print(completeness_warning, file=sys.stderr) + + combined_warning = " ".join( + message for message in [warning, completeness_warning] if message + ) + metrics_df = _add_run_metadata( + metrics_df, + is_partial_sample=is_partial_sample, + n_rows_used=len(df), + warning=combined_warning, + ) + consensus_df = _add_run_metadata( + consensus_df, + is_partial_sample=is_partial_sample, + n_rows_used=len(df), + warning=combined_warning, + ) + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + metrics_path = output_dir / "group_metrics.csv" + consensus_path = output_dir / "culture_consensus.csv" + report_path = output_dir / "parse_report.json" + + metrics_df.to_csv(metrics_path, index=False) + consensus_df.to_csv(consensus_path, index=False) + report_payload = report.__dict__ | { + "is_partial_sample": is_partial_sample, + "n_rows_used": len(df), + "warning": combined_warning, + "expected_group_count": len(expected_groups), + "computed_group_count": len(computed_groups), + "missing_groups": [list(group) for group in missing_groups], + } + with open(report_path, "w", encoding="utf-8") as f: + json.dump(report_payload, f, indent=2) + + print(f"Wrote {len(metrics_df)} group metric rows to {metrics_path}") + print(f"Wrote {len(consensus_df)} consensus rows to {consensus_path}") + print( + f"Parse report: {report.parse_errors} parse errors / " + f"{report.total_rows} rows" + ) + + +if __name__ == "__main__": + main() diff --git a/code/smoke_generation.py b/code/smoke_generation.py new file mode 100644 index 0000000..28f0372 --- /dev/null +++ b/code/smoke_generation.py @@ -0,0 +1,460 @@ +"""Small faithful/proxy MAKIEval generation smoke test via Together AI.""" + +from __future__ import annotations + +import argparse +import json +import os +import random +import re +import sys +from collections import Counter +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +from openai import OpenAI + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import analysis_beverage +import analysis_book +import analysis_clothing +import analysis_food +import analysis_music +import analysis_transportation +import entity_extraction +import prompt_construct +from data_loading import clean_entities_column, filter_entities_for_metrics, load_makieval_dataset, normalize_filters +from metrics import collect_qids, culture_specificity, diversity, granularity + + +TOGETHER_BASE_URL = "https://api.together.ai/v1" + +FAITHFUL_DECODING = { + "temperature": 0.7, + "top_p": 0.9, + "top_k": 10, + "max_tokens": 100, +} + +PAPER_MODEL_TO_TOGETHER = { + "Qwen2.5-7B-Instruct": "Qwen/Qwen2.5-7B-Instruct-Turbo", + "Llama-3.1-8B-Instruct": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + "Llama-3.3-70B-Instruct": "meta-llama/Llama-3.3-70B-Instruct-Turbo", + "Mistral-7B-Instruct-v0.1": "mistralai/Mistral-7B-Instruct-v0.1", + "aya-expanse-8b": "CohereLabs/aya-expanse-8b", + "DeepSeek-V3": "deepseek-ai/DeepSeek-V3", + "ChatGPT-4o-mini": "openai/gpt-oss-120b", +} + +MODEL_ALIASES = { + "qwen": "Qwen2.5-7B-Instruct", + "qwen2.5-7b-instruct": "Qwen2.5-7B-Instruct", + "llama3": "Llama-3.1-8B-Instruct", + "llama3_70b": "Llama-3.3-70B-Instruct", + "mistral": "Mistral-7B-Instruct-v0.1", + "aya": "aya-expanse-8b", + "deepseek": "DeepSeek-V3", + "chatgpt": "ChatGPT-4o-mini", +} + +LANGUAGE_INSTRUCTION_MAP = { + "en": "Please answer in English.", + "zh": "\u8bf7\u7528\u4e2d\u6587\u4f5c\u7b54\u3002", + "zh-tw": "\u8acb\u7528\u7e41\u9ad4\u4e2d\u6587\u4f5c\u7b54\u3002", + "de": "Bitte antworten Sie auf Deutsch.", + "it": "Si prega di rispondere in italiano.", + "yo": "J\u1ecdw\u1ecd dahun ni ede Yoruba.", + "ha": "Don Allah ku amsa da Hausa.", + "ar": "\u064a\u0631\u062c\u0649 \u0627\u0644\u0631\u062f \u0628\u0627\u0644\u0644\u063a\u0629 \u0627\u0644\u0639\u0631\u0628\u064a\u0629.", + "tr": "L\u00fctfen T\u00fcrk\u00e7e cevap verin.", + "fa": "\u0644\u0637\u0641\u0627\u064b \u0628\u0647 \u0632\u0628\u0627\u0646 \u0641\u0627\u0631\u0633\u06cc \u067e\u0627\u0633\u062e \u062f\u0647\u06cc\u062f.", + "th": "\u0e01\u0e23\u0e38\u0e13\u0e32\u0e15\u0e2d\u0e1a\u0e40\u0e1b\u0e47\u0e19\u0e20\u0e32\u0e29\u0e32\u0e44\u0e17\u0e22", + "ja": "\u65e5\u672c\u8a9e\u3067\u7b54\u3048\u3066\u304f\u3060\u3055\u3044\u3002", + "ko": "\ud55c\uad6d\uc5b4\ub85c \ub300\ub2f5\ud574\uc8fc\uc138\uc694.", + "hi": "\u0915\u0943\u092a\u092f\u093e \u0939\u093f\u0902\u0926\u0940 \u092e\u0947\u0902 \u0909\u0924\u094d\u0924\u0930 \u0926\u0947\u0902\u0964", +} + +TOPIC_FETCHERS = { + "food": analysis_food.fetch_wikidata_food_info, + "beverage": analysis_beverage.fetch_wikidata_beverage_info, + "clothing": analysis_clothing.fetch_wikidata_clothing_info, + "music": analysis_music.fetch_wikidata_music_info, + "book": analysis_book.fetch_wikidata_book_info, + "transportation": analysis_transportation.fetch_wikidata_transportation_info, +} + + +def canonical_model_name(model: str) -> str: + return MODEL_ALIASES.get(model.strip().lower(), model.strip()) + + +def normalize_topic(topic: str) -> str: + return "book" if topic.strip().lower() == "books" else topic.strip().lower() + + +def safe_slug(value: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("_") + + +def require_together_key() -> str: + key = os.environ.get("TOGETHER_API_KEY") + if not key: + key = os.environ.get("TOGETHER_API_KEY".lower()) + if not key: + raise ValueError("TOGETHER_API_KEY environment variable is required.") + return key + + +def together_client() -> OpenAI: + return OpenAI(api_key=require_together_key(), base_url=TOGETHER_BASE_URL) + + +def resolve_together_model(model: str) -> tuple[str, list[str]]: + canonical = canonical_model_name(model) + if canonical not in PAPER_MODEL_TO_TOGETHER: + allowed = ", ".join(sorted(PAPER_MODEL_TO_TOGETHER)) + raise ValueError(f"Model '{model}' is outside the paper set. Allowed: {allowed}") + + deviations = [] + together_model = PAPER_MODEL_TO_TOGETHER[canonical] + if canonical == "ChatGPT-4o-mini": + deviations.append("PROXY MODEL - ChatGPT-4o-mini is replaced by Together openai/gpt-oss-120b.") + elif canonical != together_model: + deviations.append(f"Together serving endpoint used: {together_model}.") + if together_model.endswith("-Turbo"): + deviations.append("Provider serving variant uses a Turbo endpoint, not local weights.") + return together_model, deviations + + +def add_language_instruction_if_needed(prompt: str, model: str, language: str) -> str: + canonical = canonical_model_name(model) + if canonical.startswith("Qwen") or canonical.startswith("Mistral"): + instruction = LANGUAGE_INSTRUCTION_MAP.get(language, f"Please answer in {language}.") + return prompt + instruction + return prompt + + +def load_prompt( + model: str, + topic: str, + language: str, + country: str, + use_published_prompt: bool, +) -> tuple[str, dict[str, Any]]: + filters = { + "model": canonical_model_name(model), + "topic": normalize_topic(topic), + "language": language, + "country_region": country, + } + if use_published_prompt: + df, _report = load_makieval_dataset(filters=filters, limit=1) + if not df.empty: + row = df.iloc[0].to_dict() + return str(row["prompt"]), {"source": "published_hf_prompt", "row": row} + + matches = prompt_construct.search_category_bias( + "meta_info/prompt.json", + normalize_topic(topic), + "bias", + language_filter=language, + ) + if not matches: + raise ValueError(f"No prompt template for topic={topic}, language={language}.") + prompt = matches[0]["text"].replace("{country}", country) + return prompt, {"source": "meta_info/prompt.json"} + + +def together_chat( + client: OpenAI, + model: str, + messages: list[dict[str, str]], + decoding: dict[str, Any], + *, + json_mode: bool = False, + seed: int | None = None, +) -> str: + extra_body = {"top_k": decoding["top_k"], "reasoning_effort": "low"} + if seed is not None: + extra_body["seed"] = seed + kwargs: dict[str, Any] = { + "model": model, + "messages": messages, + "temperature": decoding["temperature"], + "top_p": decoding["top_p"], + "max_tokens": decoding["max_tokens"], + "extra_body": extra_body, + } + if json_mode: + kwargs["response_format"] = {"type": "json_object"} + response = client.chat.completions.create(**kwargs) + return response.choices[0].message.content or "" + + +def generate_responses( + client: OpenAI, + prompt: str, + together_model: str, + num_responses: int, + decoding: dict[str, Any], + seed: int, +) -> list[str]: + rows = [] + for idx in range(num_responses): + messages = [{"role": "user", "content": prompt}] + rows.append( + together_chat( + client, + together_model, + messages, + decoding, + seed=seed + idx, + ) + ) + return rows + + +def parse_extraction_dict(content: str) -> dict[str, str]: + text = content.strip() + if not text: + return {} + fenced = re.search(r"```(?:json)?\s*(.*?)```", text, flags=re.DOTALL | re.IGNORECASE) + if fenced: + text = fenced.group(1).strip() + first = text.find("{") + last = text.rfind("}") + if first != -1 and last != -1 and last > first: + text = text[first : last + 1] + try: + parsed = json.loads(text) + except json.JSONDecodeError: + return {} + if not isinstance(parsed, dict): + return {} + return {str(k): str(v) for k, v in parsed.items()} + + +def extract_entities_with_together( + client: OpenAI, + responses: list[str], + topic: str, + extraction_model: str, +) -> tuple[list[list[dict[str, Any]]], str]: + decoding = {"temperature": 0.0, "top_p": 1.0, "top_k": 10, "max_tokens": 180} + extracted_rows: list[list[dict[str, Any]]] = [] + used_model = extraction_model + + for response in responses: + prompt = f"""Help me extract the words or phrases in the given text which are {entity_extraction.get_topic_hint(topic)}. +Answer in a dictionary format, with the type of the extracted text labeled. +{entity_extraction.get_example_for_topic(topic)} +Please ensure that you do not extract redundant entities. Do not provide any explanations, only output JSON like: +{{"": "type", "": "type"}} +Text: {response}""" + content = "" + errors = [] + for candidate in [extraction_model, "Qwen/Qwen2.5-7B-Instruct-Turbo"]: + try: + content = together_chat( + client, + candidate, + [ + {"role": "system", "content": "You extract cultural entities as strict JSON."}, + {"role": "user", "content": prompt}, + ], + decoding, + json_mode=True, + ) + used_model = candidate + break + except Exception as exc: + errors.append(f"{candidate}: {type(exc).__name__}") + if not content and errors: + raise RuntimeError("Together extraction failed: " + "; ".join(errors)) + parsed = parse_extraction_dict(content) + extracted_rows.append( + [ + {"entity": label, "entity_type": entity_type, "qid": None} + for label, entity_type in parsed.items() + ] + ) + return extracted_rows, used_model + + +def link_entities( + extracted_rows: list[list[dict[str, Any]]], + topic: str, + language: str, +) -> tuple[list[list[dict[str, Any]]], dict[str, str]]: + fetcher = TOPIC_FETCHERS.get(normalize_topic(topic)) + if fetcher is None: + raise ValueError(f"Unsupported topic for linking: {topic}") + + labels = sorted( + { + str(entity["entity"]) + for row in extracted_rows + for entity in row + if entity.get("entity") + } + ) + if not labels: + return extracted_rows, {} + + linked_df = fetcher(labels, language) + label_to_info: dict[str, dict[str, Any]] = {} + for record in linked_df.to_dict(orient="records"): + label_to_info[str(record.get("Original Label"))] = record + + country_lookup: dict[str, str] = {} + linked_rows: list[list[dict[str, Any]]] = [] + for row in extracted_rows: + linked_row = [] + for entity in row: + info = label_to_info.get(str(entity.get("entity")), {}) + qid = info.get("Q_ID") + qid_value = None if qid in (None, "", "NA") else str(qid) + if qid_value: + country_lookup[qid_value] = str(info.get("Origin Country", "NA")) + linked = dict(entity) + linked["qid"] = qid_value + linked["origin_country"] = info.get("Origin Country", "NA") + linked["is_category"] = info.get("is_category", "NA") + linked_row.append(linked) + linked_rows.append(linked_row) + return linked_rows, country_lookup + + +def aggregate_metrics(rows: list[dict[str, Any]], country: str, country_lookup: dict[str, str]) -> dict[str, Any]: + all_entities: list[dict[str, Any]] = [] + for row in rows: + filtered, _excluded = filter_entities_for_metrics(row.get("entities", [])) + all_entities.extend(filtered) + return { + "granularity": granularity(all_entities), + "diversity": diversity(all_entities), + "culture_specificity": culture_specificity(all_entities, country, country_lookup), + "entity_count": len(all_entities), + "unique_qids": len(collect_qids(all_entities)), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run a small MAKIEval generation smoke test.") + parser.add_argument("--model", default="Qwen2.5-7B-Instruct", help="Paper model name") + parser.add_argument("--language", required=True) + parser.add_argument("--topic", required=True) + parser.add_argument("--country", required=True) + parser.add_argument("--num-responses", type=int, default=5) + parser.add_argument("--faithful", action="store_true", default=True, help="Lock paper decoding values") + parser.add_argument("--no-faithful", dest="faithful", action="store_false") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--use-published-prompt", action="store_true", default=True) + parser.add_argument("--no-use-published-prompt", dest="use_published_prompt", action="store_false") + parser.add_argument("--device", default="api", choices=["api", "auto", "cuda", "cpu"]) + parser.add_argument("--no-extraction", action="store_true") + parser.add_argument( + "--extraction-model", + default=os.environ.get("TOGETHER_EXTRACTION_MODEL", "openai/gpt-oss-120b"), + help="Together model for proxy extraction when OPENAI_API_KEY is unavailable", + ) + parser.add_argument("--output-dir", default="results/smoke") + args = parser.parse_args() + + random.seed(args.seed) + np.random.seed(args.seed) + + canonical = canonical_model_name(args.model) + topic = normalize_topic(args.topic) + together_model, deviations = resolve_together_model(canonical) + decoding = dict(FAITHFUL_DECODING) + if not args.faithful: + deviations.append("Non-faithful mode requested; paper decoding lock disabled.") + + prompt, prompt_meta = load_prompt( + canonical, + topic, + args.language, + args.country, + args.use_published_prompt, + ) + model_prompt = add_language_instruction_if_needed(prompt, canonical, args.language) + client = together_client() + responses = generate_responses( + client, + model_prompt, + together_model, + args.num_responses, + decoding, + args.seed, + ) + + rows = [{"generated_text": text, "entities": []} for text in responses] + country_lookup: dict[str, str] = {} + extraction_status = "skipped" + extraction_model_used = None + metrics = None + + if args.no_extraction: + deviations.append("Extraction disabled with --no-extraction; metrics not computed.") + else: + deviations.append( + "OPENAI_API_KEY not used; GPT-4o-mini extraction replaced by Together proxy extraction." + ) + extracted_rows, extraction_model_used = extract_entities_with_together( + client, + responses, + topic, + args.extraction_model, + ) + linked_rows, country_lookup = link_entities(extracted_rows, topic, args.language) + for row, entities in zip(rows, linked_rows): + row["entities"] = entities + extraction_status = "completed" + metrics = aggregate_metrics(rows, args.country, country_lookup) + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / ( + f"{safe_slug(canonical)}_{safe_slug(args.language)}_" + f"{safe_slug(topic)}_{safe_slug(args.country)}.json" + ) + + payload = { + "metadata": { + "model": canonical, + "together_model": together_model, + "language": args.language, + "topic": topic, + "country": args.country, + "num_responses": args.num_responses, + "seed": args.seed, + "faithful": args.faithful, + "decoding": decoding, + "prompt_source": prompt_meta["source"], + "prompt": prompt, + "model_prompt": model_prompt, + "provider": "together", + "base_url": TOGETHER_BASE_URL, + "extraction_status": extraction_status, + "extraction_model": extraction_model_used, + "deviations": deviations, + }, + "rows": rows, + "country_lookup": country_lookup, + "metrics": metrics, + } + output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + + print(f"Wrote smoke results to {output_path}") + if metrics: + print(json.dumps(metrics, indent=2)) + else: + print("Extraction/metrics skipped.") + + +if __name__ == "__main__": + main() diff --git a/code/validate_fidelity.py b/code/validate_fidelity.py new file mode 100644 index 0000000..6fd3df7 --- /dev/null +++ b/code/validate_fidelity.py @@ -0,0 +1,310 @@ +"""Validate MAKIEval metric fidelity against paper-level reference checks.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from collections import Counter, defaultdict +from pathlib import Path +from statistics import mean +from typing import Any + +from tqdm import tqdm + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from data_loading import ( # noqa: E402 + GROUP_COLUMNS, + filter_entities_for_metrics, + iter_makieval_parquet_frames, + parse_entities_field, +) +from metrics import collect_qids, culture_consensus # noqa: E402 + +TARGET_DEEPSEEK_SLICE = { + "model": "DeepSeek-V3", + "topic": "book", + "language": "en", + "country_region": "united states", +} + +REFERENCE_NOTES = { + "deepseek_us_book": ( + "Paper Section 6.2: DeepSeek-V3, English, book, US is dominated by " + "To Kill a Mockingbird (~499/500), diversity approx 1." + ), + "diversity": ( + "Paper Table 12 order-of-magnitude: Llama-3.1-8B highest (~36-39), " + "DeepSeek lowest (~14-17)." + ), + "consensus": ( + "Paper Table 3 order-of-magnitude: model averages roughly 0.11-0.21; " + "larger models around 0.20." + ), +} + + +def _group_key(row: Any) -> tuple[str, str, str, str]: + return tuple(getattr(row, column) for column in GROUP_COLUMNS) + + +def _cache_path(cache_dir: Path, full: bool, limit: int | None) -> Path: + suffix = "full" if full else f"limit_{limit}" + return cache_dir / f"fidelity_cache_{suffix}.json" + + +def _serialize_group_qids(group_qids: dict[tuple[str, str, str, str], set[str]]) -> dict[str, list[str]]: + return {"||".join(key): sorted(qids) for key, qids in group_qids.items()} + + +def _deserialize_group_qids(raw: dict[str, list[str]]) -> dict[tuple[str, str, str, str], set[str]]: + return {tuple(key.split("||")): set(qids) for key, qids in raw.items()} + + +def _compute_cache(full: bool, limit: int | None, cache_file: Path) -> dict[str, Any]: + group_qids: dict[tuple[str, str, str, str], set[str]] = defaultdict(set) + deepseek_qid_row_counts: Counter[str] = Counter() + deepseek_rows = 0 + rows_seen = 0 + + columns = GROUP_COLUMNS + ["entities"] + for frame in tqdm( + iter_makieval_parquet_frames(columns=columns), + desc="Reading MAKIEval parquet shards", + unit="shard", + ): + if frame.empty: + continue + + for row in frame.itertuples(index=False): + if not full and limit is not None and rows_seen >= limit: + break + rows_seen += 1 + + entities, ok = parse_entities_field(getattr(row, "entities")) + if not ok: + continue + filtered_entities, _ = filter_entities_for_metrics(entities) + qids = collect_qids(filtered_entities) + + key = _group_key(row) + if dict(zip(GROUP_COLUMNS, key)) == TARGET_DEEPSEEK_SLICE: + deepseek_rows += 1 + deepseek_qid_row_counts.update(qids) + + if not qids: + continue + group_qids[key].update(qids) + + if not full and limit is not None and rows_seen >= limit: + break + + payload = { + "full": full, + "limit": limit, + "rows_seen": rows_seen, + "group_qids": _serialize_group_qids(group_qids), + "deepseek_us_book": { + "rows": deepseek_rows, + "qid_row_counts": dict(deepseek_qid_row_counts), + }, + } + cache_file.parent.mkdir(parents=True, exist_ok=True) + with open(cache_file, "w", encoding="utf-8") as f: + json.dump(payload, f) + return payload + + +def _load_or_compute_cache(args: argparse.Namespace) -> dict[str, Any]: + cache_dir = Path(args.cache_dir) + cache_file = _cache_path(cache_dir, args.full, args.limit) + if cache_file.exists() and not args.force_refresh: + with open(cache_file, encoding="utf-8") as f: + return json.load(f) + return _compute_cache(full=args.full, limit=args.limit, cache_file=cache_file) + + +def _deepseek_us_book_row(payload: dict[str, Any], group_qids: dict[tuple[str, str, str, str], set[str]]) -> dict[str, str]: + target_key = tuple(TARGET_DEEPSEEK_SLICE[column] for column in GROUP_COLUMNS) + qids = group_qids.get(target_key, set()) + diversity_value = len(qids) + + slice_payload = payload.get("deepseek_us_book", {}) + qid_counts = Counter(slice_payload.get("qid_row_counts", {})) + total_rows = int(slice_payload.get("rows", 0)) + dominant_qid = "NA" + dominant_count = 0 + if qid_counts: + dominant_qid, dominant_count = qid_counts.most_common(1)[0] + + dominant_rate = dominant_count / total_rows if total_rows else 0.0 + status = "PASS" if diversity_value <= 2 and dominant_rate >= 0.9 else "MISMATCH" + return { + "metric": "DeepSeek US book dominance", + "slice": "DeepSeek-V3 / book / en / US", + "computed": ( + f"diversity={diversity_value}; dominant={dominant_qid} " + f"{dominant_count}/{total_rows} ({dominant_rate:.1%})" + ), + "paper_reference": REFERENCE_NOTES["deepseek_us_book"], + "status": status, + "comment": "PASS threshold: diversity <= 2 and dominant QID row frequency >= 90%.", + } + + +def _average_diversity_rows(group_qids: dict[tuple[str, str, str, str], set[str]]) -> list[dict[str, str]]: + values_by_model: dict[str, list[int]] = defaultdict(list) + for (model, _topic, _language, _country), qids in group_qids.items(): + values_by_model[model].append(len(qids)) + + averages = {model: mean(values) for model, values in values_by_model.items() if values} + if not averages: + return [] + + highest_model = max(averages, key=averages.get) + lowest_model = min(averages, key=averages.get) + rows = [] + for model, value in sorted(averages.items()): + if model == "Llama-3.1-8B-Instruct": + status = "PASS" if highest_model == model and 25 <= value <= 50 else "MISMATCH" + comment = f"Highest model observed: {highest_model}." + elif model == "DeepSeek-V3": + status = "PASS" if lowest_model == model and value <= 25 else "MISMATCH" + comment = f"Lowest model observed: {lowest_model}." + else: + status = "PASS" if 5 <= value <= 60 else "MISMATCH" + comment = "Broad order-of-magnitude check; no exact per-model target asserted." + + rows.append( + { + "metric": "Average diversity", + "slice": model, + "computed": f"{value:.2f}", + "paper_reference": REFERENCE_NOTES["diversity"], + "status": status, + "comment": comment, + } + ) + return rows + + +def _average_consensus_rows(group_qids: dict[tuple[str, str, str, str], set[str]]) -> list[dict[str, str]]: + by_model_topic_country: dict[tuple[str, str, str], dict[str, set[str]]] = defaultdict(dict) + for (model, topic, language, country), qids in group_qids.items(): + by_model_topic_country[(model, topic, country)][language] = qids + + consensus_by_model: dict[str, list[float]] = defaultdict(list) + for (model, _topic, _country), lang_qids in by_model_topic_country.items(): + languages = sorted(lang_qids) + for idx, lang_a in enumerate(languages): + for lang_b in languages[idx + 1 :]: + consensus_by_model[model].append( + culture_consensus(lang_qids[lang_a], lang_qids[lang_b]) + ) + + rows = [] + for model, values in sorted(consensus_by_model.items()): + if not values: + continue + value = mean(values) + status = "PASS" if 0.08 <= value <= 0.25 else "MISMATCH" + rows.append( + { + "metric": "Average consensus", + "slice": model, + "computed": f"{value:.3f}", + "paper_reference": REFERENCE_NOTES["consensus"], + "status": status, + "comment": "Band check only; exact values may differ with released linking.", + } + ) + return rows + + +def _markdown_table(rows: list[dict[str, str]]) -> str: + headers = ["metric", "slice", "computed", "paper_reference", "status", "comment"] + lines = [ + "| " + " | ".join(headers) + " |", + "| " + " | ".join(["---"] * len(headers)) + " |", + ] + for row in rows: + values = [row.get(header, "").replace("|", "\\|") for header in headers] + lines.append("| " + " | ".join(values) + " |") + return "\n".join(lines) + + +def _write_report(rows: list[dict[str, str]], output_path: Path, payload: dict[str, Any]) -> None: + output_path.parent.mkdir(parents=True, exist_ok=True) + body = [ + "# MAKIEval Fidelity Report", + "", + "Source: `code/validate_fidelity.py`.", + "", + f"- Full dataset mode: `{payload.get('full')}`", + f"- Rows scanned: `{payload.get('rows_seen')}`", + f"- Group count with non-null metric QIDs: `{len(payload.get('group_qids', {}))}`", + "", + _markdown_table(rows), + "", + ] + output_path.write_text("\n".join(body), encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Validate MAKIEval metric fidelity checks.") + parser.add_argument( + "--full", + action="store_true", + help="Scan the full Hugging Face parquet release.", + ) + parser.add_argument( + "--limit", + type=int, + default=100000, + help="Development limit used only without --full.", + ) + parser.add_argument( + "--cache-dir", + default="results/fidelity", + help="Directory for slice/group-QID cache files.", + ) + parser.add_argument( + "--force-refresh", + action="store_true", + help="Ignore existing cache and rescan parquet shards.", + ) + parser.add_argument( + "--output", + default="docs/FIDELITY_REPORT.md", + help="Markdown report path.", + ) + args = parser.parse_args() + + if not args.full: + print( + "WARNING: validate_fidelity.py without --full is a development smoke test; " + "paper-facing checks require --full.", + file=sys.stderr, + ) + + payload = _load_or_compute_cache(args) + group_qids = _deserialize_group_qids(payload.get("group_qids", {})) + rows = [_deepseek_us_book_row(payload, group_qids)] + rows.extend(_average_diversity_rows(group_qids)) + rows.extend(_average_consensus_rows(group_qids)) + + output_path = Path(args.output) + _write_report(rows, output_path, payload) + + statuses = Counter(row["status"] for row in rows) + print(f"Wrote fidelity report to {output_path}") + print( + "Summary: " + + ", ".join(f"{status}={count}" for status, count in sorted(statuses.items())) + ) + + +if __name__ == "__main__": + main() diff --git a/docs/DATA_QUALITY_REPORT.md b/docs/DATA_QUALITY_REPORT.md new file mode 100644 index 0000000..18c4e65 --- /dev/null +++ b/docs/DATA_QUALITY_REPORT.md @@ -0,0 +1,122 @@ +# MAKIEval Published Data Quality Report + +Scope: published HF dataset, version may differ from paper (per author). + +- Mode: `sample=200 per model-language-topic` +- Rows analyzed for quality checks: `109200` +- Observed `(model, topic, language, country_region)` cells: `10740` + +## Summary + +| Scope | Rows | Repeated n-gram | Repeated sentence | Empty entities | Language mismatch | missing_qid_cultural_only | missing_qid_all_types | Suspicious rows | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| Overall | 109200 | 0.11% | 0.10% | 5.16% | 2.76% | 44.67% | 64.23% | 15 | + +## Slice Breakdown + +| Model / Language / Topic | Rows | Repeated n-gram | Repeated sentence | Empty entities | Language mismatch | missing_qid_cultural_only | missing_qid_all_types | Suspicious rows | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| Qwen2.5-7B-Instruct / th / beverage | 200 | 0.00% | 0.00% | 13.50% | 47.50% | 50.87% | 78.67% | 0 | +| Qwen2.5-7B-Instruct / th / music | 200 | 0.00% | 0.00% | 9.50% | 37.50% | 74.02% | 85.42% | 0 | +| Mistral-7B-Instruct-v0.1 / it / beverage | 200 | 0.50% | 0.00% | 6.00% | 36.50% | 40.00% | 60.31% | 0 | +| Qwen2.5-7B-Instruct / ar / music | 200 | 0.00% | 0.00% | 5.00% | 36.50% | 70.05% | 87.22% | 0 | +| Mistral-7B-Instruct-v0.1 / it / food | 200 | 0.50% | 0.50% | 4.00% | 31.00% | 56.21% | 68.19% | 0 | +| Qwen2.5-7B-Instruct / fa / clothing | 200 | 0.00% | 1.50% | 11.50% | 30.50% | 80.56% | 86.51% | 0 | +| Qwen2.5-7B-Instruct / ar / food | 200 | 0.50% | 0.00% | 4.50% | 30.00% | 74.06% | 86.42% | 0 | +| Qwen2.5-7B-Instruct / es / transportation | 200 | 0.00% | 0.00% | 0.00% | 29.00% | 48.75% | 62.05% | 0 | +| Qwen2.5-7B-Instruct / ko / transportation | 200 | 0.00% | 0.00% | 3.50% | 28.00% | 48.36% | 74.71% | 0 | +| Qwen2.5-7B-Instruct / th / transportation | 200 | 0.50% | 0.00% | 14.50% | 27.50% | 41.85% | 54.25% | 0 | +| Qwen2.5-7B-Instruct / th / book | 200 | 0.00% | 0.00% | 7.50% | 27.50% | 81.25% | 88.94% | 0 | +| Mistral-7B-Instruct-v0.1 / de / food | 200 | 0.50% | 0.00% | 0.00% | 27.50% | 37.39% | 54.48% | 0 | +| Qwen2.5-7B-Instruct / en / beverage | 200 | 0.00% | 0.00% | 3.00% | 26.50% | 26.40% | 67.05% | 0 | +| Mistral-7B-Instruct-v0.1 / es / transportation | 200 | 0.50% | 0.50% | 2.00% | 25.50% | 14.84% | 48.54% | 0 | +| Mistral-7B-Instruct-v0.1 / de / book | 200 | 0.00% | 0.00% | 1.00% | 25.50% | 39.45% | 64.71% | 0 | +| Mistral-7B-Instruct-v0.1 / it / clothing | 200 | 0.00% | 1.00% | 2.50% | 25.00% | 59.30% | 69.18% | 0 | +| Qwen2.5-7B-Instruct / it / transportation | 200 | 2.00% | 0.00% | 17.00% | 23.00% | 19.06% | 50.00% | 0 | +| Mistral-7B-Instruct-v0.1 / de / beverage | 200 | 0.00% | 0.50% | 1.50% | 22.50% | 38.40% | 61.76% | 0 | +| Mistral-7B-Instruct-v0.1 / de / music | 200 | 0.00% | 0.00% | 2.00% | 22.00% | 34.51% | 60.47% | 0 | +| Qwen2.5-7B-Instruct / ko / clothing | 200 | 0.00% | 0.00% | 2.00% | 22.00% | 69.09% | 77.82% | 0 | +| Mistral-7B-Instruct-v0.1 / it / book | 200 | 0.00% | 0.00% | 8.50% | 21.50% | 46.82% | 67.23% | 0 | +| Qwen2.5-7B-Instruct / de / clothing | 200 | 0.00% | 0.00% | 7.50% | 19.50% | 49.23% | 56.54% | 0 | +| Mistral-7B-Instruct-v0.1 / es / music | 200 | 0.50% | 0.00% | 30.50% | 19.00% | 50.89% | 73.89% | 0 | +| Qwen2.5-7B-Instruct / ar / transportation | 200 | 0.00% | 0.00% | 11.00% | 19.00% | 38.20% | 76.06% | 0 | +| Qwen2.5-7B-Instruct / th / food | 200 | 0.00% | 0.00% | 2.50% | 18.50% | 77.47% | 88.75% | 0 | +| Mistral-7B-Instruct-v0.1 / th / transportation | 200 | 2.50% | 1.50% | 40.50% | 18.00% | 42.86% | 93.30% | 0 | +| Mistral-7B-Instruct-v0.1 / it / music | 200 | 0.00% | 0.50% | 37.00% | 18.00% | 37.57% | 61.49% | 0 | +| Qwen2.5-7B-Instruct / tr / music | 200 | 2.50% | 0.00% | 3.00% | 18.00% | 36.98% | 47.61% | 0 | +| Mistral-7B-Instruct-v0.1 / es / clothing | 200 | 0.50% | 0.00% | 2.00% | 17.50% | 53.35% | 64.30% | 0 | +| Qwen2.5-7B-Instruct / es / food | 200 | 0.00% | 0.00% | 7.50% | 17.00% | 52.06% | 71.83% | 0 | + +## Missing QID By Topic + +| Topic | cultural_entities | missing_qid_cultural_only | missing_qid_cultural_only_rate | all_entities | missing_qid_all_types | missing_qid_all_types_rate | +|---|---:|---:|---:|---:|---:|---:| +| beverage | 27862 | 9514 | 34.15% | 54389 | 34331 | 63.12% | +| book | 26346 | 11530 | 43.76% | 53852 | 38373 | 71.26% | +| clothing | 83695 | 47559 | 56.82% | 99994 | 63442 | 63.45% | +| food | 63228 | 28352 | 44.84% | 96032 | 59236 | 61.68% | +| music | 25148 | 10911 | 43.39% | 49540 | 34847 | 70.34% | +| transportation | 40664 | 11386 | 28.00% | 75580 | 45548 | 60.26% | + +## Surface Form To QID Inconsistency + +| Normalized label | Distinct QIDs | Top QIDs | +|---|---:|---| +| 水 | 17 | Q132780 (12), Q54366215 (11), Q87533299 (8), Q4527378 (8), Q27016 (7) | +| drive | 17 | Q5457713 (14), Q1125653 (8), Q15081971 (8), Q5307897 (6), Q3039511 (4) | +| wasser | 15 | Q68834339 (4), Q110508848 (3), Q283 (3), Q21504880 (2), Q769808 (2) | +| bar | 14 | Q187456 (30), Q87533831 (20), Q17015569 (16), Q2138087 (11), Q103510 (8) | +| tango | 13 | Q14390274 (160), Q1980003 (48), Q338450 (18), Q109615012 (12), Q1001165 (3) | +| mate | 13 | Q203540 (100), Q870803 (41), Q122373714 (24), Q16290653 (9), Q20819848 (8) | +| roman | 13 | Q8261 (51), Q2178269 (6), Q2723443 (6), Q2164608 (5), Q484023 (5) | +| عربية | 13 | Q13955 (20), Q20301934 (4), Q1828555 (3), Q131583317 (3), Q28900199 (3) | +| fisch | 13 | Q152 (6), Q132229323 (6), Q121345135 (6), Q110695502 (5), Q1419666 (4) | +| قطار | 12 | Q870 (336), Q20404111 (41), Q28900655 (40), Q131364791 (39), Q20404092 (35) | +| トラム | 12 | Q2060132 (25), Q2036588 (12), Q839557 (8), Q1753254 (6), Q3085094 (2) | +| german | 12 | Q57477652 (5), Q2734574 (4), Q348514 (4), Q24088311 (2), Q28706955 (1) | +| restaurant | 11 | Q123385071 (20), Q11707 (11), Q123385011 (9), Q21183823 (9), Q11666766 (8) | +| stil | 11 | Q18254095 (8), Q21146257 (7), Q2313235 (7), Q2203264 (4), P149 (4) | +| buch | 11 | Q254025 (8), Q60559 (7), Q18527846 (5), Q64853 (4), Q55021352 (4) | +| cafe | 10 | Q8486 (588), Q30022 (39), Q60741434 (35), Q3058833 (32), Q153697 (9) | +| bts | 10 | Q298548 (124), Q4971165 (31), Q13580495 (26), Q828379 (20), Q4998432 (7) | +| busse | 10 | Q415540 (34), Q1017684 (34), Q15241652 (20), Q65231992 (17), Q18923859 (14) | +| don quixote | 10 | Q4166021 (23), Q944201 (15), Q5823029 (11), Q151815 (7), Q17258133 (7) | +| italian | 10 | Q11286685 (40), Q2022782 (8), Q231213 (7), Q110594072 (7), Q233810 (4) | + +## Repeated n-gram examples + +- Qwen2.5-7B-Instruct / beverage / ar / iran: ـ ++) ~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~... +- Qwen2.5-7B-Instruct / food / ar / united arab emirates: مuser في إحدى المطاعmsmsmsmsmsmsmmsmsmsmsmsm姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m姆 mum m... +- Mistral-7B-Instruct-v0.1 / clothing / ar / south korea: يرجى الرد ================ يرجى الرد باللغة العربية. كوريا الجنوبية + +## Repeated sentence examples + +- Mistral-7B-Instruct-v0.1 / clothing / ar / united kingdom: يرجى الرد باللغة العربية ملابس يرتديها شخص من المملكة المتحدة. يرجى الرد باللغة العربية. يرجى الرد باللغة العربية. يرج +- Mistral-7B-Instruct-v0.1 / clothing / ar / japan: يرجى الرد باللغة العربية يرجى الرد باللغة العربية. في اليابان شخص يرتديها أيضا. يرجى الرد باللغة العربية. في اليابان +- Mistral-7B-Instruct-v0.1 / clothing / ar / united kingdom: يرجى الرد باللغة العربية.يرجى الرد باللغة العربية.يرجى الرد باللغة العربية.يرجى الرد باللغة العربية.يرجى الرد باللغة العربية + +## Empty entity examples + +- Qwen2.5-7B-Instruct / beverage / ar / iran: ـ ++) ~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~... +- Aya-Expanse-8B / music / ar / india: يجلس راج في زاوية هادئة من منزله، يغمر نفسه في موسيقاه المفضلة. تملأ النغمات العذبة المكان، بينما تتدفق الكلمات بلغة هندية قديمة. تتأرجح أنفاسه مع الإيقاعات الساحرة، وينغمس في ذ... +- Aya-Expanse-8B / music / ar / china: يجلس لي، الشاب الصيني الهادئ، في غرفته المليئة بالضوضاء العالية للمدينة. لكنه يجد ملاذه في سماعاته، حيث ينغمس في موسيقاه المفضلة. الألحان الهادئة تعبر عنه وتريح روحه. يغمض عينيه... + +## Language mismatch examples + +- Qwen2.5-7B-Instruct / beverage / ar / canada (detected=zh): ُنَّشْرِي نصًا قصيرًا如下是生成的阿拉伯文内容，按照您的要求翻译成了中文： 在多伦多的一家咖啡馆里，杰克正享受着他的拿铁咖啡，阳光透过窗户洒在他的脸上。 +- Qwen2.5-7B-Instruct / beverage / ar / south korea (detected=zh): 是韩国人，您可能是想让韩语助手写一个关于韩国人的短句。不过既然您的要求是用阿拉伯语写的，那么我就按照您的语言需求来创作。下面是一个大约50词的短句： كانت سونغ جين تشرب الشاي الأخضر كل صباح، فهي تؤمن بفوائده الصحية العديدة ورونقه المميز. +- Qwen2.5-7B-Instruct / beverage / ar / taiwan (detected=zh): ύστερα في يوم صيفي حار، توقف张先生在炎炎夏日的午后，走进一家小茶馆，点了一杯冰绿茶解渴。 + +## Suspicious extraction review candidates + +- Qwen2.5-7B-Instruct / beverage / ar / india (الكافيين (drink_name, Q60235)): َّuser في مدينة مومباي، يشرب سرداريالا الكافيين من كوب قهوة الصباح في الشارع، مليء بالحيوية والطاقة. +- Qwen2.5-7B-Instruct / beverage / ar / united arab emirates (الكافيين (drink_name, Q60235)): ْ[System Response] في أحد المقاهي بدولة الإمارات، شربت سعاد الكافيين من قهوة عربية رقيقة، مليئة برائحة النutmeg والزنجبيل، مختلطة مع ذكريات الصيف الحارق في دبي. +- DeepSeek-V3 / beverage / ar / australia (الكافيين (beverage_category, Q60235)): في مقهى صغير بمدينة سيدني، يجلس "جاك" تحت أشعة الشمس الدافئة. بيده فنجان قهوة "فلات وايت" رغوته ذهبية، يرتشفها ببطء بينما يتأمل الأمواج على الشاطئ. ابتسامة رضا تعلو وجهه، فالقوة... + +## Notes + +- Language detection uses `langid` when installed, with a script-based fallback. +- Suspicious extraction rows are rule-based review candidates, not confirmed false positives. +- `missing_qid_cultural_only` excludes `place`, `person_name`, `listener_name`, and `reader_name` via `data_loading.filter_entities_for_metrics`; this is the Table 10-comparable missing-QID rate. +- `missing_qid_all_types` keeps every extracted entity type for broader extraction/linking completeness diagnostics. +- Missing-QID rates are compared descriptively against the paper's 26-35% range; released data may use a different snapshot. diff --git a/docs/DEVIATIONS.md b/docs/DEVIATIONS.md new file mode 100644 index 0000000..e5614cf --- /dev/null +++ b/docs/DEVIATIONS.md @@ -0,0 +1,64 @@ +# Deviations from upstream / paper + +This fork documents reproducibility additions for the public +`mainlp/MAKIEval` release and notes where the released Hugging Face data cannot +be compared directly to paper-level numbers. + +## Metrics layer + +| Item | Status | Notes | +|------|--------|-------| +| Metric computation | **Added** | Upstream did not include a runnable implementation for granularity, diversity, culture specificity, or culture consensus. This branch adds `code/metrics.py`, `code/data_loading.py`, and `code/run_metrics.py`. | +| Fidelity validation | **Added** | `code/validate_fidelity.py` scans Hugging Face parquet shards and writes `docs/FIDELITY_REPORT.md` with paper-facing sanity checks. MISMATCH rows are reported rather than hidden because released linking can differ from paper-side attribution. | +| Published-data quality audit | **Added** | `code/quality_report.py` audits the released Hugging Face rows without generation, GPU, or API keys and writes `docs/DATA_QUALITY_REPORT.md`. Language detection is approximate and rule-based checks are review candidates, not manual labels. | +| Smoke reproduction comparison | **Added** | `code/smoke_generation.py` and `code/compare_to_published.py` run a small Together-backed generation/extraction/linking path and compare it with the matching published HF slice. This is intentionally small-sample and not paper-scale. | + +## Repository hygiene + +| Item | Status | Notes | +|------|--------|-------| +| API keys | **Changed** | Hardcoded placeholder/default API keys are removed from `code/lm_utils.py`. OpenAI and DeepSeek clients now read `OPENAI_API_KEY` and `DEEPSEEK_API_KEY` from the environment. | +| `entity_extraction.py` syntax | **Fixed** | Restores importability by replacing the invalid `OpenAI(api_key )=` line with an environment-backed client initialization. | +| Together API | **Added** | Small smoke generation can use `TOGETHER_API_KEY` through an OpenAI-compatible client. Keys stay in the environment and are not committed. | + +## Metric specifics + +- **Null QIDs:** Excluded from diversity, consensus QID sets, and + culture-specificity denominators. +- **Culture specificity without lookup:** `run_metrics.py` reports `0.0` + unless a `--country-lookup` JSON is supplied; the Hugging Face rows do not + include origin-country metadata per entity. +- **Excluded entity types:** `place`, `person_name`, `listener_name`, + `reader_name`. +- **Metric CLI default:** `run_metrics.py` defaults to a deterministic + `head(5000)` limit for quick smoke tests. This is biased toward early Hugging + Face rows and is **not comparable to paper diversity/consensus values**; pass + `--full` for all matching rows. Partial outputs include `is_partial_sample`, + `n_rows_used`, and warning metadata. +- **Partial group completeness:** `run_metrics.py` scans filtered parquet group + columns to compare actually present Hugging Face + `(model, topic, language, country_region)` groups against computed groups and + warns when partial sampling drops complete groups. +- **Figure 2 ZH/consensus note:** The EN and DE Figure 2 book columns are + reproducible from stated QIDs and origin attributions in + `tests/test_metrics.py`. The ZH column / consensus=0.5 value could not be + reproduced from the stated EN/DE entity attributions alone; the corresponding + test is marked `xfail` rather than inventing an expected value. +- **Together serving variants:** When smoke reproduction uses Together-hosted + endpoints such as `Qwen/Qwen2.5-7B-Instruct-Turbo`, the run is provider-backed + and not a local-weight execution. Reports record the Together endpoint. +- **Proxy extraction without OpenAI:** The paper uses GPT-4o-mini for + extraction. If `OPENAI_API_KEY` is intentionally unavailable, this branch can + use a Together extraction model and records the run as proxy extraction. Those + metrics are not exact paper-faithful extraction results. +- **Live Wikidata rate limits:** Smoke linking uses the existing live SPARQL + analysis modules. Wikidata 429/rate-limit responses can reduce linked QIDs or + origin-country metadata in small smoke runs; this is reported rather than + hidden. + +## README corrections + +- Dataset lists **7 LLMs** (not 13). +- Hugging Face release has **approximately 5.25M rows**; the paper mentions + 85.8M generated texts. At the response level, the expected order of magnitude + is about 6M rows (1,716 prompts x 500 responses x 7 models). diff --git a/docs/FIDELITY_REPORT.md b/docs/FIDELITY_REPORT.md new file mode 100644 index 0000000..217fb27 --- /dev/null +++ b/docs/FIDELITY_REPORT.md @@ -0,0 +1,25 @@ +# MAKIEval Fidelity Report + +Source: `code/validate_fidelity.py`. + +- Full dataset mode: `True` +- Rows scanned: `5247018` +- Group count with non-null metric QIDs: `10740` + +| metric | slice | computed | paper_reference | status | comment | +| --- | --- | --- | --- | --- | --- | +| DeepSeek US book dominance | DeepSeek-V3 / book / en / US | diversity=6; dominant=Q212340 373/374 (99.7%) | Paper Section 6.2: DeepSeek-V3, English, book, US is dominated by To Kill a Mockingbird (~499/500), diversity approx 1. | MISMATCH | PASS threshold: diversity <= 2 and dominant QID row frequency >= 90%. | +| Average diversity | Aya-Expanse-8B | 49.74 | Paper Table 12 order-of-magnitude: Llama-3.1-8B highest (~36-39), DeepSeek lowest (~14-17). | PASS | Broad order-of-magnitude check; no exact per-model target asserted. | +| Average diversity | ChatGPT-4o-mini | 45.59 | Paper Table 12 order-of-magnitude: Llama-3.1-8B highest (~36-39), DeepSeek lowest (~14-17). | PASS | Broad order-of-magnitude check; no exact per-model target asserted. | +| Average diversity | DeepSeek-V3 | 31.13 | Paper Table 12 order-of-magnitude: Llama-3.1-8B highest (~36-39), DeepSeek lowest (~14-17). | MISMATCH | Lowest model observed: DeepSeek-V3. | +| Average diversity | Llama-3.1-8B-Instruct | 93.15 | Paper Table 12 order-of-magnitude: Llama-3.1-8B highest (~36-39), DeepSeek lowest (~14-17). | MISMATCH | Highest model observed: Llama-3.1-8B-Instruct. | +| Average diversity | Llama-3.3-70B-Instruct | 52.53 | Paper Table 12 order-of-magnitude: Llama-3.1-8B highest (~36-39), DeepSeek lowest (~14-17). | PASS | Broad order-of-magnitude check; no exact per-model target asserted. | +| Average diversity | Mistral-7B-Instruct-v0.1 | 76.82 | Paper Table 12 order-of-magnitude: Llama-3.1-8B highest (~36-39), DeepSeek lowest (~14-17). | MISMATCH | Broad order-of-magnitude check; no exact per-model target asserted. | +| Average diversity | Qwen2.5-7B-Instruct | 52.08 | Paper Table 12 order-of-magnitude: Llama-3.1-8B highest (~36-39), DeepSeek lowest (~14-17). | PASS | Broad order-of-magnitude check; no exact per-model target asserted. | +| Average consensus | Aya-Expanse-8B | 0.085 | Paper Table 3 order-of-magnitude: model averages roughly 0.11-0.21; larger models around 0.20. | PASS | Band check only; exact values may differ with released linking. | +| Average consensus | ChatGPT-4o-mini | 0.101 | Paper Table 3 order-of-magnitude: model averages roughly 0.11-0.21; larger models around 0.20. | PASS | Band check only; exact values may differ with released linking. | +| Average consensus | DeepSeek-V3 | 0.102 | Paper Table 3 order-of-magnitude: model averages roughly 0.11-0.21; larger models around 0.20. | PASS | Band check only; exact values may differ with released linking. | +| Average consensus | Llama-3.1-8B-Instruct | 0.077 | Paper Table 3 order-of-magnitude: model averages roughly 0.11-0.21; larger models around 0.20. | MISMATCH | Band check only; exact values may differ with released linking. | +| Average consensus | Llama-3.3-70B-Instruct | 0.101 | Paper Table 3 order-of-magnitude: model averages roughly 0.11-0.21; larger models around 0.20. | PASS | Band check only; exact values may differ with released linking. | +| Average consensus | Mistral-7B-Instruct-v0.1 | 0.036 | Paper Table 3 order-of-magnitude: model averages roughly 0.11-0.21; larger models around 0.20. | MISMATCH | Band check only; exact values may differ with released linking. | +| Average consensus | Qwen2.5-7B-Instruct | 0.075 | Paper Table 3 order-of-magnitude: model averages roughly 0.11-0.21; larger models around 0.20. | MISMATCH | Band check only; exact values may differ with released linking. | diff --git a/docs/REPRODUCTION_COMPARISON.md b/docs/REPRODUCTION_COMPARISON.md new file mode 100644 index 0000000..97c80a2 --- /dev/null +++ b/docs/REPRODUCTION_COMPARISON.md @@ -0,0 +1,36 @@ +# MAKIEval Smoke Reproduction Comparison + +Scope: small-sample, not paper-scale. Published data may be an older snapshot per the author. + +- Model: `Qwen2.5-7B-Instruct` +- Together endpoint: `Qwen/Qwen2.5-7B-Instruct-Turbo` +- Slice: `en / book / United States` +- Our N: `3` +- Published N used: `3` (limit=3) +- Extraction status: `completed` + +## Deviations + +- Together serving endpoint used: Qwen/Qwen2.5-7B-Instruct-Turbo. +- Provider serving variant uses a Turbo endpoint, not local weights. +- OPENAI_API_KEY not used; GPT-4o-mini extraction replaced by Together proxy extraction. + +## Metric Comparison + +| Metric | Ours | Published | Status | Comment | +|---|---:|---:|---|---| +| granularity | 0.8333 | 1.0000 | DIVERGE | Small N; exact match is not expected. | +| diversity | 1.0000 | 2.0000 | DIVERGE | Small N; exact match is not expected. | +| culture_specificity | 0.0000 | 0.0000 | N/A (origin lookup not supplied) | Needs --country-origin-map; not evaluated here. | +| entity_count | 6.0000 | 9.0000 | DIVERGE | Small N; exact match is not expected. | +| unique_qids | 1.0000 | 2.0000 | DIVERGE | Small N; exact match is not expected. | +| qid_set_jaccard | 0.0000 | n/a | DIVERGE | Entity-set overlap between our slice and published slice. | +| dominant_qid | Q508865 (2) | Q212340 (2) | DIVERGE | Mode QID comparison. | +| degenerate_rate | 0.0000 | 0.0000 | MATCH | Rule-based repeated text check. | +| language_leakage_rate | 0.0000 | 0.0000 | MATCH | langid/script detector. | + +## Notes + +- Diversity and consensus are N-sensitive; this comparison is directional only. +- Published culture specificity is `0.0` here unless per-QID origin lookup metadata is supplied. +- GPT-4o-mini extraction was not used because OPENAI_API_KEY is intentionally unavailable. diff --git a/docs/RESULTS.md b/docs/RESULTS.md new file mode 100644 index 0000000..ea883e0 --- /dev/null +++ b/docs/RESULTS.md @@ -0,0 +1,101 @@ +# Reproducibility and Data-Quality Audit — Results + +This document summarizes what the reproducibility branch adds on top of the public +MAKIEval snapshot, and the findings from running the new tooling against the released +Hugging Face dataset (`Raoyuan/MAKIEval`). + +Scope note: the public code is an earlier snapshot than the authors' working version, and +it is not confirmed whether the released dataset matches the exact version used for the +paper. Numbers below are measured on a 200-row-per-cell sample of the released dataset +(`--sample 200 --seed 42`) unless stated otherwise, and are reported descriptively rather +than as corrections to the paper. + +## What was added + +- A runnable implementation of the four paper metrics (granularity, diversity, culture + specificity, culture consensus) computed directly from the released dataset, with unit + tests and a Figure 2 reproduction test. +- A fidelity check (`validate_fidelity.py`) that re-derives selected paper observations + from the released data. +- A published-data quality audit (`quality_report.py`). +- An end-to-end smoke reproduction over the Together API (`smoke_generation.py` plus + `compare_to_published.py`). +- Repository hygiene: environment-based API keys, `requirements.txt`, license, and README + reproduction instructions. + +## Metric layer and fidelity check + +The metrics follow the paper definitions (Eq. 1 for granularity; unique-QID counting for +diversity; target-country share for culture specificity; Jaccard over QID sets for culture +consensus), and exclude `place`, `person_name`, `listener_name`, and `reader_name` from +metric computation. Unit tests pin the Figure 2 English and German columns +(diversity 2 and 3; granularity 1.0; specificity 1.0 and 0.667). The Figure 2 consensus +value of 0.5 could not be derived from the entity attributions printed in the figure +(the stated sets give a Jaccard of 0.25), so that single case is recorded as an expected +failure rather than forced to pass. + +The fidelity check reproduces the qualitative mode-collapse finding from Section 6.2: +for DeepSeek on English book prompts about the United States, one title dominates 99.7% +of generations in the released data. The exact diversity integer differs from the paper's +"close to 1" because diversity is a set-cardinality count and the released slice contains +a few additional rare titles; this is reported as a divergence rather than reconciled by +changing the metric. + +## Published-data quality audit + +### Unmatched cultural entities by topic + +After restricting to cultural entity types (the same exclusions the metrics use), the +missing-QID rate is close to the paper's Table 10 for some topics (beverage, transportation) +and higher for others (clothing, book, food, music). Counting all entity types — including +person names and places, which carry no QID by design — roughly doubles the rate, which is +why the all-types number is reported only as a completeness diagnostic. + +![Missing-QID rate by topic](figures/missing_qid_by_topic.png) + +### Surface-form to QID fragmentation + +The same normalized surface form is frequently linked to many different QIDs (for example, +the word for "water" in Chinese and German each map to 15–17 distinct QIDs). This +fragmentation inflates diversity and depresses consensus, and is a useful target for the +linking step. + +![Surface-form to QID fragmentation](figures/surface_form_fragmentation.png) + +### Language mismatch + +Overall, 2.8% of generations are in a language other than the prompt language, but this is +concentrated in specific model-language slices — chiefly Qwen and Mistral on non-English +prompts — which is consistent with the behavior the paper describes in Appendix B. + +![Language mismatch by slice](figures/language_mismatch_top_slices.png) + +Other observations: 5.16% of rows have an empty entity list, and rule-based checks flag a +small fraction of degenerate generations (repeated character or sentence loops); examples +are included in `DATA_QUALITY_REPORT.md`. + +## End-to-end smoke reproduction + +A small reproduction was run through the Together API to confirm the pipeline runs end to +end (prompt → generation → extraction → linking → metrics). This is a pipeline-integrity +check, not a faithful reproduction: it used the Together `Qwen2.5-7B-Instruct-Turbo` +serving endpoint rather than local paper weights, replaced the paper's GPT-4o-mini +extraction with a Together proxy model (no `OPENAI_API_KEY` was used), and generated only a +few responses per cell. The comparison in `REPRODUCTION_COMPARISON.md` is therefore +directional only; diversity and consensus are sample-size sensitive at this scale. + +## Caveats + +- Quality numbers use a 200-row-per-cell sample; pass `--full` to scan all rows. +- The released dataset may be an earlier snapshot than the paper run; the missing-QID gap + for some topics may reflect that rather than a measurement difference. This is an open + question for the authors. +- Live Wikidata SPARQL was rate-limited during the smoke run, which limited origin-based + specificity in that path. + +## Regenerating + +``` +python code/quality_report.py --full --seed 42 # refresh DATA_QUALITY_REPORT.md +python scripts/make_figures.py # refresh docs/figures/*.png +``` diff --git a/docs/figures/language_mismatch_top_slices.png b/docs/figures/language_mismatch_top_slices.png new file mode 100644 index 0000000..6ce8427 Binary files /dev/null and b/docs/figures/language_mismatch_top_slices.png differ diff --git a/docs/figures/missing_qid_by_topic.png b/docs/figures/missing_qid_by_topic.png new file mode 100644 index 0000000..a718246 Binary files /dev/null and b/docs/figures/missing_qid_by_topic.png differ diff --git a/docs/figures/surface_form_fragmentation.png b/docs/figures/surface_form_fragmentation.png new file mode 100644 index 0000000..afadb10 Binary files /dev/null and b/docs/figures/surface_form_fragmentation.png differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2d0bf33 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +datasets>=2.19.0,<4.0.0 +pandas>=2.0.0,<3.0.0 +pyarrow>=15.0.0,<25.0.0 +SPARQLWrapper>=2.0.0,<3.0.0 +openai>=1.30.0,<3.0.0 +torch>=2.1.0,<3.0.0 +transformers>=4.40.0,<5.0.0 +tqdm>=4.66.0,<5.0.0 +pytest>=8.0.0,<10.0.0 +numpy>=1.26.0,<3.0.0 +wikipedia>=1.4.0,<2.0.0 +requests>=2.32.0,<3.0.0 +langid>=1.1.6,<2.0.0 +matplotlib>=3.8.0,<4.0.0 diff --git a/scripts/make_figures.py b/scripts/make_figures.py new file mode 100644 index 0000000..c9cb9cc --- /dev/null +++ b/scripts/make_figures.py @@ -0,0 +1,147 @@ +"""Generate result figures for the MAKIEval reproducibility audit. + +Numbers are taken verbatim from docs/DATA_QUALITY_REPORT.md (sample=200, seed=42) +and the paper's Table 10. Re-run after regenerating the report to refresh figures. +""" +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.ticker import PercentFormatter + +# Neutral, print-friendly palette +C_AUDIT = "#2F6DB5" # this audit (cultural-only) +C_PAPER = "#B5852F" # paper Table 10 +C_ALL = "#9AA7B4" # all-types (context) +GRID = "#D9DEE3" +plt.rcParams.update({ + "font.size": 11, + "axes.edgecolor": "#444", + "axes.linewidth": 0.8, + "figure.dpi": 150, +}) + +def style(ax): + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) + ax.set_axisbelow(True) + + +# --------------------------------------------------------------------------- +# Figure 1: Missing-QID rate by topic — cultural-only audit vs paper Table 10 +# --------------------------------------------------------------------------- +topics = ["Beverage", "Transportation", "Music", "Book", "Food", "Clothing"] +audit_cultural = [34.15, 28.00, 43.39, 43.76, 44.84, 56.82] # this audit, cultural-only +paper_t10 = [30.59, 25.98, 31.11, 33.25, 32.18, 35.92] # paper Table 10 +all_types = [63.12, 60.26, 70.34, 71.26, 61.68, 63.45] # all entity types (context) + +x = range(len(topics)) +w = 0.38 +fig, ax = plt.subplots(figsize=(9, 4.6)) +b1 = ax.bar([i - w/2 for i in x], audit_cultural, w, label="This audit (cultural entities only)", color=C_AUDIT) +b2 = ax.bar([i + w/2 for i in x], paper_t10, w, label="Paper, Table 10", color=C_PAPER) +# all-types as light reference markers +ax.scatter(list(x), all_types, marker="_", s=420, color=C_ALL, linewidths=2.4, + label="All entity types (incl. names/places)", zorder=5) +for rects in (b1, b2): + for r in rects: + ax.annotate(f"{r.get_height():.0f}%", (r.get_x()+r.get_width()/2, r.get_height()), + ha="center", va="bottom", fontsize=8.5, xytext=(0, 1), textcoords="offset points") +ax.set_ylabel("Missing-QID rate") +ax.set_title("Unmatched cultural entities by topic: released data vs. paper", + fontsize=12.5, fontweight="bold", pad=10) +ax.set_xticks(list(x)); ax.set_xticklabels(topics) +ax.yaxis.set_major_formatter(PercentFormatter()) +ax.set_ylim(0, 80) +ax.grid(axis="y", color=GRID) +ax.legend(frameon=False, fontsize=9, loc="upper left", ncol=1) +style(ax) +fig.text(0.01, -0.02, + "Cultural-only excludes person_name, place, listener_name, reader_name (the metric exclusions). " + "Measured on a 200-row-per-cell sample of the released HF dataset.", + fontsize=7.8, color="#555") +fig.tight_layout() +fig.savefig("docs/figures/missing_qid_by_topic.png", bbox_inches="tight") +plt.close(fig) +print("wrote docs/figures/missing_qid_by_topic.png") + + +# --------------------------------------------------------------------------- +# Figure 2: Surface-form to QID fragmentation (top labels) +# --------------------------------------------------------------------------- +# label gloss (romanized so the figure renders without CJK/Arabic fonts), distinct QIDs +frag = [ + ("shuǐ / 'water' (zh)", 17), + ("drive (en)", 17), + ("wasser / 'water' (de)", 15), + ("bar (en/de)", 14), + ("tango (es)", 13), + ("mate (es)", 13), + ("roman / 'novel' (de)", 13), + ("ʿarabiyya (ar)", 13), + ("fisch / 'fish' (de)", 13), + ("qiṭār / 'train' (ar)", 12), + ("toramu / 'tram' (ja)", 12), + ("german (en)", 12), +] +frag = frag[::-1] +labels = [f[0] for f in frag]; vals = [f[1] for f in frag] +fig, ax = plt.subplots(figsize=(8.4, 5)) +bars = ax.barh(labels, vals, color=C_AUDIT, height=0.66) +for b in bars: + ax.annotate(f"{int(b.get_width())}", (b.get_width(), b.get_y()+b.get_height()/2), + ha="left", va="center", fontsize=9, xytext=(3, 0), textcoords="offset points") +ax.set_xlabel("Number of distinct Wikidata QIDs linked to the same surface form") +ax.set_title("Surface-form to QID fragmentation (top labels)", + fontsize=12.5, fontweight="bold", pad=10) +ax.set_xlim(0, 20) +ax.grid(axis="x", color=GRID) +style(ax) +fig.text(0.01, -0.03, + "A single normalized label resolving to many QIDs inflates Diversity and lowers Consensus. " + "Glosses are romanized; counts from the released HF dataset (sample=200/cell).", + fontsize=7.8, color="#555") +fig.tight_layout() +fig.savefig("docs/figures/surface_form_fragmentation.png", bbox_inches="tight") +plt.close(fig) +print("wrote docs/figures/surface_form_fragmentation.png") + + +# --------------------------------------------------------------------------- +# Figure 3: Language mismatch (prompt language != generated language) — worst slices +# --------------------------------------------------------------------------- +leak = [ + ("Qwen2.5-7B / th / beverage", 47.5), + ("Qwen2.5-7B / th / music", 37.5), + ("Mistral-7B / it / beverage", 36.5), + ("Qwen2.5-7B / ar / music", 36.5), + ("Mistral-7B / it / food", 31.0), + ("Qwen2.5-7B / fa / clothing", 30.5), + ("Qwen2.5-7B / ar / food", 30.0), + ("Qwen2.5-7B / es / transportation", 29.0), + ("Qwen2.5-7B / ko / transportation", 28.0), + ("Qwen2.5-7B / th / book", 27.5), +] +leak = leak[::-1] +labels = [f[0] for f in leak]; vals = [f[1] for f in leak] +fig, ax = plt.subplots(figsize=(8.6, 4.8)) +bars = ax.barh(labels, vals, color="#9C4F2F", height=0.62) +for b in bars: + ax.annotate(f"{b.get_width():.1f}%", (b.get_width(), b.get_y()+b.get_height()/2), + ha="left", va="center", fontsize=9, xytext=(3, 0), textcoords="offset points") +ax.axvline(2.76, color="#444", linestyle="--", linewidth=1) +ax.annotate("dataset overall: 2.8%", (2.76, 0.2), xytext=(6, 0), textcoords="offset points", + fontsize=8.5, color="#444", va="bottom") +ax.set_xlabel("Share of generations whose detected language differs from the prompt language") +ax.set_title("Language mismatch is concentrated in specific model-language slices", + fontsize=12, fontweight="bold", pad=10) +ax.set_xlim(0, 52) +ax.grid(axis="x", color=GRID) +style(ax) +fig.text(0.01, -0.03, + "Worst 10 slices shown; Qwen and Mistral on non-English prompts dominate, " + "consistent with the paper's Appendix B note. Detector: langid (sample=200/cell).", + fontsize=7.8, color="#555") +fig.tight_layout() +fig.savefig("docs/figures/language_mismatch_top_slices.png", bbox_inches="tight") +plt.close(fig) +print("wrote docs/figures/language_mismatch_top_slices.png") diff --git a/tests/test_data_loading.py b/tests/test_data_loading.py new file mode 100644 index 0000000..9a0dac2 --- /dev/null +++ b/tests/test_data_loading.py @@ -0,0 +1,68 @@ +import os +import sys + +import pandas as pd + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "code")) + +import data_loading + + +def test_load_expected_group_keys_exact_uses_present_filtered_groups(monkeypatch): + frames = [ + pd.DataFrame( + [ + { + "model": "Model-A", + "topic": "book", + "language": "en", + "country_region": "united states", + }, + { + "model": "Model-A", + "topic": "book", + "language": "en", + "country_region": "united states", + }, + { + "model": "Model-A", + "topic": "book", + "language": "de", + "country_region": "germany", + }, + ] + ), + pd.DataFrame( + [ + { + "model": "Model-A", + "topic": "book", + "language": "zh", + "country_region": "china", + } + ] + ), + ] + + def fake_iter_makieval_parquet_frames(split, filters, columns): + assert split == "train" + assert filters == {"model": "Model-A", "topic": "book"} + assert columns == data_loading.GROUP_COLUMNS + yield from frames + + monkeypatch.setattr( + data_loading, + "iter_makieval_parquet_frames", + fake_iter_makieval_parquet_frames, + ) + + groups = data_loading.load_expected_group_keys( + filters={"model": "Model-A", "topic": "books"}, + exact=True, + ) + + assert groups == [ + ("Model-A", "book", "de", "germany"), + ("Model-A", "book", "en", "united states"), + ("Model-A", "book", "zh", "china"), + ] diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..b246718 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,181 @@ +"""Unit tests for MAKIEval metrics (paper Figure 2 / Section 4).""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "code")) + +from data_loading import filter_entities_for_metrics +from metrics import ( + culture_consensus, + culture_specificity, + diversity, + granularity, +) + + +# Figure 2 US-books example: three extracted entities with one repeated title. +FIGURE2_ENTITIES = [ + {"entity": "To Kill a Mockingbird", "entity_type": "book_title", "qid": "Q1114275"}, + {"entity": "To Kill a Mockingbird", "entity_type": "book_title", "qid": "Q1114275"}, + {"entity": "American literature", "entity_type": "book_genre", "qid": "Q36279"}, +] + + +def test_diversity_figure2_us_books(): + assert diversity(FIGURE2_ENTITIES) == 2 + + +def test_granularity_figure2_us_books(): + # Two specific book titles + one genre category -> (1+1+0)/3 + assert granularity(FIGURE2_ENTITIES) == pytest.approx(2 / 3) + + +def test_culture_specificity_us_match(): + lookup = { + "Q1114275": "United States", + "Q36279": "United States", + } + score = culture_specificity(FIGURE2_ENTITIES, "United States", lookup) + assert score == pytest.approx(1.0) + + +def test_culture_specificity_partial_match(): + entities = [ + {"entity": "To Kill a Mockingbird", "entity_type": "book_title", "qid": "Q1114275"}, + {"entity": "American literature", "entity_type": "book_genre", "qid": "Q36279"}, + ] + lookup = { + "Q1114275": "United States", + "Q36279": "United Kingdom", + } + score = culture_specificity(entities, "United States", lookup) + assert score == pytest.approx(0.5) + + +def test_culture_consensus_jaccard(): + qids_a = {"Q1", "Q2", "Q3"} + qids_b = {"Q2", "Q3", "Q4"} + assert culture_consensus(qids_a, qids_b) == pytest.approx(2 / 4) + + +def test_culture_consensus_empty_union(): + assert culture_consensus([], []) == 0.0 + + +def test_diversity_excludes_null_qids(): + entities = [ + {"entity": "tea", "entity_type": "drink_name", "qid": None}, + {"entity": "coffee", "entity_type": "drink_name", "qid": "Q8486"}, + ] + assert diversity(entities) == 1 + + +def test_granularity_category_only(): + entities = [ + {"entity": "vegetables", "entity_type": "ingredient_category", "qid": "Q11004"}, + {"entity": "stir-fry", "entity_type": "dish_category", "qid": "Q381160"}, + ] + assert granularity(entities) == 0.0 + + +def test_jaccard_exact_paper_formula_cases(): + assert culture_consensus({"Q1", "Q2"}, {"Q2", "Q3"}) == pytest.approx(1 / 3) + assert culture_consensus({"Q1", "Q2"}, {"Q1", "Q2"}) == pytest.approx(1.0) + assert culture_consensus({"Q1"}, {"Q2"}) == pytest.approx(0.0) + # Empty-union policy: Jaccard is defined as 0.0 for two empty QID sets. + assert culture_consensus(set(), set()) == pytest.approx(0.0) + + +def test_diversity_counts_unique_qids_and_excludes_nulls(): + entities = [ + {"entity": "a", "entity_type": "drink_name", "qid": "Q1"}, + {"entity": "a again", "entity_type": "drink_name", "qid": "Q1"}, + {"entity": "b", "entity_type": "drink_name", "qid": "Q2"}, + {"entity": "missing", "entity_type": "drink_name", "qid": None}, + {"entity": "na", "entity_type": "drink_name", "qid": "NA"}, + ] + assert diversity(entities) == 2 + + +def test_granularity_specific_category_and_mixed_eq1(): + specific = [ + {"entity": "ramen", "entity_type": "dish", "qid": "Q1"}, + {"entity": "tea", "entity_type": "drink_name", "qid": "Q2"}, + ] + categories = [ + {"entity": "soups", "entity_type": "dish_category", "qid": "Q3"}, + {"entity": "genres", "entity_type": "book_genre", "qid": "Q4"}, + ] + assert granularity(specific) == pytest.approx(1.0) + assert granularity(categories) == pytest.approx(0.0) + assert granularity(specific + categories) == pytest.approx(0.5) + + +def test_culture_specificity_known_origin_ratio(): + entities = [ + {"entity": "us item", "entity_type": "book_title", "qid": "Q1"}, + {"entity": "uk item", "entity_type": "book_title", "qid": "Q2"}, + {"entity": "missing origin", "entity_type": "book_title", "qid": "Q3"}, + ] + lookup = {"Q1": "United States", "Q2": "United Kingdom"} + assert culture_specificity(entities, "US", lookup) == pytest.approx(1 / 2) + + +def test_figure2_en_books_reproduction(): + entities = [ + {"entity": "The Great Gatsby", "entity_type": "book_title", "qid": "Q214371"}, + {"entity": "To Kill a Mockingbird", "entity_type": "book_title", "qid": "Q212340"}, + {"entity": "To Kill a Mockingbird", "entity_type": "book_title", "qid": "Q212340"}, + ] + lookup = {"Q214371": "United States", "Q212340": "United States"} + assert diversity(entities) == 2 + assert granularity(entities) == pytest.approx(1.0) + assert culture_specificity(entities, "US", lookup) == pytest.approx(1.0) + + +def test_figure2_de_books_reproduction(): + entities = [ + {"entity": "Wer die Nachtigall stört", "entity_type": "book_title", "qid": "Q212340"}, + {"entity": "Der alte Mann und das Meer", "entity_type": "book_title", "qid": "Q26505"}, + {"entity": "Der Hobbit", "entity_type": "book_title", "qid": "Q74287"}, + ] + lookup = { + "Q212340": "United States", + "Q26505": "United States", + "Q74287": "United Kingdom", + } + assert diversity(entities) == 3 + assert granularity(entities) == pytest.approx(1.0) + assert culture_specificity(entities, "United States", lookup) == pytest.approx(2 / 3) + + +@pytest.mark.xfail( + reason=( + "Figure 2 ZH-column / consensus=0.5 cannot be derived from the stated " + "EN/DE entity attributions here; EN-DE Jaccard from stated QIDs is 0.25." + ) +) +def test_figure2_consensus_reference_not_directly_reproducible(): + en_qids = {"Q214371", "Q212340"} + de_qids = {"Q212340", "Q26505", "Q74287"} + assert culture_consensus(en_qids, de_qids) == pytest.approx(0.5) + + +def test_metric_entity_filter_excludes_non_cultural_entity_types(): + entities = [ + {"entity": "Lagos", "entity_type": "place", "qid": None}, + {"entity": "Ali", "entity_type": "person_name", "qid": None}, + {"entity": "listener", "entity_type": "listener_name", "qid": None}, + {"entity": "reader", "entity_type": "reader_name", "qid": None}, + {"entity": "tea", "entity_type": "drink_name", "qid": "Q6097"}, + ] + kept, excluded = filter_entities_for_metrics(entities) + assert excluded == 4 + assert kept == [{"entity": "tea", "entity_type": "drink_name", "qid": "Q6097"}] + + +def test_consensus_excludes_null_qids(): + assert culture_consensus(["Q1", None, ""], ["Q1", "Q2", None]) == pytest.approx(1 / 2) diff --git a/tests/test_quality_report.py b/tests/test_quality_report.py new file mode 100644 index 0000000..7b12712 --- /dev/null +++ b/tests/test_quality_report.py @@ -0,0 +1,48 @@ +import json +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "code")) + +from quality_report import QualityState, build_report, process_row + + +def test_missing_qid_counts_separate_all_types_from_cultural_only(): + state = QualityState() + row = { + "model": "Model-A", + "topic": "book", + "language": "en", + "country_region": "united states", + "generated_text": "", + "entities": json.dumps( + [ + {"entity": "Lagos", "entity_type": "place", "qid": None}, + {"entity": "Ali", "entity_type": "person_name", "qid": ""}, + {"entity": "tea", "entity_type": "drink_name", "qid": None}, + {"entity": "novel", "entity_type": "book_title", "qid": "Q1"}, + ] + ), + } + + process_row(state, row) + + assert state.overall.entity_count_all_types == 4 + assert state.overall.missing_qid_all_types == 3 + assert state.overall.entity_count_cultural_only == 2 + assert state.overall.missing_qid_cultural_only == 1 + + topic_stats = state.by_topic["book"] + assert topic_stats.missing_qid_all_types == 3 + assert topic_stats.missing_qid_cultural_only == 1 + + +def test_quality_report_marks_cultural_metric_as_table_10_comparable(): + state = QualityState() + state.overall.rows = 1 + + report = build_report(state, "test") + + assert "missing_qid_cultural_only" in report + assert "missing_qid_all_types" in report + assert "Table 10-comparable" in report