Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions scripts/quality/eval_smoke.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,14 @@ cd "${PROJECT_ROOT}"

WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/eval-smoke.XXXXXX")"
TEMP_INPUT_CREATED=0
PRESERVE_BUNDLE="${EVAL_SMOKE_PRESERVE_BUNDLE:-0}"

cleanup() {
rm -rf "${WORK_DIR}"
if [[ "${TEMP_INPUT_CREATED}" == "1" ]]; then
rm -f "${INPUT_PATH}"
if [[ "${PRESERVE_BUNDLE}" != "1" ]]; then
rm -rf "${WORK_DIR}"
if [[ "${TEMP_INPUT_CREATED}" == "1" ]]; then
rm -f "${INPUT_PATH}"
fi
fi
}

Expand Down Expand Up @@ -78,8 +81,12 @@ def main() -> None:
inputs_record = {"id": "smoke", "output": normalized_text}
expected_record = {"id": "smoke", "expected": normalized_text}

inputs_path.write_text(json.dumps(inputs_record) + "\n", encoding="utf-8")
expected_path.write_text(json.dumps(expected_record) + "\n", encoding="utf-8")
inputs_path.write_text(
json.dumps(inputs_record, ensure_ascii=False) + "\n", encoding="utf-8"
)
expected_path.write_text(
json.dumps(expected_record, ensure_ascii=False) + "\n", encoding="utf-8"
)


if __name__ == "__main__":
Expand Down
21 changes: 21 additions & 0 deletions tests/scripts/quality/test_eval_smoke.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,7 @@ def test_eval_smoke_pipeline_invokes_stubs(
metrics_path = tmp_path / "metrics.json"
monkeypatch.setenv("EVAL_SMOKE_RECORD_PATH", str(record_path))
monkeypatch.setenv("EVAL_SMOKE_METRICS_PATH", str(metrics_path))
monkeypatch.setenv("EVAL_SMOKE_PRESERVE_BUNDLE", "1")

script_path = PROJECT_ROOT / "scripts" / "quality" / "eval_smoke.sh"
assert script_path.exists(), "eval_smoke.sh must exist for the smoke test"
Expand Down Expand Up @@ -237,6 +238,26 @@ def test_eval_smoke_pipeline_invokes_stubs(
expected_payload = json.loads(expected_payload_line.split("=", 1)[1])
assert inputs_payload == [{"id": "smoke", "output": normalized_text}]
assert expected_payload == [{"id": "smoke", "expected": normalized_text}]

inputs_path = Path(inputs_line.split("=", 1)[1])
expected_path = Path(expected_line.split("=", 1)[1])
assert inputs_path.exists()
assert expected_path.exists()

inputs_records = [
json.loads(line)
for line in inputs_path.read_text(encoding="utf-8").splitlines()
if line.strip()
]
expected_records = [
json.loads(line)
for line in expected_path.read_text(encoding="utf-8").splitlines()
if line.strip()
]
assert inputs_records == [{"id": "smoke", "output": normalized_text}]
assert expected_records == [{"id": "smoke", "expected": normalized_text}]

shutil.rmtree(inputs_path.parent, ignore_errors=True)
assert "bert_score" in lines
assert "rouge" in lines

Expand Down