From acd13801492c35e9820b7b3527695ae1d5947f95 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 12:03:09 +0300 Subject: [PATCH 01/79] Guard replay requires for planner input --- Makefile | 25 +- examples/demo_qa/batch.py | 17 +- examples/demo_qa/chat_repl.py | 10 +- examples/demo_qa/fixture_cli.py | 175 ++++++++++ examples/demo_qa/runner.py | 108 ++++++- examples/demo_qa/tests/test_demo_qa_batch.py | 2 +- src/fetchgraph/cli.py | 63 ++++ src/fetchgraph/core/context.py | 12 +- src/fetchgraph/planning/normalize/__init__.py | 14 +- .../planning/normalize/plan_normalizer.py | 91 ++++-- src/fetchgraph/replay/__init__.py | 11 + src/fetchgraph/replay/export.py | 302 ++++++++++++++++++ src/fetchgraph/replay/handlers/__init__.py | 5 + .../replay/handlers/plan_normalize.py | 61 ++++ src/fetchgraph/replay/log.py | 41 +++ src/fetchgraph/replay/runtime.py | 13 + src/fetchgraph/replay/snapshots.py | 23 ++ tests/fixtures/replay_points/.gitkeep | 1 + .../test_relational_normalizer_regression.py | 2 +- tests/test_replay_fixtures.py | 93 ++++++ 20 files changed, 1026 insertions(+), 43 deletions(-) create mode 100644 examples/demo_qa/fixture_cli.py create mode 100644 src/fetchgraph/cli.py create mode 100644 src/fetchgraph/replay/__init__.py create mode 100644 src/fetchgraph/replay/export.py create mode 100644 src/fetchgraph/replay/handlers/__init__.py create mode 100644 src/fetchgraph/replay/handlers/plan_normalize.py create mode 100644 src/fetchgraph/replay/log.py create mode 100644 src/fetchgraph/replay/runtime.py create mode 100644 src/fetchgraph/replay/snapshots.py create mode 100644 tests/fixtures/replay_points/.gitkeep create mode 100644 tests/test_replay_fixtures.py diff --git a/Makefile b/Makefile index 31458374..49dc3b1e 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,13 @@ OUT ?= $(DATA)/.runs/results.jsonl TAG ?= NOTE ?= CASE ?= +RUN_ID ?= +REPLAY_ID ?= plan_normalize.spec_v1 +WITH ?= +SPEC_IDX ?= +PROVIDER ?= +OUT_DIR ?= +ALL ?= LIMIT ?= 50 CHANGES ?= 10 NEW_TAG ?= @@ -99,7 +106,7 @@ LIMIT_FLAG := $(if $(strip $(LIMIT)),--limit $(LIMIT),) batch batch-tag batch-failed batch-failed-from \ batch-missed batch-missed-from batch-failed-tag batch-missed-tag \ batch-fail-fast batch-max-fails \ - stats history-case report-tag report-tag-changes tags tag-rm case-run case-open compare compare-tag + stats history-case report-tag report-tag-changes tags tag-rm case-run case-open fixture compare compare-tag # ============================================================================== # help (на русском) @@ -146,6 +153,7 @@ help: @echo " make tags [PATTERN=*] DATA=... - показать список тегов" @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" + @echo " make fixture CASE=agg_01 [TAG=...] [RUN_ID=...] [REPLAY_ID=plan_normalize.spec_v1] [WITH=requires] [SPEC_IDX=0] [PROVIDER=relational] [OUT_DIR=tests/fixtures/replay_points] [ALL=1]" @echo "" @echo "Уборка:" @echo " make tag-rm TAG=... [DRY=1] [PURGE_RUNS=1] [PRUNE_HISTORY=1] [PRUNE_CASE_HISTORY=1]" @@ -330,6 +338,16 @@ case-open: check @test -n "$(strip $(CASE))" || (echo "Нужно задать CASE=case_42" && exit 1) @$(CLI) case open "$(CASE)" --data "$(DATA)" +# 11) Replay fixtures +fixture: check + @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture CASE=agg_01" && exit 1) + @$(PYTHON) -m examples.demo_qa.fixture_cli --case "$(CASE)" --data "$(DATA)" \ + $(TAG_FLAG) $(if $(strip $(RUN_ID)),--run-id "$(RUN_ID)",) $(if $(strip $(REPLAY_ID)),--id "$(REPLAY_ID)",) \ + $(if $(strip $(SPEC_IDX)),--spec-idx "$(SPEC_IDX)",) $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ + $(if $(strip $(OUT_DIR)),--out-dir "$(OUT_DIR)",) \ + $(if $(strip $(ALL)),--all,) \ + $(if $(filter requires,$(WITH)),--with-requires,) + # compare (diff.md + junit) compare: check @test -n "$(strip $(BASE))" || (echo "Нужно задать BASE=.../results_prev.jsonl" && exit 1) @@ -359,8 +377,3 @@ compare-tag: check tag-rm: @test -n "$(strip $(TAG))" || (echo "TAG обязателен: make tag-rm TAG=..." && exit 1) @TAG="$(TAG)" DATA="$(DATA)" PURGE_RUNS="$(PURGE_RUNS)" PRUNE_HISTORY="$(PRUNE_HISTORY)" PRUNE_CASE_HISTORY="$(PRUNE_CASE_HISTORY)" DRY="$(DRY)" $(PYTHON) -m scripts.tag_rm - - - - - diff --git a/examples/demo_qa/batch.py b/examples/demo_qa/batch.py index 47074173..2df3b830 100644 --- a/examples/demo_qa/batch.py +++ b/examples/demo_qa/batch.py @@ -1018,7 +1018,14 @@ def handle_batch(args) -> int: for case in cases: current_case_id = case.id try: - result = run_one(case, runner, artifacts_root, plan_only=args.plan_only, event_logger=event_logger) + result = run_one( + case, + runner, + artifacts_root, + plan_only=args.plan_only, + event_logger=event_logger, + schema_path=schema_path, + ) except KeyboardInterrupt: interrupted = True interrupted_at_case_id = current_case_id @@ -1390,7 +1397,13 @@ def handle_case_run(args) -> int: llm = build_llm(settings) runner = build_agent(llm, provider) - result = run_one(cases[args.case_id], runner, artifacts_root, plan_only=args.plan_only) + result = run_one( + cases[args.case_id], + runner, + artifacts_root, + plan_only=args.plan_only, + schema_path=args.schema, + ) write_results(results_path, [result]) counts = summarize([result]) bad = bad_statuses("bad", False) diff --git a/examples/demo_qa/chat_repl.py b/examples/demo_qa/chat_repl.py index 5d3feb65..e33e00d9 100644 --- a/examples/demo_qa/chat_repl.py +++ b/examples/demo_qa/chat_repl.py @@ -112,7 +112,15 @@ def start_repl( artifacts: RunArtifacts | None = None try: case = Case(id=run_id, question=line, tags=[]) - result = run_one(case, runner, runs_root, plan_only=False, event_logger=event_logger, run_dir=run_dir) + result = run_one( + case, + runner, + runs_root, + plan_only=False, + event_logger=event_logger, + run_dir=run_dir, + schema_path=None, + ) plan_obj = _load_json(Path(result.artifacts_dir) / "plan.json") ctx_obj = _load_json(Path(result.artifacts_dir) / "context.json") or {} artifacts = RunArtifacts( diff --git a/examples/demo_qa/fixture_cli.py b/examples/demo_qa/fixture_cli.py new file mode 100644 index 00000000..daef31d9 --- /dev/null +++ b/examples/demo_qa/fixture_cli.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Iterable, Optional + +from fetchgraph.replay.export import export_replay_fixture, export_replay_fixtures + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + p = argparse.ArgumentParser(description="DemoQA: export replay fixtures from .runs") + p.add_argument("--case", required=True, help="Case id to extract from runs") + p.add_argument("--tag", default=None, help="Optional run tag filter") + p.add_argument("--run-id", default=None, help="Exact run_id to select") + p.add_argument("--id", default="plan_normalize.spec_v1", help="Replay point id to extract") + p.add_argument("--spec-idx", type=int, default=None, help="Filter replay_point by meta.spec_idx") + p.add_argument("--provider", default=None, help="Filter replay_point by meta.provider") + p.add_argument("--with-requires", action="store_true", help="Export replay bundle with dependencies") + p.add_argument("--all", action="store_true", help="Export all matching replay points") + + p.add_argument("--data", type=Path, default=None, help="Data dir containing .runs (default: cwd)") + p.add_argument("--runs-dir", type=Path, default=None, help="Explicit .runs/runs dir (overrides --data)") + p.add_argument( + "--out-dir", + type=Path, + default=Path("tests") / "fixtures" / "replay_points", + help="Output directory for replay fixtures", + ) + return p.parse_args(argv) + + +def _load_json(path: Path) -> Optional[dict]: + if not path.exists(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except Exception: + return None + + +def _iter_run_folders(runs_root: Path) -> Iterable[Path]: + if not runs_root.exists(): + return [] + for entry in runs_root.iterdir(): + if entry.is_dir(): + yield entry + + +def _load_run_meta(run_folder: Path) -> dict: + return (_load_json(run_folder / "run_meta.json") or {}) or (_load_json(run_folder / "summary.json") or {}) + + +def _parse_run_id_from_name(run_folder: Path) -> str | None: + parts = run_folder.name.split("_") + return parts[-1] if parts else None + + +def _case_dirs(run_folder: Path, case_id: str) -> list[Path]: + cases_root = run_folder / "cases" + if not cases_root.exists(): + return [] + return sorted(cases_root.glob(f"{case_id}_*")) + + +def _pick_latest(paths: Iterable[Path]) -> Optional[Path]: + candidates = list(paths) + if not candidates: + return None + return max(candidates, key=lambda p: p.stat().st_mtime) + + +def _is_missed_case(case_dir: Path) -> bool: + status = _load_json(case_dir / "status.json") or {} + status_value = str(status.get("status") or "").lower() + if status_value in {"missed"}: + return True + if status_value and status_value != "skipped": + return False + reason = str(status.get("reason") or "").lower() + if "missed" in reason or "missing" in reason: + return True + return bool(status.get("missed")) + + +def _resolve_runs_root(args: argparse.Namespace) -> Path: + if args.runs_dir: + return args.runs_dir + if args.data: + return args.data / ".runs" / "runs" + return Path(".runs") / "runs" + + +def _resolve_run_folder(args: argparse.Namespace, runs_root: Path) -> Path: + if args.run_id: + for run_folder in _iter_run_folders(runs_root): + meta = _load_run_meta(run_folder) + run_id = meta.get("run_id") or _parse_run_id_from_name(run_folder) + if run_id == args.run_id: + return run_folder + raise SystemExit(f"run_id={args.run_id!r} not found in {runs_root}") + + tag = args.tag + candidates = [] + for run_folder in _iter_run_folders(runs_root): + meta = _load_run_meta(run_folder) + if tag: + if meta.get("tag") != tag: + continue + case_dirs = _case_dirs(run_folder, args.case) + if not case_dirs: + continue + latest_case = _pick_latest(case_dirs) + if latest_case and _is_missed_case(latest_case): + continue + candidates.append(run_folder) + + latest = _pick_latest(candidates) + if latest is None: + raise SystemExit(f"No runs found for case={args.case!r} (tag={tag!r}) in {runs_root}") + return latest + + +def _find_case_run_dir(run_folder: Path, case_id: str) -> Path: + latest = _pick_latest(_case_dirs(run_folder, case_id)) + if latest is None: + raise SystemExit(f"No case run dir found for case={case_id!r} in {run_folder}") + return latest + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + runs_root = _resolve_runs_root(args) + run_folder = _resolve_run_folder(args, runs_root) + case_run_dir = _find_case_run_dir(run_folder, args.case) + events_path = case_run_dir / "events.jsonl" + if not events_path.exists(): + raise SystemExit(f"events.jsonl not found at {events_path}") + + if args.all: + export_replay_fixtures( + events_path=events_path, + run_dir=case_run_dir, + out_dir=args.out_dir, + replay_id=args.id, + spec_idx=args.spec_idx, + provider=args.provider, + with_requires=args.with_requires, + source_extra={ + "case_id": args.case, + "tag": args.tag, + "picked": "run_id" if args.run_id else "latest_non_missed", + }, + ) + else: + export_replay_fixture( + events_path=events_path, + run_dir=case_run_dir, + out_dir=args.out_dir, + replay_id=args.id, + spec_idx=args.spec_idx, + provider=args.provider, + with_requires=args.with_requires, + source_extra={ + "case_id": args.case, + "tag": args.tag, + "picked": "run_id" if args.run_id else "latest_non_missed", + }, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/demo_qa/runner.py b/examples/demo_qa/runner.py index f78ce2aa..58fb7785 100644 --- a/examples/demo_qa/runner.py +++ b/examples/demo_qa/runner.py @@ -1,8 +1,10 @@ from __future__ import annotations import datetime +import hashlib import json import re +import shutil import statistics import time import uuid @@ -11,7 +13,9 @@ from typing import Dict, Iterable, List, Mapping, NotRequired, TypedDict from fetchgraph.core import create_generic_agent +from fetchgraph.core.context import BaseGraphAgent from fetchgraph.core.models import TaskProfile +from fetchgraph.replay.snapshots import snapshot_provider_catalog from fetchgraph.utils import set_run_id @@ -114,7 +118,7 @@ def saver(feature_name: str, parsed: object) -> None: ], ) - self.agent = create_generic_agent( + self.agent: BaseGraphAgent = create_generic_agent( llm_invoke=llm, providers={provider.name: provider}, saver=saver, @@ -129,6 +133,7 @@ def run_question( *, plan_only: bool = False, event_logger: EventLogger | None = None, + schema_ref: str | None = None, ) -> RunArtifacts: set_run_id(run_id) artifacts = RunArtifacts(run_id=run_id, run_dir=run_dir, question=case.question, plan_only=plan_only) @@ -137,8 +142,19 @@ def run_question( try: if event_logger: event_logger.emit({"type": "plan_started", "case_id": case.id}) + _emit_planner_input( + event_logger, + case=case, + agent=self, + schema_ref=schema_ref, + plan_only=plan_only, + ) plan_started = time.perf_counter() - plan = self.agent._plan(case.question) # type: ignore[attr-defined] + if event_logger: + self.agent.set_event_logger(event_logger) + else: + self.agent.set_event_logger(None) + plan = self.agent._plan(case.question) artifacts.timings.plan_s = time.perf_counter() - plan_started artifacts.plan = plan.model_dump() @@ -184,6 +200,70 @@ def _save_json(path: Path, payload: object) -> None: path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") +def _hash_file(path: Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(8192), b""): + hasher.update(chunk) + return f"sha256:{hasher.hexdigest()}" + + +def _provider_catalog_snapshot(agent: object) -> Dict[str, object]: + plan_normalizer = getattr(agent, "plan_normalizer", None) + provider_catalog = getattr(plan_normalizer, "provider_catalog", {}) if plan_normalizer else {} + return snapshot_provider_catalog(provider_catalog) + + +def _emit_schema_snapshot(event_logger: EventLogger, run_dir: Path, schema_path: Path) -> str: + run_dir.mkdir(parents=True, exist_ok=True) + suffix = schema_path.suffix.lstrip(".") or "txt" + snapshot_name = f"schema_snapshot.{suffix}" + snapshot_path = run_dir / snapshot_name + shutil.copy2(schema_path, snapshot_path) + file_hash = _hash_file(snapshot_path) + event_logger.emit( + { + "type": "replay_resource", + "v": 1, + "id": "schema_v1", + "meta": {"format": suffix}, + "data": None, + "data_ref": {"file": snapshot_name, "hash": file_hash}, + "hash": file_hash, + } + ) + return "schema_v1" + + +def _emit_planner_input( + event_logger: EventLogger, + *, + case: Case, + agent: AgentRunner, + schema_ref: str | None, + plan_only: bool, +) -> None: + input_payload: Dict[str, object] = { + "feature_name": case.id, + "user_query": case.question, + "options": {"plan_only": plan_only}, + } + if schema_ref: + input_payload["schema_ref"] = schema_ref + provider_catalog = _provider_catalog_snapshot(agent.agent) + if provider_catalog: + input_payload["provider_catalog"] = provider_catalog + event_logger.emit( + { + "type": "planner_input", + "v": 1, + "id": "planner_input_v1", + "case_id": case.id, + "input": input_payload, + } + ) + + def save_artifacts(artifacts: RunArtifacts) -> None: artifacts.run_dir.mkdir(parents=True, exist_ok=True) if artifacts.plan is not None: @@ -299,6 +379,7 @@ def run_one( plan_only: bool = False, event_logger: EventLogger | None = None, run_dir: Path | None = None, + schema_path: Path | None = None, ) -> RunResult: if run_dir is None: run_id = uuid.uuid4().hex[:8] @@ -309,6 +390,9 @@ def run_one( case_logger = event_logger.for_case(case.id, run_dir / "events.jsonl") if event_logger else None if case_logger: case_logger.emit({"type": "case_started", "case_id": case.id, "run_dir": str(run_dir)}) + schema_ref: str | None = None + if case_logger and schema_path and schema_path.exists(): + schema_ref = _emit_schema_snapshot(case_logger, run_dir, schema_path) if case.skip: run_dir.mkdir(parents=True, exist_ok=True) _save_text(run_dir / "skipped.txt", "Skipped by request") @@ -333,7 +417,14 @@ def run_one( case_logger.emit({"type": "case_finished", "case_id": case.id, "status": "skipped"}) return result - artifacts = runner.run_question(case, run_id, run_dir, plan_only=plan_only, event_logger=case_logger) + artifacts = runner.run_question( + case, + run_id, + run_dir, + plan_only=plan_only, + event_logger=case_logger, + schema_ref=schema_ref, + ) save_artifacts(artifacts) expected_check = None if plan_only else _match_expected(case, artifacts.answer) @@ -791,9 +882,10 @@ def format_status_line(result: RunResult) -> str: class EventLogger: - def __init__(self, path: Path | None, run_id: str): + def __init__(self, path: Path | None, run_id: str, case_id: str | None = None): self.path = path self.run_id = run_id + self.case_id = case_id if path: path.parent.mkdir(parents=True, exist_ok=True) @@ -802,13 +894,13 @@ def emit(self, event: Dict[str, object]) -> None: return now = datetime.datetime.now(datetime.timezone.utc) payload = {"timestamp": now.isoformat().replace("+00:00", "Z"), "run_id": self.run_id, **event} + if self.case_id and "case_id" not in payload: + payload["case_id"] = self.case_id with self.path.open("a", encoding="utf-8") as f: f.write(json.dumps(payload, ensure_ascii=False) + "\n") - def for_case(self, case_id: str, path: Path | None = None) -> "EventLogger": - if path is None: - return self - return EventLogger(path, self.run_id) + def for_case(self, case_id: str, path: Path) -> "EventLogger": + return EventLogger(path, self.run_id, case_id) DiffCaseChange = TypedDict( diff --git a/examples/demo_qa/tests/test_demo_qa_batch.py b/examples/demo_qa/tests/test_demo_qa_batch.py index fa163a8b..48bebfe8 100644 --- a/examples/demo_qa/tests/test_demo_qa_batch.py +++ b/examples/demo_qa/tests/test_demo_qa_batch.py @@ -441,7 +441,7 @@ def test_format_healed_explain_includes_key_lines() -> None: assert any("run_id=r2" in line and "status=ok" in line for line in lines) -def _stubbed_run_one(case, runner, artifacts_root, *, plan_only=False, event_logger=None): +def _stubbed_run_one(case, runner, artifacts_root, *, plan_only=False, event_logger=None, schema_path=None): run_dir = artifacts_root / f"{case.id}_stub" run_dir.mkdir(parents=True, exist_ok=True) return RunResult( diff --git a/src/fetchgraph/cli.py b/src/fetchgraph/cli.py new file mode 100644 index 00000000..84ac6a22 --- /dev/null +++ b/src/fetchgraph/cli.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from fetchgraph.replay.export import export_replay_fixture, export_replay_fixtures + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Fetchgraph utilities") + sub = parser.add_subparsers(dest="command", required=True) + + fixture = sub.add_parser("fixture", help="Export replay fixture from events.jsonl") + fixture.add_argument("--events", type=Path, required=True, help="Path to events.jsonl") + fixture.add_argument("--run-dir", type=Path, default=None, help="Case run dir (needed for --with-requires)") + fixture.add_argument("--id", default="plan_normalize.spec_v1", help="Replay point id to extract") + fixture.add_argument("--spec-idx", type=int, default=None, help="Filter replay_point by meta.spec_idx") + fixture.add_argument( + "--provider", + default=None, + help="Filter replay_point by meta.provider (case-insensitive)", + ) + fixture.add_argument("--with-requires", action="store_true", help="Export replay bundle with dependencies") + fixture.add_argument("--all", action="store_true", help="Export all matching replay points") + fixture.add_argument( + "--out-dir", + type=Path, + default=Path("tests") / "fixtures" / "replay_points", + help="Output directory for replay fixtures", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + if args.command == "fixture": + if args.all: + export_replay_fixtures( + events_path=args.events, + run_dir=args.run_dir, + out_dir=args.out_dir, + replay_id=args.id, + spec_idx=args.spec_idx, + provider=args.provider, + with_requires=args.with_requires, + ) + else: + export_replay_fixture( + events_path=args.events, + run_dir=args.run_dir, + out_dir=args.out_dir, + replay_id=args.id, + spec_idx=args.spec_idx, + provider=args.provider, + with_requires=args.with_requires, + ) + return 0 + raise SystemExit(f"Unknown command: {args.command}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/fetchgraph/core/context.py b/src/fetchgraph/core/context.py index 0a0c08b8..0c3e8589 100644 --- a/src/fetchgraph/core/context.py +++ b/src/fetchgraph/core/context.py @@ -26,6 +26,7 @@ SupportsFilter, Verifier, ) +from ..replay.log import EventLoggerLike from .utils import load_pkg_text, render_prompt logger = logging.getLogger(__name__) @@ -334,6 +335,7 @@ def __init__( task_profile: Optional[TaskProfile] = None, llm_refetch: Optional[Callable[[str, Dict[str, str], Plan], str]] = None, max_refetch_iters: int = 1, + event_logger: EventLoggerLike | None = None, ): self.llm_plan = llm_plan self.llm_synth = llm_synth @@ -349,6 +351,9 @@ def __init__( self.plan_normalizer = plan_normalizer or PlanNormalizer.from_providers( providers ) + self.event_logger = event_logger + if self.plan_normalizer is not None: + self.plan_normalizer.event_logger = event_logger self.baseline = baseline or [] if self.plan_normalizer is not None and self.baseline: normalized_specs = self.plan_normalizer.normalize_specs( @@ -437,6 +442,11 @@ def run(self, feature_name: str) -> Any: return parsed # ---- steps ---- + def set_event_logger(self, event_logger: EventLoggerLike | None) -> None: + self.event_logger = event_logger + if self.plan_normalizer is not None: + self.plan_normalizer.event_logger = event_logger + def _plan(self, feature_name: str) -> Plan: t0 = time.perf_counter() lite_ctx = self._lite_context(feature_name) @@ -456,7 +466,7 @@ def _plan(self, feature_name: str) -> Plan: else: plan = Plan.model_validate_json(plan_raw.text) if self.plan_normalizer is not None: - plan = self.plan_normalizer.normalize(plan) + plan = self.plan_normalizer.normalize(plan, event_logger=self.event_logger) elapsed = time.perf_counter() - t0 logger.info( "Planning finished for feature_name=%r in %.3fs " diff --git a/src/fetchgraph/planning/normalize/__init__.py b/src/fetchgraph/planning/normalize/__init__.py index 1e4b3944..7706d3ea 100644 --- a/src/fetchgraph/planning/normalize/__init__.py +++ b/src/fetchgraph/planning/normalize/__init__.py @@ -1,5 +1,15 @@ """Normalization utilities for planning.""" -from .plan_normalizer import NormalizedPlan, PlanNormalizer, PlanNormalizerOptions +from .plan_normalizer import ( + NormalizedPlan, + PlanNormalizer, + PlanNormalizerOptions, + SelectorNormalizationRule, +) -__all__ = ["NormalizedPlan", "PlanNormalizer", "PlanNormalizerOptions"] \ No newline at end of file +__all__ = [ + "NormalizedPlan", + "PlanNormalizer", + "PlanNormalizerOptions", + "SelectorNormalizationRule", +] diff --git a/src/fetchgraph/planning/normalize/plan_normalizer.py b/src/fetchgraph/planning/normalize/plan_normalizer.py index 11c3fa24..f14874ae 100644 --- a/src/fetchgraph/planning/normalize/plan_normalizer.py +++ b/src/fetchgraph/planning/normalize/plan_normalizer.py @@ -1,8 +1,9 @@ from __future__ import annotations +import copy import json import logging -from dataclasses import dataclass +from dataclasses import asdict, dataclass from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple from pydantic import Field, TypeAdapter, ValidationError @@ -12,6 +13,7 @@ from ...relational.models import RelationalRequest from ...relational.normalize import normalize_relational_selectors from ...relational.providers.base import RelationalDataProvider +from ...replay.log import EventLoggerLike, log_replay_point logger = logging.getLogger(__name__) @@ -33,6 +35,7 @@ class PlanNormalizerOptions: @dataclass(frozen=True) class SelectorNormalizationRule: + kind: str | None = None validator: TypeAdapter[Any] normalize_selectors: Callable[[Any], Any] @@ -44,12 +47,14 @@ def __init__( schema_registry: Optional[Dict[str, Dict[str, Any]]] = None, normalizer_registry: Optional[Dict[str, SelectorNormalizationRule]] = None, options: Optional[PlanNormalizerOptions] = None, + event_logger: EventLoggerLike | None = None, ) -> None: self.provider_catalog = dict(provider_catalog) self.schema_registry = schema_registry or {} self.normalizer_registry = normalizer_registry or {} self.options = options or PlanNormalizerOptions() self._provider_aliases = self._build_provider_aliases(self.provider_catalog) + self.event_logger = event_logger @classmethod def from_providers( @@ -78,6 +83,7 @@ def from_providers( schema_registry[key] = info.selectors_schema if isinstance(prov, RelationalDataProvider): normalizer_registry[key] = SelectorNormalizationRule( + kind="relational_v1", validator=TypeAdapter(RelationalRequest), normalize_selectors=normalize_relational_selectors, ) @@ -88,7 +94,7 @@ def from_providers( options=options, ) - def normalize(self, plan: Plan) -> NormalizedPlan: + def normalize(self, plan: Plan, *, event_logger: EventLoggerLike | None = None) -> NormalizedPlan: notes: List[str] = [] required_context = self._normalize_required_context(plan.required_context, notes) context_plan = self._normalize_context_plan(plan.context_plan, notes) @@ -98,7 +104,7 @@ def normalize(self, plan: Plan) -> NormalizedPlan: # Baseline/plan merge owns "ensure required providers exist" logic, # and must do it in a baseline-safe way (never overriding baseline selectors/mode). - context_plan = self._normalize_specs(context_plan, notes) + context_plan = self._normalize_specs(context_plan, notes, event_logger=event_logger) adr_queries = self._normalize_text_list(plan.adr_queries, notes, "adr_queries") constraints = self._normalize_text_list( @@ -121,9 +127,10 @@ def normalize_specs( specs: Iterable[ContextFetchSpec], *, notes: Optional[List[str]] = None, + event_logger: EventLoggerLike | None = None, ) -> List[ContextFetchSpec]: local_notes: List[str] = [] - normalized = self._normalize_specs(specs, local_notes) + normalized = self._normalize_specs(specs, local_notes, event_logger=event_logger) if notes is not None: notes.extend(local_notes) if local_notes: @@ -134,39 +141,81 @@ def normalize_specs( return normalized def _normalize_specs( - self, specs: Iterable[ContextFetchSpec], notes: List[str] + self, + specs: Iterable[ContextFetchSpec], + notes: List[str], + *, + event_logger: EventLoggerLike | None = None, ) -> List[ContextFetchSpec]: normalized: List[ContextFetchSpec] = [] - for spec in specs: + replay_logger = event_logger or self.event_logger + for spec_idx, spec in enumerate(specs): rule = self.normalizer_registry.get(spec.provider) if rule is None: normalized.append(spec) continue - orig = spec.selectors - before_ok = self._validate_selectors(rule.validator, orig) + selectors_before = copy.deepcopy(spec.selectors) + before_ok = self._validate_selectors(rule.validator, selectors_before) decision = "keep_original_valid" if before_ok else "keep_original_still_invalid" - use = orig + use = selectors_before after_ok = before_ok if not before_ok: - candidate = rule.normalize_selectors(orig) + candidate = rule.normalize_selectors(copy.deepcopy(selectors_before)) after_ok = self._validate_selectors(rule.validator, candidate) if after_ok: decision = "use_normalized_fixed" use = candidate - elif candidate != orig: + elif candidate != selectors_before: decision = "use_normalized_unvalidated" use = candidate - notes.append( - self._format_selectors_note( - spec.provider, - before_ok, - after_ok, - decision, - selectors_before=orig, - selectors_after=use, - ) + note = self._format_selectors_note( + spec.provider, + before_ok, + after_ok, + decision, + selectors_before=selectors_before, + selectors_after=use, ) - if use is orig: + notes.append(note) + rule_kind = rule.kind + if replay_logger and rule_kind: + input_payload = { + "spec": { + "provider": spec.provider, + "mode": spec.mode, + "selectors": selectors_before, + }, + "options": asdict(self.options), + "normalizer_rules": {spec.provider: rule_kind}, + } + expected_payload = { + "out_spec": { + "provider": spec.provider, + "mode": spec.mode, + "selectors": use, + } + } + requires = None + if getattr(replay_logger, "case_id", None): + requires = ["planner_input_v1"] + log_replay_point( + replay_logger, + id="plan_normalize.spec_v1", + meta={ + "provider": spec.provider, + "mode": spec.mode, + "spec_idx": spec_idx, + }, + input=input_payload, + expected=expected_payload, + requires=requires, + diag={ + "selectors_valid_before": before_ok, + "selectors_valid_after": after_ok, + }, + note=note, + ) + if use == selectors_before: normalized.append(spec) continue data = spec.model_dump() diff --git a/src/fetchgraph/replay/__init__.py b/src/fetchgraph/replay/__init__.py new file mode 100644 index 00000000..c09445c1 --- /dev/null +++ b/src/fetchgraph/replay/__init__.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from .log import EventLoggerLike, log_replay_point +from .runtime import REPLAY_HANDLERS, ReplayContext + +__all__ = [ + "EventLoggerLike", + "REPLAY_HANDLERS", + "ReplayContext", + "log_replay_point", +] diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py new file mode 100644 index 00000000..f22b435c --- /dev/null +++ b/src/fetchgraph/replay/export.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterable + + +def iter_events(path: Path) -> Iterable[tuple[int, dict]]: + with path.open("r", encoding="utf-8") as handle: + for idx, line in enumerate(handle): + line = line.strip() + if not line: + continue + try: + yield idx, json.loads(line) + except json.JSONDecodeError: + continue + + +def canonical_json(payload: object) -> str: + return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def fixture_name(event_id: str, input_payload: dict) -> str: + digest = hashlib.sha256((event_id + canonical_json(input_payload)).encode("utf-8")).hexdigest() + return f"{event_id}__{digest[:8]}.json" + + +def _copy_resource_file(run_dir: Path, fixture_root: Path, root_name: str, data_ref: dict) -> dict: + file_name = data_ref.get("file") + if not file_name: + return data_ref + src_path = run_dir / file_name + if not src_path.exists(): + raise SystemExit(f"Resource file {src_path} not found for replay bundle") + target_name = f"{root_name}__{Path(file_name).name}" + dest_path = fixture_root / target_name + shutil.copy2(src_path, dest_path) + updated = dict(data_ref) + updated["file"] = target_name + return updated + + +@dataclass(frozen=True) +class ExportSelection: + event_index: int + event: dict + + +@dataclass(frozen=True) +class ExportContext: + resources: Dict[str, dict] + extras: Dict[str, dict] + + +def _match_meta(event: dict, *, spec_idx: int | None, provider: str | None) -> bool: + meta = event.get("meta") + if not isinstance(meta, dict): + return False + if spec_idx is not None and meta.get("spec_idx") != spec_idx: + return False + if provider is not None: + p = meta.get("provider") + if not isinstance(p, str): + return False + if p.lower() != provider.lower(): + return False + return True + + +def select_replay_points( + events_path: Path, + *, + replay_id: str, + spec_idx: int | None = None, + provider: str | None = None, +) -> tuple[list[ExportSelection], ExportContext]: + selected: list[ExportSelection] = [] + resources: Dict[str, dict] = {} + extras: Dict[str, dict] = {} + + for idx, event in iter_events(events_path): + etype = event.get("type") + eid = event.get("id") + + if etype == "replay_resource" and isinstance(eid, str): + resources[eid] = event + continue + + if etype == "planner_input" and isinstance(eid, str): + extras[eid] = event + continue + + if etype == "replay_point" and eid == replay_id: + if _match_meta(event, spec_idx=spec_idx, provider=provider): + selected.append(ExportSelection(event_index=idx, event=event)) + + if not selected: + details = [] + if spec_idx is not None: + details.append(f"spec_idx={spec_idx}") + if provider is not None: + details.append(f"provider={provider!r}") + detail_str = f" (filters: {', '.join(details)})" if details else "" + raise SystemExit(f"No replay_point id={replay_id!r} found in {events_path}{detail_str}") + return selected, ExportContext(resources=resources, extras=extras) + + +def write_fixture(event: dict, *, out_dir: Path, source: dict) -> Path: + out_dir.mkdir(parents=True, exist_ok=True) + event_id = event["id"] + out_path = out_dir / fixture_name(event_id, event["input"]) + if out_path.exists(): + print(f"Fixture already exists: {out_path}") + return out_path + + payload = dict(event) + payload["source"] = source + out_path.write_text(canonical_json(payload), encoding="utf-8") + print(f"Wrote fixture: {out_path}") + return out_path + + +def write_bundle( + root_event: dict, + *, + out_dir: Path, + run_dir: Path, + resources: Dict[str, dict], + extras: Dict[str, dict], + source: dict, + root_source: dict, +) -> Path: + out_dir.mkdir(parents=True, exist_ok=True) + fixture_file = fixture_name(root_event["id"], root_event["input"]) + out_path = out_dir / fixture_file + if out_path.exists(): + print(f"Fixture already exists: {out_path}") + return out_path + + root_name = Path(fixture_file).stem + updated_resources: Dict[str, dict] = {} + for rid, resource in resources.items(): + rp = dict(resource) + if "data_ref" in rp and isinstance(rp["data_ref"], dict): + rp["data_ref"] = _copy_resource_file(run_dir, out_dir, root_name, rp["data_ref"]) + updated_resources[rid] = rp + + payload = { + "type": "replay_bundle", + "v": 1, + "root": dict(root_event), + "resources": updated_resources, + "extras": dict(extras), + "source": dict(source), + } + payload["root"]["source"] = dict(root_source) + + out_path.write_text(canonical_json(payload), encoding="utf-8") + print(f"Wrote fixture bundle: {out_path}") + return out_path + + +def export_replay_fixture( + *, + events_path: Path, + out_dir: Path, + replay_id: str, + spec_idx: int | None = None, + provider: str | None = None, + with_requires: bool = False, + run_dir: Path | None = None, + source_extra: dict | None = None, + allow_multiple: bool = False, +) -> Path: + selections, ctx = select_replay_points( + events_path, + replay_id=replay_id, + spec_idx=spec_idx, + provider=provider, + ) + if len(selections) > 1 and not allow_multiple: + raise SystemExit( + "Multiple replay points matched. Provide --spec-idx/--provider or use --all to export all." + ) + if len(selections) > 1 and allow_multiple: + raise SystemExit("Use export_replay_fixtures for multiple selections.") + selection = selections[0] + event = selection.event + + common_source = { + "events_path": str(events_path), + "event_index": selection.event_index, + **(source_extra or {}), + } + + if not with_requires: + if event.get("requires"): + print("Warning: replay_point has requires; export without --with-requires may be incomplete.") + return write_fixture(event, out_dir=out_dir, source=common_source) + + requires = event.get("requires") or [] + resolved_resources: Dict[str, dict] = {} + resolved_extras: Dict[str, dict] = {} + + for rid in requires: + if rid in ctx.resources: + resolved_resources[rid] = ctx.resources[rid] + continue + if rid in ctx.extras: + resolved_extras[rid] = ctx.extras[rid] + continue + raise SystemExit(f"Required dependency {rid!r} not found in {events_path}") + + if run_dir is None: + raise SystemExit("--run-dir is required for --with-requires (to copy resource files)") + + root_source = { + "events_path": str(events_path), + "event_index": selection.event_index, + "run_dir": str(run_dir), + **(source_extra or {}), + } + + return write_bundle( + event, + out_dir=out_dir, + run_dir=run_dir, + resources=resolved_resources, + extras=resolved_extras, + source=common_source, + root_source=root_source, + ) + + +def export_replay_fixtures( + *, + events_path: Path, + out_dir: Path, + replay_id: str, + spec_idx: int | None = None, + provider: str | None = None, + with_requires: bool = False, + run_dir: Path | None = None, + source_extra: dict | None = None, +) -> list[Path]: + selections, ctx = select_replay_points( + events_path, + replay_id=replay_id, + spec_idx=spec_idx, + provider=provider, + ) + paths: list[Path] = [] + for selection in selections: + event = selection.event + common_source = { + "events_path": str(events_path), + "event_index": selection.event_index, + **(source_extra or {}), + } + if not with_requires: + if event.get("requires"): + print("Warning: replay_point has requires; export without --with-requires may be incomplete.") + paths.append(write_fixture(event, out_dir=out_dir, source=common_source)) + continue + + requires = event.get("requires") or [] + resolved_resources: Dict[str, dict] = {} + resolved_extras: Dict[str, dict] = {} + for rid in requires: + if rid in ctx.resources: + resolved_resources[rid] = ctx.resources[rid] + continue + if rid in ctx.extras: + resolved_extras[rid] = ctx.extras[rid] + continue + raise SystemExit(f"Required dependency {rid!r} not found in {events_path}") + + if run_dir is None: + raise SystemExit("--run-dir is required for --with-requires (to copy resource files)") + + root_source = { + "events_path": str(events_path), + "event_index": selection.event_index, + "run_dir": str(run_dir), + **(source_extra or {}), + } + paths.append( + write_bundle( + event, + out_dir=out_dir, + run_dir=run_dir, + resources=resolved_resources, + extras=resolved_extras, + source=common_source, + root_source=root_source, + ) + ) + return paths diff --git a/src/fetchgraph/replay/handlers/__init__.py b/src/fetchgraph/replay/handlers/__init__.py new file mode 100644 index 00000000..55f3c2a5 --- /dev/null +++ b/src/fetchgraph/replay/handlers/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from .plan_normalize import replay_plan_normalize_spec_v1 + +__all__ = ["replay_plan_normalize_spec_v1"] diff --git a/src/fetchgraph/replay/handlers/plan_normalize.py b/src/fetchgraph/replay/handlers/plan_normalize.py new file mode 100644 index 00000000..ef2a0af8 --- /dev/null +++ b/src/fetchgraph/replay/handlers/plan_normalize.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from typing import Dict + +from pydantic import TypeAdapter + +from ..core.models import ContextFetchSpec, ProviderInfo +from ..planning.normalize import PlanNormalizer, PlanNormalizerOptions, SelectorNormalizationRule +from ..relational.models import RelationalRequest +from ..relational.normalize import normalize_relational_selectors +from ..runtime import REPLAY_HANDLERS, ReplayContext + + +def replay_plan_normalize_spec_v1(inp: dict, ctx: ReplayContext) -> dict: + spec_dict = dict(inp["spec"]) + options = PlanNormalizerOptions(**inp["options"]) + rules = inp.get("normalizer_rules") or inp.get("normalizer_registry") or {} + provider = spec_dict["provider"] + provider_catalog: Dict[str, ProviderInfo] = {} + planner = ctx.extras.get("planner_input_v1") or {} + planner_input = planner.get("input") if isinstance(planner, dict) else {} + catalog_raw = {} + if isinstance(planner_input, dict): + catalog_raw = planner_input.get("provider_catalog") or {} + if provider in catalog_raw and isinstance(catalog_raw[provider], dict): + provider_catalog[provider] = ProviderInfo(**catalog_raw[provider]) + else: + provider_catalog[provider] = ProviderInfo(name=provider, capabilities=[]) + + rule_kind = rules.get(provider) + normalizer_registry: Dict[str, SelectorNormalizationRule] = {} + if rule_kind == "relational_v1": + normalizer_registry[provider] = SelectorNormalizationRule( + kind="relational_v1", + validator=TypeAdapter(RelationalRequest), + normalize_selectors=normalize_relational_selectors, + ) + + normalizer = PlanNormalizer( + provider_catalog, + normalizer_registry=normalizer_registry, + options=options, + ) + + spec = ContextFetchSpec(**spec_dict) + notes: list[str] = [] + out_specs = normalizer.normalize_specs([spec], notes=notes) + out = out_specs[0] + + out_spec = { + "provider": out.provider, + "mode": out.mode, + "selectors": out.selectors, + } + return { + "out_spec": out_spec, + "notes_last": notes[-1] if notes else None, + } + + +REPLAY_HANDLERS["plan_normalize.spec_v1"] = replay_plan_normalize_spec_v1 diff --git a/src/fetchgraph/replay/log.py b/src/fetchgraph/replay/log.py new file mode 100644 index 00000000..afe1a9b9 --- /dev/null +++ b/src/fetchgraph/replay/log.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from typing import Dict, Protocol + + +class EventLoggerLike(Protocol): + def emit(self, event: Dict[str, object]) -> None: ... + + +def log_replay_point( + logger: EventLoggerLike, + *, + id: str, + meta: dict, + input: dict, + expected: dict, + requires: list[str] | None = None, + diag: dict | None = None, + note: str | None = None, + error: str | None = None, + extra: dict | None = None, +) -> None: + event: Dict[str, object] = { + "type": "replay_point", + "v": 1, + "id": id, + "meta": meta, + "input": input, + "expected": expected, + } + if requires is not None: + event["requires"] = requires + if diag is not None: + event["diag"] = diag + if note is not None: + event["note"] = note + if error is not None: + event["error"] = error + if extra: + event.update(extra) + logger.emit(event) diff --git a/src/fetchgraph/replay/runtime.py b/src/fetchgraph/replay/runtime.py new file mode 100644 index 00000000..c1c1d217 --- /dev/null +++ b/src/fetchgraph/replay/runtime.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Callable, Dict + + +@dataclass(frozen=True) +class ReplayContext: + resources: Dict[str, dict] = field(default_factory=dict) + extras: Dict[str, dict] = field(default_factory=dict) + + +REPLAY_HANDLERS: Dict[str, Callable[[dict, ReplayContext], dict]] = {} diff --git a/src/fetchgraph/replay/snapshots.py b/src/fetchgraph/replay/snapshots.py new file mode 100644 index 00000000..fe49d0c4 --- /dev/null +++ b/src/fetchgraph/replay/snapshots.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from typing import Dict, Mapping + +from ..core.models import ProviderInfo + + +def snapshot_provider_info(info: ProviderInfo) -> Dict[str, object]: + payload: Dict[str, object] = {"name": info.name} + if info.capabilities: + payload["capabilities"] = list(info.capabilities) + if info.selectors_schema: + payload["selectors_schema"] = info.selectors_schema + return payload + + +def snapshot_provider_catalog(provider_catalog: Mapping[str, object]) -> Dict[str, object]: + snapshot: Dict[str, object] = {} + for key, info in provider_catalog.items(): + if not isinstance(info, ProviderInfo): + continue + snapshot[key] = snapshot_provider_info(info) + return snapshot diff --git a/tests/fixtures/replay_points/.gitkeep b/tests/fixtures/replay_points/.gitkeep new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tests/fixtures/replay_points/.gitkeep @@ -0,0 +1 @@ + diff --git a/tests/test_relational_normalizer_regression.py b/tests/test_relational_normalizer_regression.py index cec371ee..607d5b6b 100644 --- a/tests/test_relational_normalizer_regression.py +++ b/tests/test_relational_normalizer_regression.py @@ -173,6 +173,7 @@ def _build_plan_normalizer(providers: Set[str]) -> PlanNormalizer: } relational_rule = SelectorNormalizationRule( + kind="relational_v1", validator=TypeAdapter(RelationalRequest), normalize_selectors=normalize_relational_selectors, ) @@ -325,4 +326,3 @@ def test_regression_fixtures_invalid_inputs_are_fixed_by_plan_normalizer(case: T f"Selectors(before): {json.dumps(spec.selectors, ensure_ascii=False)[:2000]}\n" f"Selectors(after): {json.dumps(out.selectors, ensure_ascii=False)[:2000]}" ) - diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py new file mode 100644 index 00000000..9bd7c7a8 --- /dev/null +++ b/tests/test_replay_fixtures.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import difflib +import json +from pathlib import Path +from typing import Iterable + +import pytest + +import fetchgraph.replay.handlers.plan_normalize # noqa: F401 + +from fetchgraph.replay.runtime import REPLAY_HANDLERS, ReplayContext + +FIXTURES_ROOT = Path(__file__).parent / "fixtures" / "replay_points" + + +def _iter_fixture_paths() -> Iterable[Path]: + if not FIXTURES_ROOT.exists(): + return [] + return sorted(FIXTURES_ROOT.glob("*.json")) + + +def _format_json(payload: object) -> str: + return json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + + +def _truncate(text: str, limit: int = 2000) -> str: + if len(text) <= limit: + return text + return f"{text[:limit]}\n... (truncated {len(text) - limit} chars)" + + +def _selectors_diff(expected: object, actual: object) -> str: + expected_text = _format_json(expected).splitlines() + actual_text = _format_json(actual).splitlines() + diff = "\n".join( + difflib.unified_diff(expected_text, actual_text, fromfile="expected", tofile="actual", lineterm="") + ) + return _truncate(diff) + + +def _load_fixture(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def _parse_fixture(event: dict) -> tuple[dict, ReplayContext]: + if event.get("type") == "replay_bundle": + ctx = ReplayContext( + resources=event.get("resources") or {}, + extras=event.get("extras") or {}, + ) + return event["root"], ctx + return event, ReplayContext() + + +def _fixture_paths() -> list[Path]: + paths = list(_iter_fixture_paths()) + if not paths: + pytest.skip("No replay fixtures found in tests/fixtures/replay_points") + return paths + + +@pytest.mark.parametrize("path", _fixture_paths(), ids=lambda p: p.name) +def test_replay_fixture(path: Path) -> None: + raw = _load_fixture(path) + event, ctx = _parse_fixture(raw) + assert event.get("type") == "replay_point" + event_id = event.get("id") + assert event_id in REPLAY_HANDLERS + + handler = REPLAY_HANDLERS[event_id] + result = handler(event["input"], ctx) + expected = event["expected"] + + actual_spec = result.get("out_spec") + expected_spec = expected.get("out_spec") + assert isinstance(expected_spec, dict) + assert isinstance(actual_spec, dict) + if actual_spec != expected_spec: + meta = _format_json(event.get("meta")) + note = event.get("note") + diff = _selectors_diff(expected_spec.get("selectors"), actual_spec.get("selectors")) + pytest.fail( + "\n".join( + [ + f"Replay mismatch for {path.name}", + f"meta: {meta}", + f"note: {note}", + "selectors diff:", + diff, + ] + ) + ) From 6c6c7b534745ff5a0b1bf00af5555c4765196929 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 12:28:21 +0300 Subject: [PATCH 02/79] Make ALL flag boolean in fixture target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 49dc3b1e..4907fe1f 100644 --- a/Makefile +++ b/Makefile @@ -345,7 +345,7 @@ fixture: check $(TAG_FLAG) $(if $(strip $(RUN_ID)),--run-id "$(RUN_ID)",) $(if $(strip $(REPLAY_ID)),--id "$(REPLAY_ID)",) \ $(if $(strip $(SPEC_IDX)),--spec-idx "$(SPEC_IDX)",) $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ $(if $(strip $(OUT_DIR)),--out-dir "$(OUT_DIR)",) \ - $(if $(strip $(ALL)),--all,) \ + $(if $(filter 1 true yes on,$(ALL)),--all,) \ $(if $(filter requires,$(WITH)),--with-requires,) # compare (diff.md + junit) From 509fbe48d55aed6a207c22deeeaa8fced2b8a6a9 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 12:33:16 +0300 Subject: [PATCH 03/79] Fix replay plan normalize imports --- src/fetchgraph/planning/normalize/plan_normalizer.py | 2 +- src/fetchgraph/replay/handlers/plan_normalize.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/fetchgraph/planning/normalize/plan_normalizer.py b/src/fetchgraph/planning/normalize/plan_normalizer.py index f14874ae..4a26c372 100644 --- a/src/fetchgraph/planning/normalize/plan_normalizer.py +++ b/src/fetchgraph/planning/normalize/plan_normalizer.py @@ -35,9 +35,9 @@ class PlanNormalizerOptions: @dataclass(frozen=True) class SelectorNormalizationRule: - kind: str | None = None validator: TypeAdapter[Any] normalize_selectors: Callable[[Any], Any] + kind: str | None = None class PlanNormalizer: diff --git a/src/fetchgraph/replay/handlers/plan_normalize.py b/src/fetchgraph/replay/handlers/plan_normalize.py index ef2a0af8..8f94b6f1 100644 --- a/src/fetchgraph/replay/handlers/plan_normalize.py +++ b/src/fetchgraph/replay/handlers/plan_normalize.py @@ -4,10 +4,10 @@ from pydantic import TypeAdapter -from ..core.models import ContextFetchSpec, ProviderInfo -from ..planning.normalize import PlanNormalizer, PlanNormalizerOptions, SelectorNormalizationRule -from ..relational.models import RelationalRequest -from ..relational.normalize import normalize_relational_selectors +from ...core.models import ContextFetchSpec, ProviderInfo +from ...planning.normalize import PlanNormalizer, PlanNormalizerOptions, SelectorNormalizationRule +from ...relational.models import RelationalRequest +from ...relational.normalize import normalize_relational_selectors from ..runtime import REPLAY_HANDLERS, ReplayContext From e60e312e46fae5b109196a6a9ae135adb623e5a4 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 12:40:09 +0300 Subject: [PATCH 04/79] Allow module-level skip for replay fixtures --- tests/test_replay_fixtures.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index 9bd7c7a8..c5a69a85 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -56,7 +56,10 @@ def _parse_fixture(event: dict) -> tuple[dict, ReplayContext]: def _fixture_paths() -> list[Path]: paths = list(_iter_fixture_paths()) if not paths: - pytest.skip("No replay fixtures found in tests/fixtures/replay_points") + pytest.skip( + "No replay fixtures found in tests/fixtures/replay_points", + allow_module_level=True, + ) return paths From d73c690cc1204747810ab13633435bdf86f89962 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 23 Jan 2026 14:39:46 +0300 Subject: [PATCH 05/79] version inc - tracer --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b3e82dec..b83b91d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "fetchgraph" -version = "0.2.1" +version = "0.2.2" description = "Graph-like planning → context fetching → synthesis agent (library-style)." readme = "README.md" requires-python = ">=3.11" From 809897d28df801822b12c8d6b33dca0935212523 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 14:41:39 +0300 Subject: [PATCH 06/79] Clarify fixture help defaults --- Makefile | 5 +- ...0119_133537_692233_orders_._plan_trace.txt | 0 ...9_133537_708677_customers_._plan_trace.txt | 0 ...133537_736816_order_items_._plan_trace.txt | 0 ...ders.order_date_YYYY-MM-DD._plan_trace.txt | 0 ...ders.order_date_YYYY-MM-DD._plan_trace.txt | 0 ..._sum_orders.order_total_2_._plan_trace.txt | 0 ..._avg_orders.order_total_2_._plan_trace.txt | 0 ...dian_orders.order_total_2_._plan_trace.txt | 0 ..._min_orders.order_total_2_._plan_trace.txt | 0 ..._max_orders.order_total_2_._plan_trace.txt | 0 ...5165_order_id_order_total_._plan_trace.txt | 0 ...0859_order_id_order_total_._plan_trace.txt | 0 ...9_133537_928468_cancelled_._plan_trace.txt | 0 ...9_133537_944943_delivered_._plan_trace.txt | 0 ...119_133537_961602_pending_._plan_trace.txt | 0 ..._133537_978273_processing_._plan_trace.txt | 0 ...119_133537_995082_shipped_._plan_trace.txt | 0 ...0119_133538_011607_online_._plan_trace.txt | 0 ...119_133538_028336_partner_._plan_trace.txt | 0 ...60119_133538_055979_phone_._plan_trace.txt | 0 ...0119_133538_084494_retail_._plan_trace.txt | 0 ...260119_133538_114093_2022_._plan_trace.txt | 0 ...260119_133538_142347_2023_._plan_trace.txt | 0 ...260119_133538_157966_2024_._plan_trace.txt | 0 ...0119_133538_174624_YYYY-MM._plan_trace.txt | 0 ...0119_133538_194420_YYYY-MM._plan_trace.txt | 0 ...119_133538_211755_2022-03_._plan_trace.txt | 0 ...119_133538_229173_2022-07_._plan_trace.txt | 0 ...items.line_total_category_._plan_trace.txt | 0 ...529_sum_line_total_toys_2_._plan_trace.txt | 0 ...390_sum_line_total_toys_2_._plan_trace.txt | 0 ...92_sum_line_total_books_2_._plan_trace.txt | 0 ..._line_total_electronics_2_._plan_trace.txt | 0 ...um_line_total_furniture_2_._plan_trace.txt | 0 ...e_total_office_supplies_2_._plan_trace.txt | 0 ...sum_line_total_outdoors_2_._plan_trace.txt | 0 ...uct_id_max_products.price_._plan_trace.txt | 0 ...457942_max_products.price_._plan_trace.txt | 0 ...uct_id_min_products.price_._plan_trace.txt | 0 ...504992_min_products.price_._plan_trace.txt | 0 ...um_order_items.line_total_._plan_trace.txt | 0 ...duct_id-_sum_line_total_2_._plan_trace.txt | 0 ...um_order_items.quantity_-_._plan_trace.txt | 0 ..._sum_order_items.quantity_._plan_trace.txt | 0 ...rder_items_per_order_id_1_._plan_trace.txt | 0 ...93992_San_Antonio_2022-03_._plan_trace.txt | 0 ...ipped_San_Antonio_2022-03_._plan_trace.txt | 0 ...32318_Los_Angeles_2023-08_._plan_trace.txt | 0 ...41_448371_customer_id_323_._plan_trace.txt | 0 ...umer_corporate_home_office._plan_trace.txt | 0 ...customer_id_323_YYYY-MM-DD._plan_trace.txt | 0 ...41_500165_customer_id_323_._plan_trace.txt | 0 ..._sum_orders.order_total_2_._plan_trace.txt | 0 ...customer_id_323_YYYY-MM-DD._plan_trace.txt | 0 ...57_042633_customer_id_536_._plan_trace.txt | 0 ...umer_corporate_home_office._plan_trace.txt | 0 ...customer_id_536_YYYY-MM-DD._plan_trace.txt | 0 ...06_123642_customer_id_536_._plan_trace.txt | 0 ..._sum_orders.order_total_2_._plan_trace.txt | 0 ...customer_id_536_YYYY-MM-DD._plan_trace.txt | 0 ...07_536957_customer_id_692_._plan_trace.txt | 0 ...umer_corporate_home_office._plan_trace.txt | 0 ...customer_id_692_YYYY-MM-DD._plan_trace.txt | 0 ...38_280588_customer_id_692_._plan_trace.txt | 0 ..._sum_orders.order_total_2_._plan_trace.txt | 0 ...customer_id_692_YYYY-MM-DD._plan_trace.txt | 0 ...48_740364_customer_id_722_._plan_trace.txt | 0 ...umer_corporate_home_office._plan_trace.txt | 0 ...customer_id_722_YYYY-MM-DD._plan_trace.txt | 0 ...07_431102_customer_id_722_._plan_trace.txt | 0 ..._sum_orders.order_total_2_._plan_trace.txt | 0 ...customer_id_722_YYYY-MM-DD._plan_trace.txt | 0 ...19_274457_customer_id_725_._plan_trace.txt | 0 ...umer_corporate_home_office._plan_trace.txt | 0 ...customer_id_725_YYYY-MM-DD._plan_trace.txt | 0 .../known_bad}/agg_003_plan_trace.txt | 0 .../known_bad}/agg_005_plan_trace.txt | 0 .../known_bad}/agg_035_plan_trace.txt | 0 .../known_bad}/agg_036_plan_trace.txt | 0 .../known_bad}/geo_001_plan_trace.txt | 0 .../known_bad}/items_002_plan_trace.txt | 0 .../known_bad}/qa_001_plan_trace.txt | 0 .../replay_points/{ => fixed}/.gitkeep | 0 .../fixtures/replay_points/known_bad/.gitkeep | 0 .../test_relational_normalizer_regression.py | 72 +++++++------------ tests/test_replay_fixtures.py | 24 +++++-- 87 files changed, 46 insertions(+), 55 deletions(-) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0001_20260119_133537_692233_orders_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0002_20260119_133537_708677_customers_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0004_20260119_133537_736816_order_items_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0006_20260119_133537_770370_min_orders.order_date_YYYY-MM-DD._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0007_20260119_133537_787946_max_orders.order_date_YYYY-MM-DD._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0008_20260119_133537_808662_sum_orders.order_total_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0009_20260119_133537_829192_avg_orders.order_total_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0010_20260119_133537_844853_median_orders.order_total_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0011_20260119_133537_860976_min_orders.order_total_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0012_20260119_133537_878573_max_orders.order_total_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0013_20260119_133537_895165_order_id_order_total_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0014_20260119_133537_910859_order_id_order_total_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0015_20260119_133537_928468_cancelled_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0016_20260119_133537_944943_delivered_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0017_20260119_133537_961602_pending_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0018_20260119_133537_978273_processing_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0019_20260119_133537_995082_shipped_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0020_20260119_133538_011607_online_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0021_20260119_133538_028336_partner_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0022_20260119_133538_055979_phone_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0023_20260119_133538_084494_retail_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0024_20260119_133538_114093_2022_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0025_20260119_133538_142347_2023_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0026_20260119_133538_157966_2024_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0027_20260119_133538_174624_YYYY-MM._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0028_20260119_133538_194420_YYYY-MM._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0029_20260119_133538_211755_2022-03_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0030_20260119_133538_229173_2022-07_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0031_20260119_133538_245323_order_items.line_total_category_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0032_20260119_133539_259529_sum_line_total_toys_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0033_20260119_133539_279390_sum_line_total_toys_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0034_20260119_133539_305892_sum_line_total_books_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0035_20260119_133539_332861_sum_line_total_electronics_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0036_20260119_133539_360048_sum_line_total_furniture_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0037_20260119_133539_389073_sum_line_total_office_supplies_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0038_20260119_133539_411584_sum_line_total_outdoors_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0039_20260119_133539_433448_product_id_max_products.price_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0040_20260119_133539_457942_max_products.price_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0041_20260119_133539_480567_product_id_min_products.price_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0042_20260119_133539_504992_min_products.price_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0043_20260119_133539_527837_product_id_sum_order_items.line_total_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0044_20260119_133540_291264_product_id-_sum_line_total_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0045_20260119_133540_822643_sum_order_items.quantity_-_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0046_20260119_133541_342609_sum_order_items.quantity_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0048_20260119_133541_375247_1_order_items_per_order_id_1_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0049_20260119_133541_393992_San_Antonio_2022-03_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0050_20260119_133541_413772_shipped_San_Antonio_2022-03_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0051_20260119_133541_432318_Los_Angeles_2023-08_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0052_20260119_133541_448371_customer_id_323_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0053_20260119_133541_464736_customer_id_323_consumer_corporate_home_office._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0054_20260119_133541_480392_customers.signup_date_customer_id_323_YYYY-MM-DD._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0055_20260119_133541_500165_customer_id_323_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0056_20260119_163550_586309_customer_id_323_sum_orders.order_total_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0057_20260119_163556_526136_customer_id_323_YYYY-MM-DD._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0058_20260119_163557_042633_customer_id_536_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0059_20260119_163557_528349_customer_id_536_consumer_corporate_home_office._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0060_20260119_163604_515336_customers.signup_date_customer_id_536_YYYY-MM-DD._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0061_20260119_163606_123642_customer_id_536_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0062_20260119_163606_140388_customer_id_536_sum_orders.order_total_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0063_20260119_163606_850367_customer_id_536_YYYY-MM-DD._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0064_20260119_163607_536957_customer_id_692_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0065_20260119_163608_189367_customer_id_692_consumer_corporate_home_office._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0066_20260119_163608_690578_customers.signup_date_customer_id_692_YYYY-MM-DD._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0067_20260119_163638_280588_customer_id_692_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0068_20260119_163638_300137_customer_id_692_sum_orders.order_total_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0069_20260119_163638_822771_customer_id_692_YYYY-MM-DD._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0070_20260119_163648_740364_customer_id_722_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0071_20260119_163653_414936_customer_id_722_consumer_corporate_home_office._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0072_20260119_163658_005036_customers.signup_date_customer_id_722_YYYY-MM-DD._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0073_20260119_163707_431102_customer_id_722_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0074_20260119_163710_235823_customer_id_722_sum_orders.order_total_2_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0075_20260119_163715_021803_customer_id_722_YYYY-MM-DD._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0076_20260119_163719_274457_customer_id_725_._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0077_20260119_163722_195616_customer_id_725_consumer_corporate_home_office._plan_trace.txt (100%) rename tests/fixtures/{regressions_fixed/fetchgraph_plans => plan_traces/fixed}/0078_20260119_163726_540394_customers.signup_date_customer_id_725_YYYY-MM-DD._plan_trace.txt (100%) rename tests/fixtures/{regressions_known_bad/fetchgraph_plans => plan_traces/known_bad}/agg_003_plan_trace.txt (100%) rename tests/fixtures/{regressions_known_bad/fetchgraph_plans => plan_traces/known_bad}/agg_005_plan_trace.txt (100%) rename tests/fixtures/{regressions_known_bad/fetchgraph_plans => plan_traces/known_bad}/agg_035_plan_trace.txt (100%) rename tests/fixtures/{regressions_known_bad/fetchgraph_plans => plan_traces/known_bad}/agg_036_plan_trace.txt (100%) rename tests/fixtures/{regressions_known_bad/fetchgraph_plans => plan_traces/known_bad}/geo_001_plan_trace.txt (100%) rename tests/fixtures/{regressions_known_bad/fetchgraph_plans => plan_traces/known_bad}/items_002_plan_trace.txt (100%) rename tests/fixtures/{regressions_known_bad/fetchgraph_plans => plan_traces/known_bad}/qa_001_plan_trace.txt (100%) rename tests/fixtures/replay_points/{ => fixed}/.gitkeep (100%) create mode 100644 tests/fixtures/replay_points/known_bad/.gitkeep diff --git a/Makefile b/Makefile index 4907fe1f..eef5013f 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,8 @@ REPLAY_ID ?= plan_normalize.spec_v1 WITH ?= SPEC_IDX ?= PROVIDER ?= -OUT_DIR ?= +BUCKET ?= fixed +OUT_DIR ?= tests/fixtures/replay_points/$(BUCKET) ALL ?= LIMIT ?= 50 CHANGES ?= 10 @@ -153,7 +154,7 @@ help: @echo " make tags [PATTERN=*] DATA=... - показать список тегов" @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" - @echo " make fixture CASE=agg_01 [TAG=...] [RUN_ID=...] [REPLAY_ID=plan_normalize.spec_v1] [WITH=requires] [SPEC_IDX=0] [PROVIDER=relational] [OUT_DIR=tests/fixtures/replay_points] [ALL=1]" + @echo " make fixture CASE=agg_01 [TAG=...] [RUN_ID=...] [REPLAY_ID=plan_normalize.spec_v1] [WITH=requires] [SPEC_IDX=0] [PROVIDER=relational] [BUCKET=fixed|known_bad] [OUT_DIR=tests/fixtures/replay_points/$$(BUCKET)] [ALL=1]" @echo "" @echo "Уборка:" @echo " make tag-rm TAG=... [DRY=1] [PURGE_RUNS=1] [PRUNE_HISTORY=1] [PRUNE_CASE_HISTORY=1]" diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0001_20260119_133537_692233_orders_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0001_20260119_133537_692233_orders_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0001_20260119_133537_692233_orders_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0001_20260119_133537_692233_orders_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0002_20260119_133537_708677_customers_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0002_20260119_133537_708677_customers_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0002_20260119_133537_708677_customers_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0002_20260119_133537_708677_customers_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0004_20260119_133537_736816_order_items_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0004_20260119_133537_736816_order_items_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0004_20260119_133537_736816_order_items_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0004_20260119_133537_736816_order_items_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0006_20260119_133537_770370_min_orders.order_date_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0006_20260119_133537_770370_min_orders.order_date_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0006_20260119_133537_770370_min_orders.order_date_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0006_20260119_133537_770370_min_orders.order_date_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0007_20260119_133537_787946_max_orders.order_date_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0007_20260119_133537_787946_max_orders.order_date_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0007_20260119_133537_787946_max_orders.order_date_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0007_20260119_133537_787946_max_orders.order_date_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0008_20260119_133537_808662_sum_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0008_20260119_133537_808662_sum_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0008_20260119_133537_808662_sum_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0008_20260119_133537_808662_sum_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0009_20260119_133537_829192_avg_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0009_20260119_133537_829192_avg_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0009_20260119_133537_829192_avg_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0009_20260119_133537_829192_avg_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0010_20260119_133537_844853_median_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0010_20260119_133537_844853_median_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0010_20260119_133537_844853_median_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0010_20260119_133537_844853_median_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0011_20260119_133537_860976_min_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0011_20260119_133537_860976_min_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0011_20260119_133537_860976_min_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0011_20260119_133537_860976_min_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0012_20260119_133537_878573_max_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0012_20260119_133537_878573_max_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0012_20260119_133537_878573_max_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0012_20260119_133537_878573_max_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0013_20260119_133537_895165_order_id_order_total_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0013_20260119_133537_895165_order_id_order_total_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0013_20260119_133537_895165_order_id_order_total_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0013_20260119_133537_895165_order_id_order_total_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0014_20260119_133537_910859_order_id_order_total_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0014_20260119_133537_910859_order_id_order_total_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0014_20260119_133537_910859_order_id_order_total_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0014_20260119_133537_910859_order_id_order_total_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0015_20260119_133537_928468_cancelled_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0015_20260119_133537_928468_cancelled_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0015_20260119_133537_928468_cancelled_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0015_20260119_133537_928468_cancelled_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0016_20260119_133537_944943_delivered_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0016_20260119_133537_944943_delivered_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0016_20260119_133537_944943_delivered_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0016_20260119_133537_944943_delivered_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0017_20260119_133537_961602_pending_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0017_20260119_133537_961602_pending_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0017_20260119_133537_961602_pending_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0017_20260119_133537_961602_pending_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0018_20260119_133537_978273_processing_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0018_20260119_133537_978273_processing_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0018_20260119_133537_978273_processing_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0018_20260119_133537_978273_processing_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0019_20260119_133537_995082_shipped_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0019_20260119_133537_995082_shipped_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0019_20260119_133537_995082_shipped_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0019_20260119_133537_995082_shipped_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0020_20260119_133538_011607_online_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0020_20260119_133538_011607_online_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0020_20260119_133538_011607_online_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0020_20260119_133538_011607_online_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0021_20260119_133538_028336_partner_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0021_20260119_133538_028336_partner_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0021_20260119_133538_028336_partner_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0021_20260119_133538_028336_partner_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0022_20260119_133538_055979_phone_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0022_20260119_133538_055979_phone_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0022_20260119_133538_055979_phone_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0022_20260119_133538_055979_phone_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0023_20260119_133538_084494_retail_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0023_20260119_133538_084494_retail_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0023_20260119_133538_084494_retail_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0023_20260119_133538_084494_retail_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0024_20260119_133538_114093_2022_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0024_20260119_133538_114093_2022_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0024_20260119_133538_114093_2022_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0024_20260119_133538_114093_2022_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0025_20260119_133538_142347_2023_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0025_20260119_133538_142347_2023_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0025_20260119_133538_142347_2023_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0025_20260119_133538_142347_2023_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0026_20260119_133538_157966_2024_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0026_20260119_133538_157966_2024_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0026_20260119_133538_157966_2024_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0026_20260119_133538_157966_2024_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0027_20260119_133538_174624_YYYY-MM._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0027_20260119_133538_174624_YYYY-MM._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0027_20260119_133538_174624_YYYY-MM._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0027_20260119_133538_174624_YYYY-MM._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0028_20260119_133538_194420_YYYY-MM._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0028_20260119_133538_194420_YYYY-MM._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0028_20260119_133538_194420_YYYY-MM._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0028_20260119_133538_194420_YYYY-MM._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0029_20260119_133538_211755_2022-03_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0029_20260119_133538_211755_2022-03_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0029_20260119_133538_211755_2022-03_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0029_20260119_133538_211755_2022-03_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0030_20260119_133538_229173_2022-07_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0030_20260119_133538_229173_2022-07_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0030_20260119_133538_229173_2022-07_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0030_20260119_133538_229173_2022-07_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0031_20260119_133538_245323_order_items.line_total_category_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0031_20260119_133538_245323_order_items.line_total_category_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0031_20260119_133538_245323_order_items.line_total_category_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0031_20260119_133538_245323_order_items.line_total_category_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0032_20260119_133539_259529_sum_line_total_toys_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0032_20260119_133539_259529_sum_line_total_toys_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0032_20260119_133539_259529_sum_line_total_toys_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0032_20260119_133539_259529_sum_line_total_toys_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0033_20260119_133539_279390_sum_line_total_toys_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0033_20260119_133539_279390_sum_line_total_toys_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0033_20260119_133539_279390_sum_line_total_toys_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0033_20260119_133539_279390_sum_line_total_toys_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0034_20260119_133539_305892_sum_line_total_books_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0034_20260119_133539_305892_sum_line_total_books_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0034_20260119_133539_305892_sum_line_total_books_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0034_20260119_133539_305892_sum_line_total_books_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0035_20260119_133539_332861_sum_line_total_electronics_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0035_20260119_133539_332861_sum_line_total_electronics_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0035_20260119_133539_332861_sum_line_total_electronics_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0035_20260119_133539_332861_sum_line_total_electronics_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0036_20260119_133539_360048_sum_line_total_furniture_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0036_20260119_133539_360048_sum_line_total_furniture_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0036_20260119_133539_360048_sum_line_total_furniture_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0036_20260119_133539_360048_sum_line_total_furniture_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0037_20260119_133539_389073_sum_line_total_office_supplies_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0037_20260119_133539_389073_sum_line_total_office_supplies_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0037_20260119_133539_389073_sum_line_total_office_supplies_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0037_20260119_133539_389073_sum_line_total_office_supplies_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0038_20260119_133539_411584_sum_line_total_outdoors_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0038_20260119_133539_411584_sum_line_total_outdoors_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0038_20260119_133539_411584_sum_line_total_outdoors_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0038_20260119_133539_411584_sum_line_total_outdoors_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0039_20260119_133539_433448_product_id_max_products.price_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0039_20260119_133539_433448_product_id_max_products.price_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0039_20260119_133539_433448_product_id_max_products.price_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0039_20260119_133539_433448_product_id_max_products.price_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0040_20260119_133539_457942_max_products.price_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0040_20260119_133539_457942_max_products.price_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0040_20260119_133539_457942_max_products.price_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0040_20260119_133539_457942_max_products.price_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0041_20260119_133539_480567_product_id_min_products.price_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0041_20260119_133539_480567_product_id_min_products.price_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0041_20260119_133539_480567_product_id_min_products.price_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0041_20260119_133539_480567_product_id_min_products.price_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0042_20260119_133539_504992_min_products.price_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0042_20260119_133539_504992_min_products.price_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0042_20260119_133539_504992_min_products.price_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0042_20260119_133539_504992_min_products.price_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0043_20260119_133539_527837_product_id_sum_order_items.line_total_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0043_20260119_133539_527837_product_id_sum_order_items.line_total_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0043_20260119_133539_527837_product_id_sum_order_items.line_total_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0043_20260119_133539_527837_product_id_sum_order_items.line_total_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0044_20260119_133540_291264_product_id-_sum_line_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0044_20260119_133540_291264_product_id-_sum_line_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0044_20260119_133540_291264_product_id-_sum_line_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0044_20260119_133540_291264_product_id-_sum_line_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0045_20260119_133540_822643_sum_order_items.quantity_-_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0045_20260119_133540_822643_sum_order_items.quantity_-_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0045_20260119_133540_822643_sum_order_items.quantity_-_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0045_20260119_133540_822643_sum_order_items.quantity_-_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0046_20260119_133541_342609_sum_order_items.quantity_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0046_20260119_133541_342609_sum_order_items.quantity_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0046_20260119_133541_342609_sum_order_items.quantity_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0046_20260119_133541_342609_sum_order_items.quantity_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0048_20260119_133541_375247_1_order_items_per_order_id_1_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0048_20260119_133541_375247_1_order_items_per_order_id_1_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0048_20260119_133541_375247_1_order_items_per_order_id_1_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0048_20260119_133541_375247_1_order_items_per_order_id_1_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0049_20260119_133541_393992_San_Antonio_2022-03_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0049_20260119_133541_393992_San_Antonio_2022-03_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0049_20260119_133541_393992_San_Antonio_2022-03_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0049_20260119_133541_393992_San_Antonio_2022-03_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0050_20260119_133541_413772_shipped_San_Antonio_2022-03_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0050_20260119_133541_413772_shipped_San_Antonio_2022-03_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0050_20260119_133541_413772_shipped_San_Antonio_2022-03_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0050_20260119_133541_413772_shipped_San_Antonio_2022-03_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0051_20260119_133541_432318_Los_Angeles_2023-08_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0051_20260119_133541_432318_Los_Angeles_2023-08_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0051_20260119_133541_432318_Los_Angeles_2023-08_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0051_20260119_133541_432318_Los_Angeles_2023-08_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0052_20260119_133541_448371_customer_id_323_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0052_20260119_133541_448371_customer_id_323_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0052_20260119_133541_448371_customer_id_323_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0052_20260119_133541_448371_customer_id_323_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0053_20260119_133541_464736_customer_id_323_consumer_corporate_home_office._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0053_20260119_133541_464736_customer_id_323_consumer_corporate_home_office._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0053_20260119_133541_464736_customer_id_323_consumer_corporate_home_office._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0053_20260119_133541_464736_customer_id_323_consumer_corporate_home_office._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0054_20260119_133541_480392_customers.signup_date_customer_id_323_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0054_20260119_133541_480392_customers.signup_date_customer_id_323_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0054_20260119_133541_480392_customers.signup_date_customer_id_323_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0054_20260119_133541_480392_customers.signup_date_customer_id_323_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0055_20260119_133541_500165_customer_id_323_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0055_20260119_133541_500165_customer_id_323_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0055_20260119_133541_500165_customer_id_323_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0055_20260119_133541_500165_customer_id_323_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0056_20260119_163550_586309_customer_id_323_sum_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0056_20260119_163550_586309_customer_id_323_sum_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0056_20260119_163550_586309_customer_id_323_sum_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0056_20260119_163550_586309_customer_id_323_sum_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0057_20260119_163556_526136_customer_id_323_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0057_20260119_163556_526136_customer_id_323_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0057_20260119_163556_526136_customer_id_323_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0057_20260119_163556_526136_customer_id_323_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0058_20260119_163557_042633_customer_id_536_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0058_20260119_163557_042633_customer_id_536_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0058_20260119_163557_042633_customer_id_536_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0058_20260119_163557_042633_customer_id_536_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0059_20260119_163557_528349_customer_id_536_consumer_corporate_home_office._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0059_20260119_163557_528349_customer_id_536_consumer_corporate_home_office._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0059_20260119_163557_528349_customer_id_536_consumer_corporate_home_office._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0059_20260119_163557_528349_customer_id_536_consumer_corporate_home_office._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0060_20260119_163604_515336_customers.signup_date_customer_id_536_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0060_20260119_163604_515336_customers.signup_date_customer_id_536_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0060_20260119_163604_515336_customers.signup_date_customer_id_536_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0060_20260119_163604_515336_customers.signup_date_customer_id_536_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0061_20260119_163606_123642_customer_id_536_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0061_20260119_163606_123642_customer_id_536_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0061_20260119_163606_123642_customer_id_536_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0061_20260119_163606_123642_customer_id_536_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0062_20260119_163606_140388_customer_id_536_sum_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0062_20260119_163606_140388_customer_id_536_sum_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0062_20260119_163606_140388_customer_id_536_sum_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0062_20260119_163606_140388_customer_id_536_sum_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0063_20260119_163606_850367_customer_id_536_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0063_20260119_163606_850367_customer_id_536_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0063_20260119_163606_850367_customer_id_536_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0063_20260119_163606_850367_customer_id_536_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0064_20260119_163607_536957_customer_id_692_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0064_20260119_163607_536957_customer_id_692_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0064_20260119_163607_536957_customer_id_692_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0064_20260119_163607_536957_customer_id_692_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0065_20260119_163608_189367_customer_id_692_consumer_corporate_home_office._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0065_20260119_163608_189367_customer_id_692_consumer_corporate_home_office._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0065_20260119_163608_189367_customer_id_692_consumer_corporate_home_office._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0065_20260119_163608_189367_customer_id_692_consumer_corporate_home_office._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0066_20260119_163608_690578_customers.signup_date_customer_id_692_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0066_20260119_163608_690578_customers.signup_date_customer_id_692_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0066_20260119_163608_690578_customers.signup_date_customer_id_692_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0066_20260119_163608_690578_customers.signup_date_customer_id_692_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0067_20260119_163638_280588_customer_id_692_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0067_20260119_163638_280588_customer_id_692_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0067_20260119_163638_280588_customer_id_692_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0067_20260119_163638_280588_customer_id_692_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0068_20260119_163638_300137_customer_id_692_sum_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0068_20260119_163638_300137_customer_id_692_sum_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0068_20260119_163638_300137_customer_id_692_sum_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0068_20260119_163638_300137_customer_id_692_sum_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0069_20260119_163638_822771_customer_id_692_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0069_20260119_163638_822771_customer_id_692_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0069_20260119_163638_822771_customer_id_692_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0069_20260119_163638_822771_customer_id_692_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0070_20260119_163648_740364_customer_id_722_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0070_20260119_163648_740364_customer_id_722_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0070_20260119_163648_740364_customer_id_722_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0070_20260119_163648_740364_customer_id_722_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0071_20260119_163653_414936_customer_id_722_consumer_corporate_home_office._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0071_20260119_163653_414936_customer_id_722_consumer_corporate_home_office._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0071_20260119_163653_414936_customer_id_722_consumer_corporate_home_office._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0071_20260119_163653_414936_customer_id_722_consumer_corporate_home_office._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0072_20260119_163658_005036_customers.signup_date_customer_id_722_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0072_20260119_163658_005036_customers.signup_date_customer_id_722_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0072_20260119_163658_005036_customers.signup_date_customer_id_722_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0072_20260119_163658_005036_customers.signup_date_customer_id_722_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0073_20260119_163707_431102_customer_id_722_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0073_20260119_163707_431102_customer_id_722_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0073_20260119_163707_431102_customer_id_722_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0073_20260119_163707_431102_customer_id_722_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0074_20260119_163710_235823_customer_id_722_sum_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0074_20260119_163710_235823_customer_id_722_sum_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0074_20260119_163710_235823_customer_id_722_sum_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0074_20260119_163710_235823_customer_id_722_sum_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0075_20260119_163715_021803_customer_id_722_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0075_20260119_163715_021803_customer_id_722_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0075_20260119_163715_021803_customer_id_722_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0075_20260119_163715_021803_customer_id_722_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0076_20260119_163719_274457_customer_id_725_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0076_20260119_163719_274457_customer_id_725_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0076_20260119_163719_274457_customer_id_725_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0076_20260119_163719_274457_customer_id_725_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0077_20260119_163722_195616_customer_id_725_consumer_corporate_home_office._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0077_20260119_163722_195616_customer_id_725_consumer_corporate_home_office._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0077_20260119_163722_195616_customer_id_725_consumer_corporate_home_office._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0077_20260119_163722_195616_customer_id_725_consumer_corporate_home_office._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0078_20260119_163726_540394_customers.signup_date_customer_id_725_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0078_20260119_163726_540394_customers.signup_date_customer_id_725_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0078_20260119_163726_540394_customers.signup_date_customer_id_725_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0078_20260119_163726_540394_customers.signup_date_customer_id_725_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_003_plan_trace.txt b/tests/fixtures/plan_traces/known_bad/agg_003_plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_003_plan_trace.txt rename to tests/fixtures/plan_traces/known_bad/agg_003_plan_trace.txt diff --git a/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_005_plan_trace.txt b/tests/fixtures/plan_traces/known_bad/agg_005_plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_005_plan_trace.txt rename to tests/fixtures/plan_traces/known_bad/agg_005_plan_trace.txt diff --git a/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_035_plan_trace.txt b/tests/fixtures/plan_traces/known_bad/agg_035_plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_035_plan_trace.txt rename to tests/fixtures/plan_traces/known_bad/agg_035_plan_trace.txt diff --git a/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_036_plan_trace.txt b/tests/fixtures/plan_traces/known_bad/agg_036_plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_036_plan_trace.txt rename to tests/fixtures/plan_traces/known_bad/agg_036_plan_trace.txt diff --git a/tests/fixtures/regressions_known_bad/fetchgraph_plans/geo_001_plan_trace.txt b/tests/fixtures/plan_traces/known_bad/geo_001_plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_known_bad/fetchgraph_plans/geo_001_plan_trace.txt rename to tests/fixtures/plan_traces/known_bad/geo_001_plan_trace.txt diff --git a/tests/fixtures/regressions_known_bad/fetchgraph_plans/items_002_plan_trace.txt b/tests/fixtures/plan_traces/known_bad/items_002_plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_known_bad/fetchgraph_plans/items_002_plan_trace.txt rename to tests/fixtures/plan_traces/known_bad/items_002_plan_trace.txt diff --git a/tests/fixtures/regressions_known_bad/fetchgraph_plans/qa_001_plan_trace.txt b/tests/fixtures/plan_traces/known_bad/qa_001_plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_known_bad/fetchgraph_plans/qa_001_plan_trace.txt rename to tests/fixtures/plan_traces/known_bad/qa_001_plan_trace.txt diff --git a/tests/fixtures/replay_points/.gitkeep b/tests/fixtures/replay_points/fixed/.gitkeep similarity index 100% rename from tests/fixtures/replay_points/.gitkeep rename to tests/fixtures/replay_points/fixed/.gitkeep diff --git a/tests/fixtures/replay_points/known_bad/.gitkeep b/tests/fixtures/replay_points/known_bad/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_relational_normalizer_regression.py b/tests/test_relational_normalizer_regression.py index 607d5b6b..ad99cc2f 100644 --- a/tests/test_relational_normalizer_regression.py +++ b/tests/test_relational_normalizer_regression.py @@ -38,17 +38,8 @@ def _iter_json_objects_from_trace_text(text: str) -> Iterable[Dict[str, Any]]: yield obj -def _find_fixtures_dir() -> Optional[Path]: - """ - Ищем папку `fixtures` вверх по дереву от текущего test-файла. - Это устойчиво к вложенности tests/... - """ - here = Path(__file__).resolve() - for parent in here.parents: - cand = parent / "fixtures" - if cand.exists(): - return cand - return None +def _fixtures_root() -> Path: + return Path(__file__).parent / "fixtures" / "plan_traces" @dataclass(frozen=True) @@ -69,24 +60,32 @@ def case_id(self) -> str: def _load_trace_cases_from_fixtures() -> List[TraceCase]: """ Ищет по бакетам: - - fixtures/regressions_fixed/fetchgraph_plans.zip - - fixtures/regressions_fixed/fetchgraph_plans/*.txt - - fixtures/regressions_known_bad/fetchgraph_plans.zip - - fixtures/regressions_known_bad/fetchgraph_plans/*.txt + - fixtures/plan_traces/fixed/*.txt + - fixtures/plan_traces/known_bad/*.txt + - fixtures/plan_traces/{bucket}/fetchgraph_plans.zip (опционально) Достаёт спецификации из stage=before_normalize (plan.context_plan[*]). """ - fixtures_dir = _find_fixtures_dir() - if fixtures_dir is None: - pytest.skip("No fixtures dir found (expected .../fixtures).", allow_module_level=True) + fixtures_root = _fixtures_root() + if not fixtures_root.exists(): + pytest.skip( + "No plan fixtures found in tests/fixtures/plan_traces.", + allow_module_level=True, + ) return [] cases: List[TraceCase] = [] - buckets = ["regressions_fixed", "regressions_known_bad"] + buckets = ["fixed", "known_bad"] for bucket in buckets: - zip_path = fixtures_dir / bucket / "fetchgraph_plans.zip" - dir_path = fixtures_dir / bucket / "fetchgraph_plans" + bucket_dir = fixtures_root / bucket + txt_paths = list(sorted(bucket_dir.glob("*_plan_trace.txt"))) + if txt_paths: + for p in txt_paths: + text = p.read_text(encoding="utf-8", errors="replace") + cases.extend(_extract_before_specs(trace_name=p.name, text=text, bucket=bucket)) + continue + zip_path = bucket_dir / "fetchgraph_plans.zip" if zip_path.exists(): with zipfile.ZipFile(zip_path) as zf: for name in sorted(zf.namelist()): @@ -94,32 +93,11 @@ def _load_trace_cases_from_fixtures() -> List[TraceCase]: continue text = zf.read(name).decode("utf-8", errors="replace") cases.extend(_extract_before_specs(trace_name=name, text=text, bucket=bucket)) - continue - - if dir_path.exists(): - for p in sorted(dir_path.glob("*_plan_trace.txt")): - text = p.read_text(encoding="utf-8", errors="replace") - cases.extend(_extract_before_specs(trace_name=p.name, text=text, bucket=bucket)) - - legacy_zip = fixtures_dir / "fetchgraph_plans.zip" - legacy_dir = fixtures_dir / "fetchgraph_plans" - if legacy_zip.exists(): - with zipfile.ZipFile(legacy_zip) as zf: - for name in sorted(zf.namelist()): - if not name.endswith("_plan_trace.txt"): - continue - text = zf.read(name).decode("utf-8", errors="replace") - cases.extend(_extract_before_specs(trace_name=name, text=text, bucket="regressions_fixed")) - if legacy_dir.exists(): - for p in sorted(legacy_dir.glob("*_plan_trace.txt")): - text = p.read_text(encoding="utf-8", errors="replace") - cases.extend(_extract_before_specs(trace_name=p.name, text=text, bucket="regressions_fixed")) if not cases: pytest.skip( - "No plan fixtures found. Put fetchgraph_plans.zip into fixtures/regressions_fixed " - "or fixtures/regressions_known_bad, or unpack it to " - "fixtures/regressions_fixed/fetchgraph_plans or fixtures/regressions_known_bad/fetchgraph_plans.", + "No plan fixtures found. Put *_plan_trace.txt into " + "tests/fixtures/plan_traces/{fixed,known_bad} or add fetchgraph_plans.zip there.", allow_module_level=True, ) return cases @@ -216,15 +194,15 @@ def _parse_note(note: str) -> Dict[str, Any]: CASES = _load_trace_cases_from_fixtures() -# 1) Контрактный тест запускаем ТОЛЬКО на regressions_fixed -CASES_FIXED = [c for c in CASES if c.bucket == "regressions_fixed"] +# 1) Контрактный тест запускаем ТОЛЬКО на fixed +CASES_FIXED = [c for c in CASES if c.bucket == "fixed"] # 2) Для общего набора — автоматически проставляем marks по bucket CASES_ALL = [ pytest.param( c, id=c.case_id, - marks=(pytest.mark.known_bad,) if c.bucket == "regressions_known_bad" else (), + marks=(pytest.mark.known_bad,) if c.bucket == "known_bad" else (), ) for c in CASES ] diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index c5a69a85..b1ff2c47 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -12,12 +12,20 @@ from fetchgraph.replay.runtime import REPLAY_HANDLERS, ReplayContext FIXTURES_ROOT = Path(__file__).parent / "fixtures" / "replay_points" +_BUCKETS = ("fixed", "known_bad") -def _iter_fixture_paths() -> Iterable[Path]: +def _iter_fixture_paths() -> Iterable[tuple[Path, str]]: if not FIXTURES_ROOT.exists(): return [] - return sorted(FIXTURES_ROOT.glob("*.json")) + paths: list[tuple[Path, str]] = [] + for bucket in _BUCKETS: + bucket_dir = FIXTURES_ROOT / bucket + if not bucket_dir.exists(): + continue + for path in sorted(bucket_dir.rglob("*.json")): + paths.append((path, bucket)) + return paths def _format_json(payload: object) -> str: @@ -53,17 +61,21 @@ def _parse_fixture(event: dict) -> tuple[dict, ReplayContext]: return event, ReplayContext() -def _fixture_paths() -> list[Path]: +def _fixture_paths() -> list[pytest.ParameterSet]: paths = list(_iter_fixture_paths()) if not paths: pytest.skip( - "No replay fixtures found in tests/fixtures/replay_points", + "No replay fixtures found in tests/fixtures/replay_points/{fixed,known_bad}", allow_module_level=True, ) - return paths + params: list[pytest.ParameterSet] = [] + for path, bucket in paths: + marks = (pytest.mark.known_bad,) if bucket == "known_bad" else () + params.append(pytest.param(path, id=path.name, marks=marks)) + return params -@pytest.mark.parametrize("path", _fixture_paths(), ids=lambda p: p.name) +@pytest.mark.parametrize("path", _fixture_paths()) def test_replay_fixture(path: Path) -> None: raw = _load_fixture(path) event, ctx = _parse_fixture(raw) From 7e1c909d2450471b2ff80ac90ff3d219c73db82a Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 14:43:27 +0300 Subject: [PATCH 07/79] Fix replay fixture typing --- tests/test_replay_fixtures.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index b1ff2c47..251876c9 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -15,17 +15,17 @@ _BUCKETS = ("fixed", "known_bad") -def _iter_fixture_paths() -> Iterable[tuple[Path, str]]: +def _iter_fixture_paths() -> Iterable[tuple[str, Path]]: if not FIXTURES_ROOT.exists(): return [] - paths: list[tuple[Path, str]] = [] + paths: list[tuple[str, Path]] = [] for bucket in _BUCKETS: bucket_dir = FIXTURES_ROOT / bucket if not bucket_dir.exists(): continue - for path in sorted(bucket_dir.rglob("*.json")): - paths.append((path, bucket)) - return paths + for path in bucket_dir.rglob("*.json"): + paths.append((bucket, path)) + return sorted(paths) def _format_json(payload: object) -> str: @@ -69,9 +69,9 @@ def _fixture_paths() -> list[pytest.ParameterSet]: allow_module_level=True, ) params: list[pytest.ParameterSet] = [] - for path, bucket in paths: + for bucket, path in paths: marks = (pytest.mark.known_bad,) if bucket == "known_bad" else () - params.append(pytest.param(path, id=path.name, marks=marks)) + params.append(pytest.param(path, id=f"{bucket}/{path.name}", marks=marks)) return params From 2d46fa29ba876b4d7b654f4dbac20a5f3369162c Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:05:54 +0300 Subject: [PATCH 08/79] Validate replay fixtures for relational selectors --- tests/test_replay_fixtures.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index 251876c9..6c66bdc3 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -6,13 +6,16 @@ from typing import Iterable import pytest +from pydantic import TypeAdapter import fetchgraph.replay.handlers.plan_normalize # noqa: F401 +from fetchgraph.relational.models import RelationalRequest from fetchgraph.replay.runtime import REPLAY_HANDLERS, ReplayContext FIXTURES_ROOT = Path(__file__).parent / "fixtures" / "replay_points" _BUCKETS = ("fixed", "known_bad") +_REL_ADAPTER = TypeAdapter(RelationalRequest) def _iter_fixture_paths() -> Iterable[tuple[str, Path]]: @@ -75,8 +78,13 @@ def _fixture_paths() -> list[pytest.ParameterSet]: return params +def _bucket_from_path(path: Path) -> str: + return path.relative_to(FIXTURES_ROOT).parts[0] + + @pytest.mark.parametrize("path", _fixture_paths()) def test_replay_fixture(path: Path) -> None: + bucket = _bucket_from_path(path) raw = _load_fixture(path) event, ctx = _parse_fixture(raw) assert event.get("type") == "replay_point" @@ -106,3 +114,9 @@ def test_replay_fixture(path: Path) -> None: ] ) ) + + if event_id == "plan_normalize.spec_v1": + provider = actual_spec.get("provider") or event["input"]["spec"]["provider"] + rule_kind = (event["input"].get("normalizer_rules") or {}).get(provider) + if rule_kind == "relational_v1": + _REL_ADAPTER.validate_python(actual_spec["selectors"]) From 5408b022b3c2d30d59fc81f9067815df6c25ee46 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:09:39 +0300 Subject: [PATCH 09/79] Add fixture management commands --- Makefile | 17 +- examples/demo_qa/fixture_tools.py | 333 ++++++++++++++++++++++++++++++ 2 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 examples/demo_qa/fixture_tools.py diff --git a/Makefile b/Makefile index eef5013f..34c98fd4 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,7 @@ DEFAULT_CASES := examples/demo_qa/cases/retail_cases.json VENV ?= .venv PYTHON ?= $(if $(wildcard $(VENV)/bin/python),$(VENV)/bin/python,python) CLI := $(PYTHON) -m examples.demo_qa.cli +CLI_FIXT := $(PYTHON) -m examples.demo_qa.fixture_tools # ============================================================================== # 4) Пути demo_qa (можно переопределять через CLI или в $(CONFIG)) @@ -46,6 +47,8 @@ OUT ?= $(DATA)/.runs/results.jsonl TAG ?= NOTE ?= CASE ?= +NAME ?= +PATTERN ?= RUN_ID ?= REPLAY_ID ?= plan_normalize.spec_v1 WITH ?= @@ -53,11 +56,11 @@ SPEC_IDX ?= PROVIDER ?= BUCKET ?= fixed OUT_DIR ?= tests/fixtures/replay_points/$(BUCKET) +SCOPE ?= both ALL ?= LIMIT ?= 50 CHANGES ?= 10 NEW_TAG ?= -PATTERN ?= TAGS_FORMAT ?= table TAGS_COLOR ?= auto @@ -78,6 +81,7 @@ PURGE_RUNS ?= 0 PRUNE_HISTORY ?= 0 PRUNE_CASE_HISTORY ?= 0 DRY ?= 0 +MOVE_TRACES ?= 0 # ============================================================================== # 6) Настройки LLM-конфига (редактирование/просмотр) @@ -107,7 +111,7 @@ LIMIT_FLAG := $(if $(strip $(LIMIT)),--limit $(LIMIT),) batch batch-tag batch-failed batch-failed-from \ batch-missed batch-missed-from batch-failed-tag batch-missed-tag \ batch-fail-fast batch-max-fails \ - stats history-case report-tag report-tag-changes tags tag-rm case-run case-open fixture compare compare-tag + stats history-case report-tag report-tag-changes tags tag-rm case-run case-open fixture fixture-rm fixture-fix compare compare-tag # ============================================================================== # help (на русском) @@ -155,6 +159,8 @@ help: @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" @echo " make fixture CASE=agg_01 [TAG=...] [RUN_ID=...] [REPLAY_ID=plan_normalize.spec_v1] [WITH=requires] [SPEC_IDX=0] [PROVIDER=relational] [BUCKET=fixed|known_bad] [OUT_DIR=tests/fixtures/replay_points/$$(BUCKET)] [ALL=1]" + @echo " make fixture-rm NAME=... [PATTERN=...] [BUCKET=fixed|known_bad] [SCOPE=replay|traces|both] [DRY=1]" + @echo " make fixture-fix NAME=... [PATTERN=...] [CASE=...] [MOVE_TRACES=1] [DRY=1]" @echo "" @echo "Уборка:" @echo " make tag-rm TAG=... [DRY=1] [PURGE_RUNS=1] [PRUNE_HISTORY=1] [PRUNE_CASE_HISTORY=1]" @@ -349,6 +355,13 @@ fixture: check $(if $(filter 1 true yes on,$(ALL)),--all,) \ $(if $(filter requires,$(WITH)),--with-requires,) +# 12) Fixture tools +fixture-rm: check + @$(CLI_FIXT) rm --name "$(NAME)" --pattern "$(PATTERN)" --bucket "$(BUCKET)" --scope "$(SCOPE)" --dry "$(DRY)" + +fixture-fix: check + @$(CLI_FIXT) fix --name "$(NAME)" --pattern "$(PATTERN)" --case "$(CASE)" --move-traces "$(MOVE_TRACES)" --dry "$(DRY)" + # compare (diff.md + junit) compare: check @test -n "$(strip $(BASE))" || (echo "Нужно задать BASE=.../results_prev.jsonl" && exit 1) diff --git a/examples/demo_qa/fixture_tools.py b/examples/demo_qa/fixture_tools.py new file mode 100644 index 00000000..7785ef2b --- /dev/null +++ b/examples/demo_qa/fixture_tools.py @@ -0,0 +1,333 @@ +from __future__ import annotations + +import argparse +import fnmatch +import json +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Iterable, Optional + +REPO_ROOT = Path(__file__).resolve().parents[2] +REPLAY_ROOT = REPO_ROOT / "tests" / "fixtures" / "replay_points" +TRACE_ROOT = REPO_ROOT / "tests" / "fixtures" / "plan_traces" +BUCKETS = ("fixed", "known_bad") + + +def _normalize(value: Optional[str]) -> Optional[str]: + if value is None: + return None + value = value.strip() + return value or None + + +def _is_git_repo() -> bool: + result = subprocess.run( + ["git", "rev-parse", "--is-inside-work-tree"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + return result.returncode == 0 and result.stdout.strip() == "true" + + +def _git_tracked(path: Path) -> bool: + result = subprocess.run( + ["git", "ls-files", "--error-unmatch", str(path)], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + return result.returncode == 0 + + +def _remove_file(path: Path, *, use_git: bool, dry_run: bool) -> None: + if dry_run: + print(f"DRY: remove {path}") + return + if use_git and _git_tracked(path): + try: + subprocess.run(["git", "rm", "-f", str(path)], cwd=REPO_ROOT, check=True) + return + except subprocess.CalledProcessError: + pass + path.unlink(missing_ok=True) + + +def _move_file(src: Path, dst: Path, *, use_git: bool, dry_run: bool) -> None: + if dry_run: + print(f"DRY: move {src} -> {dst}") + return + dst.parent.mkdir(parents=True, exist_ok=True) + if use_git and _git_tracked(src): + try: + subprocess.run(["git", "mv", str(src), str(dst)], cwd=REPO_ROOT, check=True) + return + except subprocess.CalledProcessError: + pass + shutil.move(str(src), str(dst)) + + +def _matches_filters(path: Path, *, name: Optional[str], pattern: Optional[str]) -> bool: + if name and path.name != name and path.stem != name: + return False + if pattern and not fnmatch.fnmatch(path.name, pattern): + return False + return True + + +def _iter_replay_paths(bucket: Optional[str]) -> Iterable[Path]: + buckets = [bucket] if bucket else BUCKETS + for bkt in buckets: + root = REPLAY_ROOT / bkt + if not root.exists(): + continue + yield from root.rglob("*.json") + + +def _iter_trace_paths(bucket: Optional[str]) -> Iterable[Path]: + buckets = [bucket] if bucket else BUCKETS + for bkt in buckets: + root = TRACE_ROOT / bkt + if not root.exists(): + continue + yield from root.glob("*_plan_trace.txt") + + +def _relative(path: Path) -> str: + try: + return str(path.relative_to(REPO_ROOT)) + except ValueError: + return str(path) + + +def _bucket_from_path(path: Path, root: Path) -> str: + try: + return path.relative_to(root).parts[0] + except ValueError: + return "unknown" + + +def _load_case_id(path: Path) -> Optional[str]: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return None + if data.get("type") == "replay_bundle": + root = data.get("root") or {} + else: + root = data + case_id = root.get("case_id") + return case_id if isinstance(case_id, str) else None + + +def _select_with_case(paths: list[Path], case_id: str) -> list[Path]: + name_matches = [path for path in paths if case_id in path.name] + if name_matches: + return name_matches + return [path for path in paths if _load_case_id(path) == case_id] + + +def _validate_name_or_pattern(name: Optional[str], pattern: Optional[str]) -> None: + if not name and not pattern: + raise ValueError("Нужно задать хотя бы NAME или PATTERN.") + + +def _validate_name_pattern_case(name: Optional[str], pattern: Optional[str], case_id: Optional[str]) -> None: + if not name and not pattern and not case_id: + raise ValueError("Нужно задать хотя бы NAME, PATTERN или CASE.") + + +def _collect_candidates( + *, + scope: str, + bucket: Optional[str], + name: Optional[str], + pattern: Optional[str], +) -> list[Path]: + candidates: list[Path] = [] + if scope in ("replay", "both"): + for path in _iter_replay_paths(bucket): + if _matches_filters(path, name=name, pattern=pattern): + candidates.append(path) + if scope in ("traces", "both"): + for path in _iter_trace_paths(bucket): + if _matches_filters(path, name=name, pattern=pattern): + candidates.append(path) + return sorted(candidates, key=lambda p: _relative(p)) + + +def cmd_rm(args: argparse.Namespace) -> int: + name = _normalize(args.name) + pattern = _normalize(args.pattern) + bucket = _normalize(args.bucket) + scope = _normalize(args.scope) or "both" + dry_run = bool(args.dry) + + if bucket and bucket not in BUCKETS: + print(f"Неизвестный BUCKET: {bucket}", file=sys.stderr) + return 1 + if scope not in ("replay", "traces", "both"): + print(f"Неизвестный SCOPE: {scope}", file=sys.stderr) + return 1 + + try: + _validate_name_or_pattern(name, pattern) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 1 + + candidates = _collect_candidates(scope=scope, bucket=bucket, name=name, pattern=pattern) + if not candidates: + print("Ничего не найдено.", file=sys.stderr) + return 1 + + use_git = _is_git_repo() + print("Found files to remove:") + for path in candidates: + print(f"- {_bucket_from_path(path, REPLAY_ROOT if 'replay_points' in str(path) else TRACE_ROOT)}: {_relative(path)}") + + if dry_run: + print(f"DRY: would remove {len(candidates)} files.") + return 0 + + for path in candidates: + _remove_file(path, use_git=use_git, dry_run=False) + + print(f"Removed {len(candidates)} files.") + return 0 + + +def _collect_replay_known_bad( + *, + name: Optional[str], + pattern: Optional[str], + case_id: Optional[str], +) -> list[Path]: + candidates = [path for path in _iter_replay_paths("known_bad") if _matches_filters(path, name=name, pattern=pattern)] + if case_id: + candidates = _select_with_case(candidates, case_id) + return sorted(candidates, key=lambda p: _relative(p)) + + +def _collect_traces_for_case(case_id: str) -> list[Path]: + root = TRACE_ROOT / "known_bad" + if not root.exists(): + return [] + return sorted(root.glob(f"*{case_id}*plan_trace*.txt"), key=lambda p: _relative(p)) + + +def cmd_fix(args: argparse.Namespace) -> int: + name = _normalize(args.name) + pattern = _normalize(args.pattern) + case_id = _normalize(args.case) + move_traces = bool(args.move_traces) + dry_run = bool(args.dry) + + try: + _validate_name_pattern_case(name, pattern, case_id) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 1 + + candidates = _collect_replay_known_bad(name=name, pattern=pattern, case_id=case_id) + if not candidates: + print("Ничего не найдено в known_bad.", file=sys.stderr) + return 1 + + dests = [] + conflicts = [] + for src in candidates: + dst = REPLAY_ROOT / "fixed" / src.name + if dst.exists(): + conflicts.append(dst) + dests.append((src, dst)) + if conflicts: + print("Конфликт имён в fixed:", file=sys.stderr) + for conflict in conflicts: + print(f"- {_relative(conflict)}", file=sys.stderr) + return 1 + + use_git = _is_git_repo() + print("Found replay fixtures to promote:") + for src in candidates: + dst = REPLAY_ROOT / "fixed" / src.name + print(f"- {_relative(src)} -> {_relative(dst)}") + + if dry_run: + print(f"DRY: would promote {len(candidates)} replay fixtures.") + if move_traces: + print("DRY: plan traces would also be promoted if found.") + return 0 + + for src, dst in dests: + _move_file(src, dst, use_git=use_git, dry_run=False) + + print(f"Promoted {len(candidates)} replay fixtures to fixed.") + + if move_traces: + case_ids: set[str] = set() + for src in candidates: + if case_id: + case_ids.add(case_id) + continue + loaded_case_id = _load_case_id(src) + if loaded_case_id: + case_ids.add(loaded_case_id) + + promoted_traces: list[Path] = [] + for cid in sorted(case_ids): + promoted_traces.extend(_collect_traces_for_case(cid)) + + if not promoted_traces: + print("No plan traces found to promote.") + return 0 + + for trace in promoted_traces: + dst = TRACE_ROOT / "fixed" / trace.name + if dst.exists(): + print(f"Конфликт trace в fixed: {_relative(dst)}", file=sys.stderr) + return 1 + + for trace in promoted_traces: + _move_file(trace, TRACE_ROOT / "fixed" / trace.name, use_git=use_git, dry_run=False) + + print(f"Also promoted {len(promoted_traces)} plan traces.") + + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Tools for managing test fixtures.") + sub = parser.add_subparsers(dest="command", required=True) + + rm_parser = sub.add_parser("rm", help="Remove fixtures by name or pattern.") + rm_parser.add_argument("--name", default="") + rm_parser.add_argument("--pattern", default="") + rm_parser.add_argument("--bucket", default="") + rm_parser.add_argument("--scope", default="both") + rm_parser.add_argument("--dry", type=int, default=0) + rm_parser.set_defaults(func=cmd_rm) + + fix_parser = sub.add_parser("fix", help="Promote known_bad fixtures to fixed.") + fix_parser.add_argument("--name", default="") + fix_parser.add_argument("--pattern", default="") + fix_parser.add_argument("--case", dest="case", default="") + fix_parser.add_argument("--move-traces", type=int, default=0) + fix_parser.add_argument("--dry", type=int, default=0) + fix_parser.set_defaults(func=cmd_fix) + + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) From 17fc8eab9193cedadcdc32d7f4d6f53bc08cf6a8 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:35:31 +0300 Subject: [PATCH 10/79] Decouple fixture commands from check --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 34c98fd4..ef3c82d8 100644 --- a/Makefile +++ b/Makefile @@ -356,10 +356,10 @@ fixture: check $(if $(filter requires,$(WITH)),--with-requires,) # 12) Fixture tools -fixture-rm: check +fixture-rm: @$(CLI_FIXT) rm --name "$(NAME)" --pattern "$(PATTERN)" --bucket "$(BUCKET)" --scope "$(SCOPE)" --dry "$(DRY)" -fixture-fix: check +fixture-fix: @$(CLI_FIXT) fix --name "$(NAME)" --pattern "$(PATTERN)" --case "$(CASE)" --move-traces "$(MOVE_TRACES)" --dry "$(DRY)" # compare (diff.md + junit) From 35c825edb2609d99ec2fe1431ec48a7533660a27 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:35:44 +0300 Subject: [PATCH 11/79] Fix fixture tools git paths and case ids --- examples/demo_qa/fixture_tools.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/examples/demo_qa/fixture_tools.py b/examples/demo_qa/fixture_tools.py index 7785ef2b..f7d5e73a 100644 --- a/examples/demo_qa/fixture_tools.py +++ b/examples/demo_qa/fixture_tools.py @@ -33,9 +33,13 @@ def _is_git_repo() -> bool: return result.returncode == 0 and result.stdout.strip() == "true" +def _git_path(path: Path) -> str: + return str(path.relative_to(REPO_ROOT)) + + def _git_tracked(path: Path) -> bool: result = subprocess.run( - ["git", "ls-files", "--error-unmatch", str(path)], + ["git", "ls-files", "--error-unmatch", _git_path(path)], cwd=REPO_ROOT, capture_output=True, text=True, @@ -50,7 +54,7 @@ def _remove_file(path: Path, *, use_git: bool, dry_run: bool) -> None: return if use_git and _git_tracked(path): try: - subprocess.run(["git", "rm", "-f", str(path)], cwd=REPO_ROOT, check=True) + subprocess.run(["git", "rm", "-f", _git_path(path)], cwd=REPO_ROOT, check=True) return except subprocess.CalledProcessError: pass @@ -64,7 +68,7 @@ def _move_file(src: Path, dst: Path, *, use_git: bool, dry_run: bool) -> None: dst.parent.mkdir(parents=True, exist_ok=True) if use_git and _git_tracked(src): try: - subprocess.run(["git", "mv", str(src), str(dst)], cwd=REPO_ROOT, check=True) + subprocess.run(["git", "mv", _git_path(src), _git_path(dst)], cwd=REPO_ROOT, check=True) return except subprocess.CalledProcessError: pass @@ -263,21 +267,19 @@ def cmd_fix(args: argparse.Namespace) -> int: print("DRY: plan traces would also be promoted if found.") return 0 + case_ids: set[str] = set() + if move_traces: + for src in candidates: + resolved_case_id = case_id or _load_case_id(src) + if resolved_case_id: + case_ids.add(resolved_case_id) + for src, dst in dests: _move_file(src, dst, use_git=use_git, dry_run=False) print(f"Promoted {len(candidates)} replay fixtures to fixed.") if move_traces: - case_ids: set[str] = set() - for src in candidates: - if case_id: - case_ids.add(case_id) - continue - loaded_case_id = _load_case_id(src) - if loaded_case_id: - case_ids.add(loaded_case_id) - promoted_traces: list[Path] = [] for cid in sorted(case_ids): promoted_traces.extend(_collect_traces_for_case(cid)) From a76f976ae03775c2ebeb4ab6f729beefb6a9b14f Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:43:41 +0300 Subject: [PATCH 12/79] Remove unused replay bucket helper --- tests/test_replay_fixtures.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index 6c66bdc3..e8d52919 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -6,6 +6,7 @@ from typing import Iterable import pytest +from _pytest.mark.structures import ParameterSet from pydantic import TypeAdapter import fetchgraph.replay.handlers.plan_normalize # noqa: F401 @@ -64,27 +65,22 @@ def _parse_fixture(event: dict) -> tuple[dict, ReplayContext]: return event, ReplayContext() -def _fixture_paths() -> list[pytest.ParameterSet]: +def _fixture_paths() -> list[ParameterSet]: paths = list(_iter_fixture_paths()) if not paths: pytest.skip( "No replay fixtures found in tests/fixtures/replay_points/{fixed,known_bad}", allow_module_level=True, ) - params: list[pytest.ParameterSet] = [] + params: list[ParameterSet] = [] for bucket, path in paths: marks = (pytest.mark.known_bad,) if bucket == "known_bad" else () params.append(pytest.param(path, id=f"{bucket}/{path.name}", marks=marks)) return params -def _bucket_from_path(path: Path) -> str: - return path.relative_to(FIXTURES_ROOT).parts[0] - - @pytest.mark.parametrize("path", _fixture_paths()) def test_replay_fixture(path: Path) -> None: - bucket = _bucket_from_path(path) raw = _load_fixture(path) event, ctx = _parse_fixture(raw) assert event.get("type") == "replay_point" From b3eba81f7917c22592a1b53289756feaaf5ed2e8 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 16:05:49 +0300 Subject: [PATCH 13/79] Improve replay fixture debug output --- tests/test_replay_fixtures.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index e8d52919..cb2b4874 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -2,6 +2,7 @@ import difflib import json +import os from pathlib import Path from typing import Iterable @@ -17,6 +18,7 @@ FIXTURES_ROOT = Path(__file__).parent / "fixtures" / "replay_points" _BUCKETS = ("fixed", "known_bad") _REL_ADAPTER = TypeAdapter(RelationalRequest) +DEBUG_REPLAY = os.getenv("DEBUG_REPLAY", "").lower() in ("1", "true", "yes", "on") def _iter_fixture_paths() -> Iterable[tuple[str, Path]]: @@ -79,6 +81,11 @@ def _fixture_paths() -> list[ParameterSet]: return params +def _rerun_hint(path: Path) -> str: + bucket = path.relative_to(FIXTURES_ROOT).parts[0] + return f"pytest -vv {__file__}::test_replay_fixture[{bucket}/{path.name}] -s" + + @pytest.mark.parametrize("path", _fixture_paths()) def test_replay_fixture(path: Path) -> None: raw = _load_fixture(path) @@ -95,16 +102,25 @@ def test_replay_fixture(path: Path) -> None: expected_spec = expected.get("out_spec") assert isinstance(expected_spec, dict) assert isinstance(actual_spec, dict) + if DEBUG_REPLAY: + print(f"\n=== DEBUG {path} ===") + print("meta:", _format_json(event.get("meta"))) + print("note:", event.get("note")) + print("input:", _truncate(_format_json(event.get("input")), 8000)) if actual_spec != expected_spec: meta = _format_json(event.get("meta")) note = event.get("note") + inp = _truncate(_format_json(event.get("input")), limit=8000) diff = _selectors_diff(expected_spec.get("selectors"), actual_spec.get("selectors")) pytest.fail( "\n".join( [ f"Replay mismatch for {path.name}", + f"rerun: {_rerun_hint(path)}", f"meta: {meta}", f"note: {note}", + "input:", + inp, "selectors diff:", diff, ] From 344b3017712598463d40b2f068c46886272772fc Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 23 Jan 2026 16:15:58 +0300 Subject: [PATCH 14/79] =?UTF-8?q?=D0=BD=D0=BE=D0=B2=D1=8B=D0=B5=20=D1=82?= =?UTF-8?q?=D1=80=D0=B5=D0=B9=D1=81=D1=8B=20=D1=81=20=D0=BF=D0=B0=D0=B4?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F=D0=BC=D0=B8=20(=D0=BF=D0=B5=D1=80?= =?UTF-8?q?=D0=B5=D0=BC=D0=B5=D1=89=D0=B0=D0=B5=D0=BC)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../known_bad/agg_003_plan_trace.txt | 35 ----- .../known_bad/agg_005_plan_trace.txt | 49 ------- .../known_bad/agg_035_plan_trace.txt | 38 ----- .../known_bad/agg_036_plan_trace.txt | 38 ----- .../known_bad/geo_001_plan_trace.txt | 55 ------- .../known_bad/items_002_plan_trace.txt | 54 ------- .../known_bad/qa_001_plan_trace.txt | 134 ------------------ .../plan_normalize.spec_v1__027f241b.json | 1 + .../plan_normalize.spec_v1__172bff2e.json | 1 + .../plan_normalize.spec_v1__b6b6420e.json | 1 + .../plan_normalize.spec_v1__5f883b82.json | 1 + .../plan_normalize.spec_v1__b18ec08f.json | 1 + .../plan_normalize.spec_v1__b4d64d21.json | 1 + .../plan_normalize.spec_v1__dfd64ee7.json | 1 + 14 files changed, 7 insertions(+), 403 deletions(-) delete mode 100644 tests/fixtures/plan_traces/known_bad/agg_003_plan_trace.txt delete mode 100644 tests/fixtures/plan_traces/known_bad/agg_005_plan_trace.txt delete mode 100644 tests/fixtures/plan_traces/known_bad/agg_035_plan_trace.txt delete mode 100644 tests/fixtures/plan_traces/known_bad/agg_036_plan_trace.txt delete mode 100644 tests/fixtures/plan_traces/known_bad/geo_001_plan_trace.txt delete mode 100644 tests/fixtures/plan_traces/known_bad/items_002_plan_trace.txt delete mode 100644 tests/fixtures/plan_traces/known_bad/qa_001_plan_trace.txt create mode 100644 tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__027f241b.json create mode 100644 tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__172bff2e.json create mode 100644 tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__b6b6420e.json create mode 100644 tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__5f883b82.json create mode 100644 tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b18ec08f.json create mode 100644 tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b4d64d21.json create mode 100644 tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__dfd64ee7.json diff --git a/tests/fixtures/plan_traces/known_bad/agg_003_plan_trace.txt b/tests/fixtures/plan_traces/known_bad/agg_003_plan_trace.txt deleted file mode 100644 index d565b440..00000000 --- a/tests/fixtures/plan_traces/known_bad/agg_003_plan_trace.txt +++ /dev/null @@ -1,35 +0,0 @@ -{ - "stage": "before_normalize", - "plan": { - "required_context": [ - "demo_qa" - ], - "context_plan": [ - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "entity": "products", - "aggregations": [ - { - "field": "product_id", - "agg": "count", - "alias": "total_products" - } - ], - "group_by": [], - "filters": null - }, - "max_tokens": 100 - } - ], - "adr_queries": [], - "constraints": [], - "entities": [], - "dtos": [], - "normalization_notes": [ - "{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"error\", \"selectors_validate_after\": \"error\", \"selectors_normalization_decision\": \"use_normalized_unvalidated\", \"selectors_before\": {\"op\": \"aggregate\", \"entity\": \"products\", \"aggregations\": [{\"field\": \"product_id\", \"agg\": \"count\", \"alias\": \"total_products\"}]}, \"selectors_after\": {\"op\": \"query\", \"entity\": \"products\", \"aggregations\": [{\"field\": \"product_id\", \"agg\": \"count\", \"alias\": \"total_products\"}], \"group_by\": [], \"filters\": null}}" - ] - } -} diff --git a/tests/fixtures/plan_traces/known_bad/agg_005_plan_trace.txt b/tests/fixtures/plan_traces/known_bad/agg_005_plan_trace.txt deleted file mode 100644 index d9e5a796..00000000 --- a/tests/fixtures/plan_traces/known_bad/agg_005_plan_trace.txt +++ /dev/null @@ -1,49 +0,0 @@ -{ - "stage": "before_normalize", - "plan": { - "required_context": [ - "demo_qa" - ], - "context_plan": [ - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "customers", - "select": [ - { - "expr": "customers.customer_id", - "alias": "customer_id" - } - ], - "relations": [ - "orders_to_customers" - ], - "aggregations": [ - { - "field": "customer_id", - "agg": "count_distinct", - "alias": "unique_customers" - } - ], - "filters": { - "type": "comparison", - "entity": "orders", - "field": "order_id", - "op": "is_not_null" - }, - "group_by": [] - }, - "max_tokens": 1000 - } - ], - "adr_queries": [], - "constraints": [], - "entities": [], - "dtos": [], - "normalization_notes": [ - "{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"error\", \"selectors_validate_after\": \"error\", \"selectors_normalization_decision\": \"use_normalized_unvalidated\", \"selectors_before\": {\"op\": \"query\", \"root_entity\": \"customers\", \"select\": [{\"expr\": \"customers.customer_id\", \"alias\": \"customer_id\"}], \"relations\": [\"orders_to_customers\"], \"aggregations\": [{\"field\": \"customer_id\", \"agg\": \"count_distinct\", \"alias\": \"unique_customers\"}], \"filters\": {\"type\": \"comparison\", \"entity\": \"orders\", \"field\": \"order_id\", \"op\": \"is_not_null\"}}, \"selectors_after\": {\"op\": \"query\", \"root_entity\": \"customers\", \"select\": [{\"expr\": \"customers.customer_id\", \"alias\": \"customer_id\"}], \"relations\": [\"orders_to_customers\"], \"aggregations\": [{\"field\": \"customer_id\", \"agg\": \"count_distinct\", \"alias\": \"unique_customers\"}], \"filters\": {\"type\": \"comparison\", \"entity\": \"orders\", \"field\": \"order_id\", \"op\": \"is_not_null\"}, \"group_by\": []}}" - ] - } -} diff --git a/tests/fixtures/plan_traces/known_bad/agg_035_plan_trace.txt b/tests/fixtures/plan_traces/known_bad/agg_035_plan_trace.txt deleted file mode 100644 index ac3ab166..00000000 --- a/tests/fixtures/plan_traces/known_bad/agg_035_plan_trace.txt +++ /dev/null @@ -1,38 +0,0 @@ -{ - "stage": "before_normalize", - "plan": { - "required_context": [ - "demo_qa" - ], - "context_plan": [ - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "orders", - "aggregations": [ - { - "field": "order_id", - "agg": "count" - } - ], - "filters": { - "type": "comparison", - "field": "order_date", - "op": "between", - "value": [ - "2022-03-01", - "2022-03-31" - ] - } - }, - "max_tokens": 2000 - } - ], - "adr_queries": null, - "constraints": null, - "entities": [], - "dtos": [] - } -} diff --git a/tests/fixtures/plan_traces/known_bad/agg_036_plan_trace.txt b/tests/fixtures/plan_traces/known_bad/agg_036_plan_trace.txt deleted file mode 100644 index ac3ab166..00000000 --- a/tests/fixtures/plan_traces/known_bad/agg_036_plan_trace.txt +++ /dev/null @@ -1,38 +0,0 @@ -{ - "stage": "before_normalize", - "plan": { - "required_context": [ - "demo_qa" - ], - "context_plan": [ - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "orders", - "aggregations": [ - { - "field": "order_id", - "agg": "count" - } - ], - "filters": { - "type": "comparison", - "field": "order_date", - "op": "between", - "value": [ - "2022-03-01", - "2022-03-31" - ] - } - }, - "max_tokens": 2000 - } - ], - "adr_queries": null, - "constraints": null, - "entities": [], - "dtos": [] - } -} diff --git a/tests/fixtures/plan_traces/known_bad/geo_001_plan_trace.txt b/tests/fixtures/plan_traces/known_bad/geo_001_plan_trace.txt deleted file mode 100644 index 55373c32..00000000 --- a/tests/fixtures/plan_traces/known_bad/geo_001_plan_trace.txt +++ /dev/null @@ -1,55 +0,0 @@ -{ - "stage": "before_normalize", - "plan": { - "required_context": [ - "demo_qa" - ], - "context_plan": [ - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "orders", - "select": [ - { - "expr": "count(order_id)", - "alias": "order_count" - } - ], - "filters": { - "type": "logical", - "op": "and", - "clauses": [ - { - "type": "comparison", - "entity": "customers", - "field": "city", - "op": "=", - "value": "San Antonio" - }, - { - "type": "comparison", - "field": "order_date", - "op": "starts_with", - "value": "2022-03" - } - ] - }, - "relations": [ - "orders_to_customers" - ], - "case_sensitivity": false - }, - "max_tokens": 500 - } - ], - "adr_queries": [], - "constraints": [], - "entities": [], - "dtos": [], - "normalization_notes": [ - "{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}" - ] - } -} diff --git a/tests/fixtures/plan_traces/known_bad/items_002_plan_trace.txt b/tests/fixtures/plan_traces/known_bad/items_002_plan_trace.txt deleted file mode 100644 index 711aae8c..00000000 --- a/tests/fixtures/plan_traces/known_bad/items_002_plan_trace.txt +++ /dev/null @@ -1,54 +0,0 @@ - - -{ - "stage": "before_normalize", - "plan": { - "required_context": [ - "demo_qa" - ], - "context_plan": [ - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "order_items", - "group_by": [ - { - "field": "order_id", - "alias": "order_id" - } - ], - "aggregations": [ - { - "field": "order_item_id", - "agg": "count", - "alias": "item_count" - } - ], - "filters": { - "type": "logical", - "op": "and", - "clauses": [ - { - "type": "comparison", - "entity": "order_items", - "field": "order_id", - "op": "is_not_null" - } - ] - }, - "case_sensitivity": false - }, - "max_tokens": 2000 - } - ], - "adr_queries": [], - "constraints": [], - "entities": [], - "dtos": [], - "normalization_notes": [ - "{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"error\", \"selectors_validate_after\": \"error\", \"selectors_normalization_decision\": \"keep_original_still_invalid\", \"selectors_before\": {\"op\": \"query\", \"root_entity\": \"order_items\", \"group_by\": [{\"field\": \"order_id\", \"alias\": \"order_id\"}], \"aggregations\": [{\"field\": \"order_item_id\", \"agg\": \"count\", \"alias\": \"item_count\"}], \"filters\": {\"type\": \"logical\", \"op\": \"and\", \"clauses\": [{\"type\": \"comparison\", \"entity\": \"order_items\", \"field\": \"order_id\", \"op\": \"is_not_null\"}]}, \"case_sensitivity\": false}, \"selectors_after\": {\"op\": \"query\", \"root_entity\": \"order_items\", \"group_by\": [{\"field\": \"order_id\", \"alias\": \"order_id\"}], \"aggregations\": [{\"field\": \"order_item_id\", \"agg\": \"count\", \"alias\": \"item_count\"}], \"filters\": {\"type\": \"logical\", \"op\": \"and\", \"clauses\": [{\"type\": \"comparison\", \"entity\": \"order_items\", \"field\": \"order_id\", \"op\": \"is_not_null\"}]}, \"case_sensitivity\": false}}" - ] - } -} diff --git a/tests/fixtures/plan_traces/known_bad/qa_001_plan_trace.txt b/tests/fixtures/plan_traces/known_bad/qa_001_plan_trace.txt deleted file mode 100644 index 033a172e..00000000 --- a/tests/fixtures/plan_traces/known_bad/qa_001_plan_trace.txt +++ /dev/null @@ -1,134 +0,0 @@ -{ - "stage": "before_normalize", - "plan": { - "required_context": [ - "demo_qa" - ], - "context_plan": [ - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "customers", - "select": [ - { - "expr": "city", - "alias": "city" - }, - { - "expr": "segment", - "alias": "segment" - }, - { - "expr": "signup_date", - "alias": "signup_date" - } - ], - "filters": { - "type": "comparison", - "field": "customer_id", - "op": "=", - "value": 42 - }, - "relations": [ - "orders_to_customers" - ], - "group_by": [ - { - "field": "customer_id", - "alias": "customer_id" - } - ], - "aggregations": [ - { - "field": "order_id", - "agg": "count", - "alias": "order_count" - }, - { - "field": "order_total", - "agg": "sum", - "alias": "total_spent" - } - ], - "limit": 1 - }, - "max_tokens": null - }, - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "orders", - "select": [ - { - "expr": "order_date", - "alias": "last_order_date" - }, - { - "expr": "order_status", - "alias": "last_order_status" - }, - { - "expr": "order_channel", - "alias": "last_order_channel" - } - ], - "filters": { - "type": "comparison", - "field": "customer_id", - "op": "=", - "value": 42 - }, - "limit": 1, - "order_by": [ - { - "field": "order_date", - "direction": "desc" - } - ] - }, - "max_tokens": null - }, - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "order_items", - "select": [ - { - "expr": "line_total", - "alias": "line_total" - } - ], - "filters": { - "type": "comparison", - "field": "order_id", - "op": "in", - "value": "(SELECT order_id FROM orders WHERE customer_id = 42)" - }, - "limit": 5, - "order_by": [ - { - "field": "line_total", - "direction": "desc" - } - ] - }, - "max_tokens": null - } - ], - "adr_queries": [], - "constraints": [], - "entities": [], - "dtos": [], - "normalization_notes": [ - "{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}", - "{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}", - "{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}" - ] - } -} diff --git a/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__027f241b.json b/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__027f241b.json new file mode 100644 index 00000000..43add217 --- /dev/null +++ b/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__027f241b.json @@ -0,0 +1 @@ +{"extras":{"planner_input_v1":{"case_id":"agg_036","id":"planner_input_v1","input":{"feature_name":"agg_036","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Сколько заказов было в 2022-07? Ответь только числом."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:54.791922Z","type":"planner_input","v":1}},"resources":{},"root":{"case_id":"agg_036","diag":{"selectors_valid_after":true,"selectors_valid_before":true},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"filters":{"clauses":[{"entity":"orders","field":"order_date","op":">=","type":"comparison","value":"2022-07-01"},{"entity":"orders","field":"order_date","op":"<","type":"comparison","value":"2022-08-01"}],"op":"and","type":"logical"},"op":"query","root_entity":"orders","select":[{"alias":"order_count","expr":"count(*)"}]}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"filters":{"clauses":[{"entity":"orders","field":"order_date","op":">=","type":"comparison","value":"2022-07-01"},{"entity":"orders","field":"order_date","op":"<","type":"comparison","value":"2022-08-01"}],"op":"and","type":"logical"},"op":"query","root_entity":"orders","select":[{"alias":"order_count","expr":"count(*)"}]}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}","requires":["planner_input_v1"],"run_id":"9df66f32","source":{"case_id":"agg_036","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_036_c2239c95/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_036_c2239c95","tag":null},"timestamp":"2026-01-23T10:25:54.802564Z","type":"replay_point","v":1},"source":{"case_id":"agg_036","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_036_c2239c95/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file diff --git a/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__172bff2e.json b/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__172bff2e.json new file mode 100644 index 00000000..66ae3877 --- /dev/null +++ b/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__172bff2e.json @@ -0,0 +1 @@ +{"extras":{"planner_input_v1":{"case_id":"agg_035","id":"planner_input_v1","input":{"feature_name":"agg_035","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Сколько заказов было в 2022-03? Ответь только числом."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:54.764315Z","type":"planner_input","v":1}},"resources":{},"root":{"case_id":"agg_035","diag":{"selectors_valid_after":true,"selectors_valid_before":true},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"order_count","field":"order_id"}],"filters":{"clauses":[{"field":"order_date","op":">=","type":"comparison","value":"2022-03-01"},{"field":"order_date","op":"<=","type":"comparison","value":"2022-03-31"}],"op":"and","type":"logical"},"op":"query","root_entity":"orders","select":[{"expr":"order_count"}]}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"order_count","field":"order_id"}],"filters":{"clauses":[{"field":"order_date","op":">=","type":"comparison","value":"2022-03-01"},{"field":"order_date","op":"<=","type":"comparison","value":"2022-03-31"}],"op":"and","type":"logical"},"op":"query","root_entity":"orders","select":[{"expr":"order_count"}]}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}","requires":["planner_input_v1"],"run_id":"9df66f32","source":{"case_id":"agg_035","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_035_8dbaf955/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_035_8dbaf955","tag":null},"timestamp":"2026-01-23T10:25:54.776593Z","type":"replay_point","v":1},"source":{"case_id":"agg_035","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_035_8dbaf955/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file diff --git a/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__b6b6420e.json b/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__b6b6420e.json new file mode 100644 index 00000000..2c2e0764 --- /dev/null +++ b/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__b6b6420e.json @@ -0,0 +1 @@ +{"extras":{"planner_input_v1":{"case_id":"qa_001","id":"planner_input_v1","input":{"feature_name":"qa_001","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Расскажи всё о клиенте 42: профиль (city, segment, signup_date), количество заказов, общая сумма, дата/статус/канал последнего заказа и 5 самых дорогих покупок (по line_total)."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:59.312378Z","type":"planner_input","v":1}},"resources":{},"root":{"case_id":"qa_001","diag":{"selectors_valid_after":true,"selectors_valid_before":true},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"order_count","field":"order_id"},{"agg":"sum","alias":"total_spent","field":"order_total"}],"filters":{"field":"customer_id","op":"=","type":"comparison","value":42},"group_by":[{"alias":"customer_id","field":"customer_id"}],"limit":1,"op":"query","relations":["orders_to_customers"],"root_entity":"customers","select":[{"alias":"city","expr":"city"},{"alias":"segment","expr":"segment"},{"alias":"signup_date","expr":"signup_date"}]}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"order_count","field":"order_id"},{"agg":"sum","alias":"total_spent","field":"order_total"}],"filters":{"field":"customer_id","op":"=","type":"comparison","value":42},"group_by":[{"alias":"customer_id","field":"customer_id"}],"limit":1,"op":"query","relations":["orders_to_customers"],"root_entity":"customers","select":[{"alias":"city","expr":"city"},{"alias":"segment","expr":"segment"},{"alias":"signup_date","expr":"signup_date"}]}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}","requires":["planner_input_v1"],"run_id":"9df66f32","source":{"case_id":"qa_001","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/qa_001_713263cb/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/qa_001_713263cb","tag":null},"timestamp":"2026-01-23T10:25:59.327950Z","type":"replay_point","v":1},"source":{"case_id":"qa_001","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/qa_001_713263cb/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file diff --git a/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__5f883b82.json b/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__5f883b82.json new file mode 100644 index 00000000..aeb5a899 --- /dev/null +++ b/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__5f883b82.json @@ -0,0 +1 @@ +{"extras":{"planner_input_v1":{"case_id":"items_002","id":"planner_input_v1","input":{"feature_name":"items_002","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Сколько заказов содержат больше 3 позиций (кол-во order_items по order_id > 3)? Ответь только числом."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:55.690244Z","type":"planner_input","v":1}},"resources":{},"root":{"case_id":"items_002","diag":{"selectors_valid_after":false,"selectors_valid_before":false},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"item_count","field":"order_item_id"}],"case_sensitivity":false,"filters":{"clauses":[{"entity":"order_items","field":"order_id","op":"is_not_null","type":"comparison"}],"op":"and","type":"logical"},"group_by":[{"alias":"order_id","field":"order_id"}],"op":"query","root_entity":"order_items"}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"item_count","field":"order_item_id"}],"case_sensitivity":false,"filters":{"clauses":[{"entity":"order_items","field":"order_id","op":"is_not_null","type":"comparison"}],"op":"and","type":"logical"},"group_by":[{"alias":"order_id","field":"order_id"}],"op":"query","root_entity":"order_items"}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"error\", \"selectors_validate_after\": \"error\", \"selectors_normalization_decision\": \"keep_original_still_invalid\", \"selectors_before\": {\"op\": \"query\", \"root_entity\": \"order_items\", \"group_by\": [{\"field\": \"order_id\", \"alias\": \"order_id\"}], \"aggregations\": [{\"field\": \"order_item_id\", \"agg\": \"count\", \"alias\": \"item_count\"}], \"filters\": {\"type\": \"logical\", \"op\": \"and\", \"clauses\": [{\"type\": \"comparison\", \"entity\": \"order_items\", \"field\": \"order_id\", \"op\": \"is_not_null\"}]}, \"case_sensitivity\": false}, \"selectors_after\": {\"op\": \"query\", \"root_entity\": \"order_items\", \"group_by\": [{\"field\": \"order_id\", \"alias\": \"order_id\"}], \"aggregations\": [{\"field\": \"order_item_id\", \"agg\": \"count\", \"alias\": \"item_count\"}], \"filters\": {\"type\": \"logical\", \"op\": \"and\", \"clauses\": [{\"type\": \"comparison\", \"entity\": \"order_items\", \"field\": \"order_id\", \"op\": \"is_not_null\"}]}, \"case_sensitivity\": false}}","requires":["planner_input_v1"],"run_id":"9df66f32","source":{"case_id":"items_002","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/items_002_2b870ad2/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/items_002_2b870ad2","tag":null},"timestamp":"2026-01-23T10:25:55.699897Z","type":"replay_point","v":1},"source":{"case_id":"items_002","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/items_002_2b870ad2/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file diff --git a/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b18ec08f.json b/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b18ec08f.json new file mode 100644 index 00000000..0456b8be --- /dev/null +++ b/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b18ec08f.json @@ -0,0 +1 @@ +{"extras":{"planner_input_v1":{"case_id":"agg_005","id":"planner_input_v1","input":{"feature_name":"agg_005","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Сколько уникальных клиентов сделали хотя бы один заказ? Ответь только числом."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:54.062206Z","type":"planner_input","v":1}},"resources":{},"root":{"case_id":"agg_005","diag":{"selectors_valid_after":false,"selectors_valid_before":false},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count_distinct","alias":"unique_customers","field":"customer_id"}],"filters":{"entity":"orders","field":"order_id","op":"is_not_null","type":"comparison"},"group_by":[],"op":"query","relations":["orders_to_customers"],"root_entity":"customers","select":[{"alias":"customer_id","expr":"customers.customer_id"}]}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count_distinct","alias":"unique_customers","field":"customer_id"}],"filters":{"entity":"orders","field":"order_id","op":"is_not_null","type":"comparison"},"op":"query","relations":["orders_to_customers"],"root_entity":"customers","select":[{"alias":"customer_id","expr":"customers.customer_id"}]}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"error\", \"selectors_validate_after\": \"error\", \"selectors_normalization_decision\": \"use_normalized_unvalidated\", \"selectors_before\": {\"op\": \"query\", \"root_entity\": \"customers\", \"select\": [{\"expr\": \"customers.customer_id\", \"alias\": \"customer_id\"}], \"relations\": [\"orders_to_customers\"], \"aggregations\": [{\"field\": \"customer_id\", \"agg\": \"count_distinct\", \"alias\": \"unique_customers\"}], \"filters\": {\"type\": \"comparison\", \"entity\": \"orders\", \"field\": \"order_id\", \"op\": \"is_not_null\"}}, \"selectors_after\": {\"op\": \"query\", \"root_entity\": \"customers\", \"select\": [{\"expr\": \"customers.customer_id\", \"alias\": \"customer_id\"}], \"relations\": [\"orders_to_customers\"], \"aggregations\": [{\"field\": \"customer_id\", \"agg\": \"count_distinct\", \"alias\": \"unique_customers\"}], \"filters\": {\"type\": \"comparison\", \"entity\": \"orders\", \"field\": \"order_id\", \"op\": \"is_not_null\"}, \"group_by\": []}}","requires":["planner_input_v1"],"run_id":"9df66f32","source":{"case_id":"agg_005","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_005_b2da5838/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_005_b2da5838","tag":null},"timestamp":"2026-01-23T10:25:54.077062Z","type":"replay_point","v":1},"source":{"case_id":"agg_005","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_005_b2da5838/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file diff --git a/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b4d64d21.json b/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b4d64d21.json new file mode 100644 index 00000000..363a159c --- /dev/null +++ b/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b4d64d21.json @@ -0,0 +1 @@ +{"extras":{"planner_input_v1":{"case_id":"geo_001","id":"planner_input_v1","input":{"feature_name":"geo_001","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Сколько заказов было у клиентов из города San Antonio в месяце 2022-03? Ответь только числом."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:55.718816Z","type":"planner_input","v":1}},"resources":{},"root":{"case_id":"geo_001","diag":{"selectors_valid_after":true,"selectors_valid_before":true},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"case_sensitivity":false,"filters":{"clauses":[{"entity":"customers","field":"city","op":"=","type":"comparison","value":"San Antonio"},{"field":"order_date","op":"starts_with","type":"comparison","value":"2022-03"}],"op":"and","type":"logical"},"op":"query","relations":["orders_to_customers"],"root_entity":"orders","select":[{"alias":"order_count","expr":"count(order_id)"}]}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"case_sensitivity":false,"filters":{"clauses":[{"entity":"customers","field":"city","op":"=","type":"comparison","value":"San Antonio"},{"field":"order_date","op":"starts_with","type":"comparison","value":"2022-03"}],"op":"and","type":"logical"},"op":"query","relations":["orders_to_customers"],"root_entity":"orders","select":[{"alias":"order_count","expr":"count(order_id)"}]}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}","requires":["planner_input_v1"],"run_id":"9df66f32","source":{"case_id":"geo_001","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/geo_001_f83d8b50/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/geo_001_f83d8b50","tag":null},"timestamp":"2026-01-23T10:25:55.730263Z","type":"replay_point","v":1},"source":{"case_id":"geo_001","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/geo_001_f83d8b50/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file diff --git a/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__dfd64ee7.json b/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__dfd64ee7.json new file mode 100644 index 00000000..85cc7c94 --- /dev/null +++ b/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__dfd64ee7.json @@ -0,0 +1 @@ +{"extras":{"planner_input_v1":{"case_id":"agg_003","id":"planner_input_v1","input":{"feature_name":"agg_003","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Сколько всего товаров (products) в датасете? Ответь только числом."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:54.010082Z","type":"planner_input","v":1}},"resources":{},"root":{"case_id":"agg_003","diag":{"selectors_valid_after":false,"selectors_valid_before":false},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"total_products","field":"product_id"}],"entity":"products","filters":null,"group_by":[],"op":"query"}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"total_products","field":"product_id"}],"entity":"products","op":"aggregate"}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"error\", \"selectors_validate_after\": \"error\", \"selectors_normalization_decision\": \"use_normalized_unvalidated\", \"selectors_before\": {\"op\": \"aggregate\", \"entity\": \"products\", \"aggregations\": [{\"field\": \"product_id\", \"agg\": \"count\", \"alias\": \"total_products\"}]}, \"selectors_after\": {\"op\": \"query\", \"entity\": \"products\", \"aggregations\": [{\"field\": \"product_id\", \"agg\": \"count\", \"alias\": \"total_products\"}], \"group_by\": [], \"filters\": null}}","requires":["planner_input_v1"],"run_id":"9df66f32","source":{"case_id":"agg_003","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_003_22b4ce14/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_003_22b4ce14","tag":null},"timestamp":"2026-01-23T10:25:54.022209Z","type":"replay_point","v":1},"source":{"case_id":"agg_003","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_003_22b4ce14/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file From c5bcb45f93cee60faa7aa1d3b2739610e2a89dec Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 16:34:26 +0300 Subject: [PATCH 15/79] Precheck trace conflicts and rollback fixture moves --- examples/demo_qa/fixture_tools.py | 90 ++++++++++++++++++++++--------- 1 file changed, 65 insertions(+), 25 deletions(-) diff --git a/examples/demo_qa/fixture_tools.py b/examples/demo_qa/fixture_tools.py index f7d5e73a..1bf7c90a 100644 --- a/examples/demo_qa/fixture_tools.py +++ b/examples/demo_qa/fixture_tools.py @@ -75,6 +75,29 @@ def _move_file(src: Path, dst: Path, *, use_git: bool, dry_run: bool) -> None: shutil.move(str(src), str(dst)) +def _rollback_moves(moves_done: list[tuple[Path, Path]], *, use_git: bool) -> None: + """ + Best-effort rollback of already executed moves. + moves_done: list of (src, dst) that were successfully moved src -> dst. + Rollback tries to move dst -> src in reverse order. + """ + if not moves_done: + return + print("Rolling back already moved files...", file=sys.stderr) + for src, dst in reversed(moves_done): + try: + if not dst.exists(): + print(f"ROLLBACK: skip (missing) {dst}", file=sys.stderr) + continue + if src.exists(): + print(f"ROLLBACK: skip (src exists) {src} <- {dst}", file=sys.stderr) + continue + _move_file(dst, src, use_git=use_git, dry_run=False) + print(f"ROLLBACK: {dst} -> {src}", file=sys.stderr) + except Exception as exc: + print(f"ROLLBACK ERROR: failed {dst} -> {src}: {exc}", file=sys.stderr) + + def _matches_filters(path: Path, *, name: Optional[str], pattern: Optional[str]) -> bool: if name and path.name != name and path.stem != name: return False @@ -261,45 +284,62 @@ def cmd_fix(args: argparse.Namespace) -> int: dst = REPLAY_ROOT / "fixed" / src.name print(f"- {_relative(src)} -> {_relative(dst)}") - if dry_run: - print(f"DRY: would promote {len(candidates)} replay fixtures.") - if move_traces: - print("DRY: plan traces would also be promoted if found.") - return 0 - case_ids: set[str] = set() + promoted_traces: list[Path] = [] + trace_dests: list[tuple[Path, Path]] = [] if move_traces: for src in candidates: resolved_case_id = case_id or _load_case_id(src) if resolved_case_id: case_ids.add(resolved_case_id) - for src, dst in dests: - _move_file(src, dst, use_git=use_git, dry_run=False) - - print(f"Promoted {len(candidates)} replay fixtures to fixed.") - - if move_traces: - promoted_traces: list[Path] = [] for cid in sorted(case_ids): promoted_traces.extend(_collect_traces_for_case(cid)) - if not promoted_traces: - print("No plan traces found to promote.") - return 0 - for trace in promoted_traces: - dst = TRACE_ROOT / "fixed" / trace.name - if dst.exists(): - print(f"Конфликт trace в fixed: {_relative(dst)}", file=sys.stderr) - return 1 + trace_dests.append((trace, TRACE_ROOT / "fixed" / trace.name)) - for trace in promoted_traces: - _move_file(trace, TRACE_ROOT / "fixed" / trace.name, use_git=use_git, dry_run=False) + trace_conflicts = [dst for _, dst in trace_dests if dst.exists()] + if trace_conflicts: + print("Конфликт trace в fixed:", file=sys.stderr) + for conflict in trace_conflicts: + print(f"- {_relative(conflict)}", file=sys.stderr) + return 1 - print(f"Also promoted {len(promoted_traces)} plan traces.") + if dry_run: + print(f"DRY: would promote {len(candidates)} replay fixtures.") + if move_traces: + if not promoted_traces: + print("DRY: no plan traces found to promote.") + else: + print(f"DRY: would also promote {len(promoted_traces)} plan traces:") + for src, dst in trace_dests: + print(f"- {_relative(src)} -> {_relative(dst)}") + return 0 - return 0 + moves_done: list[tuple[Path, Path]] = [] + try: + for src, dst in dests: + _move_file(src, dst, use_git=use_git, dry_run=False) + moves_done.append((src, dst)) + + print(f"Promoted {len(candidates)} replay fixtures to fixed.") + + if move_traces: + if not promoted_traces: + print("No plan traces found to promote.") + return 0 + for src, dst in trace_dests: + _move_file(src, dst, use_git=use_git, dry_run=False) + moves_done.append((src, dst)) + print(f"Also promoted {len(promoted_traces)} plan traces.") + + return 0 + except Exception as exc: + print("ERROR: fix failed during move. State may be partial; attempting rollback.", file=sys.stderr) + print(f"Cause: {exc}", file=sys.stderr) + _rollback_moves(moves_done, use_git=use_git) + return 1 def build_parser() -> argparse.ArgumentParser: From 6b4cef087750d7b20310433b980ebbcf37499ce8 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 17:13:41 +0300 Subject: [PATCH 16/79] Improve fixture tool discovery --- examples/demo_qa/fixture_tools.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/examples/demo_qa/fixture_tools.py b/examples/demo_qa/fixture_tools.py index 1bf7c90a..d04ca514 100644 --- a/examples/demo_qa/fixture_tools.py +++ b/examples/demo_qa/fixture_tools.py @@ -121,7 +121,7 @@ def _iter_trace_paths(bucket: Optional[str]) -> Iterable[Path]: root = TRACE_ROOT / bkt if not root.exists(): continue - yield from root.glob("*_plan_trace.txt") + yield from root.rglob("*_plan_trace.txt") def _relative(path: Path) -> str: @@ -147,7 +147,12 @@ def _load_case_id(path: Path) -> Optional[str]: root = data.get("root") or {} else: root = data - case_id = root.get("case_id") + case_id = ( + root.get("case_id") + or (root.get("meta") or {}).get("case_id") + or (data.get("meta") or {}).get("case_id") + or (data.get("input") or {}).get("case_id") + ) return case_id if isinstance(case_id, str) else None @@ -215,7 +220,8 @@ def cmd_rm(args: argparse.Namespace) -> int: use_git = _is_git_repo() print("Found files to remove:") for path in candidates: - print(f"- {_bucket_from_path(path, REPLAY_ROOT if 'replay_points' in str(path) else TRACE_ROOT)}: {_relative(path)}") + root = REPLAY_ROOT if path.is_relative_to(REPLAY_ROOT) else TRACE_ROOT + print(f"- {_bucket_from_path(path, root)}: {_relative(path)}") if dry_run: print(f"DRY: would remove {len(candidates)} files.") From 456618de7efa254901322ee9b37738e7d592bd05 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 17:33:25 +0300 Subject: [PATCH 17/79] Use git mv for tracked rollbacks --- examples/demo_qa/fixture_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/demo_qa/fixture_tools.py b/examples/demo_qa/fixture_tools.py index d04ca514..847301e6 100644 --- a/examples/demo_qa/fixture_tools.py +++ b/examples/demo_qa/fixture_tools.py @@ -66,7 +66,7 @@ def _move_file(src: Path, dst: Path, *, use_git: bool, dry_run: bool) -> None: print(f"DRY: move {src} -> {dst}") return dst.parent.mkdir(parents=True, exist_ok=True) - if use_git and _git_tracked(src): + if use_git and (_git_tracked(src) or _git_tracked(dst)): try: subprocess.run(["git", "mv", _git_path(src), _git_path(dst)], cwd=REPO_ROOT, check=True) return From 921de378e900c7d1127d262de1853dfe414cc2e6 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 17:43:12 +0300 Subject: [PATCH 18/79] Allow unbucketed replay fixtures --- tests/test_replay_fixtures.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index cb2b4874..aacc386c 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -25,6 +25,8 @@ def _iter_fixture_paths() -> Iterable[tuple[str, Path]]: if not FIXTURES_ROOT.exists(): return [] paths: list[tuple[str, Path]] = [] + for path in FIXTURES_ROOT.glob("*.json"): + paths.append(("root", path)) for bucket in _BUCKETS: bucket_dir = FIXTURES_ROOT / bucket if not bucket_dir.exists(): From fd18fb475faab10a3e4de1f80e1e67d7149932ca Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 18:14:26 +0300 Subject: [PATCH 19/79] Report promoted fixture paths --- examples/demo_qa/fixture_tools.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/demo_qa/fixture_tools.py b/examples/demo_qa/fixture_tools.py index 847301e6..75d588b7 100644 --- a/examples/demo_qa/fixture_tools.py +++ b/examples/demo_qa/fixture_tools.py @@ -273,10 +273,15 @@ def cmd_fix(args: argparse.Namespace) -> int: dests = [] conflicts = [] + dst_seen: dict[Path, Path] = {} for src in candidates: - dst = REPLAY_ROOT / "fixed" / src.name + rel_path = src.relative_to(REPLAY_ROOT / "known_bad") + dst = REPLAY_ROOT / "fixed" / rel_path + if dst in dst_seen and dst_seen[dst] != src: + conflicts.append(dst) if dst.exists(): conflicts.append(dst) + dst_seen[dst] = src dests.append((src, dst)) if conflicts: print("Конфликт имён в fixed:", file=sys.stderr) @@ -286,8 +291,7 @@ def cmd_fix(args: argparse.Namespace) -> int: use_git = _is_git_repo() print("Found replay fixtures to promote:") - for src in candidates: - dst = REPLAY_ROOT / "fixed" / src.name + for src, dst in dests: print(f"- {_relative(src)} -> {_relative(dst)}") case_ids: set[str] = set() From d50d1104118c60c8301981e62ae2cf8840218e88 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 18:33:08 +0300 Subject: [PATCH 20/79] Fix rerun hint for root fixtures --- tests/test_replay_fixtures.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index aacc386c..5f2ece39 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -79,17 +79,17 @@ def _fixture_paths() -> list[ParameterSet]: params: list[ParameterSet] = [] for bucket, path in paths: marks = (pytest.mark.known_bad,) if bucket == "known_bad" else () - params.append(pytest.param(path, id=f"{bucket}/{path.name}", marks=marks)) + params.append(pytest.param((bucket, path), id=f"{bucket}/{path.name}", marks=marks)) return params -def _rerun_hint(path: Path) -> str: - bucket = path.relative_to(FIXTURES_ROOT).parts[0] +def _rerun_hint(bucket: str, path: Path) -> str: return f"pytest -vv {__file__}::test_replay_fixture[{bucket}/{path.name}] -s" -@pytest.mark.parametrize("path", _fixture_paths()) -def test_replay_fixture(path: Path) -> None: +@pytest.mark.parametrize("fixture_info", _fixture_paths()) +def test_replay_fixture(fixture_info: tuple[str, Path]) -> None: + bucket, path = fixture_info raw = _load_fixture(path) event, ctx = _parse_fixture(raw) assert event.get("type") == "replay_point" @@ -118,7 +118,7 @@ def test_replay_fixture(path: Path) -> None: "\n".join( [ f"Replay mismatch for {path.name}", - f"rerun: {_rerun_hint(path)}", + f"rerun: {_rerun_hint(bucket, path)}", f"meta: {meta}", f"note: {note}", "input:", From a6da9e3b91e6caab2a358c1de4074a10fc96984f Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Fri, 23 Jan 2026 18:43:07 +0300 Subject: [PATCH 21/79] Drop zip plan-trace loading --- .../test_relational_normalizer_regression.py | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/tests/test_relational_normalizer_regression.py b/tests/test_relational_normalizer_regression.py index ad99cc2f..1a961adc 100644 --- a/tests/test_relational_normalizer_regression.py +++ b/tests/test_relational_normalizer_regression.py @@ -3,7 +3,6 @@ import copy import json import re -import zipfile from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Set @@ -62,8 +61,6 @@ def _load_trace_cases_from_fixtures() -> List[TraceCase]: Ищет по бакетам: - fixtures/plan_traces/fixed/*.txt - fixtures/plan_traces/known_bad/*.txt - - fixtures/plan_traces/{bucket}/fetchgraph_plans.zip (опционально) - Достаёт спецификации из stage=before_normalize (plan.context_plan[*]). """ fixtures_root = _fixtures_root() @@ -78,26 +75,14 @@ def _load_trace_cases_from_fixtures() -> List[TraceCase]: buckets = ["fixed", "known_bad"] for bucket in buckets: bucket_dir = fixtures_root / bucket - txt_paths = list(sorted(bucket_dir.glob("*_plan_trace.txt"))) - if txt_paths: - for p in txt_paths: - text = p.read_text(encoding="utf-8", errors="replace") - cases.extend(_extract_before_specs(trace_name=p.name, text=text, bucket=bucket)) - continue - - zip_path = bucket_dir / "fetchgraph_plans.zip" - if zip_path.exists(): - with zipfile.ZipFile(zip_path) as zf: - for name in sorted(zf.namelist()): - if not name.endswith("_plan_trace.txt"): - continue - text = zf.read(name).decode("utf-8", errors="replace") - cases.extend(_extract_before_specs(trace_name=name, text=text, bucket=bucket)) + for p in sorted(bucket_dir.glob("*_plan_trace.txt")): + text = p.read_text(encoding="utf-8", errors="replace") + cases.extend(_extract_before_specs(trace_name=p.name, text=text, bucket=bucket)) if not cases: pytest.skip( "No plan fixtures found. Put *_plan_trace.txt into " - "tests/fixtures/plan_traces/{fixed,known_bad} or add fetchgraph_plans.zip there.", + "tests/fixtures/plan_traces/{fixed,known_bad}.", allow_module_level=True, ) return cases From 4f3e6ae1921d94a95ac440e97d75038ab7911489 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 23 Jan 2026 19:16:27 +0300 Subject: [PATCH 22/79] ruff fixes --- src/fetchgraph/core/context.py | 2 +- src/fetchgraph/relational/normalize.py | 2 +- src/fetchgraph/replay/handlers/plan_normalize.py | 6 +++++- tests/test_replay_fixtures.py | 1 - 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/fetchgraph/core/context.py b/src/fetchgraph/core/context.py index 0c3e8589..265978e9 100644 --- a/src/fetchgraph/core/context.py +++ b/src/fetchgraph/core/context.py @@ -7,6 +7,7 @@ from ..parsing.plan_parser import PlanParser from ..planning.normalize import PlanNormalizer +from ..replay.log import EventLoggerLike from .models import ( BaselineSpec, ContextFetchSpec, @@ -26,7 +27,6 @@ SupportsFilter, Verifier, ) -from ..replay.log import EventLoggerLike from .utils import load_pkg_text, render_prompt logger = logging.getLogger(__name__) diff --git a/src/fetchgraph/relational/normalize.py b/src/fetchgraph/relational/normalize.py index 6ef0a7d1..6d6e4d0b 100644 --- a/src/fetchgraph/relational/normalize.py +++ b/src/fetchgraph/relational/normalize.py @@ -1,8 +1,8 @@ from __future__ import annotations import re -from typing import Any, Dict, Optional from collections.abc import Callable, MutableMapping +from typing import Any, Dict, Optional from .types import SelectorsDict diff --git a/src/fetchgraph/replay/handlers/plan_normalize.py b/src/fetchgraph/replay/handlers/plan_normalize.py index 8f94b6f1..ecea6c5f 100644 --- a/src/fetchgraph/replay/handlers/plan_normalize.py +++ b/src/fetchgraph/replay/handlers/plan_normalize.py @@ -5,7 +5,11 @@ from pydantic import TypeAdapter from ...core.models import ContextFetchSpec, ProviderInfo -from ...planning.normalize import PlanNormalizer, PlanNormalizerOptions, SelectorNormalizationRule +from ...planning.normalize import ( + PlanNormalizer, + PlanNormalizerOptions, + SelectorNormalizationRule, +) from ...relational.models import RelationalRequest from ...relational.normalize import normalize_relational_selectors from ..runtime import REPLAY_HANDLERS, ReplayContext diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index 5f2ece39..276ff7b1 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -11,7 +11,6 @@ from pydantic import TypeAdapter import fetchgraph.replay.handlers.plan_normalize # noqa: F401 - from fetchgraph.relational.models import RelationalRequest from fetchgraph.replay.runtime import REPLAY_HANDLERS, ReplayContext From 46284fb16781911c4d0d55f82a3bfdff81ebec13 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 10:10:20 +0300 Subject: [PATCH 23/79] fixture_tools: restore JSON on migrate rollback, add resource handling and `WITH_RESOURCES` flag (#114) * Restore fixture JSON on migrate rollback * Warn on nested fixtures during migrate * Clean up untracked files after git rm * Fail on invalid replay fixture JSON --- Makefile | 14 +- examples/demo_qa/fixture_tools.py | 318 +++++++++++++++++- src/fetchgraph/replay/export.py | 59 +++- src/fetchgraph/replay/runtime.py | 8 + .../plan_normalize.spec_v1__027f241b.json | 2 +- .../sample_resource.json | 1 + tests/test_replay_fixtures.py | 97 +++++- tests/test_replay_runtime.py | 15 + 8 files changed, 485 insertions(+), 29 deletions(-) create mode 100644 tests/fixtures/replay_points/fixed/resources/plan_normalize.spec_v1__027f241b/sample_resource.json create mode 100644 tests/test_replay_runtime.py diff --git a/Makefile b/Makefile index ef3c82d8..9279ae8e 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,7 @@ PROVIDER ?= BUCKET ?= fixed OUT_DIR ?= tests/fixtures/replay_points/$(BUCKET) SCOPE ?= both +WITH_RESOURCES ?= 1 ALL ?= LIMIT ?= 50 CHANGES ?= 10 @@ -111,7 +112,7 @@ LIMIT_FLAG := $(if $(strip $(LIMIT)),--limit $(LIMIT),) batch batch-tag batch-failed batch-failed-from \ batch-missed batch-missed-from batch-failed-tag batch-missed-tag \ batch-fail-fast batch-max-fails \ - stats history-case report-tag report-tag-changes tags tag-rm case-run case-open fixture fixture-rm fixture-fix compare compare-tag + stats history-case report-tag report-tag-changes tags tag-rm case-run case-open fixture fixture-rm fixture-fix fixture-migrate compare compare-tag # ============================================================================== # help (на русском) @@ -159,8 +160,10 @@ help: @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" @echo " make fixture CASE=agg_01 [TAG=...] [RUN_ID=...] [REPLAY_ID=plan_normalize.spec_v1] [WITH=requires] [SPEC_IDX=0] [PROVIDER=relational] [BUCKET=fixed|known_bad] [OUT_DIR=tests/fixtures/replay_points/$$(BUCKET)] [ALL=1]" - @echo " make fixture-rm NAME=... [PATTERN=...] [BUCKET=fixed|known_bad] [SCOPE=replay|traces|both] [DRY=1]" - @echo " make fixture-fix NAME=... [PATTERN=...] [CASE=...] [MOVE_TRACES=1] [DRY=1]" + @echo " fixtures layout: replay_points//.json, resources: replay_points//resources//..." + @echo " make fixture-rm NAME=... [PATTERN=...] [BUCKET=fixed|known_bad] [SCOPE=replay|traces|both] [WITH_RESOURCES=1] [DRY=1] (удаляет fixture и resources)" + @echo " make fixture-fix NAME=... [PATTERN=...] [CASE=...] [MOVE_TRACES=1] [DRY=1] (переносит fixture и resources)" + @echo " make fixture-migrate [BUCKET=fixed|known_bad] [DRY=1] (миграция ресурсов в resources//)" @echo "" @echo "Уборка:" @echo " make tag-rm TAG=... [DRY=1] [PURGE_RUNS=1] [PRUNE_HISTORY=1] [PRUNE_CASE_HISTORY=1]" @@ -357,11 +360,14 @@ fixture: check # 12) Fixture tools fixture-rm: - @$(CLI_FIXT) rm --name "$(NAME)" --pattern "$(PATTERN)" --bucket "$(BUCKET)" --scope "$(SCOPE)" --dry "$(DRY)" + @$(CLI_FIXT) rm --name "$(NAME)" --pattern "$(PATTERN)" --bucket "$(BUCKET)" --scope "$(SCOPE)" --with-resources "$(WITH_RESOURCES)" --dry "$(DRY)" fixture-fix: @$(CLI_FIXT) fix --name "$(NAME)" --pattern "$(PATTERN)" --case "$(CASE)" --move-traces "$(MOVE_TRACES)" --dry "$(DRY)" +fixture-migrate: + @$(CLI_FIXT) migrate --bucket "$(BUCKET)" --dry "$(DRY)" + # compare (diff.md + junit) compare: check @test -n "$(strip $(BASE))" || (echo "Нужно задать BASE=.../results_prev.jsonl" && exit 1) diff --git a/examples/demo_qa/fixture_tools.py b/examples/demo_qa/fixture_tools.py index 75d588b7..f075b26e 100644 --- a/examples/demo_qa/fixture_tools.py +++ b/examples/demo_qa/fixture_tools.py @@ -48,6 +48,24 @@ def _git_tracked(path: Path) -> bool: return result.returncode == 0 +def _git_has_tracked(path: Path) -> bool: + result = subprocess.run( + ["git", "ls-files", "-z", "--", _git_path(path)], + cwd=REPO_ROOT, + capture_output=True, + check=False, + ) + return result.returncode == 0 and bool(result.stdout) + + +def _is_relative_to(path: Path, base: Path) -> bool: + try: + path.relative_to(base) + return True + except ValueError: + return False + + def _remove_file(path: Path, *, use_git: bool, dry_run: bool) -> None: if dry_run: print(f"DRY: remove {path}") @@ -61,6 +79,30 @@ def _remove_file(path: Path, *, use_git: bool, dry_run: bool) -> None: path.unlink(missing_ok=True) +def _remove_tree(path: Path, *, use_git: bool, dry_run: bool) -> None: + if not path.exists(): + return + if dry_run: + if use_git and _git_has_tracked(path): + print(f"DRY: git rm -r -f {_git_path(path)}") + print(f"DRY: rmtree (cleanup untracked) {path}") + else: + print(f"DRY: remove tree {path}") + return + + # If directory contains tracked files, remove them via git rm first. + if use_git and _git_has_tracked(path): + try: + subprocess.run(["git", "rm", "-r", "-f", _git_path(path)], cwd=REPO_ROOT, check=True) + except subprocess.CalledProcessError: + # Fall back to filesystem removal below. + pass + + # git rm deletes only tracked files; remove any remaining untracked files/dirs. + if path.exists(): + shutil.rmtree(path, ignore_errors=True) + + def _move_file(src: Path, dst: Path, *, use_git: bool, dry_run: bool) -> None: if dry_run: print(f"DRY: move {src} -> {dst}") @@ -75,16 +117,30 @@ def _move_file(src: Path, dst: Path, *, use_git: bool, dry_run: bool) -> None: shutil.move(str(src), str(dst)) -def _rollback_moves(moves_done: list[tuple[Path, Path]], *, use_git: bool) -> None: +def _move_tree(src: Path, dst: Path, *, use_git: bool, dry_run: bool) -> None: + if dry_run: + print(f"DRY: move tree {src} -> {dst}") + return + dst.parent.mkdir(parents=True, exist_ok=True) + if use_git and (_git_has_tracked(src) or _git_has_tracked(dst)): + try: + subprocess.run(["git", "mv", _git_path(src), _git_path(dst)], cwd=REPO_ROOT, check=True) + return + except subprocess.CalledProcessError: + pass + shutil.move(str(src), str(dst)) + + +def _rollback_moves(moves_done: list[tuple[Path, Path, bool]], *, use_git: bool) -> None: """ Best-effort rollback of already executed moves. - moves_done: list of (src, dst) that were successfully moved src -> dst. + moves_done: list of (src, dst, is_dir) that were successfully moved src -> dst. Rollback tries to move dst -> src in reverse order. """ if not moves_done: return print("Rolling back already moved files...", file=sys.stderr) - for src, dst in reversed(moves_done): + for src, dst, is_dir in reversed(moves_done): try: if not dst.exists(): print(f"ROLLBACK: skip (missing) {dst}", file=sys.stderr) @@ -92,7 +148,10 @@ def _rollback_moves(moves_done: list[tuple[Path, Path]], *, use_git: bool) -> No if src.exists(): print(f"ROLLBACK: skip (src exists) {src} <- {dst}", file=sys.stderr) continue - _move_file(dst, src, use_git=use_git, dry_run=False) + if is_dir: + _move_tree(dst, src, use_git=use_git, dry_run=False) + else: + _move_file(dst, src, use_git=use_git, dry_run=False) print(f"ROLLBACK: {dst} -> {src}", file=sys.stderr) except Exception as exc: print(f"ROLLBACK ERROR: failed {dst} -> {src}: {exc}", file=sys.stderr) @@ -112,7 +171,10 @@ def _iter_replay_paths(bucket: Optional[str]) -> Iterable[Path]: root = REPLAY_ROOT / bkt if not root.exists(): continue - yield from root.rglob("*.json") + for path in root.rglob("*.json"): + if "resources" in path.parts: + continue + yield path def _iter_trace_paths(bucket: Optional[str]) -> Iterable[Path]: @@ -131,6 +193,54 @@ def _relative(path: Path) -> str: return str(path) +def _resource_key_for_fixture(fixture_path: Path) -> str: + if _is_relative_to(fixture_path, REPLAY_ROOT): + rel = fixture_path.relative_to(REPLAY_ROOT) + if len(rel.parts) >= 2: + bucket = rel.parts[0] + fixture_rel = fixture_path.relative_to(REPLAY_ROOT / bucket).with_suffix("") + return "__".join(fixture_rel.parts) + return fixture_path.stem + + +def _resources_dir_for_fixture(fixture_path: Path) -> Path: + if _is_relative_to(fixture_path, REPLAY_ROOT): + rel = fixture_path.relative_to(REPLAY_ROOT) + if len(rel.parts) >= 2: + bucket = rel.parts[0] + return REPLAY_ROOT / bucket / "resources" / _resource_key_for_fixture(fixture_path) + return fixture_path.parent / "resources" / _resource_key_for_fixture(fixture_path) + + +def _canonical_json(payload: object) -> str: + return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _resource_destination( + fixture_path: Path, + file_name: str, + resource_key: str, + used_paths: set[Path], +) -> Path: + rel_path = Path(file_name) + if rel_path.is_absolute() or ".." in rel_path.parts: + raise ValueError(f"Invalid resource path: {file_name}") + base_dir = Path("resources") / _resource_key_for_fixture(fixture_path) + if rel_path.parent != Path("."): + dest_rel = base_dir / rel_path + else: + dest_rel = base_dir / rel_path.name + if dest_rel in used_paths: + prefix = resource_key or "resource" + dest_rel = base_dir / f"{prefix}__{rel_path.name}" + suffix = 1 + while dest_rel in used_paths: + dest_rel = base_dir / f"{prefix}_{suffix}__{rel_path.name}" + suffix += 1 + used_paths.add(dest_rel) + return dest_rel + + def _bucket_from_path(path: Path, root: Path) -> str: try: return path.relative_to(root).parts[0] @@ -198,6 +308,7 @@ def cmd_rm(args: argparse.Namespace) -> int: bucket = _normalize(args.bucket) scope = _normalize(args.scope) or "both" dry_run = bool(args.dry) + with_resources = bool(args.with_resources) if bucket and bucket not in BUCKETS: print(f"Неизвестный BUCKET: {bucket}", file=sys.stderr) @@ -220,17 +331,34 @@ def cmd_rm(args: argparse.Namespace) -> int: use_git = _is_git_repo() print("Found files to remove:") for path in candidates: - root = REPLAY_ROOT if path.is_relative_to(REPLAY_ROOT) else TRACE_ROOT + root = REPLAY_ROOT if _is_relative_to(path, REPLAY_ROOT) else TRACE_ROOT print(f"- {_bucket_from_path(path, root)}: {_relative(path)}") + if with_resources and _is_relative_to(path, REPLAY_ROOT): + resources_dir = _resources_dir_for_fixture(path) + if resources_dir.exists(): + print(f" - resources: {_relative(resources_dir)}") if dry_run: print(f"DRY: would remove {len(candidates)} files.") return 0 + resource_dirs: list[Path] = [] + if with_resources and scope in ("replay", "both"): + for path in candidates: + if _is_relative_to(path, REPLAY_ROOT): + resources_dir = _resources_dir_for_fixture(path) + if resources_dir.exists(): + resource_dirs.append(resources_dir) + + for resource_dir in sorted(set(resource_dirs), key=lambda p: _relative(p)): + _remove_tree(resource_dir, use_git=use_git, dry_run=False) for path in candidates: _remove_file(path, use_git=use_git, dry_run=False) - print(f"Removed {len(candidates)} files.") + if resource_dirs: + print(f"Removed {len(candidates)} files and {len(set(resource_dirs))} resource trees.") + else: + print(f"Removed {len(candidates)} files.") return 0 @@ -272,6 +400,7 @@ def cmd_fix(args: argparse.Namespace) -> int: return 1 dests = [] + resource_moves: list[tuple[Path, Path]] = [] conflicts = [] dst_seen: dict[Path, Path] = {} for src in candidates: @@ -283,6 +412,12 @@ def cmd_fix(args: argparse.Namespace) -> int: conflicts.append(dst) dst_seen[dst] = src dests.append((src, dst)) + src_resources = _resources_dir_for_fixture(src) + if src_resources.exists(): + dst_resources = _resources_dir_for_fixture(dst) + if dst_resources.exists(): + conflicts.append(dst_resources) + resource_moves.append((src_resources, dst_resources)) if conflicts: print("Конфликт имён в fixed:", file=sys.stderr) for conflict in conflicts: @@ -293,6 +428,8 @@ def cmd_fix(args: argparse.Namespace) -> int: print("Found replay fixtures to promote:") for src, dst in dests: print(f"- {_relative(src)} -> {_relative(dst)}") + for src, dst in resource_moves: + print(f" - resources: {_relative(src)} -> {_relative(dst)}") case_ids: set[str] = set() promoted_traces: list[Path] = [] @@ -327,11 +464,14 @@ def cmd_fix(args: argparse.Namespace) -> int: print(f"- {_relative(src)} -> {_relative(dst)}") return 0 - moves_done: list[tuple[Path, Path]] = [] + moves_done: list[tuple[Path, Path, bool]] = [] try: for src, dst in dests: _move_file(src, dst, use_git=use_git, dry_run=False) - moves_done.append((src, dst)) + moves_done.append((src, dst, False)) + for src, dst in resource_moves: + _move_tree(src, dst, use_git=use_git, dry_run=False) + moves_done.append((src, dst, True)) print(f"Promoted {len(candidates)} replay fixtures to fixed.") @@ -341,7 +481,7 @@ def cmd_fix(args: argparse.Namespace) -> int: return 0 for src, dst in trace_dests: _move_file(src, dst, use_git=use_git, dry_run=False) - moves_done.append((src, dst)) + moves_done.append((src, dst, False)) print(f"Also promoted {len(promoted_traces)} plan traces.") return 0 @@ -352,6 +492,158 @@ def cmd_fix(args: argparse.Namespace) -> int: return 1 +def cmd_migrate(args: argparse.Namespace) -> int: + bucket = _normalize(args.bucket) + dry_run = bool(args.dry) + if bucket and bucket not in BUCKETS: + print(f"Неизвестный BUCKET: {bucket}", file=sys.stderr) + return 1 + + buckets = [bucket] if bucket else BUCKETS + use_git = _is_git_repo() + total_moves = 0 + total_updates = 0 + + for bkt in buckets: + root = REPLAY_ROOT / bkt + if not root.exists(): + continue + fixture_candidates = [path for path in root.rglob("*.json") if "resources" not in path.parts] + for fixture_path in sorted(fixture_candidates, key=lambda p: _relative(p)): + if fixture_path.parent != root: + print( + "Skip nested fixture path; expected fixtures at bucket root: " + f"{_relative(fixture_path)}", + file=sys.stderr, + ) + continue + original_text: str | None = None + try: + original_text = fixture_path.read_text(encoding="utf-8") + data = json.loads(original_text) + except (json.JSONDecodeError, OSError) as exc: + print(f"Skip invalid json {_relative(fixture_path)}: {exc}", file=sys.stderr) + continue + if data.get("type") != "replay_bundle": + continue + resources = data.get("resources") + if not isinstance(resources, dict): + continue + used_paths: set[Path] = set() + moves: list[tuple[Path, Path]] = [] + src_to_dest_rel: dict[Path, Path] = {} + changes = False + conflicts: list[Path] = [] + for rid, resource in resources.items(): + if not isinstance(resource, dict): + continue + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + continue + rel_path = Path(file_name) + if rel_path.is_absolute() or ".." in rel_path.parts: + print( + f"Skip invalid resource path {_relative(fixture_path)}: {file_name}", + file=sys.stderr, + ) + continue + if ( + len(rel_path.parts) >= 2 + and rel_path.parts[0] == "resources" + and rel_path.parts[1] == _resource_key_for_fixture(fixture_path) + ): + used_paths.add(rel_path) + continue + src_path = fixture_path.parent / file_name + if not src_path.exists(): + print( + f"Missing resource for {_relative(fixture_path)}: {_relative(src_path)}", + file=sys.stderr, + ) + continue + if src_path in src_to_dest_rel: + existing_rel = src_to_dest_rel[src_path] + updated_resource = dict(resource) + updated_data_ref = dict(data_ref) + updated_data_ref["file"] = existing_rel.as_posix() + updated_resource["data_ref"] = updated_data_ref + resources[rid] = updated_resource + changes = True + continue + + try: + dest_rel = _resource_destination(fixture_path, file_name, rid, used_paths) + except ValueError as exc: + print(f"Skip resource in {_relative(fixture_path)}: {exc}", file=sys.stderr) + continue + + dst_path = fixture_path.parent / dest_rel + if dst_path.exists(): + conflicts.append(dst_path) + continue + + # De-duplicate moves so shared resources are only relocated once. + moves.append((src_path, dst_path)) + src_to_dest_rel[src_path] = dest_rel + updated_resource = dict(resource) + updated_data_ref = dict(data_ref) + updated_data_ref["file"] = dest_rel.as_posix() + updated_resource["data_ref"] = updated_data_ref + resources[rid] = updated_resource + changes = True + + if conflicts: + print(f"Conflicts for {_relative(fixture_path)}:", file=sys.stderr) + for conflict in conflicts: + print(f"- {_relative(conflict)}", file=sys.stderr) + continue + + if not moves and not changes: + continue + + print(f"Migrate resources for {_relative(fixture_path)}:") + for src, dst in moves: + print(f"- {_relative(src)} -> {_relative(dst)}") + + if dry_run: + continue + + moves_done: list[tuple[Path, Path, bool]] = [] + try: + for src, dst in moves: + _move_file(src, dst, use_git=use_git, dry_run=False) + moves_done.append((src, dst, False)) + + if changes: + data["resources"] = resources + fixture_path.write_text(_canonical_json(data), encoding="utf-8") + total_updates += 1 + total_moves += len(moves) + except Exception as exc: + print( + f"ERROR: migrate failed for {_relative(fixture_path)}: {exc}", + file=sys.stderr, + ) + _rollback_moves(moves_done, use_git=use_git) + if original_text is not None: + try: + fixture_path.write_text(original_text, encoding="utf-8") + except Exception as rollback_exc: + print( + f"ROLLBACK ERROR: failed to restore {_relative(fixture_path)}: {rollback_exc}", + file=sys.stderr, + ) + + if dry_run: + print("DRY: migration scan complete.") + else: + print(f"Migrated {total_moves} resources across {total_updates} fixtures.") + return 0 + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Tools for managing test fixtures.") sub = parser.add_subparsers(dest="command", required=True) @@ -361,6 +653,7 @@ def build_parser() -> argparse.ArgumentParser: rm_parser.add_argument("--pattern", default="") rm_parser.add_argument("--bucket", default="") rm_parser.add_argument("--scope", default="both") + rm_parser.add_argument("--with-resources", type=int, default=1) rm_parser.add_argument("--dry", type=int, default=0) rm_parser.set_defaults(func=cmd_rm) @@ -372,6 +665,11 @@ def build_parser() -> argparse.ArgumentParser: fix_parser.add_argument("--dry", type=int, default=0) fix_parser.set_defaults(func=cmd_fix) + migrate_parser = sub.add_parser("migrate", help="Migrate replay bundle resources into resources//") + migrate_parser.add_argument("--bucket", default="") + migrate_parser.add_argument("--dry", type=int, default=0) + migrate_parser.set_defaults(func=cmd_migrate) + return parser diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py index f22b435c..57c37fed 100644 --- a/src/fetchgraph/replay/export.py +++ b/src/fetchgraph/replay/export.py @@ -29,18 +29,59 @@ def fixture_name(event_id: str, input_payload: dict) -> str: return f"{event_id}__{digest[:8]}.json" -def _copy_resource_file(run_dir: Path, fixture_root: Path, root_name: str, data_ref: dict) -> dict: +def _resource_target_path( + *, + fixture_stem: str, + file_name: str, + resource_key: str, + used_paths: set[Path], +) -> Path: + rel_path = Path(file_name) + if rel_path.is_absolute(): + raise SystemExit(f"Resource file path must be relative: {file_name}") + if ".." in rel_path.parts: + raise SystemExit(f"Resource file path must not traverse parents: {file_name}") + base_dir = Path("resources") / fixture_stem + if rel_path.parent != Path("."): + dest_rel = base_dir / rel_path + else: + dest_rel = base_dir / rel_path.name + if dest_rel in used_paths: + prefix = resource_key or "resource" + dest_rel = base_dir / f"{prefix}__{rel_path.name}" + suffix = 1 + while dest_rel in used_paths: + dest_rel = base_dir / f"{prefix}_{suffix}__{rel_path.name}" + suffix += 1 + used_paths.add(dest_rel) + return dest_rel + + +def _copy_resource_file( + run_dir: Path, + out_dir: Path, + fixture_stem: str, + resource_key: str, + used_paths: set[Path], + data_ref: dict, +) -> dict: file_name = data_ref.get("file") if not file_name: return data_ref src_path = run_dir / file_name if not src_path.exists(): raise SystemExit(f"Resource file {src_path} not found for replay bundle") - target_name = f"{root_name}__{Path(file_name).name}" - dest_path = fixture_root / target_name + dest_rel = _resource_target_path( + fixture_stem=fixture_stem, + file_name=file_name, + resource_key=resource_key, + used_paths=used_paths, + ) + dest_path = out_dir / dest_rel + dest_path.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(src_path, dest_path) updated = dict(data_ref) - updated["file"] = target_name + updated["file"] = dest_rel.as_posix() return updated @@ -143,10 +184,18 @@ def write_bundle( root_name = Path(fixture_file).stem updated_resources: Dict[str, dict] = {} + used_paths: set[Path] = set() for rid, resource in resources.items(): rp = dict(resource) if "data_ref" in rp and isinstance(rp["data_ref"], dict): - rp["data_ref"] = _copy_resource_file(run_dir, out_dir, root_name, rp["data_ref"]) + rp["data_ref"] = _copy_resource_file( + run_dir, + out_dir, + root_name, + rid, + used_paths, + rp["data_ref"], + ) updated_resources[rid] = rp payload = { diff --git a/src/fetchgraph/replay/runtime.py b/src/fetchgraph/replay/runtime.py index c1c1d217..f13e13c9 100644 --- a/src/fetchgraph/replay/runtime.py +++ b/src/fetchgraph/replay/runtime.py @@ -1,6 +1,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from pathlib import Path from typing import Callable, Dict @@ -8,6 +9,13 @@ class ReplayContext: resources: Dict[str, dict] = field(default_factory=dict) extras: Dict[str, dict] = field(default_factory=dict) + base_dir: Path | None = None + + def resolve_resource_path(self, resource_path: str | Path) -> Path: + path = Path(resource_path) + if path.is_absolute() or self.base_dir is None: + return path + return self.base_dir / path REPLAY_HANDLERS: Dict[str, Callable[[dict, ReplayContext], dict]] = {} diff --git a/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__027f241b.json b/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__027f241b.json index 43add217..6606e172 100644 --- a/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__027f241b.json +++ b/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__027f241b.json @@ -1 +1 @@ -{"extras":{"planner_input_v1":{"case_id":"agg_036","id":"planner_input_v1","input":{"feature_name":"agg_036","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Сколько заказов было в 2022-07? Ответь только числом."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:54.791922Z","type":"planner_input","v":1}},"resources":{},"root":{"case_id":"agg_036","diag":{"selectors_valid_after":true,"selectors_valid_before":true},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"filters":{"clauses":[{"entity":"orders","field":"order_date","op":">=","type":"comparison","value":"2022-07-01"},{"entity":"orders","field":"order_date","op":"<","type":"comparison","value":"2022-08-01"}],"op":"and","type":"logical"},"op":"query","root_entity":"orders","select":[{"alias":"order_count","expr":"count(*)"}]}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"filters":{"clauses":[{"entity":"orders","field":"order_date","op":">=","type":"comparison","value":"2022-07-01"},{"entity":"orders","field":"order_date","op":"<","type":"comparison","value":"2022-08-01"}],"op":"and","type":"logical"},"op":"query","root_entity":"orders","select":[{"alias":"order_count","expr":"count(*)"}]}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}","requires":["planner_input_v1"],"run_id":"9df66f32","source":{"case_id":"agg_036","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_036_c2239c95/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_036_c2239c95","tag":null},"timestamp":"2026-01-23T10:25:54.802564Z","type":"replay_point","v":1},"source":{"case_id":"agg_036","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_036_c2239c95/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file +{"extras":{"planner_input_v1":{"case_id":"agg_036","id":"planner_input_v1","input":{"feature_name":"agg_036","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Сколько заказов было в 2022-07? Ответь только числом."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:54.791922Z","type":"planner_input","v":1}},"resources":{"resource_file_demo":{"data_ref":{"file":"resources/plan_normalize.spec_v1__027f241b/sample_resource.json"},"id":"resource_file_demo","kind":"file","type":"replay_resource"}},"root":{"case_id":"agg_036","diag":{"selectors_valid_after":true,"selectors_valid_before":true},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"filters":{"clauses":[{"entity":"orders","field":"order_date","op":">=","type":"comparison","value":"2022-07-01"},{"entity":"orders","field":"order_date","op":"<","type":"comparison","value":"2022-08-01"}],"op":"and","type":"logical"},"op":"query","root_entity":"orders","select":[{"alias":"order_count","expr":"count(*)"}]}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"filters":{"clauses":[{"entity":"orders","field":"order_date","op":">=","type":"comparison","value":"2022-07-01"},{"entity":"orders","field":"order_date","op":"<","type":"comparison","value":"2022-08-01"}],"op":"and","type":"logical"},"op":"query","root_entity":"orders","select":[{"alias":"order_count","expr":"count(*)"}]}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}","requires":["planner_input_v1","resource_file_demo"],"run_id":"9df66f32","source":{"case_id":"agg_036","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_036_c2239c95/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_036_c2239c95","tag":null},"timestamp":"2026-01-23T10:25:54.802564Z","type":"replay_point","v":1},"source":{"case_id":"agg_036","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_036_c2239c95/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file diff --git a/tests/fixtures/replay_points/fixed/resources/plan_normalize.spec_v1__027f241b/sample_resource.json b/tests/fixtures/replay_points/fixed/resources/plan_normalize.spec_v1__027f241b/sample_resource.json new file mode 100644 index 00000000..15390e6e --- /dev/null +++ b/tests/fixtures/replay_points/fixed/resources/plan_normalize.spec_v1__027f241b/sample_resource.json @@ -0,0 +1 @@ +{"message": "demo resource"} \ No newline at end of file diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index 276ff7b1..c6ba7468 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -20,19 +20,51 @@ DEBUG_REPLAY = os.getenv("DEBUG_REPLAY", "").lower() in ("1", "true", "yes", "on") -def _iter_fixture_paths() -> Iterable[tuple[str, Path]]: +def _is_replay_payload(payload: object) -> bool: + if not isinstance(payload, dict): + return False + if payload.get("type") == "replay_point": + return True + if payload.get("type") == "replay_bundle": + root = payload.get("root") or {} + return isinstance(root, dict) and root.get("type") == "replay_point" + return False + + +def _iter_fixture_paths() -> tuple[list[tuple[str, Path]], list[Path], list[tuple[Path, str]]]: if not FIXTURES_ROOT.exists(): - return [] + return [], [], [] paths: list[tuple[str, Path]] = [] + ignored: list[Path] = [] + invalid_json: list[tuple[Path, str]] = [] for path in FIXTURES_ROOT.glob("*.json"): - paths.append(("root", path)) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + invalid_json.append((path, f"{exc.msg} (line {exc.lineno}, col {exc.colno})")) + continue + if _is_replay_payload(payload): + paths.append(("root", path)) + else: + ignored.append(path) for bucket in _BUCKETS: bucket_dir = FIXTURES_ROOT / bucket if not bucket_dir.exists(): continue for path in bucket_dir.rglob("*.json"): - paths.append((bucket, path)) - return sorted(paths) + if "resources" in path.parts: + ignored.append(path) + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + invalid_json.append((path, f"{exc.msg} (line {exc.lineno}, col {exc.colno})")) + continue + if _is_replay_payload(payload): + paths.append((bucket, path)) + else: + ignored.append(path) + return sorted(paths), sorted(ignored), sorted(invalid_json) def _format_json(payload: object) -> str: @@ -58,23 +90,30 @@ def _load_fixture(path: Path) -> dict: return json.loads(path.read_text(encoding="utf-8")) -def _parse_fixture(event: dict) -> tuple[dict, ReplayContext]: +def _parse_fixture(event: dict, *, base_dir: Path) -> tuple[dict, ReplayContext]: if event.get("type") == "replay_bundle": ctx = ReplayContext( resources=event.get("resources") or {}, extras=event.get("extras") or {}, + base_dir=base_dir, ) return event["root"], ctx - return event, ReplayContext() + return event, ReplayContext(base_dir=base_dir) def _fixture_paths() -> list[ParameterSet]: - paths = list(_iter_fixture_paths()) + paths, ignored, invalid_json = _iter_fixture_paths() if not paths: pytest.skip( "No replay fixtures found in tests/fixtures/replay_points/{fixed,known_bad}", allow_module_level=True, ) + if DEBUG_REPLAY and ignored: + ignored_list = "\n".join(f"- {path}" for path in ignored) + print(f"\n=== DEBUG ignored json files ===\n{ignored_list}") + if DEBUG_REPLAY and invalid_json: + invalid_list = "\n".join(f"- {path}: {error}" for path, error in invalid_json) + print(f"\n=== DEBUG invalid json files ===\n{invalid_list}") params: list[ParameterSet] = [] for bucket, path in paths: marks = (pytest.mark.known_bad,) if bucket == "known_bad" else () @@ -90,7 +129,7 @@ def _rerun_hint(bucket: str, path: Path) -> str: def test_replay_fixture(fixture_info: tuple[str, Path]) -> None: bucket, path = fixture_info raw = _load_fixture(path) - event, ctx = _parse_fixture(raw) + event, ctx = _parse_fixture(raw, base_dir=path.parent) assert event.get("type") == "replay_point" event_id = event.get("id") assert event_id in REPLAY_HANDLERS @@ -133,3 +172,43 @@ def test_replay_fixture(fixture_info: tuple[str, Path]) -> None: rule_kind = (event["input"].get("normalizer_rules") or {}).get(provider) if rule_kind == "relational_v1": _REL_ADAPTER.validate_python(actual_spec["selectors"]) + + +def test_replay_fixture_resources_exist() -> None: + paths, _, _ = _iter_fixture_paths() + resource_checks: list[tuple[Path, Path]] = [] + for _, path in paths: + raw = _load_fixture(path) + event, ctx = _parse_fixture(raw, base_dir=path.parent) + if raw.get("type") != "replay_bundle": + continue + resources = raw.get("resources") or {} + if not isinstance(resources, dict): + continue + for resource in resources.values(): + if not isinstance(resource, dict): + continue + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + continue + resolved = ctx.resolve_resource_path(file_name) + resource_checks.append((path, resolved)) + + if not resource_checks: + pytest.skip("No replay fixtures with file resources found.") + + missing = [(fixture, resource) for fixture, resource in resource_checks if not resource.exists()] + if missing: + details = "\n".join(f"- {fixture}: {resource}" for fixture, resource in missing) + pytest.fail(f"Missing replay resources:\n{details}") + + +def test_replay_fixture_json_valid() -> None: + _, _, invalid_json = _iter_fixture_paths() + if not invalid_json: + return + details = "\n".join(f"- {path}: {error}" for path, error in invalid_json) + pytest.fail(f"Invalid replay fixture JSON:\n{details}") diff --git a/tests/test_replay_runtime.py b/tests/test_replay_runtime.py new file mode 100644 index 00000000..ce03ba2a --- /dev/null +++ b/tests/test_replay_runtime.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from pathlib import Path + +from fetchgraph.replay.runtime import ReplayContext + + +def test_resolve_resource_path(tmp_path) -> None: + ctx = ReplayContext(base_dir=tmp_path) + + rel_path = Path("resources/sample.json") + assert ctx.resolve_resource_path(rel_path) == tmp_path / rel_path + + abs_path = Path("/var/data/sample.json") + assert ctx.resolve_resource_path(abs_path) == abs_path From d2cefbd66ca97d3bd2c43c5eda83dba6fff8972f Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 25 Jan 2026 10:50:22 +0300 Subject: [PATCH 24/79] =?UTF-8?q?=D0=B4=D0=BE=D0=BA=D1=83=D0=BC=D0=B5?= =?UTF-8?q?=D0=BD=D1=82=D0=B0=D1=86=D0=B8=D1=8F=20=D0=BF=D0=BE=20=D1=82?= =?UTF-8?q?=D1=80=D0=B5=D0=B9=D1=81=D0=B5=D1=80=D1=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/fetchgraph/replay/fetchgraph_tracer.md | 314 +++++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 src/fetchgraph/replay/fetchgraph_tracer.md diff --git a/src/fetchgraph/replay/fetchgraph_tracer.md b/src/fetchgraph/replay/fetchgraph_tracer.md new file mode 100644 index 00000000..ff56ef36 --- /dev/null +++ b/src/fetchgraph/replay/fetchgraph_tracer.md @@ -0,0 +1,314 @@ +# Fetchgraph Tracer: Event Log, Replay Points и фикстуры + +> Этот документ описывает “трейсер” (событийный лог + механизм воспроизведения/реплея) по коду из приложенных модулей: +> - `log.py` — контракт логгера событий и хелпер `log_replay_point` +> - `runtime.py` — `ReplayContext` и реестр `REPLAY_HANDLERS` +> - `snapshots.py` — снапшоты для `ProviderInfo` / каталога провайдеров +> - `plan_normalize.py` — пример replay-обработчика `plan_normalize.spec_v1` +> - `export.py` — экспорт replay-point’ов в фикстуры / бандлы (копирование ресурсов) + +--- + +## 1) Зачем это нужно + +Трейсер решает две связанные задачи: + +1) **Диагностика и воспроизводимость** + В рантайме Fetchgraph можно записывать “точки реплея” (replay points) — маленькие, детерминированные фрагменты вычислений: вход → ожидаемый выход (+ метаданные). + +2) **Автоматическая генерация регрессионных тестов (fixtures)** + Из потока событий (`events.jsonl`) можно автоматически выгрузить фикстуры и гонять “реплей” без LLM/внешних зависимостей: просто вызвать нужный обработчик из `REPLAY_HANDLERS` и сравнить результат с `expected`. + +--- + +## 2) Ключевые понятия + +### Event stream (JSONL) +События пишутся в файл (или другой sink) как **JSON Lines**: *одна JSON-строка = одно событие*. + +Пример общего вида (как минимум это делает `EventLogger` из demo runner’а): +```json +{"timestamp":"2026-01-25T00:00:00Z","run_id":"abcd1234","type":"...","...": "..."} +``` + +### Replay point +Событие типа `replay_point` описывает собтыие, которое можно воспроизвести: +- `id` — строковый идентификатор точки (обычно совпадает с ключом обработчика в `REPLAY_HANDLERS`) +- `meta` — фильтруемые метаданные (например `spec_idx`, `provider`) +- `input` — входные данные для реплея +- `expected` — ожидаемый результат +- `requires` *(опционально)* — список зависимостей (ресурсы/доп. события), необходимых для реплея + +Минимальный пример: +```json +{ + "type": "replay_point", + "v": 1, + "id": "plan_normalize.spec_v1", + "meta": {"spec_idx": 0, "provider": "sql"}, + "input": {"spec": {...}, "options": {...}}, + "expected": {"out_spec": {...}, "notes_last": "..."}, + "requires": ["planner_input_v1"] +} +``` + +### Replay resource / extras +`export.py` умеет подтягивать зависимости из event stream двух типов: +- `type="replay_resource"` — ресурсы (часто файлы), которые могут понадобиться при реплее +- `type="planner_input"` — дополнительные входы/контекст (“extras”), которые обработчик может использовать + +Важно: для `extras` ключом является `event["id"]` (например `"planner_input_v1"`), а обработчики потом читают это из `ctx.extras[...]`. + +--- + +## 3) Ответственность модулей и классов + +### `EventLoggerLike` (`log.py`) +Минимальный контракт “куда писать события”: + +```py +class EventLoggerLike(Protocol): + def emit(self, event: Dict[str, object]) -> None: ... +``` + +Идея: рантайм может писать события в “настоящий” `EventLog`/`EventLogger`, а тесты/утилиты — принимать любой sink, который реализует `emit`. + +### `log_replay_point` (`log.py`) +Унифицированный способ записать `replay_point`: + +- гарантирует базовую форму события (`type`, `v`, `id`, `meta`, `input`, `expected`) +- опционально добавляет: `requires`, `diag`, `note`, `error`, `extra` + +Рекомендуемый паттерн: **для всех реплейных точек использовать только этот хелпер**, чтобы формат не “расползался”. + +--- + +### `ReplayContext` и `REPLAY_HANDLERS` (`runtime.py`) + +#### `ReplayContext` +Контейнер для зависимостей и контекста реплея: +```py +@dataclass(frozen=True) +class ReplayContext: + resources: Dict[str, dict] = ... + extras: Dict[str, dict] = ... + base_dir: Path | None = None + + def resolve_resource_path(self, resource_path: str | Path) -> Path: + ... +``` + +- `resources`: “словарь ресурсов” по id (обычно события `replay_resource`) +- `extras`: “словарь доп. данных” по id (обычно события `planner_input`) +- `base_dir`: база для резолва относительных путей ресурсов (полезно для бандлов фикстур) + +#### `REPLAY_HANDLERS` +Глобальный реестр обработчиков: +```py +REPLAY_HANDLERS: Dict[str, Callable[[dict, ReplayContext], dict]] = {} +``` + +Ключ: `replay_point.id` +Значение: функция `handler(input_dict, ctx) -> output_dict` + +--- + +### `snapshots.py` +Снапшоты данных о провайдерах, чтобы реплей был более стабильным: + +- `snapshot_provider_info(info: ProviderInfo) -> Dict[str, object]` +- `snapshot_provider_catalog(provider_catalog: Mapping[str, object]) -> Dict[str, object]` + +Смысл: вместо того чтобы зависеть от “живых” объектов, сохраняем стабильный JSON-слепок. + +--- + +### Пример обработчика: `plan_normalize.spec_v1` (`plan_normalize.py`) +Функция: +```py +def replay_plan_normalize_spec_v1(inp: dict, ctx: ReplayContext) -> dict: ... +REPLAY_HANDLERS["plan_normalize.spec_v1"] = replay_plan_normalize_spec_v1 +``` + +Что делает (по коду): +1) Берёт `inp["spec"]` и `inp["options"]` +2) Строит `PlanNormalizerOptions` +3) Вытаскивает правила нормализации из `normalizer_rules` или `normalizer_registry` +4) Пытается восстановить `ProviderInfo` из `ctx.extras["planner_input_v1"]["input"]["provider_catalog"][provider]` + - если не получилось — создаёт `ProviderInfo(name=provider, capabilities=[])` +5) Собирает `PlanNormalizer` и нормализует один `ContextFetchSpec` +6) Возвращает: + - `out_spec` (provider/mode/selectors) + - `notes_last` (последняя заметка нормализатора) + +--- + +### Экспорт фикстур: `export.py` + +#### Что делает `export.py` +1) Читает `events.jsonl` построчно (`iter_events`) +2) Находит `replay_point` с нужным `id` (+ фильтры `spec_idx`/`provider`) +3) Опционально подтягивает зависимости из `requires`: + - `replay_resource` события → `ctx.resources` + - `planner_input` события → `ctx.extras` +4) Пишет фикстуру в `out_dir`: + - **простая фикстура** (`write_fixture`) — только replay_point + `source` + - **bundle** (`write_bundle`) — root replay_point + resources + extras + `source`, + и при необходимости копирует файлы ресурсов в структуру `resources//...` + +#### Имена фикстур +Имя вычисляется стабильно: +```py +fixture_name = f"{event_id}__{sha256(event_id + canonical_json(input))[:8]}.json" +``` + +--- + +## 4) Как этим пользоваться + +### 4.1 В рантайме: как логировать replay-point’ы + +1) Подготовить объект, реализующий `EventLoggerLike.emit(...)`. + +Пример (упрощённый) — JSONL writer: +```py +from pathlib import Path +import json, datetime + +class JsonlEventLog: + def __init__(self, path: Path): + self.path = path + self.path.parent.mkdir(parents=True, exist_ok=True) + + def emit(self, event: dict) -> None: + payload = {"timestamp": datetime.datetime.utcnow().isoformat() + "Z", **event} + with self.path.open("a", encoding="utf-8") as f: + f.write(json.dumps(payload, ensure_ascii=False) + "\n") +``` + +2) В точке, где хотите получать реплейный тест, вызвать `log_replay_point(...)`: + +```py +from fetchgraph.tracer import log_replay_point # путь зависит от того, где пакет лежит у вас + +log_replay_point( + logger=event_log, + id="plan_normalize.spec_v1", + meta={"spec_idx": i, "provider": spec.provider}, + input={"spec": spec.model_dump(), "options": options.model_dump(), "normalizer_rules": rules}, + expected={"out_spec": out_spec, "notes_last": notes_last}, + requires=["planner_input_v1"], # если обработчику нужен контекст +) +``` + +3) Если есть внешние файлы/ресурсы, которые нужно будет копировать в bundle — записывайте отдельные события `replay_resource` +(формат в коде не зафиксирован типами, но `export.py` ожидает хотя бы `type="replay_resource"`, `id=str`, и опционально `data_ref.file`): +```py +event_log.emit({ + "type": "replay_resource", + "id": "catalog_csv", + "data_ref": {"file": "relative/path/to/catalog.csv"} +}) +``` + +--- + +### 4.2 Реплей: как выполнить фикстуру + +1) Импортировать модули обработчиков, чтобы они зарегистрировались в `REPLAY_HANDLERS` +(сейчас регистрация — через side-effect: модуль при импорте пишет в dict). + +Например: +```py +from fetchgraph.tracer.runtime import REPLAY_HANDLERS, ReplayContext +import fetchgraph.tracer.plan_normalize # важно: импорт модуля регистрирует обработчик +``` + +2) Загрузить фикстуру (или bundle) и построить `ReplayContext`. + +- Для простой фикстуры: +```py +fixture = json.load(open(path, "r", encoding="utf-8")) +ctx = ReplayContext(resources={}, extras={}, base_dir=path.parent) +handler = REPLAY_HANDLERS[fixture["id"]] +out = handler(fixture["input"], ctx) +assert out == fixture["expected"] +``` + +- Для bundle: +```py +bundle = json.load(open(path, "r", encoding="utf-8")) +root = bundle["root"] +ctx = ReplayContext( + resources=bundle.get("resources", {}), + extras=bundle.get("extras", {}), + base_dir=path.parent, +) +handler = REPLAY_HANDLERS[root["id"]] +out = handler(root["input"], ctx) +assert out == root["expected"] +``` + +--- + +### 4.3 Экспорт фикстур из events.jsonl + +Python API (как в `export.py`): + +- **Одна фикстура**: +```py +from fetchgraph.tracer.export import export_replay_fixture +export_replay_fixture( + events_path=Path("_demo_data/.../events.jsonl"), + out_dir=Path("tests/fixtures/replay"), + replay_id="plan_normalize.spec_v1", + spec_idx=0, + provider="sql", + with_requires=True, + run_dir=Path("_demo_data/.../run_dir"), +) +``` + +- **Все совпадения (если один `id` встречается много раз)**: +```py +from fetchgraph.tracer.export import export_replay_fixtures +paths = export_replay_fixtures( + events_path=..., + out_dir=..., + replay_id="plan_normalize.spec_v1", + with_requires=False, +) +``` + +--- + +## 5) Рекомендации по pytest-фикстурам + +Минимальный удобный слой (совет): + +- `load_replay_fixture(path) -> (root_event, ctx)` +- `run_replay(root_event, ctx) -> out` + +Чтобы каждый тест был “одной строкой”. + +Пример (псевдокод): +```py +@pytest.mark.parametrize("fixture_path", glob("tests/fixtures/replay/*.json")) +def test_replay_fixture(fixture_path): + root, ctx = load_fixture_and_ctx(fixture_path) + handler = REPLAY_HANDLERS[root["id"]] + assert handler(root["input"], ctx) == root["expected"] +``` + +--- + +## 6) “Короткая памятка” для разработчика + +1) В коде, где хотите регрессию, используйте `log_replay_point(...)`. +2) Если реплей требует контекст/файлы: + - логируйте `planner_input` (extras) и/или `replay_resource` + - добавляйте их `id` в `requires` +3) Для тестов: + - экспортируйте фикстуры из `events.jsonl` через `export.py` + - импортируйте модули обработчиков (чтобы заполнить `REPLAY_HANDLERS`) + - грузите фикстуру, строите `ReplayContext`, вызывайте handler, сравнивайте с `expected` From 9ea28656e4f383a07f2b69d730908677d8c7007d Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 11:42:03 +0300 Subject: [PATCH 25/79] Document replay case fixture regeneration --- Makefile | 18 +- pyproject.toml | 3 + src/fetchgraph/cli.py | 27 +- .../planning/normalize/plan_normalizer.py | 13 +- src/fetchgraph/replay/__init__.py | 8 +- src/fetchgraph/replay/export.py | 407 +++++++----------- src/fetchgraph/replay/log.py | 86 +++- src/fetchgraph/replay/runtime.py | 19 + src/fetchgraph/tracer/__init__.py | 13 + src/fetchgraph/tracer/cli.py | 56 +++ src/fetchgraph/tracer/export.py | 19 + src/fetchgraph/tracer/handlers/__init__.py | 5 + src/fetchgraph/tracer/log.py | 5 + src/fetchgraph/tracer/runtime.py | 5 + src/fetchgraph/tracer/validators.py | 17 + .../known_bad => replay_cases}/.gitkeep | 0 tests/fixtures/replay_cases/README.md | 4 + tests/fixtures/replay_cases/fixed/.gitkeep | 0 .../fixtures/replay_cases/known_bad/.gitkeep | 0 tests/fixtures/replay_points/fixed/.gitkeep | 1 - .../plan_normalize.spec_v1__027f241b.json | 1 - .../plan_normalize.spec_v1__172bff2e.json | 1 - .../plan_normalize.spec_v1__b6b6420e.json | 1 - .../sample_resource.json | 1 - .../plan_normalize.spec_v1__5f883b82.json | 1 - .../plan_normalize.spec_v1__b18ec08f.json | 1 - .../plan_normalize.spec_v1__b4d64d21.json | 1 - .../plan_normalize.spec_v1__dfd64ee7.json | 1 - tests/test_replay_fixtures.py | 229 ++-------- 29 files changed, 448 insertions(+), 495 deletions(-) create mode 100644 src/fetchgraph/tracer/__init__.py create mode 100644 src/fetchgraph/tracer/cli.py create mode 100644 src/fetchgraph/tracer/export.py create mode 100644 src/fetchgraph/tracer/handlers/__init__.py create mode 100644 src/fetchgraph/tracer/log.py create mode 100644 src/fetchgraph/tracer/runtime.py create mode 100644 src/fetchgraph/tracer/validators.py rename tests/fixtures/{replay_points/known_bad => replay_cases}/.gitkeep (100%) create mode 100644 tests/fixtures/replay_cases/README.md create mode 100644 tests/fixtures/replay_cases/fixed/.gitkeep create mode 100644 tests/fixtures/replay_cases/known_bad/.gitkeep delete mode 100644 tests/fixtures/replay_points/fixed/.gitkeep delete mode 100644 tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__027f241b.json delete mode 100644 tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__172bff2e.json delete mode 100644 tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__b6b6420e.json delete mode 100644 tests/fixtures/replay_points/fixed/resources/plan_normalize.spec_v1__027f241b/sample_resource.json delete mode 100644 tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__5f883b82.json delete mode 100644 tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b18ec08f.json delete mode 100644 tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b4d64d21.json delete mode 100644 tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__dfd64ee7.json diff --git a/Makefile b/Makefile index 9279ae8e..807f3eae 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,8 @@ WITH ?= SPEC_IDX ?= PROVIDER ?= BUCKET ?= fixed -OUT_DIR ?= tests/fixtures/replay_points/$(BUCKET) +OUT_DIR ?= tests/fixtures/replay_cases/$(BUCKET) +RUN_DIR ?= SCOPE ?= both WITH_RESOURCES ?= 1 ALL ?= @@ -112,7 +113,7 @@ LIMIT_FLAG := $(if $(strip $(LIMIT)),--limit $(LIMIT),) batch batch-tag batch-failed batch-failed-from \ batch-missed batch-missed-from batch-failed-tag batch-missed-tag \ batch-fail-fast batch-max-fails \ - stats history-case report-tag report-tag-changes tags tag-rm case-run case-open fixture fixture-rm fixture-fix fixture-migrate compare compare-tag + stats history-case report-tag report-tag-changes tags tag-rm case-run case-open fixture fixture-rm fixture-fix fixture-migrate tracer-export compare compare-tag # ============================================================================== # help (на русском) @@ -159,8 +160,8 @@ help: @echo " make tags [PATTERN=*] DATA=... - показать список тегов" @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" - @echo " make fixture CASE=agg_01 [TAG=...] [RUN_ID=...] [REPLAY_ID=plan_normalize.spec_v1] [WITH=requires] [SPEC_IDX=0] [PROVIDER=relational] [BUCKET=fixed|known_bad] [OUT_DIR=tests/fixtures/replay_points/$$(BUCKET)] [ALL=1]" - @echo " fixtures layout: replay_points//.json, resources: replay_points//resources//..." + @echo " make fixture CASE=agg_01 [TAG=...] [RUN_ID=...] [REPLAY_ID=plan_normalize.spec_v1] [WITH=requires] [SPEC_IDX=0] [PROVIDER=relational] [BUCKET=fixed|known_bad] [OUT_DIR=tests/fixtures/replay_cases/$$(BUCKET)] [ALL=1]" + @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources//..." @echo " make fixture-rm NAME=... [PATTERN=...] [BUCKET=fixed|known_bad] [SCOPE=replay|traces|both] [WITH_RESOURCES=1] [DRY=1] (удаляет fixture и resources)" @echo " make fixture-fix NAME=... [PATTERN=...] [CASE=...] [MOVE_TRACES=1] [DRY=1] (переносит fixture и resources)" @echo " make fixture-migrate [BUCKET=fixed|known_bad] [DRY=1] (миграция ресурсов в resources//)" @@ -358,6 +359,15 @@ fixture: check $(if $(filter 1 true yes on,$(ALL)),--all,) \ $(if $(filter requires,$(WITH)),--with-requires,) +tracer-export: + @test -n "$(strip $(ID))" || (echo "ID обязателен: make tracer-export ID=plan_normalize.spec_v1" && exit 1) + @test -n "$(strip $(EVENTS))" || (echo "EVENTS обязателен: make tracer-export EVENTS=path/to/events.jsonl" && exit 1) + @test -n "$(strip $(OUT))" || (echo "OUT обязателен: make tracer-export OUT=path/to/out_dir" && exit 1) + @fetchgraph-tracer export-case-bundle --events "$(EVENTS)" --out "$(OUT)" --id "$(ID)" \ + $(if $(strip $(SPEC_IDX)),--spec-idx $(SPEC_IDX),) \ + $(if $(strip $(PROVIDER)),--provider $(PROVIDER),) \ + $(if $(strip $(RUN_DIR)),--run-dir $(RUN_DIR),) + # 12) Fixture tools fixture-rm: @$(CLI_FIXT) rm --name "$(NAME)" --pattern "$(PATTERN)" --bucket "$(BUCKET)" --scope "$(SCOPE)" --with-resources "$(WITH_RESOURCES)" --dry "$(DRY)" diff --git a/pyproject.toml b/pyproject.toml index b83b91d9..0b2fb8a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,3 +47,6 @@ Homepage = "https://github.com/AlexanderOnischenko/fetchgraph" Repository = "https://github.com/AlexanderOnischenko/fetchgraph" Documentation = "https://github.com/AlexanderOnischenko/fetchgraph#readme" Issues = "https://github.com/AlexanderOnischenko/fetchgraph/issues" + +[project.scripts] +fetchgraph-tracer = "fetchgraph.tracer.cli:main" diff --git a/src/fetchgraph/cli.py b/src/fetchgraph/cli.py index 84ac6a22..dff40227 100644 --- a/src/fetchgraph/cli.py +++ b/src/fetchgraph/cli.py @@ -4,30 +4,29 @@ import sys from pathlib import Path -from fetchgraph.replay.export import export_replay_fixture, export_replay_fixtures +from fetchgraph.replay.export import export_replay_case_bundle, export_replay_case_bundles def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Fetchgraph utilities") sub = parser.add_subparsers(dest="command", required=True) - fixture = sub.add_parser("fixture", help="Export replay fixture from events.jsonl") + fixture = sub.add_parser("fixture", help="Export replay case bundle from events.jsonl") fixture.add_argument("--events", type=Path, required=True, help="Path to events.jsonl") - fixture.add_argument("--run-dir", type=Path, default=None, help="Case run dir (needed for --with-requires)") - fixture.add_argument("--id", default="plan_normalize.spec_v1", help="Replay point id to extract") - fixture.add_argument("--spec-idx", type=int, default=None, help="Filter replay_point by meta.spec_idx") + fixture.add_argument("--run-dir", type=Path, default=None, help="Case run dir (needed for file resources)") + fixture.add_argument("--id", default="plan_normalize.spec_v1", help="Replay case id to extract") + fixture.add_argument("--spec-idx", type=int, default=None, help="Filter replay_case by meta.spec_idx") fixture.add_argument( "--provider", default=None, - help="Filter replay_point by meta.provider (case-insensitive)", + help="Filter replay_case by meta.provider (case-insensitive)", ) - fixture.add_argument("--with-requires", action="store_true", help="Export replay bundle with dependencies") fixture.add_argument("--all", action="store_true", help="Export all matching replay points") fixture.add_argument( "--out-dir", type=Path, - default=Path("tests") / "fixtures" / "replay_points", - help="Output directory for replay fixtures", + default=Path("tests") / "fixtures" / "replay_cases", + help="Output directory for replay case bundles", ) return parser.parse_args(argv) @@ -36,24 +35,22 @@ def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) if args.command == "fixture": if args.all: - export_replay_fixtures( + export_replay_case_bundles( events_path=args.events, - run_dir=args.run_dir, out_dir=args.out_dir, replay_id=args.id, spec_idx=args.spec_idx, provider=args.provider, - with_requires=args.with_requires, + run_dir=args.run_dir, ) else: - export_replay_fixture( + export_replay_case_bundle( events_path=args.events, - run_dir=args.run_dir, out_dir=args.out_dir, replay_id=args.id, spec_idx=args.spec_idx, provider=args.provider, - with_requires=args.with_requires, + run_dir=args.run_dir, ) return 0 raise SystemExit(f"Unknown command: {args.command}") diff --git a/src/fetchgraph/planning/normalize/plan_normalizer.py b/src/fetchgraph/planning/normalize/plan_normalizer.py index 4a26c372..0cf025f2 100644 --- a/src/fetchgraph/planning/normalize/plan_normalizer.py +++ b/src/fetchgraph/planning/normalize/plan_normalizer.py @@ -13,7 +13,7 @@ from ...relational.models import RelationalRequest from ...relational.normalize import normalize_relational_selectors from ...relational.providers.base import RelationalDataProvider -from ...replay.log import EventLoggerLike, log_replay_point +from ...replay.log import EventLoggerLike, log_replay_case logger = logging.getLogger(__name__) @@ -188,17 +188,18 @@ def _normalize_specs( "options": asdict(self.options), "normalizer_rules": {spec.provider: rule_kind}, } - expected_payload = { + observed_payload = { "out_spec": { "provider": spec.provider, "mode": spec.mode, "selectors": use, - } + }, + "notes_last": note, } requires = None if getattr(replay_logger, "case_id", None): - requires = ["planner_input_v1"] - log_replay_point( + requires = [{"kind": "extra", "id": "planner_input_v1"}] + log_replay_case( replay_logger, id="plan_normalize.spec_v1", meta={ @@ -207,7 +208,7 @@ def _normalize_specs( "spec_idx": spec_idx, }, input=input_payload, - expected=expected_payload, + observed=observed_payload, requires=requires, diag={ "selectors_valid_before": before_ok, diff --git a/src/fetchgraph/replay/__init__.py b/src/fetchgraph/replay/__init__.py index c09445c1..80da066e 100644 --- a/src/fetchgraph/replay/__init__.py +++ b/src/fetchgraph/replay/__init__.py @@ -1,11 +1,13 @@ from __future__ import annotations -from .log import EventLoggerLike, log_replay_point -from .runtime import REPLAY_HANDLERS, ReplayContext +from .log import EventLoggerLike, log_replay_case +from .runtime import REPLAY_HANDLERS, ReplayContext, load_case_bundle, run_case __all__ = [ "EventLoggerLike", "REPLAY_HANDLERS", "ReplayContext", - "log_replay_point", + "load_case_bundle", + "log_replay_case", + "run_case", ] diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py index 57c37fed..c3a365bb 100644 --- a/src/fetchgraph/replay/export.py +++ b/src/fetchgraph/replay/export.py @@ -10,93 +10,31 @@ def iter_events(path: Path) -> Iterable[tuple[int, dict]]: with path.open("r", encoding="utf-8") as handle: - for idx, line in enumerate(handle): + for idx, line in enumerate(handle, start=1): line = line.strip() if not line: continue try: yield idx, json.loads(line) - except json.JSONDecodeError: - continue + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON on line {idx} in {path}: {exc.msg}") from exc def canonical_json(payload: object) -> str: return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) -def fixture_name(event_id: str, input_payload: dict) -> str: +def case_bundle_name(event_id: str, input_payload: dict) -> str: digest = hashlib.sha256((event_id + canonical_json(input_payload)).encode("utf-8")).hexdigest() - return f"{event_id}__{digest[:8]}.json" - - -def _resource_target_path( - *, - fixture_stem: str, - file_name: str, - resource_key: str, - used_paths: set[Path], -) -> Path: - rel_path = Path(file_name) - if rel_path.is_absolute(): - raise SystemExit(f"Resource file path must be relative: {file_name}") - if ".." in rel_path.parts: - raise SystemExit(f"Resource file path must not traverse parents: {file_name}") - base_dir = Path("resources") / fixture_stem - if rel_path.parent != Path("."): - dest_rel = base_dir / rel_path - else: - dest_rel = base_dir / rel_path.name - if dest_rel in used_paths: - prefix = resource_key or "resource" - dest_rel = base_dir / f"{prefix}__{rel_path.name}" - suffix = 1 - while dest_rel in used_paths: - dest_rel = base_dir / f"{prefix}_{suffix}__{rel_path.name}" - suffix += 1 - used_paths.add(dest_rel) - return dest_rel - - -def _copy_resource_file( - run_dir: Path, - out_dir: Path, - fixture_stem: str, - resource_key: str, - used_paths: set[Path], - data_ref: dict, -) -> dict: - file_name = data_ref.get("file") - if not file_name: - return data_ref - src_path = run_dir / file_name - if not src_path.exists(): - raise SystemExit(f"Resource file {src_path} not found for replay bundle") - dest_rel = _resource_target_path( - fixture_stem=fixture_stem, - file_name=file_name, - resource_key=resource_key, - used_paths=used_paths, - ) - dest_path = out_dir / dest_rel - dest_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src_path, dest_path) - updated = dict(data_ref) - updated["file"] = dest_rel.as_posix() - return updated + return f"{event_id}__{digest[:8]}.case.json" @dataclass(frozen=True) class ExportSelection: - event_index: int + line: int event: dict -@dataclass(frozen=True) -class ExportContext: - resources: Dict[str, dict] - extras: Dict[str, dict] - - def _match_meta(event: dict, *, spec_idx: int | None, provider: str | None) -> bool: meta = event.get("meta") if not isinstance(meta, dict): @@ -112,240 +50,225 @@ def _match_meta(event: dict, *, spec_idx: int | None, provider: str | None) -> b return True -def select_replay_points( +def _select_replay_cases( events_path: Path, *, replay_id: str, spec_idx: int | None = None, provider: str | None = None, -) -> tuple[list[ExportSelection], ExportContext]: +) -> list[ExportSelection]: selected: list[ExportSelection] = [] - resources: Dict[str, dict] = {} - extras: Dict[str, dict] = {} - - for idx, event in iter_events(events_path): - etype = event.get("type") - eid = event.get("id") - - if etype == "replay_resource" and isinstance(eid, str): - resources[eid] = event + for line, event in iter_events(events_path): + if event.get("type") != "replay_case": continue - - if etype == "planner_input" and isinstance(eid, str): - extras[eid] = event + if event.get("id") != replay_id: continue + if _match_meta(event, spec_idx=spec_idx, provider=provider): + selected.append(ExportSelection(line=line, event=event)) + return selected - if etype == "replay_point" and eid == replay_id: - if _match_meta(event, spec_idx=spec_idx, provider=provider): - selected.append(ExportSelection(event_index=idx, event=event)) - if not selected: - details = [] - if spec_idx is not None: - details.append(f"spec_idx={spec_idx}") - if provider is not None: - details.append(f"provider={provider!r}") - detail_str = f" (filters: {', '.join(details)})" if details else "" - raise SystemExit(f"No replay_point id={replay_id!r} found in {events_path}{detail_str}") - return selected, ExportContext(resources=resources, extras=extras) - - -def write_fixture(event: dict, *, out_dir: Path, source: dict) -> Path: - out_dir.mkdir(parents=True, exist_ok=True) - event_id = event["id"] - out_path = out_dir / fixture_name(event_id, event["input"]) - if out_path.exists(): - print(f"Fixture already exists: {out_path}") - return out_path +def collect_requires( + events_path: Path, + requires: list[dict], +) -> tuple[dict[str, dict], dict[str, dict]]: + extras: Dict[str, dict] = {} + resources: Dict[str, dict] = {} - payload = dict(event) - payload["source"] = source - out_path.write_text(canonical_json(payload), encoding="utf-8") - print(f"Wrote fixture: {out_path}") - return out_path + if not requires: + return resources, extras + for _, event in iter_events(events_path): + event_type = event.get("type") + event_id = event.get("id") + if event_type == "planner_input" and isinstance(event_id, str): + extras[event_id] = event + elif event_type == "replay_resource" and isinstance(event_id, str): + resources[event_id] = event -def write_bundle( - root_event: dict, + resolved_resources: Dict[str, dict] = {} + resolved_extras: Dict[str, dict] = {} + for req in requires: + kind = req.get("kind") + rid = req.get("id") + if kind == "extra": + if rid in extras: + resolved_extras[rid] = extras[rid] + else: + raise KeyError(f"Missing required extra {rid} in {events_path}") + elif kind == "resource": + if rid in resources: + resolved_resources[rid] = resources[rid] + else: + raise KeyError(f"Missing required resource {rid} in {events_path}") + else: + raise KeyError(f"Unknown require kind {kind!r} in {events_path}") + + return resolved_resources, resolved_extras + + +def write_case_bundle( + out_path: Path, *, - out_dir: Path, - run_dir: Path, + root_case: dict, resources: Dict[str, dict], extras: Dict[str, dict], source: dict, - root_source: dict, -) -> Path: - out_dir.mkdir(parents=True, exist_ok=True) - fixture_file = fixture_name(root_event["id"], root_event["input"]) - out_path = out_dir / fixture_file - if out_path.exists(): - print(f"Fixture already exists: {out_path}") - return out_path - - root_name = Path(fixture_file).stem - updated_resources: Dict[str, dict] = {} - used_paths: set[Path] = set() - for rid, resource in resources.items(): - rp = dict(resource) - if "data_ref" in rp and isinstance(rp["data_ref"], dict): - rp["data_ref"] = _copy_resource_file( - run_dir, - out_dir, - root_name, - rid, - used_paths, - rp["data_ref"], - ) - updated_resources[rid] = rp - +) -> None: payload = { - "type": "replay_bundle", + "schema": "fetchgraph.tracer.case_bundle", "v": 1, - "root": dict(root_event), - "resources": updated_resources, - "extras": dict(extras), - "source": dict(source), + "root": root_case, + "resources": resources, + "extras": extras, + "source": source, } - payload["root"]["source"] = dict(root_source) - + out_path.parent.mkdir(parents=True, exist_ok=True) out_path.write_text(canonical_json(payload), encoding="utf-8") - print(f"Wrote fixture bundle: {out_path}") - return out_path -def export_replay_fixture( +def copy_resource_files( + resources: dict[str, dict], + *, + run_dir: Path, + out_dir: Path, + fixture_stem: str, +) -> None: + for resource in resources.values(): + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + continue + rel_path = Path(file_name) + if rel_path.is_absolute(): + raise ValueError(f"Resource file path must be relative: {file_name}") + if ".." in rel_path.parts: + raise ValueError(f"Resource file path must not traverse parents: {file_name}") + src_path = run_dir / rel_path + if not src_path.exists(): + raise FileNotFoundError(f"Resource file {src_path} not found for replay bundle") + dest_rel = Path("resources") / fixture_stem / rel_path + dest_path = out_dir / dest_rel + dest_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_path, dest_path) + data_ref["file"] = dest_rel.as_posix() + + +def _has_resource_files(resources: dict[str, dict]) -> bool: + for resource in resources.values(): + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if isinstance(file_name, str) and file_name: + return True + return False + + +def export_replay_case_bundle( *, events_path: Path, out_dir: Path, replay_id: str, spec_idx: int | None = None, provider: str | None = None, - with_requires: bool = False, run_dir: Path | None = None, - source_extra: dict | None = None, - allow_multiple: bool = False, ) -> Path: - selections, ctx = select_replay_points( + selections = _select_replay_cases( events_path, replay_id=replay_id, spec_idx=spec_idx, provider=provider, ) - if len(selections) > 1 and not allow_multiple: - raise SystemExit( - "Multiple replay points matched. Provide --spec-idx/--provider or use --all to export all." + if not selections: + details = [] + if spec_idx is not None: + details.append(f"spec_idx={spec_idx}") + if provider is not None: + details.append(f"provider={provider!r}") + detail_str = f" (filters: {', '.join(details)})" if details else "" + raise LookupError(f"No replay_case id={replay_id!r} found in {events_path}{detail_str}") + if len(selections) > 1: + raise LookupError( + "Multiple replay_case entries matched; use export_replay_case_bundles to export all." ) - if len(selections) > 1 and allow_multiple: - raise SystemExit("Use export_replay_fixtures for multiple selections.") - selection = selections[0] - event = selection.event - - common_source = { - "events_path": str(events_path), - "event_index": selection.event_index, - **(source_extra or {}), - } - if not with_requires: - if event.get("requires"): - print("Warning: replay_point has requires; export without --with-requires may be incomplete.") - return write_fixture(event, out_dir=out_dir, source=common_source) - - requires = event.get("requires") or [] - resolved_resources: Dict[str, dict] = {} - resolved_extras: Dict[str, dict] = {} - - for rid in requires: - if rid in ctx.resources: - resolved_resources[rid] = ctx.resources[rid] - continue - if rid in ctx.extras: - resolved_extras[rid] = ctx.extras[rid] - continue - raise SystemExit(f"Required dependency {rid!r} not found in {events_path}") + selection = selections[0] + root_event = selection.event + requires = root_event.get("requires") or [] - if run_dir is None: - raise SystemExit("--run-dir is required for --with-requires (to copy resource files)") + resources, extras = collect_requires(events_path, requires) + fixture_name = case_bundle_name(replay_id, root_event["input"]) + if _has_resource_files(resources): + if run_dir is None: + raise ValueError("run_dir is required to export file resources") + copy_resource_files( + resources, + run_dir=run_dir, + out_dir=out_dir, + fixture_stem=fixture_name.replace(".case.json", ""), + ) - root_source = { + source = { "events_path": str(events_path), - "event_index": selection.event_index, - "run_dir": str(run_dir), - **(source_extra or {}), + "line": selection.line, + "run_id": root_event.get("run_id"), + "timestamp": root_event.get("timestamp"), } - - return write_bundle( - event, - out_dir=out_dir, - run_dir=run_dir, - resources=resolved_resources, - extras=resolved_extras, - source=common_source, - root_source=root_source, - ) + out_path = out_dir / fixture_name + write_case_bundle(out_path, root_case=root_event, resources=resources, extras=extras, source=source) + return out_path -def export_replay_fixtures( +def export_replay_case_bundles( *, events_path: Path, out_dir: Path, replay_id: str, spec_idx: int | None = None, provider: str | None = None, - with_requires: bool = False, run_dir: Path | None = None, - source_extra: dict | None = None, ) -> list[Path]: - selections, ctx = select_replay_points( + selections = _select_replay_cases( events_path, replay_id=replay_id, spec_idx=spec_idx, provider=provider, ) + if not selections: + details = [] + if spec_idx is not None: + details.append(f"spec_idx={spec_idx}") + if provider is not None: + details.append(f"provider={provider!r}") + detail_str = f" (filters: {', '.join(details)})" if details else "" + raise LookupError(f"No replay_case id={replay_id!r} found in {events_path}{detail_str}") + paths: list[Path] = [] for selection in selections: - event = selection.event - common_source = { - "events_path": str(events_path), - "event_index": selection.event_index, - **(source_extra or {}), - } - if not with_requires: - if event.get("requires"): - print("Warning: replay_point has requires; export without --with-requires may be incomplete.") - paths.append(write_fixture(event, out_dir=out_dir, source=common_source)) - continue - - requires = event.get("requires") or [] - resolved_resources: Dict[str, dict] = {} - resolved_extras: Dict[str, dict] = {} - for rid in requires: - if rid in ctx.resources: - resolved_resources[rid] = ctx.resources[rid] - continue - if rid in ctx.extras: - resolved_extras[rid] = ctx.extras[rid] - continue - raise SystemExit(f"Required dependency {rid!r} not found in {events_path}") - - if run_dir is None: - raise SystemExit("--run-dir is required for --with-requires (to copy resource files)") + root_event = selection.event + requires = root_event.get("requires") or [] + resources, extras = collect_requires(events_path, requires) + fixture_name = case_bundle_name(replay_id, root_event["input"]) + if _has_resource_files(resources): + if run_dir is None: + raise ValueError("run_dir is required to export file resources") + copy_resource_files( + resources, + run_dir=run_dir, + out_dir=out_dir, + fixture_stem=fixture_name.replace(".case.json", ""), + ) - root_source = { + source = { "events_path": str(events_path), - "event_index": selection.event_index, - "run_dir": str(run_dir), - **(source_extra or {}), + "line": selection.line, + "run_id": root_event.get("run_id"), + "timestamp": root_event.get("timestamp"), } - paths.append( - write_bundle( - event, - out_dir=out_dir, - run_dir=run_dir, - resources=resolved_resources, - extras=resolved_extras, - source=common_source, - root_source=root_source, - ) - ) + out_path = out_dir / fixture_name + write_case_bundle(out_path, root_case=root_event, resources=resources, extras=extras, source=source) + paths.append(out_path) return paths diff --git a/src/fetchgraph/replay/log.py b/src/fetchgraph/replay/log.py index afe1a9b9..982178c0 100644 --- a/src/fetchgraph/replay/log.py +++ b/src/fetchgraph/replay/log.py @@ -7,35 +7,85 @@ class EventLoggerLike(Protocol): def emit(self, event: Dict[str, object]) -> None: ... -def log_replay_point( +def log_replay_case( logger: EventLoggerLike, *, id: str, - meta: dict, input: dict, - expected: dict, - requires: list[str] | None = None, - diag: dict | None = None, + meta: dict | None = None, + observed: dict | None = None, + observed_error: dict | None = None, + requires: list[dict] | None = None, note: str | None = None, - error: str | None = None, - extra: dict | None = None, + diag: dict | None = None, ) -> None: + if not isinstance(id, str) or not id.strip(): + raise ValueError("id must be a non-empty string") + if not isinstance(input, dict): + raise ValueError("input must be a dict") + if (observed is None) == (observed_error is None): + raise ValueError("exactly one of observed or observed_error must be set") + if observed is not None and not isinstance(observed, dict): + raise ValueError("observed must be a dict when provided") + if observed_error is not None: + if not isinstance(observed_error, dict): + raise ValueError("observed_error must be a dict when provided") + if not isinstance(observed_error.get("type"), str) or not observed_error.get("type"): + raise ValueError("observed_error.type must be a non-empty string") + if not isinstance(observed_error.get("message"), str) or not observed_error.get("message"): + raise ValueError("observed_error.message must be a non-empty string") + if meta is not None and not isinstance(meta, dict): + raise ValueError("meta must be a dict when provided") + if note is not None and not isinstance(note, str): + raise ValueError("note must be a string when provided") + if diag is not None and not isinstance(diag, dict): + raise ValueError("diag must be a dict when provided") + if requires is not None: + if not isinstance(requires, list): + raise ValueError("requires must be a list when provided") + for req in requires: + if not isinstance(req, dict): + raise ValueError("requires entries must be dicts") + kind = req.get("kind") + if kind not in {"extra", "resource"}: + raise ValueError("requires.kind must be 'extra' or 'resource'") + req_id = req.get("id") + if not isinstance(req_id, str) or not req_id: + raise ValueError("requires.id must be a non-empty string") + event: Dict[str, object] = { - "type": "replay_point", - "v": 1, + "type": "replay_case", + "v": 2, "id": id, - "meta": meta, "input": input, - "expected": expected, } - if requires is not None: + if meta is not None: + event["meta"] = meta + if observed is not None: + event["observed"] = observed + if observed_error is not None: + event["observed_error"] = observed_error + if requires: event["requires"] = requires - if diag is not None: - event["diag"] = diag if note is not None: event["note"] = note - if error is not None: - event["error"] = error - if extra: - event.update(extra) + if diag is not None: + event["diag"] = diag logger.emit(event) + + +def log_replay_point( + logger: EventLoggerLike, + *, + id: str, + meta: dict, + input: dict, + expected: dict, + requires: list[str] | None = None, + diag: dict | None = None, + note: str | None = None, +) -> None: + raise ValueError( + "log_replay_point has been replaced by log_replay_case; " + "log_replay_point no longer supports expected payloads." + ) diff --git a/src/fetchgraph/replay/runtime.py b/src/fetchgraph/replay/runtime.py index f13e13c9..34a64a53 100644 --- a/src/fetchgraph/replay/runtime.py +++ b/src/fetchgraph/replay/runtime.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from dataclasses import dataclass, field from pathlib import Path from typing import Callable, Dict @@ -19,3 +20,21 @@ def resolve_resource_path(self, resource_path: str | Path) -> Path: REPLAY_HANDLERS: Dict[str, Callable[[dict, ReplayContext], dict]] = {} + + +def run_case(root: dict, ctx: ReplayContext) -> dict: + handler = REPLAY_HANDLERS[root["id"]] + return handler(root["input"], ctx) + + +def load_case_bundle(path: Path) -> tuple[dict, ReplayContext]: + data = json.loads(path.read_text(encoding="utf-8")) + if data.get("schema") != "fetchgraph.tracer.case_bundle" or data.get("v") != 1: + raise ValueError(f"Unsupported case bundle schema in {path}") + root = data["root"] + ctx = ReplayContext( + resources=data.get("resources", {}), + extras=data.get("extras", {}), + base_dir=path.parent, + ) + return root, ctx diff --git a/src/fetchgraph/tracer/__init__.py b/src/fetchgraph/tracer/__init__.py new file mode 100644 index 00000000..a0d19f20 --- /dev/null +++ b/src/fetchgraph/tracer/__init__.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from fetchgraph.replay.log import EventLoggerLike, log_replay_case +from fetchgraph.replay.runtime import REPLAY_HANDLERS, ReplayContext, load_case_bundle, run_case + +__all__ = [ + "EventLoggerLike", + "REPLAY_HANDLERS", + "ReplayContext", + "load_case_bundle", + "log_replay_case", + "run_case", +] diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py new file mode 100644 index 00000000..56168a08 --- /dev/null +++ b/src/fetchgraph/tracer/cli.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from fetchgraph.tracer.export import export_replay_case_bundle, export_replay_case_bundles + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Fetchgraph tracer utilities") + sub = parser.add_subparsers(dest="command", required=True) + + export = sub.add_parser("export-case-bundle", help="Export replay case bundle from events.jsonl") + export.add_argument("--events", type=Path, required=True, help="Path to events.jsonl") + export.add_argument("--out", type=Path, required=True, help="Output directory for bundle") + export.add_argument("--id", required=True, help="Replay case id to export") + export.add_argument("--spec-idx", type=int, default=None, help="Filter replay_case by meta.spec_idx") + export.add_argument( + "--provider", + default=None, + help="Filter replay_case by meta.provider (case-insensitive)", + ) + export.add_argument("--run-dir", type=Path, default=None, help="Run dir (required for file resources)") + export.add_argument("--all", action="store_true", help="Export all matching replay cases") + + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + if args.command == "export-case-bundle": + if args.all: + export_replay_case_bundles( + events_path=args.events, + out_dir=args.out, + replay_id=args.id, + spec_idx=args.spec_idx, + provider=args.provider, + run_dir=args.run_dir, + ) + else: + export_replay_case_bundle( + events_path=args.events, + out_dir=args.out, + replay_id=args.id, + spec_idx=args.spec_idx, + provider=args.provider, + run_dir=args.run_dir, + ) + return 0 + raise SystemExit(f"Unknown command: {args.command}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/fetchgraph/tracer/export.py b/src/fetchgraph/tracer/export.py new file mode 100644 index 00000000..785aa6da --- /dev/null +++ b/src/fetchgraph/tracer/export.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from fetchgraph.replay.export import ( + case_bundle_name, + collect_requires, + copy_resource_files, + export_replay_case_bundle, + export_replay_case_bundles, + iter_events, +) + +__all__ = [ + "case_bundle_name", + "collect_requires", + "copy_resource_files", + "export_replay_case_bundle", + "export_replay_case_bundles", + "iter_events", +] diff --git a/src/fetchgraph/tracer/handlers/__init__.py b/src/fetchgraph/tracer/handlers/__init__.py new file mode 100644 index 00000000..4c3d95cc --- /dev/null +++ b/src/fetchgraph/tracer/handlers/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from fetchgraph.replay.handlers.plan_normalize import replay_plan_normalize_spec_v1 + +__all__ = ["replay_plan_normalize_spec_v1"] diff --git a/src/fetchgraph/tracer/log.py b/src/fetchgraph/tracer/log.py new file mode 100644 index 00000000..375f1642 --- /dev/null +++ b/src/fetchgraph/tracer/log.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from fetchgraph.replay.log import EventLoggerLike, log_replay_case + +__all__ = ["EventLoggerLike", "log_replay_case"] diff --git a/src/fetchgraph/tracer/runtime.py b/src/fetchgraph/tracer/runtime.py new file mode 100644 index 00000000..9da70789 --- /dev/null +++ b/src/fetchgraph/tracer/runtime.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from fetchgraph.replay.runtime import REPLAY_HANDLERS, ReplayContext, load_case_bundle, run_case + +__all__ = ["REPLAY_HANDLERS", "ReplayContext", "load_case_bundle", "run_case"] diff --git a/src/fetchgraph/tracer/validators.py b/src/fetchgraph/tracer/validators.py new file mode 100644 index 00000000..49ab06bc --- /dev/null +++ b/src/fetchgraph/tracer/validators.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from pydantic import TypeAdapter + +from fetchgraph.relational.models import RelationalRequest + + +def validate_plan_normalize_spec_v1(out: dict) -> None: + if not isinstance(out, dict): + raise AssertionError("Output must be a dict") + out_spec = out.get("out_spec") + if not isinstance(out_spec, dict): + raise AssertionError("Output must contain out_spec dict") + selectors = out_spec.get("selectors") + if selectors is None: + raise AssertionError("out_spec.selectors is required") + TypeAdapter(RelationalRequest).validate_python(selectors) diff --git a/tests/fixtures/replay_points/known_bad/.gitkeep b/tests/fixtures/replay_cases/.gitkeep similarity index 100% rename from tests/fixtures/replay_points/known_bad/.gitkeep rename to tests/fixtures/replay_cases/.gitkeep diff --git a/tests/fixtures/replay_cases/README.md b/tests/fixtures/replay_cases/README.md new file mode 100644 index 00000000..1f84c95c --- /dev/null +++ b/tests/fixtures/replay_cases/README.md @@ -0,0 +1,4 @@ +# Replay case fixtures + +This directory is intentionally empty in the repo. Replay case bundles are generated +locally from events and should be recreated as needed for tests. diff --git a/tests/fixtures/replay_cases/fixed/.gitkeep b/tests/fixtures/replay_cases/fixed/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/replay_cases/known_bad/.gitkeep b/tests/fixtures/replay_cases/known_bad/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/replay_points/fixed/.gitkeep b/tests/fixtures/replay_points/fixed/.gitkeep deleted file mode 100644 index 8b137891..00000000 --- a/tests/fixtures/replay_points/fixed/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__027f241b.json b/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__027f241b.json deleted file mode 100644 index 6606e172..00000000 --- a/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__027f241b.json +++ /dev/null @@ -1 +0,0 @@ -{"extras":{"planner_input_v1":{"case_id":"agg_036","id":"planner_input_v1","input":{"feature_name":"agg_036","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Сколько заказов было в 2022-07? Ответь только числом."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:54.791922Z","type":"planner_input","v":1}},"resources":{"resource_file_demo":{"data_ref":{"file":"resources/plan_normalize.spec_v1__027f241b/sample_resource.json"},"id":"resource_file_demo","kind":"file","type":"replay_resource"}},"root":{"case_id":"agg_036","diag":{"selectors_valid_after":true,"selectors_valid_before":true},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"filters":{"clauses":[{"entity":"orders","field":"order_date","op":">=","type":"comparison","value":"2022-07-01"},{"entity":"orders","field":"order_date","op":"<","type":"comparison","value":"2022-08-01"}],"op":"and","type":"logical"},"op":"query","root_entity":"orders","select":[{"alias":"order_count","expr":"count(*)"}]}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"filters":{"clauses":[{"entity":"orders","field":"order_date","op":">=","type":"comparison","value":"2022-07-01"},{"entity":"orders","field":"order_date","op":"<","type":"comparison","value":"2022-08-01"}],"op":"and","type":"logical"},"op":"query","root_entity":"orders","select":[{"alias":"order_count","expr":"count(*)"}]}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}","requires":["planner_input_v1","resource_file_demo"],"run_id":"9df66f32","source":{"case_id":"agg_036","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_036_c2239c95/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_036_c2239c95","tag":null},"timestamp":"2026-01-23T10:25:54.802564Z","type":"replay_point","v":1},"source":{"case_id":"agg_036","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_036_c2239c95/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file diff --git a/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__172bff2e.json b/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__172bff2e.json deleted file mode 100644 index 66ae3877..00000000 --- a/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__172bff2e.json +++ /dev/null @@ -1 +0,0 @@ -{"extras":{"planner_input_v1":{"case_id":"agg_035","id":"planner_input_v1","input":{"feature_name":"agg_035","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Сколько заказов было в 2022-03? Ответь только числом."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:54.764315Z","type":"planner_input","v":1}},"resources":{},"root":{"case_id":"agg_035","diag":{"selectors_valid_after":true,"selectors_valid_before":true},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"order_count","field":"order_id"}],"filters":{"clauses":[{"field":"order_date","op":">=","type":"comparison","value":"2022-03-01"},{"field":"order_date","op":"<=","type":"comparison","value":"2022-03-31"}],"op":"and","type":"logical"},"op":"query","root_entity":"orders","select":[{"expr":"order_count"}]}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"order_count","field":"order_id"}],"filters":{"clauses":[{"field":"order_date","op":">=","type":"comparison","value":"2022-03-01"},{"field":"order_date","op":"<=","type":"comparison","value":"2022-03-31"}],"op":"and","type":"logical"},"op":"query","root_entity":"orders","select":[{"expr":"order_count"}]}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}","requires":["planner_input_v1"],"run_id":"9df66f32","source":{"case_id":"agg_035","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_035_8dbaf955/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_035_8dbaf955","tag":null},"timestamp":"2026-01-23T10:25:54.776593Z","type":"replay_point","v":1},"source":{"case_id":"agg_035","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_035_8dbaf955/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file diff --git a/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__b6b6420e.json b/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__b6b6420e.json deleted file mode 100644 index 2c2e0764..00000000 --- a/tests/fixtures/replay_points/fixed/plan_normalize.spec_v1__b6b6420e.json +++ /dev/null @@ -1 +0,0 @@ -{"extras":{"planner_input_v1":{"case_id":"qa_001","id":"planner_input_v1","input":{"feature_name":"qa_001","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Расскажи всё о клиенте 42: профиль (city, segment, signup_date), количество заказов, общая сумма, дата/статус/канал последнего заказа и 5 самых дорогих покупок (по line_total)."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:59.312378Z","type":"planner_input","v":1}},"resources":{},"root":{"case_id":"qa_001","diag":{"selectors_valid_after":true,"selectors_valid_before":true},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"order_count","field":"order_id"},{"agg":"sum","alias":"total_spent","field":"order_total"}],"filters":{"field":"customer_id","op":"=","type":"comparison","value":42},"group_by":[{"alias":"customer_id","field":"customer_id"}],"limit":1,"op":"query","relations":["orders_to_customers"],"root_entity":"customers","select":[{"alias":"city","expr":"city"},{"alias":"segment","expr":"segment"},{"alias":"signup_date","expr":"signup_date"}]}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"order_count","field":"order_id"},{"agg":"sum","alias":"total_spent","field":"order_total"}],"filters":{"field":"customer_id","op":"=","type":"comparison","value":42},"group_by":[{"alias":"customer_id","field":"customer_id"}],"limit":1,"op":"query","relations":["orders_to_customers"],"root_entity":"customers","select":[{"alias":"city","expr":"city"},{"alias":"segment","expr":"segment"},{"alias":"signup_date","expr":"signup_date"}]}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}","requires":["planner_input_v1"],"run_id":"9df66f32","source":{"case_id":"qa_001","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/qa_001_713263cb/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/qa_001_713263cb","tag":null},"timestamp":"2026-01-23T10:25:59.327950Z","type":"replay_point","v":1},"source":{"case_id":"qa_001","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/qa_001_713263cb/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file diff --git a/tests/fixtures/replay_points/fixed/resources/plan_normalize.spec_v1__027f241b/sample_resource.json b/tests/fixtures/replay_points/fixed/resources/plan_normalize.spec_v1__027f241b/sample_resource.json deleted file mode 100644 index 15390e6e..00000000 --- a/tests/fixtures/replay_points/fixed/resources/plan_normalize.spec_v1__027f241b/sample_resource.json +++ /dev/null @@ -1 +0,0 @@ -{"message": "demo resource"} \ No newline at end of file diff --git a/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__5f883b82.json b/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__5f883b82.json deleted file mode 100644 index aeb5a899..00000000 --- a/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__5f883b82.json +++ /dev/null @@ -1 +0,0 @@ -{"extras":{"planner_input_v1":{"case_id":"items_002","id":"planner_input_v1","input":{"feature_name":"items_002","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Сколько заказов содержат больше 3 позиций (кол-во order_items по order_id > 3)? Ответь только числом."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:55.690244Z","type":"planner_input","v":1}},"resources":{},"root":{"case_id":"items_002","diag":{"selectors_valid_after":false,"selectors_valid_before":false},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"item_count","field":"order_item_id"}],"case_sensitivity":false,"filters":{"clauses":[{"entity":"order_items","field":"order_id","op":"is_not_null","type":"comparison"}],"op":"and","type":"logical"},"group_by":[{"alias":"order_id","field":"order_id"}],"op":"query","root_entity":"order_items"}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"item_count","field":"order_item_id"}],"case_sensitivity":false,"filters":{"clauses":[{"entity":"order_items","field":"order_id","op":"is_not_null","type":"comparison"}],"op":"and","type":"logical"},"group_by":[{"alias":"order_id","field":"order_id"}],"op":"query","root_entity":"order_items"}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"error\", \"selectors_validate_after\": \"error\", \"selectors_normalization_decision\": \"keep_original_still_invalid\", \"selectors_before\": {\"op\": \"query\", \"root_entity\": \"order_items\", \"group_by\": [{\"field\": \"order_id\", \"alias\": \"order_id\"}], \"aggregations\": [{\"field\": \"order_item_id\", \"agg\": \"count\", \"alias\": \"item_count\"}], \"filters\": {\"type\": \"logical\", \"op\": \"and\", \"clauses\": [{\"type\": \"comparison\", \"entity\": \"order_items\", \"field\": \"order_id\", \"op\": \"is_not_null\"}]}, \"case_sensitivity\": false}, \"selectors_after\": {\"op\": \"query\", \"root_entity\": \"order_items\", \"group_by\": [{\"field\": \"order_id\", \"alias\": \"order_id\"}], \"aggregations\": [{\"field\": \"order_item_id\", \"agg\": \"count\", \"alias\": \"item_count\"}], \"filters\": {\"type\": \"logical\", \"op\": \"and\", \"clauses\": [{\"type\": \"comparison\", \"entity\": \"order_items\", \"field\": \"order_id\", \"op\": \"is_not_null\"}]}, \"case_sensitivity\": false}}","requires":["planner_input_v1"],"run_id":"9df66f32","source":{"case_id":"items_002","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/items_002_2b870ad2/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/items_002_2b870ad2","tag":null},"timestamp":"2026-01-23T10:25:55.699897Z","type":"replay_point","v":1},"source":{"case_id":"items_002","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/items_002_2b870ad2/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file diff --git a/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b18ec08f.json b/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b18ec08f.json deleted file mode 100644 index 0456b8be..00000000 --- a/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b18ec08f.json +++ /dev/null @@ -1 +0,0 @@ -{"extras":{"planner_input_v1":{"case_id":"agg_005","id":"planner_input_v1","input":{"feature_name":"agg_005","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Сколько уникальных клиентов сделали хотя бы один заказ? Ответь только числом."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:54.062206Z","type":"planner_input","v":1}},"resources":{},"root":{"case_id":"agg_005","diag":{"selectors_valid_after":false,"selectors_valid_before":false},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count_distinct","alias":"unique_customers","field":"customer_id"}],"filters":{"entity":"orders","field":"order_id","op":"is_not_null","type":"comparison"},"group_by":[],"op":"query","relations":["orders_to_customers"],"root_entity":"customers","select":[{"alias":"customer_id","expr":"customers.customer_id"}]}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count_distinct","alias":"unique_customers","field":"customer_id"}],"filters":{"entity":"orders","field":"order_id","op":"is_not_null","type":"comparison"},"op":"query","relations":["orders_to_customers"],"root_entity":"customers","select":[{"alias":"customer_id","expr":"customers.customer_id"}]}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"error\", \"selectors_validate_after\": \"error\", \"selectors_normalization_decision\": \"use_normalized_unvalidated\", \"selectors_before\": {\"op\": \"query\", \"root_entity\": \"customers\", \"select\": [{\"expr\": \"customers.customer_id\", \"alias\": \"customer_id\"}], \"relations\": [\"orders_to_customers\"], \"aggregations\": [{\"field\": \"customer_id\", \"agg\": \"count_distinct\", \"alias\": \"unique_customers\"}], \"filters\": {\"type\": \"comparison\", \"entity\": \"orders\", \"field\": \"order_id\", \"op\": \"is_not_null\"}}, \"selectors_after\": {\"op\": \"query\", \"root_entity\": \"customers\", \"select\": [{\"expr\": \"customers.customer_id\", \"alias\": \"customer_id\"}], \"relations\": [\"orders_to_customers\"], \"aggregations\": [{\"field\": \"customer_id\", \"agg\": \"count_distinct\", \"alias\": \"unique_customers\"}], \"filters\": {\"type\": \"comparison\", \"entity\": \"orders\", \"field\": \"order_id\", \"op\": \"is_not_null\"}, \"group_by\": []}}","requires":["planner_input_v1"],"run_id":"9df66f32","source":{"case_id":"agg_005","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_005_b2da5838/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_005_b2da5838","tag":null},"timestamp":"2026-01-23T10:25:54.077062Z","type":"replay_point","v":1},"source":{"case_id":"agg_005","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_005_b2da5838/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file diff --git a/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b4d64d21.json b/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b4d64d21.json deleted file mode 100644 index 363a159c..00000000 --- a/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__b4d64d21.json +++ /dev/null @@ -1 +0,0 @@ -{"extras":{"planner_input_v1":{"case_id":"geo_001","id":"planner_input_v1","input":{"feature_name":"geo_001","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Сколько заказов было у клиентов из города San Antonio в месяце 2022-03? Ответь только числом."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:55.718816Z","type":"planner_input","v":1}},"resources":{},"root":{"case_id":"geo_001","diag":{"selectors_valid_after":true,"selectors_valid_before":true},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"case_sensitivity":false,"filters":{"clauses":[{"entity":"customers","field":"city","op":"=","type":"comparison","value":"San Antonio"},{"field":"order_date","op":"starts_with","type":"comparison","value":"2022-03"}],"op":"and","type":"logical"},"op":"query","relations":["orders_to_customers"],"root_entity":"orders","select":[{"alias":"order_count","expr":"count(order_id)"}]}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"case_sensitivity":false,"filters":{"clauses":[{"entity":"customers","field":"city","op":"=","type":"comparison","value":"San Antonio"},{"field":"order_date","op":"starts_with","type":"comparison","value":"2022-03"}],"op":"and","type":"logical"},"op":"query","relations":["orders_to_customers"],"root_entity":"orders","select":[{"alias":"order_count","expr":"count(order_id)"}]}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}","requires":["planner_input_v1"],"run_id":"9df66f32","source":{"case_id":"geo_001","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/geo_001_f83d8b50/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/geo_001_f83d8b50","tag":null},"timestamp":"2026-01-23T10:25:55.730263Z","type":"replay_point","v":1},"source":{"case_id":"geo_001","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/geo_001_f83d8b50/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file diff --git a/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__dfd64ee7.json b/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__dfd64ee7.json deleted file mode 100644 index 85cc7c94..00000000 --- a/tests/fixtures/replay_points/known_bad/plan_normalize.spec_v1__dfd64ee7.json +++ /dev/null @@ -1 +0,0 @@ -{"extras":{"planner_input_v1":{"case_id":"agg_003","id":"planner_input_v1","input":{"feature_name":"agg_003","options":{"plan_only":false},"provider_catalog":{"demo_qa":{"capabilities":["schema","row_query","aggregate","semantic_search"],"name":"demo_qa","selectors_schema":{"oneOf":[{"description":"Request schema description.","properties":{"op":{"const":"schema","default":"schema","title":"Op","type":"string"}},"title":"SchemaRequest","type":"object"},{"description":"Request semantic search results only.","properties":{"entity":{"enum":["customers","products","orders","order_items"],"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"op":{"const":"semantic_only","default":"semantic_only","title":"Op","type":"string"},"query":{"title":"Query","type":"string"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticOnlyRequest","type":"object"},{"$defs":{"AggregationSpec":{"description":"Aggregation specification.","properties":{"agg":{"title":"Agg","type":"string"},"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"field":{"title":"Field","type":"string"}},"required":["field","agg"],"title":"AggregationSpec","type":"object"},"ComparisonFilter":{"description":"Basic comparison filter.","properties":{"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"},"op":{"title":"Op","type":"string"},"type":{"const":"comparison","default":"comparison","title":"Type","type":"string"},"value":{"title":"Value"}},"required":["field","op","value"],"title":"ComparisonFilter","type":"object"},"GroupBySpec":{"description":"Group by specification.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"entity":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Entity"},"field":{"title":"Field","type":"string"}},"required":["field"],"title":"GroupBySpec","type":"object"},"LogicalFilter":{"description":"Logical combination of filters.","properties":{"clauses":{"items":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"}]},"title":"Clauses","type":"array"},"op":{"default":"and","enum":["and","or"],"title":"Op","type":"string"},"type":{"const":"logical","default":"logical","title":"Type","type":"string"}},"title":"LogicalFilter","type":"object"},"SelectExpr":{"description":"Returned expression.","properties":{"alias":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"title":"Alias"},"expr":{"title":"Expr","type":"string"}},"required":["expr"],"title":"SelectExpr","type":"object"},"SemanticClause":{"description":"Semantic search clause for an entity.","properties":{"entity":{"title":"Entity","type":"string"},"fields":{"items":{"type":"string"},"title":"Fields","type":"array"},"mode":{"default":"filter","enum":["filter","boost"],"title":"Mode","type":"string"},"query":{"title":"Query","type":"string"},"threshold":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"title":"Threshold"},"top_k":{"default":100,"title":"Top K","type":"integer"}},"required":["entity","query"],"title":"SemanticClause","type":"object"}},"description":"Main relational query request.","properties":{"aggregations":{"items":{"$ref":"#/$defs/AggregationSpec"},"title":"Aggregations","type":"array"},"case_sensitivity":{"default":false,"title":"Case Sensitivity","type":"boolean"},"filters":{"anyOf":[{"$ref":"#/$defs/ComparisonFilter"},{"$ref":"#/$defs/LogicalFilter"},{"type":"null"}],"default":null,"title":"Filters"},"group_by":{"items":{"$ref":"#/$defs/GroupBySpec"},"title":"Group By","type":"array"},"limit":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":1000,"title":"Limit"},"offset":{"anyOf":[{"type":"integer"},{"type":"null"}],"default":0,"title":"Offset"},"op":{"const":"query","default":"query","title":"Op","type":"string"},"relations":{"items":{"enum":["orders_to_customers","items_to_orders","items_to_products"],"type":"string"},"title":"Relations","type":"array"},"root_entity":{"enum":["customers","products","orders","order_items"],"title":"Root Entity","type":"string"},"select":{"items":{"$ref":"#/$defs/SelectExpr"},"title":"Select","type":"array"},"semantic_clauses":{"items":{"$ref":"#/$defs/SemanticClause"},"title":"Semantic Clauses","type":"array"}},"required":["root_entity"],"title":"RelationalQuery","type":"object"}]}}},"schema_ref":"schema_v1","user_query":"Сколько всего товаров (products) в датасете? Ответь только числом."},"run_id":"9df66f32","timestamp":"2026-01-23T10:25:54.010082Z","type":"planner_input","v":1}},"resources":{},"root":{"case_id":"agg_003","diag":{"selectors_valid_after":false,"selectors_valid_before":false},"expected":{"out_spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"total_products","field":"product_id"}],"entity":"products","filters":null,"group_by":[],"op":"query"}}},"id":"plan_normalize.spec_v1","input":{"normalizer_rules":{"demo_qa":"relational_v1"},"options":{"allow_unknown_providers":false,"coerce_provider_case":true,"dedupe_context_plan":true,"dedupe_required_context":true,"default_mode":"full","filter_selectors_by_schema":true,"trim_text_fields":true},"spec":{"mode":"slice","provider":"demo_qa","selectors":{"aggregations":[{"agg":"count","alias":"total_products","field":"product_id"}],"entity":"products","op":"aggregate"}}},"meta":{"mode":"slice","provider":"demo_qa","spec_idx":0},"note":"{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"error\", \"selectors_validate_after\": \"error\", \"selectors_normalization_decision\": \"use_normalized_unvalidated\", \"selectors_before\": {\"op\": \"aggregate\", \"entity\": \"products\", \"aggregations\": [{\"field\": \"product_id\", \"agg\": \"count\", \"alias\": \"total_products\"}]}, \"selectors_after\": {\"op\": \"query\", \"entity\": \"products\", \"aggregations\": [{\"field\": \"product_id\", \"agg\": \"count\", \"alias\": \"total_products\"}], \"group_by\": [], \"filters\": null}}","requires":["planner_input_v1"],"run_id":"9df66f32","source":{"case_id":"agg_003","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_003_22b4ce14/events.jsonl","picked":"latest_non_missed","run_dir":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_003_22b4ce14","tag":null},"timestamp":"2026-01-23T10:25:54.022209Z","type":"replay_point","v":1},"source":{"case_id":"agg_003","event_index":4,"events_path":"_demo_data/shop/.runs/runs/20260123_132552_retail_cases/cases/agg_003_22b4ce14/events.jsonl","picked":"latest_non_missed","tag":null},"type":"replay_bundle","v":1} \ No newline at end of file diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index c6ba7468..9f1ade32 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -1,214 +1,47 @@ from __future__ import annotations -import difflib import json -import os from pathlib import Path -from typing import Iterable import pytest -from _pytest.mark.structures import ParameterSet -from pydantic import TypeAdapter +from pydantic import ValidationError -import fetchgraph.replay.handlers.plan_normalize # noqa: F401 -from fetchgraph.relational.models import RelationalRequest -from fetchgraph.replay.runtime import REPLAY_HANDLERS, ReplayContext +import fetchgraph.tracer.handlers # noqa: F401 +from fetchgraph.tracer.runtime import load_case_bundle, run_case +from fetchgraph.tracer.validators import validate_plan_normalize_spec_v1 -FIXTURES_ROOT = Path(__file__).parent / "fixtures" / "replay_points" -_BUCKETS = ("fixed", "known_bad") -_REL_ADAPTER = TypeAdapter(RelationalRequest) -DEBUG_REPLAY = os.getenv("DEBUG_REPLAY", "").lower() in ("1", "true", "yes", "on") +FIXTURES_ROOT = Path(__file__).parent / "fixtures" / "replay_cases" +KNOWN_BAD_DIR = FIXTURES_ROOT / "known_bad" +FIXED_DIR = FIXTURES_ROOT / "fixed" -def _is_replay_payload(payload: object) -> bool: - if not isinstance(payload, dict): - return False - if payload.get("type") == "replay_point": - return True - if payload.get("type") == "replay_bundle": - root = payload.get("root") or {} - return isinstance(root, dict) and root.get("type") == "replay_point" - return False +def _iter_case_paths(directory: Path) -> list[Path]: + if not directory.exists(): + return [] + return sorted(directory.glob("*.case.json")) -def _iter_fixture_paths() -> tuple[list[tuple[str, Path]], list[Path], list[tuple[Path, str]]]: - if not FIXTURES_ROOT.exists(): - return [], [], [] - paths: list[tuple[str, Path]] = [] - ignored: list[Path] = [] - invalid_json: list[tuple[Path, str]] = [] - for path in FIXTURES_ROOT.glob("*.json"): - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - invalid_json.append((path, f"{exc.msg} (line {exc.lineno}, col {exc.colno})")) - continue - if _is_replay_payload(payload): - paths.append(("root", path)) - else: - ignored.append(path) - for bucket in _BUCKETS: - bucket_dir = FIXTURES_ROOT / bucket - if not bucket_dir.exists(): - continue - for path in bucket_dir.rglob("*.json"): - if "resources" in path.parts: - ignored.append(path) - continue - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - invalid_json.append((path, f"{exc.msg} (line {exc.lineno}, col {exc.colno})")) - continue - if _is_replay_payload(payload): - paths.append((bucket, path)) - else: - ignored.append(path) - return sorted(paths), sorted(ignored), sorted(invalid_json) +def _expected_path(case_path: Path) -> Path: + if not case_path.name.endswith(".case.json"): + raise ValueError(f"Unexpected case filename: {case_path}") + return case_path.with_name(case_path.name.replace(".case.json", ".expected.json")) -def _format_json(payload: object) -> str: - return json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) +@pytest.mark.known_bad +@pytest.mark.parametrize("case_path", _iter_case_paths(KNOWN_BAD_DIR)) +def test_known_bad_cases(case_path: Path) -> None: + root, ctx = load_case_bundle(case_path) + out = run_case(root, ctx) + with pytest.raises((AssertionError, ValidationError)): + validate_plan_normalize_spec_v1(out) -def _truncate(text: str, limit: int = 2000) -> str: - if len(text) <= limit: - return text - return f"{text[:limit]}\n... (truncated {len(text) - limit} chars)" - - -def _selectors_diff(expected: object, actual: object) -> str: - expected_text = _format_json(expected).splitlines() - actual_text = _format_json(actual).splitlines() - diff = "\n".join( - difflib.unified_diff(expected_text, actual_text, fromfile="expected", tofile="actual", lineterm="") - ) - return _truncate(diff) - - -def _load_fixture(path: Path) -> dict: - return json.loads(path.read_text(encoding="utf-8")) - - -def _parse_fixture(event: dict, *, base_dir: Path) -> tuple[dict, ReplayContext]: - if event.get("type") == "replay_bundle": - ctx = ReplayContext( - resources=event.get("resources") or {}, - extras=event.get("extras") or {}, - base_dir=base_dir, - ) - return event["root"], ctx - return event, ReplayContext(base_dir=base_dir) - - -def _fixture_paths() -> list[ParameterSet]: - paths, ignored, invalid_json = _iter_fixture_paths() - if not paths: - pytest.skip( - "No replay fixtures found in tests/fixtures/replay_points/{fixed,known_bad}", - allow_module_level=True, - ) - if DEBUG_REPLAY and ignored: - ignored_list = "\n".join(f"- {path}" for path in ignored) - print(f"\n=== DEBUG ignored json files ===\n{ignored_list}") - if DEBUG_REPLAY and invalid_json: - invalid_list = "\n".join(f"- {path}: {error}" for path, error in invalid_json) - print(f"\n=== DEBUG invalid json files ===\n{invalid_list}") - params: list[ParameterSet] = [] - for bucket, path in paths: - marks = (pytest.mark.known_bad,) if bucket == "known_bad" else () - params.append(pytest.param((bucket, path), id=f"{bucket}/{path.name}", marks=marks)) - return params - - -def _rerun_hint(bucket: str, path: Path) -> str: - return f"pytest -vv {__file__}::test_replay_fixture[{bucket}/{path.name}] -s" - - -@pytest.mark.parametrize("fixture_info", _fixture_paths()) -def test_replay_fixture(fixture_info: tuple[str, Path]) -> None: - bucket, path = fixture_info - raw = _load_fixture(path) - event, ctx = _parse_fixture(raw, base_dir=path.parent) - assert event.get("type") == "replay_point" - event_id = event.get("id") - assert event_id in REPLAY_HANDLERS - - handler = REPLAY_HANDLERS[event_id] - result = handler(event["input"], ctx) - expected = event["expected"] - - actual_spec = result.get("out_spec") - expected_spec = expected.get("out_spec") - assert isinstance(expected_spec, dict) - assert isinstance(actual_spec, dict) - if DEBUG_REPLAY: - print(f"\n=== DEBUG {path} ===") - print("meta:", _format_json(event.get("meta"))) - print("note:", event.get("note")) - print("input:", _truncate(_format_json(event.get("input")), 8000)) - if actual_spec != expected_spec: - meta = _format_json(event.get("meta")) - note = event.get("note") - inp = _truncate(_format_json(event.get("input")), limit=8000) - diff = _selectors_diff(expected_spec.get("selectors"), actual_spec.get("selectors")) - pytest.fail( - "\n".join( - [ - f"Replay mismatch for {path.name}", - f"rerun: {_rerun_hint(bucket, path)}", - f"meta: {meta}", - f"note: {note}", - "input:", - inp, - "selectors diff:", - diff, - ] - ) - ) - - if event_id == "plan_normalize.spec_v1": - provider = actual_spec.get("provider") or event["input"]["spec"]["provider"] - rule_kind = (event["input"].get("normalizer_rules") or {}).get(provider) - if rule_kind == "relational_v1": - _REL_ADAPTER.validate_python(actual_spec["selectors"]) - - -def test_replay_fixture_resources_exist() -> None: - paths, _, _ = _iter_fixture_paths() - resource_checks: list[tuple[Path, Path]] = [] - for _, path in paths: - raw = _load_fixture(path) - event, ctx = _parse_fixture(raw, base_dir=path.parent) - if raw.get("type") != "replay_bundle": - continue - resources = raw.get("resources") or {} - if not isinstance(resources, dict): - continue - for resource in resources.values(): - if not isinstance(resource, dict): - continue - data_ref = resource.get("data_ref") - if not isinstance(data_ref, dict): - continue - file_name = data_ref.get("file") - if not isinstance(file_name, str) or not file_name: - continue - resolved = ctx.resolve_resource_path(file_name) - resource_checks.append((path, resolved)) - - if not resource_checks: - pytest.skip("No replay fixtures with file resources found.") - - missing = [(fixture, resource) for fixture, resource in resource_checks if not resource.exists()] - if missing: - details = "\n".join(f"- {fixture}: {resource}" for fixture, resource in missing) - pytest.fail(f"Missing replay resources:\n{details}") - - -def test_replay_fixture_json_valid() -> None: - _, _, invalid_json = _iter_fixture_paths() - if not invalid_json: - return - details = "\n".join(f"- {path}: {error}" for path, error in invalid_json) - pytest.fail(f"Invalid replay fixture JSON:\n{details}") +@pytest.mark.parametrize("case_path", _iter_case_paths(FIXED_DIR)) +def test_replay_cases_expected(case_path: Path) -> None: + expected_path = _expected_path(case_path) + if not expected_path.exists(): + pytest.skip(f"Expected fixture missing: {expected_path}") + root, ctx = load_case_bundle(case_path) + out = run_case(root, ctx) + expected = json.loads(expected_path.read_text(encoding="utf-8")) + assert out == expected From da9edfcbc19cb3ef0c1c3348cbd227160cfd928a Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 11:57:05 +0300 Subject: [PATCH 26/79] Fix replay case export filtering and tooling --- Makefile | 9 +++++---- src/fetchgraph/cli.py | 2 +- src/fetchgraph/replay/export.py | 5 +++++ src/fetchgraph/replay/log.py | 16 --------------- src/fetchgraph/tracer/validators.py | 4 ++++ tests/test_replay_fixtures.py | 31 +++++++++++++++++++++++++++++ 6 files changed, 46 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 807f3eae..c4036239 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,7 @@ SPEC_IDX ?= PROVIDER ?= BUCKET ?= fixed OUT_DIR ?= tests/fixtures/replay_cases/$(BUCKET) +TRACER_OUT_DIR ?= tests/fixtures/replay_cases/$(BUCKET) RUN_DIR ?= SCOPE ?= both WITH_RESOURCES ?= 1 @@ -160,8 +161,8 @@ help: @echo " make tags [PATTERN=*] DATA=... - показать список тегов" @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" - @echo " make fixture CASE=agg_01 [TAG=...] [RUN_ID=...] [REPLAY_ID=plan_normalize.spec_v1] [WITH=requires] [SPEC_IDX=0] [PROVIDER=relational] [BUCKET=fixed|known_bad] [OUT_DIR=tests/fixtures/replay_cases/$$(BUCKET)] [ALL=1]" - @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources//..." + @echo " make fixture CASE=agg_01 ... (legacy replay_point fixtures; use tracer-export for case bundles)" + @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources//..." @echo " make fixture-rm NAME=... [PATTERN=...] [BUCKET=fixed|known_bad] [SCOPE=replay|traces|both] [WITH_RESOURCES=1] [DRY=1] (удаляет fixture и resources)" @echo " make fixture-fix NAME=... [PATTERN=...] [CASE=...] [MOVE_TRACES=1] [DRY=1] (переносит fixture и resources)" @echo " make fixture-migrate [BUCKET=fixed|known_bad] [DRY=1] (миграция ресурсов в resources//)" @@ -362,8 +363,8 @@ fixture: check tracer-export: @test -n "$(strip $(ID))" || (echo "ID обязателен: make tracer-export ID=plan_normalize.spec_v1" && exit 1) @test -n "$(strip $(EVENTS))" || (echo "EVENTS обязателен: make tracer-export EVENTS=path/to/events.jsonl" && exit 1) - @test -n "$(strip $(OUT))" || (echo "OUT обязателен: make tracer-export OUT=path/to/out_dir" && exit 1) - @fetchgraph-tracer export-case-bundle --events "$(EVENTS)" --out "$(OUT)" --id "$(ID)" \ + @test -n "$(strip $(TRACER_OUT_DIR))" || (echo "TRACER_OUT_DIR обязателен: make tracer-export TRACER_OUT_DIR=path/to/out_dir" && exit 1) + @$(PYTHON) -m fetchgraph.tracer.cli export-case-bundle --events "$(EVENTS)" --out "$(TRACER_OUT_DIR)" --id "$(ID)" \ $(if $(strip $(SPEC_IDX)),--spec-idx $(SPEC_IDX),) \ $(if $(strip $(PROVIDER)),--provider $(PROVIDER),) \ $(if $(strip $(RUN_DIR)),--run-dir $(RUN_DIR),) diff --git a/src/fetchgraph/cli.py b/src/fetchgraph/cli.py index dff40227..c39f5ef6 100644 --- a/src/fetchgraph/cli.py +++ b/src/fetchgraph/cli.py @@ -21,7 +21,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: default=None, help="Filter replay_case by meta.provider (case-insensitive)", ) - fixture.add_argument("--all", action="store_true", help="Export all matching replay points") + fixture.add_argument("--all", action="store_true", help="Export all matching replay cases") fixture.add_argument( "--out-dir", type=Path, diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py index c3a365bb..53921cbd 100644 --- a/src/fetchgraph/replay/export.py +++ b/src/fetchgraph/replay/export.py @@ -36,6 +36,8 @@ class ExportSelection: def _match_meta(event: dict, *, spec_idx: int | None, provider: str | None) -> bool: + if spec_idx is None and provider is None: + return True meta = event.get("meta") if not isinstance(meta, dict): return False @@ -115,6 +117,9 @@ def write_case_bundle( extras: Dict[str, dict], source: dict, ) -> None: + if out_path.exists(): + print(f"Fixture already exists: {out_path}") + return payload = { "schema": "fetchgraph.tracer.case_bundle", "v": 1, diff --git a/src/fetchgraph/replay/log.py b/src/fetchgraph/replay/log.py index 982178c0..2cda3f35 100644 --- a/src/fetchgraph/replay/log.py +++ b/src/fetchgraph/replay/log.py @@ -73,19 +73,3 @@ def log_replay_case( event["diag"] = diag logger.emit(event) - -def log_replay_point( - logger: EventLoggerLike, - *, - id: str, - meta: dict, - input: dict, - expected: dict, - requires: list[str] | None = None, - diag: dict | None = None, - note: str | None = None, -) -> None: - raise ValueError( - "log_replay_point has been replaced by log_replay_case; " - "log_replay_point no longer supports expected payloads." - ) diff --git a/src/fetchgraph/tracer/validators.py b/src/fetchgraph/tracer/validators.py index 49ab06bc..01c32bc5 100644 --- a/src/fetchgraph/tracer/validators.py +++ b/src/fetchgraph/tracer/validators.py @@ -14,4 +14,8 @@ def validate_plan_normalize_spec_v1(out: dict) -> None: selectors = out_spec.get("selectors") if selectors is None: raise AssertionError("out_spec.selectors is required") + if not isinstance(selectors, dict): + raise AssertionError("out_spec.selectors must be a dict") + if "root_entity" not in selectors: + return TypeAdapter(RelationalRequest).validate_python(selectors) diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index 9f1ade32..c4b429e0 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -21,6 +21,10 @@ def _iter_case_paths(directory: Path) -> list[Path]: return sorted(directory.glob("*.case.json")) +def _iter_all_case_paths() -> list[Path]: + return _iter_case_paths(FIXED_DIR) + _iter_case_paths(KNOWN_BAD_DIR) + + def _expected_path(case_path: Path) -> Path: if not case_path.name.endswith(".case.json"): raise ValueError(f"Unexpected case filename: {case_path}") @@ -45,3 +49,30 @@ def test_replay_cases_expected(case_path: Path) -> None: out = run_case(root, ctx) expected = json.loads(expected_path.read_text(encoding="utf-8")) assert out == expected + + +def test_replay_case_resources_exist() -> None: + case_paths = _iter_all_case_paths() + if not case_paths: + pytest.skip("No replay case bundles found.") + missing: list[tuple[Path, Path]] = [] + for case_path in case_paths: + raw = json.loads(case_path.read_text(encoding="utf-8")) + resources = raw.get("resources") or {} + if not isinstance(resources, dict): + continue + for resource in resources.values(): + if not isinstance(resource, dict): + continue + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + continue + resolved = case_path.parent / file_name + if not resolved.exists(): + missing.append((case_path, resolved)) + if missing: + details = "\n".join(f"- {fixture}: {resource}" for fixture, resource in missing) + pytest.fail(f"Missing replay resources:\n{details}") From d08c2379a531440a58e592aabb5cc2968d7765c5 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 12:08:11 +0300 Subject: [PATCH 27/79] Remove legacy fixture tooling and harden exports --- Makefile | 38 +++++++-------------------------- src/fetchgraph/replay/export.py | 14 ++++++++++++ tests/test_replay_fixtures.py | 9 +++++--- 3 files changed, 28 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index c4036239..593a952e 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,6 @@ DEFAULT_CASES := examples/demo_qa/cases/retail_cases.json VENV ?= .venv PYTHON ?= $(if $(wildcard $(VENV)/bin/python),$(VENV)/bin/python,python) CLI := $(PYTHON) -m examples.demo_qa.cli -CLI_FIXT := $(PYTHON) -m examples.demo_qa.fixture_tools # ============================================================================== # 4) Пути demo_qa (можно переопределять через CLI или в $(CONFIG)) @@ -49,13 +48,11 @@ NOTE ?= CASE ?= NAME ?= PATTERN ?= -RUN_ID ?= -REPLAY_ID ?= plan_normalize.spec_v1 -WITH ?= SPEC_IDX ?= PROVIDER ?= BUCKET ?= fixed -OUT_DIR ?= tests/fixtures/replay_cases/$(BUCKET) +ID ?= +EVENTS ?= TRACER_OUT_DIR ?= tests/fixtures/replay_cases/$(BUCKET) RUN_DIR ?= SCOPE ?= both @@ -114,7 +111,7 @@ LIMIT_FLAG := $(if $(strip $(LIMIT)),--limit $(LIMIT),) batch batch-tag batch-failed batch-failed-from \ batch-missed batch-missed-from batch-failed-tag batch-missed-tag \ batch-fail-fast batch-max-fails \ - stats history-case report-tag report-tag-changes tags tag-rm case-run case-open fixture fixture-rm fixture-fix fixture-migrate tracer-export compare compare-tag + stats history-case report-tag report-tag-changes tags tag-rm case-run case-open tracer-export compare compare-tag # ============================================================================== # help (на русском) @@ -161,11 +158,9 @@ help: @echo " make tags [PATTERN=*] DATA=... - показать список тегов" @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" - @echo " make fixture CASE=agg_01 ... (legacy replay_point fixtures; use tracer-export for case bundles)" + @echo " make tracer-export ID=... EVENTS=... TRACER_OUT_DIR=... [SPEC_IDX=...] [PROVIDER=...] [RUN_DIR=...]" + @echo " (или напрямую: fetchgraph-tracer export-case-bundle --events ... --out ... --id ...)" @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources//..." - @echo " make fixture-rm NAME=... [PATTERN=...] [BUCKET=fixed|known_bad] [SCOPE=replay|traces|both] [WITH_RESOURCES=1] [DRY=1] (удаляет fixture и resources)" - @echo " make fixture-fix NAME=... [PATTERN=...] [CASE=...] [MOVE_TRACES=1] [DRY=1] (переносит fixture и resources)" - @echo " make fixture-migrate [BUCKET=fixed|known_bad] [DRY=1] (миграция ресурсов в resources//)" @echo "" @echo "Уборка:" @echo " make tag-rm TAG=... [DRY=1] [PURGE_RUNS=1] [PRUNE_HISTORY=1] [PRUNE_CASE_HISTORY=1]" @@ -223,6 +218,9 @@ show-config: @echo "SCHEMA = $(SCHEMA)" @echo "CASES = $(CASES)" @echo "OUT = $(OUT)" + @echo "EVENTS = $(EVENTS)" + @echo "ID = $(ID)" + @echo "TRACER_OUT_DIR = $(TRACER_OUT_DIR)" @echo "LLM_TOML= $(LLM_TOML)" @echo "TAG = $(TAG)" @echo "NOTE = $(NOTE)" @@ -350,16 +348,6 @@ case-open: check @test -n "$(strip $(CASE))" || (echo "Нужно задать CASE=case_42" && exit 1) @$(CLI) case open "$(CASE)" --data "$(DATA)" -# 11) Replay fixtures -fixture: check - @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture CASE=agg_01" && exit 1) - @$(PYTHON) -m examples.demo_qa.fixture_cli --case "$(CASE)" --data "$(DATA)" \ - $(TAG_FLAG) $(if $(strip $(RUN_ID)),--run-id "$(RUN_ID)",) $(if $(strip $(REPLAY_ID)),--id "$(REPLAY_ID)",) \ - $(if $(strip $(SPEC_IDX)),--spec-idx "$(SPEC_IDX)",) $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ - $(if $(strip $(OUT_DIR)),--out-dir "$(OUT_DIR)",) \ - $(if $(filter 1 true yes on,$(ALL)),--all,) \ - $(if $(filter requires,$(WITH)),--with-requires,) - tracer-export: @test -n "$(strip $(ID))" || (echo "ID обязателен: make tracer-export ID=plan_normalize.spec_v1" && exit 1) @test -n "$(strip $(EVENTS))" || (echo "EVENTS обязателен: make tracer-export EVENTS=path/to/events.jsonl" && exit 1) @@ -369,16 +357,6 @@ tracer-export: $(if $(strip $(PROVIDER)),--provider $(PROVIDER),) \ $(if $(strip $(RUN_DIR)),--run-dir $(RUN_DIR),) -# 12) Fixture tools -fixture-rm: - @$(CLI_FIXT) rm --name "$(NAME)" --pattern "$(PATTERN)" --bucket "$(BUCKET)" --scope "$(SCOPE)" --with-resources "$(WITH_RESOURCES)" --dry "$(DRY)" - -fixture-fix: - @$(CLI_FIXT) fix --name "$(NAME)" --pattern "$(PATTERN)" --case "$(CASE)" --move-traces "$(MOVE_TRACES)" --dry "$(DRY)" - -fixture-migrate: - @$(CLI_FIXT) migrate --bucket "$(BUCKET)" --dry "$(DRY)" - # compare (diff.md + junit) compare: check @test -n "$(strip $(BASE))" || (echo "Нужно задать BASE=.../results_prev.jsonl" && exit 1) diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py index 53921cbd..7241d7fe 100644 --- a/src/fetchgraph/replay/export.py +++ b/src/fetchgraph/replay/export.py @@ -1,5 +1,6 @@ from __future__ import annotations +import filecmp import hashlib import json import shutil @@ -79,6 +80,15 @@ def collect_requires( if not requires: return resources, extras + if not isinstance(requires, list): + raise ValueError("requires must be a list of objects with kind/id fields") + for req in requires: + if not isinstance(req, dict): + raise ValueError("requires must be a list of objects with kind/id fields") + if req.get("kind") not in {"extra", "resource"}: + raise ValueError("requires entries must include kind='extra' or kind='resource'") + if not isinstance(req.get("id"), str) or not req.get("id"): + raise ValueError("requires entries must include a non-empty id") for _, event in iter_events(events_path): event_type = event.get("type") @@ -156,6 +166,10 @@ def copy_resource_files( raise FileNotFoundError(f"Resource file {src_path} not found for replay bundle") dest_rel = Path("resources") / fixture_stem / rel_path dest_path = out_dir / dest_rel + if dest_path.exists(): + if filecmp.cmp(src_path, dest_path, shallow=False): + continue + raise FileExistsError(f"Resource file collision at {dest_path}") dest_path.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(src_path, dest_path) data_ref["file"] = dest_rel.as_posix() diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index c4b429e0..729cb108 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -43,11 +43,14 @@ def test_known_bad_cases(case_path: Path) -> None: @pytest.mark.parametrize("case_path", _iter_case_paths(FIXED_DIR)) def test_replay_cases_expected(case_path: Path) -> None: expected_path = _expected_path(case_path) - if not expected_path.exists(): - pytest.skip(f"Expected fixture missing: {expected_path}") root, ctx = load_case_bundle(case_path) out = run_case(root, ctx) - expected = json.loads(expected_path.read_text(encoding="utf-8")) + if expected_path.exists(): + expected = json.loads(expected_path.read_text(encoding="utf-8")) + else: + expected = root.get("observed") + if expected is None: + pytest.skip(f"Expected fixture missing and no observed data in {case_path}") assert out == expected From e477861b71efe8d59c7bf374f886c2da54b93669 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 12:21:26 +0300 Subject: [PATCH 28/79] Fix resource path update on existing files --- Makefile | 11 ++++--- src/fetchgraph/cli.py | 7 +++++ .../planning/normalize/plan_normalizer.py | 1 - src/fetchgraph/replay/export.py | 29 ++++++++++++------- src/fetchgraph/tracer/cli.py | 7 +++++ tests/test_replay_fixtures.py | 5 ++++ 6 files changed, 45 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 593a952e..5135c23e 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,7 @@ ID ?= EVENTS ?= TRACER_OUT_DIR ?= tests/fixtures/replay_cases/$(BUCKET) RUN_DIR ?= +ALLOW_BAD_JSON ?= SCOPE ?= both WITH_RESOURCES ?= 1 ALL ?= @@ -158,8 +159,8 @@ help: @echo " make tags [PATTERN=*] DATA=... - показать список тегов" @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" - @echo " make tracer-export ID=... EVENTS=... TRACER_OUT_DIR=... [SPEC_IDX=...] [PROVIDER=...] [RUN_DIR=...]" - @echo " (или напрямую: fetchgraph-tracer export-case-bundle --events ... --out ... --id ...)" + @echo " make tracer-export ID=... EVENTS=... TRACER_OUT_DIR=... [SPEC_IDX=...] [PROVIDER=...] [RUN_DIR=...] [ALLOW_BAD_JSON=1]" + @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources//..." @echo "" @echo "Уборка:" @@ -221,6 +222,7 @@ show-config: @echo "EVENTS = $(EVENTS)" @echo "ID = $(ID)" @echo "TRACER_OUT_DIR = $(TRACER_OUT_DIR)" + @echo "ALLOW_BAD_JSON = $(ALLOW_BAD_JSON)" @echo "LLM_TOML= $(LLM_TOML)" @echo "TAG = $(TAG)" @echo "NOTE = $(NOTE)" @@ -354,8 +356,9 @@ tracer-export: @test -n "$(strip $(TRACER_OUT_DIR))" || (echo "TRACER_OUT_DIR обязателен: make tracer-export TRACER_OUT_DIR=path/to/out_dir" && exit 1) @$(PYTHON) -m fetchgraph.tracer.cli export-case-bundle --events "$(EVENTS)" --out "$(TRACER_OUT_DIR)" --id "$(ID)" \ $(if $(strip $(SPEC_IDX)),--spec-idx $(SPEC_IDX),) \ - $(if $(strip $(PROVIDER)),--provider $(PROVIDER),) \ - $(if $(strip $(RUN_DIR)),--run-dir $(RUN_DIR),) + $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ + $(if $(strip $(RUN_DIR)),--run-dir "$(RUN_DIR)",) \ + $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) # compare (diff.md + junit) compare: check diff --git a/src/fetchgraph/cli.py b/src/fetchgraph/cli.py index c39f5ef6..04d54bd4 100644 --- a/src/fetchgraph/cli.py +++ b/src/fetchgraph/cli.py @@ -21,6 +21,11 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: default=None, help="Filter replay_case by meta.provider (case-insensitive)", ) + fixture.add_argument( + "--allow-bad-json", + action="store_true", + help="Skip invalid JSON lines in events.jsonl", + ) fixture.add_argument("--all", action="store_true", help="Export all matching replay cases") fixture.add_argument( "--out-dir", @@ -42,6 +47,7 @@ def main(argv: list[str] | None = None) -> int: spec_idx=args.spec_idx, provider=args.provider, run_dir=args.run_dir, + allow_bad_json=args.allow_bad_json, ) else: export_replay_case_bundle( @@ -51,6 +57,7 @@ def main(argv: list[str] | None = None) -> int: spec_idx=args.spec_idx, provider=args.provider, run_dir=args.run_dir, + allow_bad_json=args.allow_bad_json, ) return 0 raise SystemExit(f"Unknown command: {args.command}") diff --git a/src/fetchgraph/planning/normalize/plan_normalizer.py b/src/fetchgraph/planning/normalize/plan_normalizer.py index 0cf025f2..94732780 100644 --- a/src/fetchgraph/planning/normalize/plan_normalizer.py +++ b/src/fetchgraph/planning/normalize/plan_normalizer.py @@ -194,7 +194,6 @@ def _normalize_specs( "mode": spec.mode, "selectors": use, }, - "notes_last": note, } requires = None if getattr(replay_logger, "case_id", None): diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py index 7241d7fe..95568b39 100644 --- a/src/fetchgraph/replay/export.py +++ b/src/fetchgraph/replay/export.py @@ -9,7 +9,7 @@ from typing import Dict, Iterable -def iter_events(path: Path) -> Iterable[tuple[int, dict]]: +def iter_events(path: Path, *, allow_bad_json: bool = False) -> Iterable[tuple[int, dict]]: with path.open("r", encoding="utf-8") as handle: for idx, line in enumerate(handle, start=1): line = line.strip() @@ -18,6 +18,8 @@ def iter_events(path: Path) -> Iterable[tuple[int, dict]]: try: yield idx, json.loads(line) except json.JSONDecodeError as exc: + if allow_bad_json: + continue raise ValueError(f"Invalid JSON on line {idx} in {path}: {exc.msg}") from exc @@ -59,9 +61,10 @@ def _select_replay_cases( replay_id: str, spec_idx: int | None = None, provider: str | None = None, + allow_bad_json: bool = False, ) -> list[ExportSelection]: selected: list[ExportSelection] = [] - for line, event in iter_events(events_path): + for line, event in iter_events(events_path, allow_bad_json=allow_bad_json): if event.get("type") != "replay_case": continue if event.get("id") != replay_id: @@ -74,6 +77,8 @@ def _select_replay_cases( def collect_requires( events_path: Path, requires: list[dict], + *, + allow_bad_json: bool = False, ) -> tuple[dict[str, dict], dict[str, dict]]: extras: Dict[str, dict] = {} resources: Dict[str, dict] = {} @@ -90,7 +95,7 @@ def collect_requires( if not isinstance(req.get("id"), str) or not req.get("id"): raise ValueError("requires entries must include a non-empty id") - for _, event in iter_events(events_path): + for _, event in iter_events(events_path, allow_bad_json=allow_bad_json): event_type = event.get("type") event_id = event.get("id") if event_type == "planner_input" and isinstance(event_id, str): @@ -166,12 +171,12 @@ def copy_resource_files( raise FileNotFoundError(f"Resource file {src_path} not found for replay bundle") dest_rel = Path("resources") / fixture_stem / rel_path dest_path = out_dir / dest_rel - if dest_path.exists(): - if filecmp.cmp(src_path, dest_path, shallow=False): - continue - raise FileExistsError(f"Resource file collision at {dest_path}") dest_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src_path, dest_path) + if dest_path.exists(): + if not filecmp.cmp(src_path, dest_path, shallow=False): + raise FileExistsError(f"Resource file collision at {dest_path}") + else: + shutil.copy2(src_path, dest_path) data_ref["file"] = dest_rel.as_posix() @@ -194,12 +199,14 @@ def export_replay_case_bundle( spec_idx: int | None = None, provider: str | None = None, run_dir: Path | None = None, + allow_bad_json: bool = False, ) -> Path: selections = _select_replay_cases( events_path, replay_id=replay_id, spec_idx=spec_idx, provider=provider, + allow_bad_json=allow_bad_json, ) if not selections: details = [] @@ -218,7 +225,7 @@ def export_replay_case_bundle( root_event = selection.event requires = root_event.get("requires") or [] - resources, extras = collect_requires(events_path, requires) + resources, extras = collect_requires(events_path, requires, allow_bad_json=allow_bad_json) fixture_name = case_bundle_name(replay_id, root_event["input"]) if _has_resource_files(resources): if run_dir is None: @@ -249,12 +256,14 @@ def export_replay_case_bundles( spec_idx: int | None = None, provider: str | None = None, run_dir: Path | None = None, + allow_bad_json: bool = False, ) -> list[Path]: selections = _select_replay_cases( events_path, replay_id=replay_id, spec_idx=spec_idx, provider=provider, + allow_bad_json=allow_bad_json, ) if not selections: details = [] @@ -269,7 +278,7 @@ def export_replay_case_bundles( for selection in selections: root_event = selection.event requires = root_event.get("requires") or [] - resources, extras = collect_requires(events_path, requires) + resources, extras = collect_requires(events_path, requires, allow_bad_json=allow_bad_json) fixture_name = case_bundle_name(replay_id, root_event["input"]) if _has_resource_files(resources): if run_dir is None: diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index 56168a08..3e062cb3 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -22,6 +22,11 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="Filter replay_case by meta.provider (case-insensitive)", ) export.add_argument("--run-dir", type=Path, default=None, help="Run dir (required for file resources)") + export.add_argument( + "--allow-bad-json", + action="store_true", + help="Skip invalid JSON lines in events.jsonl", + ) export.add_argument("--all", action="store_true", help="Export all matching replay cases") return parser.parse_args(argv) @@ -38,6 +43,7 @@ def main(argv: list[str] | None = None) -> int: spec_idx=args.spec_idx, provider=args.provider, run_dir=args.run_dir, + allow_bad_json=args.allow_bad_json, ) else: export_replay_case_bundle( @@ -47,6 +53,7 @@ def main(argv: list[str] | None = None) -> int: spec_idx=args.spec_idx, provider=args.provider, run_dir=args.run_dir, + allow_bad_json=args.allow_bad_json, ) return 0 raise SystemExit(f"Unknown command: {args.command}") diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index 729cb108..4ff93fa4 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -25,6 +25,11 @@ def _iter_all_case_paths() -> list[Path]: return _iter_case_paths(FIXED_DIR) + _iter_case_paths(KNOWN_BAD_DIR) +def test_replay_cases_present() -> None: + if not _iter_all_case_paths(): + pytest.skip("No replay case bundles found under tests/fixtures/replay_cases") + + def _expected_path(case_path: Path) -> Path: if not case_path.name.endswith(".case.json"): raise ValueError(f"Unexpected case filename: {case_path}") From fbbc1aefb6cbecb884c7582789852b680179c2de Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 12:26:25 +0300 Subject: [PATCH 29/79] Restore fixture tools and rename replay id var --- Makefile | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 5135c23e..2df7bc78 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,7 @@ DEFAULT_CASES := examples/demo_qa/cases/retail_cases.json VENV ?= .venv PYTHON ?= $(if $(wildcard $(VENV)/bin/python),$(VENV)/bin/python,python) CLI := $(PYTHON) -m examples.demo_qa.cli +CLI_FIXT := $(PYTHON) -m examples.demo_qa.fixture_tools # ============================================================================== # 4) Пути demo_qa (можно переопределять через CLI или в $(CONFIG)) @@ -51,7 +52,7 @@ PATTERN ?= SPEC_IDX ?= PROVIDER ?= BUCKET ?= fixed -ID ?= +REPLAY_ID ?= EVENTS ?= TRACER_OUT_DIR ?= tests/fixtures/replay_cases/$(BUCKET) RUN_DIR ?= @@ -112,7 +113,8 @@ LIMIT_FLAG := $(if $(strip $(LIMIT)),--limit $(LIMIT),) batch batch-tag batch-failed batch-failed-from \ batch-missed batch-missed-from batch-failed-tag batch-missed-tag \ batch-fail-fast batch-max-fails \ - stats history-case report-tag report-tag-changes tags tag-rm case-run case-open tracer-export compare compare-tag + stats history-case report-tag report-tag-changes tags tag-rm case-run case-open tracer-export \ + fixture-rm fixture-fix fixture-migrate compare compare-tag # ============================================================================== # help (на русском) @@ -159,7 +161,7 @@ help: @echo " make tags [PATTERN=*] DATA=... - показать список тегов" @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" - @echo " make tracer-export ID=... EVENTS=... TRACER_OUT_DIR=... [SPEC_IDX=...] [PROVIDER=...] [RUN_DIR=...] [ALLOW_BAD_JSON=1]" + @echo " make tracer-export REPLAY_ID=... EVENTS=... TRACER_OUT_DIR=... [SPEC_IDX=...] [PROVIDER=...] [RUN_DIR=...] [ALLOW_BAD_JSON=1]" @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources//..." @echo "" @@ -220,7 +222,7 @@ show-config: @echo "CASES = $(CASES)" @echo "OUT = $(OUT)" @echo "EVENTS = $(EVENTS)" - @echo "ID = $(ID)" + @echo "REPLAY_ID = $(REPLAY_ID)" @echo "TRACER_OUT_DIR = $(TRACER_OUT_DIR)" @echo "ALLOW_BAD_JSON = $(ALLOW_BAD_JSON)" @echo "LLM_TOML= $(LLM_TOML)" @@ -351,15 +353,24 @@ case-open: check @$(CLI) case open "$(CASE)" --data "$(DATA)" tracer-export: - @test -n "$(strip $(ID))" || (echo "ID обязателен: make tracer-export ID=plan_normalize.spec_v1" && exit 1) + @test -n "$(strip $(REPLAY_ID))" || (echo "REPLAY_ID обязателен: make tracer-export REPLAY_ID=plan_normalize.spec_v1" && exit 1) @test -n "$(strip $(EVENTS))" || (echo "EVENTS обязателен: make tracer-export EVENTS=path/to/events.jsonl" && exit 1) @test -n "$(strip $(TRACER_OUT_DIR))" || (echo "TRACER_OUT_DIR обязателен: make tracer-export TRACER_OUT_DIR=path/to/out_dir" && exit 1) - @$(PYTHON) -m fetchgraph.tracer.cli export-case-bundle --events "$(EVENTS)" --out "$(TRACER_OUT_DIR)" --id "$(ID)" \ + @$(PYTHON) -m fetchgraph.tracer.cli export-case-bundle --events "$(EVENTS)" --out "$(TRACER_OUT_DIR)" --id "$(REPLAY_ID)" \ $(if $(strip $(SPEC_IDX)),--spec-idx $(SPEC_IDX),) \ $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ $(if $(strip $(RUN_DIR)),--run-dir "$(RUN_DIR)",) \ $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) +fixture-rm: + @$(CLI_FIXT) rm --name "$(NAME)" --pattern "$(PATTERN)" --bucket "$(BUCKET)" --scope "$(SCOPE)" --with-resources "$(WITH_RESOURCES)" --dry "$(DRY)" + +fixture-fix: + @$(CLI_FIXT) fix --name "$(NAME)" --pattern "$(PATTERN)" --case "$(CASE)" --move-traces "$(MOVE_TRACES)" --dry "$(DRY)" + +fixture-migrate: + @$(CLI_FIXT) migrate --bucket "$(BUCKET)" --dry "$(DRY)" + # compare (diff.md + junit) compare: check @test -n "$(strip $(BASE))" || (echo "Нужно задать BASE=.../results_prev.jsonl" && exit 1) From d9060b85fff1a9d2c28f7ffb01a70fde21c8fe9d Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 12:47:10 +0300 Subject: [PATCH 30/79] Add overwrite handling for replay exports --- Makefile | 7 ++- src/fetchgraph/cli.py | 7 +++ src/fetchgraph/replay/export.py | 86 ++++++++++++++++++++++++--------- src/fetchgraph/tracer/cli.py | 7 +++ 4 files changed, 83 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 2df7bc78..a52e2f64 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,7 @@ EVENTS ?= TRACER_OUT_DIR ?= tests/fixtures/replay_cases/$(BUCKET) RUN_DIR ?= ALLOW_BAD_JSON ?= +OVERWRITE ?= SCOPE ?= both WITH_RESOURCES ?= 1 ALL ?= @@ -161,7 +162,7 @@ help: @echo " make tags [PATTERN=*] DATA=... - показать список тегов" @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" - @echo " make tracer-export REPLAY_ID=... EVENTS=... TRACER_OUT_DIR=... [SPEC_IDX=...] [PROVIDER=...] [RUN_DIR=...] [ALLOW_BAD_JSON=1]" + @echo " make tracer-export REPLAY_ID=... EVENTS=... TRACER_OUT_DIR=... [SPEC_IDX=...] [PROVIDER=...] [RUN_DIR=...] [ALLOW_BAD_JSON=1] [OVERWRITE=1]" @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources//..." @echo "" @@ -225,6 +226,7 @@ show-config: @echo "REPLAY_ID = $(REPLAY_ID)" @echo "TRACER_OUT_DIR = $(TRACER_OUT_DIR)" @echo "ALLOW_BAD_JSON = $(ALLOW_BAD_JSON)" + @echo "OVERWRITE = $(OVERWRITE)" @echo "LLM_TOML= $(LLM_TOML)" @echo "TAG = $(TAG)" @echo "NOTE = $(NOTE)" @@ -360,7 +362,8 @@ tracer-export: $(if $(strip $(SPEC_IDX)),--spec-idx $(SPEC_IDX),) \ $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ $(if $(strip $(RUN_DIR)),--run-dir "$(RUN_DIR)",) \ - $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) + $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) \ + $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) fixture-rm: @$(CLI_FIXT) rm --name "$(NAME)" --pattern "$(PATTERN)" --bucket "$(BUCKET)" --scope "$(SCOPE)" --with-resources "$(WITH_RESOURCES)" --dry "$(DRY)" diff --git a/src/fetchgraph/cli.py b/src/fetchgraph/cli.py index 04d54bd4..a967f8bf 100644 --- a/src/fetchgraph/cli.py +++ b/src/fetchgraph/cli.py @@ -26,6 +26,11 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: action="store_true", help="Skip invalid JSON lines in events.jsonl", ) + fixture.add_argument( + "--overwrite", + action="store_true", + help="Overwrite existing bundles and resource copies", + ) fixture.add_argument("--all", action="store_true", help="Export all matching replay cases") fixture.add_argument( "--out-dir", @@ -48,6 +53,7 @@ def main(argv: list[str] | None = None) -> int: provider=args.provider, run_dir=args.run_dir, allow_bad_json=args.allow_bad_json, + overwrite=args.overwrite, ) else: export_replay_case_bundle( @@ -58,6 +64,7 @@ def main(argv: list[str] | None = None) -> int: provider=args.provider, run_dir=args.run_dir, allow_bad_json=args.allow_bad_json, + overwrite=args.overwrite, ) return 0 raise SystemExit(f"Unknown command: {args.command}") diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py index 95568b39..57c540a6 100644 --- a/src/fetchgraph/replay/export.py +++ b/src/fetchgraph/replay/export.py @@ -76,25 +76,13 @@ def _select_replay_cases( def collect_requires( events_path: Path, - requires: list[dict], + requires: list[dict] | list[str], *, allow_bad_json: bool = False, ) -> tuple[dict[str, dict], dict[str, dict]]: extras: Dict[str, dict] = {} resources: Dict[str, dict] = {} - if not requires: - return resources, extras - if not isinstance(requires, list): - raise ValueError("requires must be a list of objects with kind/id fields") - for req in requires: - if not isinstance(req, dict): - raise ValueError("requires must be a list of objects with kind/id fields") - if req.get("kind") not in {"extra", "resource"}: - raise ValueError("requires entries must include kind='extra' or kind='resource'") - if not isinstance(req.get("id"), str) or not req.get("id"): - raise ValueError("requires entries must include a non-empty id") - for _, event in iter_events(events_path, allow_bad_json=allow_bad_json): event_type = event.get("type") event_id = event.get("id") @@ -102,10 +90,37 @@ def collect_requires( extras[event_id] = event elif event_type == "replay_resource" and isinstance(event_id, str): resources[event_id] = event + if not requires: + return resources, extras + + normalized_requires: list[dict] + if isinstance(requires, list) and all(isinstance(req, str) for req in requires): + normalized_requires = [] + for req_id in requires: + if req_id in resources: + normalized_requires.append({"kind": "resource", "id": req_id}) + elif req_id in extras: + normalized_requires.append({"kind": "extra", "id": req_id}) + else: + raise ValueError( + f"Unknown dependency {req_id!r} in {events_path}; requires must be updated to replay_case v2." + ) + elif isinstance(requires, list): + normalized_requires = [] + for req in requires: + if not isinstance(req, dict): + raise ValueError("requires must be a list of objects with kind/id fields") + if req.get("kind") not in {"extra", "resource"}: + raise ValueError("requires entries must include kind='extra' or kind='resource'") + if not isinstance(req.get("id"), str) or not req.get("id"): + raise ValueError("requires entries must include a non-empty id") + normalized_requires.append({"kind": req["kind"], "id": req["id"]}) + else: + raise ValueError("requires must be a list of objects with kind/id fields") resolved_resources: Dict[str, dict] = {} resolved_extras: Dict[str, dict] = {} - for req in requires: + for req in normalized_requires: kind = req.get("kind") rid = req.get("id") if kind == "extra": @@ -131,8 +146,9 @@ def write_case_bundle( resources: Dict[str, dict], extras: Dict[str, dict], source: dict, + overwrite: bool = False, ) -> None: - if out_path.exists(): + if out_path.exists() and not overwrite: print(f"Fixture already exists: {out_path}") return payload = { @@ -154,7 +170,7 @@ def copy_resource_files( out_dir: Path, fixture_stem: str, ) -> None: - for resource in resources.values(): + for resource_id, resource in resources.items(): data_ref = resource.get("data_ref") if not isinstance(data_ref, dict): continue @@ -169,12 +185,16 @@ def copy_resource_files( src_path = run_dir / rel_path if not src_path.exists(): raise FileNotFoundError(f"Resource file {src_path} not found for replay bundle") - dest_rel = Path("resources") / fixture_stem / rel_path + dest_rel = Path("resources") / fixture_stem / resource_id / rel_path dest_path = out_dir / dest_rel dest_path.parent.mkdir(parents=True, exist_ok=True) if dest_path.exists(): if not filecmp.cmp(src_path, dest_path, shallow=False): - raise FileExistsError(f"Resource file collision at {dest_path}") + raise FileExistsError( + "Resource file collision at " + f"{dest_path} (resource_id={resource_id}, src={src_path}, run_dir={run_dir}, " + f"fixture_stem={fixture_stem})" + ) else: shutil.copy2(src_path, dest_path) data_ref["file"] = dest_rel.as_posix() @@ -200,6 +220,7 @@ def export_replay_case_bundle( provider: str | None = None, run_dir: Path | None = None, allow_bad_json: bool = False, + overwrite: bool = False, ) -> Path: selections = _select_replay_cases( events_path, @@ -227,14 +248,17 @@ def export_replay_case_bundle( resources, extras = collect_requires(events_path, requires, allow_bad_json=allow_bad_json) fixture_name = case_bundle_name(replay_id, root_event["input"]) + fixture_stem = fixture_name.replace(".case.json", "") if _has_resource_files(resources): if run_dir is None: raise ValueError("run_dir is required to export file resources") + if overwrite: + shutil.rmtree(out_dir / "resources" / fixture_stem, ignore_errors=True) copy_resource_files( resources, run_dir=run_dir, out_dir=out_dir, - fixture_stem=fixture_name.replace(".case.json", ""), + fixture_stem=fixture_stem, ) source = { @@ -244,7 +268,14 @@ def export_replay_case_bundle( "timestamp": root_event.get("timestamp"), } out_path = out_dir / fixture_name - write_case_bundle(out_path, root_case=root_event, resources=resources, extras=extras, source=source) + write_case_bundle( + out_path, + root_case=root_event, + resources=resources, + extras=extras, + source=source, + overwrite=overwrite, + ) return out_path @@ -257,6 +288,7 @@ def export_replay_case_bundles( provider: str | None = None, run_dir: Path | None = None, allow_bad_json: bool = False, + overwrite: bool = False, ) -> list[Path]: selections = _select_replay_cases( events_path, @@ -280,14 +312,17 @@ def export_replay_case_bundles( requires = root_event.get("requires") or [] resources, extras = collect_requires(events_path, requires, allow_bad_json=allow_bad_json) fixture_name = case_bundle_name(replay_id, root_event["input"]) + fixture_stem = fixture_name.replace(".case.json", "") if _has_resource_files(resources): if run_dir is None: raise ValueError("run_dir is required to export file resources") + if overwrite: + shutil.rmtree(out_dir / "resources" / fixture_stem, ignore_errors=True) copy_resource_files( resources, run_dir=run_dir, out_dir=out_dir, - fixture_stem=fixture_name.replace(".case.json", ""), + fixture_stem=fixture_stem, ) source = { @@ -297,6 +332,13 @@ def export_replay_case_bundles( "timestamp": root_event.get("timestamp"), } out_path = out_dir / fixture_name - write_case_bundle(out_path, root_case=root_event, resources=resources, extras=extras, source=source) + write_case_bundle( + out_path, + root_case=root_event, + resources=resources, + extras=extras, + source=source, + overwrite=overwrite, + ) paths.append(out_path) return paths diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index 3e062cb3..5de7b4a8 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -27,6 +27,11 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: action="store_true", help="Skip invalid JSON lines in events.jsonl", ) + export.add_argument( + "--overwrite", + action="store_true", + help="Overwrite existing bundles and resource copies", + ) export.add_argument("--all", action="store_true", help="Export all matching replay cases") return parser.parse_args(argv) @@ -44,6 +49,7 @@ def main(argv: list[str] | None = None) -> int: provider=args.provider, run_dir=args.run_dir, allow_bad_json=args.allow_bad_json, + overwrite=args.overwrite, ) else: export_replay_case_bundle( @@ -54,6 +60,7 @@ def main(argv: list[str] | None = None) -> int: provider=args.provider, run_dir=args.run_dir, allow_bad_json=args.allow_bad_json, + overwrite=args.overwrite, ) return 0 raise SystemExit(f"Unknown command: {args.command}") From 3a85b441eb7dde5242f4a130512040628d614631 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 12:51:00 +0300 Subject: [PATCH 31/79] Optimize replay export indexing and errors --- Makefile | 1 + src/fetchgraph/replay/export.py | 44 ++++++++++++++++++++++++++++---- src/fetchgraph/replay/runtime.py | 8 +++++- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index a52e2f64..0db282a8 100644 --- a/Makefile +++ b/Makefile @@ -165,6 +165,7 @@ help: @echo " make tracer-export REPLAY_ID=... EVENTS=... TRACER_OUT_DIR=... [SPEC_IDX=...] [PROVIDER=...] [RUN_DIR=...] [ALLOW_BAD_JSON=1] [OVERWRITE=1]" @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources//..." + @echo " fixture-rm/fix/migrate используют legacy layout (examples.demo_qa.fixture_tools)" @echo "" @echo "Уборка:" @echo " make tag-rm TAG=... [DRY=1] [PURGE_RUNS=1] [PRUNE_HISTORY=1] [PRUNE_CASE_HISTORY=1]" diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py index 57c540a6..d11bd958 100644 --- a/src/fetchgraph/replay/export.py +++ b/src/fetchgraph/replay/export.py @@ -74,9 +74,8 @@ def _select_replay_cases( return selected -def collect_requires( +def index_requires( events_path: Path, - requires: list[dict] | list[str], *, allow_bad_json: bool = False, ) -> tuple[dict[str, dict], dict[str, dict]]: @@ -90,8 +89,19 @@ def collect_requires( extras[event_id] = event elif event_type == "replay_resource" and isinstance(event_id, str): resources[event_id] = event + + return resources, extras + + +def resolve_requires( + requires: list[dict] | list[str], + *, + resources: dict[str, dict], + extras: dict[str, dict], + events_path: Path, +) -> tuple[dict[str, dict], dict[str, dict]]: if not requires: - return resources, extras + return {}, {} normalized_requires: list[dict] if isinstance(requires, list) and all(isinstance(req, str) for req in requires): @@ -139,6 +149,18 @@ def collect_requires( return resolved_resources, resolved_extras +def collect_requires( + events_path: Path, + requires: list[dict] | list[str], + *, + allow_bad_json: bool = False, +) -> tuple[dict[str, dict], dict[str, dict]]: + resources, extras = index_requires(events_path, allow_bad_json=allow_bad_json) + if not requires: + return resources, extras + return resolve_requires(requires, resources=resources, extras=extras, events_path=events_path) + + def write_case_bundle( out_path: Path, *, @@ -246,7 +268,13 @@ def export_replay_case_bundle( root_event = selection.event requires = root_event.get("requires") or [] - resources, extras = collect_requires(events_path, requires, allow_bad_json=allow_bad_json) + resources_index, extras_index = index_requires(events_path, allow_bad_json=allow_bad_json) + resources, extras = resolve_requires( + requires, + resources=resources_index, + extras=extras_index, + events_path=events_path, + ) fixture_name = case_bundle_name(replay_id, root_event["input"]) fixture_stem = fixture_name.replace(".case.json", "") if _has_resource_files(resources): @@ -306,11 +334,17 @@ def export_replay_case_bundles( detail_str = f" (filters: {', '.join(details)})" if details else "" raise LookupError(f"No replay_case id={replay_id!r} found in {events_path}{detail_str}") + resources_index, extras_index = index_requires(events_path, allow_bad_json=allow_bad_json) paths: list[Path] = [] for selection in selections: root_event = selection.event requires = root_event.get("requires") or [] - resources, extras = collect_requires(events_path, requires, allow_bad_json=allow_bad_json) + resources, extras = resolve_requires( + requires, + resources=resources_index, + extras=extras_index, + events_path=events_path, + ) fixture_name = case_bundle_name(replay_id, root_event["input"]) fixture_stem = fixture_name.replace(".case.json", "") if _has_resource_files(resources): diff --git a/src/fetchgraph/replay/runtime.py b/src/fetchgraph/replay/runtime.py index 34a64a53..f13ebfb0 100644 --- a/src/fetchgraph/replay/runtime.py +++ b/src/fetchgraph/replay/runtime.py @@ -23,7 +23,13 @@ def resolve_resource_path(self, resource_path: str | Path) -> Path: def run_case(root: dict, ctx: ReplayContext) -> dict: - handler = REPLAY_HANDLERS[root["id"]] + replay_id = root["id"] + if replay_id not in REPLAY_HANDLERS: + raise KeyError( + f"No handler for replay id={replay_id!r}. " + "Did you import fetchgraph.tracer.handlers?" + ) + handler = REPLAY_HANDLERS[replay_id] return handler(root["input"], ctx) From 17cfeb888cd89d0c366fc38442af6722d9ad6ad0 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 12:56:13 +0300 Subject: [PATCH 32/79] Improve replay bundle idempotency and diagnostics --- src/fetchgraph/replay/export.py | 72 ++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py index d11bd958..4f1c8506 100644 --- a/src/fetchgraph/replay/export.py +++ b/src/fetchgraph/replay/export.py @@ -3,11 +3,15 @@ import filecmp import hashlib import json +import logging import shutil +import textwrap from dataclasses import dataclass from pathlib import Path from typing import Dict, Iterable +logger = logging.getLogger(__name__) + def iter_events(path: Path, *, allow_bad_json: bool = False) -> Iterable[tuple[int, dict]]: with path.open("r", encoding="utf-8") as handle: @@ -170,9 +174,6 @@ def write_case_bundle( source: dict, overwrite: bool = False, ) -> None: - if out_path.exists() and not overwrite: - print(f"Fixture already exists: {out_path}") - return payload = { "schema": "fetchgraph.tracer.case_bundle", "v": 1, @@ -181,8 +182,30 @@ def write_case_bundle( "extras": extras, "source": source, } + new_text = canonical_json(payload) + if out_path.exists() and not overwrite: + try: + old_payload = json.loads(out_path.read_text(encoding="utf-8")) + old_text = canonical_json(old_payload) + except Exception as exc: + raise ValueError(f"Existing case bundle is unreadable: {out_path}: {exc}") from exc + if old_text == new_text: + logger.info("Fixture already up-to-date: %s", out_path) + return + raise FileExistsError( + textwrap.dedent( + f"""\ + Case bundle already exists and differs: {out_path} + This is fail-fast to avoid mixing fixtures from different runs. + Actions: + - delete the file, or + - choose a different --out directory, or + - rerun with --overwrite. + """ + ).rstrip() + ) out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(canonical_json(payload), encoding="utf-8") + out_path.write_text(new_text, encoding="utf-8") def copy_resource_files( @@ -192,6 +215,7 @@ def copy_resource_files( out_dir: Path, fixture_stem: str, ) -> None: + planned: dict[Path, tuple[str, Path]] = {} for resource_id, resource in resources.items(): data_ref = resource.get("data_ref") if not isinstance(data_ref, dict): @@ -206,16 +230,39 @@ def copy_resource_files( raise ValueError(f"Resource file path must not traverse parents: {file_name}") src_path = run_dir / rel_path if not src_path.exists(): - raise FileNotFoundError(f"Resource file {src_path} not found for replay bundle") + raise FileNotFoundError( + f"Missing resource file for rid={resource_id!r}: " + f"src={src_path} (run_dir={run_dir}, fixture={fixture_stem})" + ) dest_rel = Path("resources") / fixture_stem / resource_id / rel_path dest_path = out_dir / dest_rel dest_path.parent.mkdir(parents=True, exist_ok=True) + prev = planned.get(dest_path) + if prev is not None: + prev_id, prev_src = prev + if prev_id != resource_id or prev_src != src_path: + if not filecmp.cmp(prev_src, src_path, shallow=False): + raise FileExistsError( + "Resource destination collision (different contents):\n" + f" dest: {dest_path}\n" + f" fixture: {fixture_stem}\n" + f" A: rid={prev_id!r} src={prev_src}\n" + f" B: rid={resource_id!r} src={src_path}\n" + "Hint: make resource filenames unique or adjust resource layout." + ) + else: + planned[dest_path] = (resource_id, src_path) if dest_path.exists(): if not filecmp.cmp(src_path, dest_path, shallow=False): raise FileExistsError( - "Resource file collision at " - f"{dest_path} (resource_id={resource_id}, src={src_path}, run_dir={run_dir}, " - f"fixture_stem={fixture_stem})" + "Resource file collision (dest exists with different contents):\n" + f" rid: {resource_id!r}\n" + f" fixture: {fixture_stem}\n" + f" src: {src_path}\n" + f" dest: {dest_path}\n" + "Actions:\n" + " - delete destination resources directory, or\n" + " - export into a clean --out directory." ) else: shutil.copy2(src_path, dest_path) @@ -260,8 +307,13 @@ def export_replay_case_bundle( detail_str = f" (filters: {', '.join(details)})" if details else "" raise LookupError(f"No replay_case id={replay_id!r} found in {events_path}{detail_str}") if len(selections) > 1: + details = "\n".join( + f"- line {sel.line} run_id={sel.event.get('run_id')!r} timestamp={sel.event.get('timestamp')!r}" + for sel in selections[:5] + ) raise LookupError( - "Multiple replay_case entries matched; use export_replay_case_bundles to export all." + "Multiple replay_case entries matched; use export_replay_case_bundles to export all.\n" + f"{details}" ) selection = selections[0] @@ -294,6 +346,7 @@ def export_replay_case_bundle( "line": selection.line, "run_id": root_event.get("run_id"), "timestamp": root_event.get("timestamp"), + "case_id": root_event.get("case_id"), } out_path = out_dir / fixture_name write_case_bundle( @@ -364,6 +417,7 @@ def export_replay_case_bundles( "line": selection.line, "run_id": root_event.get("run_id"), "timestamp": root_event.get("timestamp"), + "case_id": root_event.get("case_id"), } out_path = out_dir / fixture_name write_case_bundle( From 24dc0422e2727d5e2c716756dd807c5fa8b4cdbf Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 13:06:32 +0300 Subject: [PATCH 33/79] Avoid mutating shared resources during export --- src/fetchgraph/replay/export.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py index 4f1c8506..643f70d1 100644 --- a/src/fetchgraph/replay/export.py +++ b/src/fetchgraph/replay/export.py @@ -1,5 +1,6 @@ from __future__ import annotations +import copy import filecmp import hashlib import json @@ -327,6 +328,8 @@ def export_replay_case_bundle( extras=extras_index, events_path=events_path, ) + resources = copy.deepcopy(resources) + extras = copy.deepcopy(extras) fixture_name = case_bundle_name(replay_id, root_event["input"]) fixture_stem = fixture_name.replace(".case.json", "") if _has_resource_files(resources): @@ -398,6 +401,8 @@ def export_replay_case_bundles( extras=extras_index, events_path=events_path, ) + resources = copy.deepcopy(resources) + extras = copy.deepcopy(extras) fixture_name = case_bundle_name(replay_id, root_event["input"]) fixture_stem = fixture_name.replace(".case.json", "") if _has_resource_files(resources): From 0fe2f6691eb74f9039eb7050422875f6f8e70385 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 13:11:57 +0300 Subject: [PATCH 34/79] Tighten export validation and makefile checks --- Makefile | 2 +- src/fetchgraph/replay/export.py | 16 ++++------------ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 0db282a8..4fa7ed71 100644 --- a/Makefile +++ b/Makefile @@ -358,7 +358,7 @@ case-open: check tracer-export: @test -n "$(strip $(REPLAY_ID))" || (echo "REPLAY_ID обязателен: make tracer-export REPLAY_ID=plan_normalize.spec_v1" && exit 1) @test -n "$(strip $(EVENTS))" || (echo "EVENTS обязателен: make tracer-export EVENTS=path/to/events.jsonl" && exit 1) - @test -n "$(strip $(TRACER_OUT_DIR))" || (echo "TRACER_OUT_DIR обязателен: make tracer-export TRACER_OUT_DIR=path/to/out_dir" && exit 1) + @# TRACER_OUT_DIR has a default; override if needed. @$(PYTHON) -m fetchgraph.tracer.cli export-case-bundle --events "$(EVENTS)" --out "$(TRACER_OUT_DIR)" --id "$(REPLAY_ID)" \ $(if $(strip $(SPEC_IDX)),--spec-idx $(SPEC_IDX),) \ $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py index 643f70d1..0b216d8d 100644 --- a/src/fetchgraph/replay/export.py +++ b/src/fetchgraph/replay/export.py @@ -154,18 +154,6 @@ def resolve_requires( return resolved_resources, resolved_extras -def collect_requires( - events_path: Path, - requires: list[dict] | list[str], - *, - allow_bad_json: bool = False, -) -> tuple[dict[str, dict], dict[str, dict]]: - resources, extras = index_requires(events_path, allow_bad_json=allow_bad_json) - if not requires: - return resources, extras - return resolve_requires(requires, resources=resources, extras=extras, events_path=events_path) - - def write_case_bundle( out_path: Path, *, @@ -218,6 +206,10 @@ def copy_resource_files( ) -> None: planned: dict[Path, tuple[str, Path]] = {} for resource_id, resource in resources.items(): + if not isinstance(resource_id, str) or not resource_id: + raise ValueError("resource_id must be a non-empty string") + if "/" in resource_id or "\\" in resource_id or ".." in Path(resource_id).parts: + raise ValueError(f"resource_id must be a safe path segment: {resource_id!r}") data_ref = resource.get("data_ref") if not isinstance(data_ref, dict): continue From 3a97075cebc59f09c091e502ff6ea348ef46d69c Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 13:44:15 +0300 Subject: [PATCH 35/79] Remove legacy fixture tooling --- Makefile | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 4fa7ed71..046017b1 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,6 @@ DEFAULT_CASES := examples/demo_qa/cases/retail_cases.json VENV ?= .venv PYTHON ?= $(if $(wildcard $(VENV)/bin/python),$(VENV)/bin/python,python) CLI := $(PYTHON) -m examples.demo_qa.cli -CLI_FIXT := $(PYTHON) -m examples.demo_qa.fixture_tools # ============================================================================== # 4) Пути demo_qa (можно переопределять через CLI или в $(CONFIG)) @@ -115,7 +114,7 @@ LIMIT_FLAG := $(if $(strip $(LIMIT)),--limit $(LIMIT),) batch-missed batch-missed-from batch-failed-tag batch-missed-tag \ batch-fail-fast batch-max-fails \ stats history-case report-tag report-tag-changes tags tag-rm case-run case-open tracer-export \ - fixture-rm fixture-fix fixture-migrate compare compare-tag + compare compare-tag # ============================================================================== # help (на русском) @@ -165,7 +164,6 @@ help: @echo " make tracer-export REPLAY_ID=... EVENTS=... TRACER_OUT_DIR=... [SPEC_IDX=...] [PROVIDER=...] [RUN_DIR=...] [ALLOW_BAD_JSON=1] [OVERWRITE=1]" @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources//..." - @echo " fixture-rm/fix/migrate используют legacy layout (examples.demo_qa.fixture_tools)" @echo "" @echo "Уборка:" @echo " make tag-rm TAG=... [DRY=1] [PURGE_RUNS=1] [PRUNE_HISTORY=1] [PRUNE_CASE_HISTORY=1]" @@ -366,14 +364,8 @@ tracer-export: $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) \ $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) -fixture-rm: - @$(CLI_FIXT) rm --name "$(NAME)" --pattern "$(PATTERN)" --bucket "$(BUCKET)" --scope "$(SCOPE)" --with-resources "$(WITH_RESOURCES)" --dry "$(DRY)" -fixture-fix: - @$(CLI_FIXT) fix --name "$(NAME)" --pattern "$(PATTERN)" --case "$(CASE)" --move-traces "$(MOVE_TRACES)" --dry "$(DRY)" -fixture-migrate: - @$(CLI_FIXT) migrate --bucket "$(BUCKET)" --dry "$(DRY)" # compare (diff.md + junit) compare: check From d0bc39e9f82cbf17b0a1415cd312fc8c5cb962de Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 13:52:19 +0300 Subject: [PATCH 36/79] Update tracer documentation for replay_case v2 --- src/fetchgraph/replay/fetchgraph_tracer.md | 356 ++++++++------------- 1 file changed, 134 insertions(+), 222 deletions(-) diff --git a/src/fetchgraph/replay/fetchgraph_tracer.md b/src/fetchgraph/replay/fetchgraph_tracer.md index ff56ef36..d7c7494b 100644 --- a/src/fetchgraph/replay/fetchgraph_tracer.md +++ b/src/fetchgraph/replay/fetchgraph_tracer.md @@ -1,314 +1,226 @@ -# Fetchgraph Tracer: Event Log, Replay Points и фикстуры +# Fetchgraph Tracer: observed-first replay cases, bundles и реплей -> Этот документ описывает “трейсер” (событийный лог + механизм воспроизведения/реплея) по коду из приложенных модулей: -> - `log.py` — контракт логгера событий и хелпер `log_replay_point` -> - `runtime.py` — `ReplayContext` и реестр `REPLAY_HANDLERS` -> - `snapshots.py` — снапшоты для `ProviderInfo` / каталога провайдеров -> - `plan_normalize.py` — пример replay-обработчика `plan_normalize.spec_v1` -> - `export.py` — экспорт replay-point’ов в фикстуры / бандлы (копирование ресурсов) +> Этот документ описывает актуальный формат трейса и реплея: +> - `log.py` — контракт логгера событий и хелпер `log_replay_case` +> - `runtime.py` — `ReplayContext`, `REPLAY_HANDLERS`, `run_case`, `load_case_bundle` +> - `handlers/*` — обработчики (регистрация в `REPLAY_HANDLERS`) +> - `export.py` — экспорт `replay_case` → case bundle (`*.case.json`) --- ## 1) Зачем это нужно -Трейсер решает две связанные задачи: +Трейсер решает две задачи: -1) **Диагностика и воспроизводимость** - В рантайме Fetchgraph можно записывать “точки реплея” (replay points) — маленькие, детерминированные фрагменты вычислений: вход → ожидаемый выход (+ метаданные). +1) **Observed-first логирование** + В рантайме пишется **input + observed outcome** (успех) **или** `observed_error` (ошибка) + зависимости для реплея. `expected` не логируется. -2) **Автоматическая генерация регрессионных тестов (fixtures)** - Из потока событий (`events.jsonl`) можно автоматически выгрузить фикстуры и гонять “реплей” без LLM/внешних зависимостей: просто вызвать нужный обработчик из `REPLAY_HANDLERS` и сравнить результат с `expected`. +2) **Реплей и регрессии** + Из `events.jsonl` экспортируются **case bundles** (root case + extras/resources + source). Реплей работает без LLM и внешних сервисов. --- ## 2) Ключевые понятия ### Event stream (JSONL) -События пишутся в файл (или другой sink) как **JSON Lines**: *одна JSON-строка = одно событие*. +События пишутся как JSONL: одна строка = одно событие. -Пример общего вида (как минимум это делает `EventLogger` из demo runner’а): +Пример общего вида: ```json {"timestamp":"2026-01-25T00:00:00Z","run_id":"abcd1234","type":"...","...": "..."} ``` -### Replay point -Событие типа `replay_point` описывает собтыие, которое можно воспроизвести: -- `id` — строковый идентификатор точки (обычно совпадает с ключом обработчика в `REPLAY_HANDLERS`) -- `meta` — фильтруемые метаданные (например `spec_idx`, `provider`) -- `input` — входные данные для реплея -- `expected` — ожидаемый результат -- `requires` *(опционально)* — список зависимостей (ресурсы/доп. события), необходимых для реплея +### Replay case (v2) +Событие `type="replay_case"`, `v=2`: +- `id`: идентификатор обработчика +- `meta`: опциональные метаданные (например `spec_idx`, `provider`) +- `input`: вход для реплея +- **ровно одно** из `observed` или `observed_error` +- `requires`: список зависимостей `[{"kind":"extra"|"resource","id":"..."}]` -Минимальный пример: +Пример: ```json { - "type": "replay_point", - "v": 1, + "type": "replay_case", + "v": 2, "id": "plan_normalize.spec_v1", "meta": {"spec_idx": 0, "provider": "sql"}, "input": {"spec": {...}, "options": {...}}, - "expected": {"out_spec": {...}, "notes_last": "..."}, - "requires": ["planner_input_v1"] + "observed": {"out_spec": {...}}, + "requires": [{"kind": "extra", "id": "planner_input_v1"}] } ``` -### Replay resource / extras -`export.py` умеет подтягивать зависимости из event stream двух типов: -- `type="replay_resource"` — ресурсы (часто файлы), которые могут понадобиться при реплее -- `type="planner_input"` — дополнительные входы/контекст (“extras”), которые обработчик может использовать - -Важно: для `extras` ключом является `event["id"]` (например `"planner_input_v1"`), а обработчики потом читают это из `ctx.extras[...]`. +### Extras / Resources +- **Extras**: события с `type="planner_input"`, ключуются по `id`. +- **Resources**: события с `type="replay_resource"`, ключуются по `id`. + - Файлы указываются в `data_ref.file` (относительный путь внутри `run_dir`). --- -## 3) Ответственность модулей и классов - -### `EventLoggerLike` (`log.py`) -Минимальный контракт “куда писать события”: +## 3) Контракт логгера и хелпер +### `EventLoggerLike` ```py class EventLoggerLike(Protocol): - def emit(self, event: Dict[str, object]) -> None: ... -``` - -Идея: рантайм может писать события в “настоящий” `EventLog`/`EventLogger`, а тесты/утилиты — принимать любой sink, который реализует `emit`. - -### `log_replay_point` (`log.py`) -Унифицированный способ записать `replay_point`: - -- гарантирует базовую форму события (`type`, `v`, `id`, `meta`, `input`, `expected`) -- опционально добавляет: `requires`, `diag`, `note`, `error`, `extra` - -Рекомендуемый паттерн: **для всех реплейных точек использовать только этот хелпер**, чтобы формат не “расползался”. - ---- - -### `ReplayContext` и `REPLAY_HANDLERS` (`runtime.py`) - -#### `ReplayContext` -Контейнер для зависимостей и контекста реплея: -```py -@dataclass(frozen=True) -class ReplayContext: - resources: Dict[str, dict] = ... - extras: Dict[str, dict] = ... - base_dir: Path | None = None - - def resolve_resource_path(self, resource_path: str | Path) -> Path: - ... -``` - -- `resources`: “словарь ресурсов” по id (обычно события `replay_resource`) -- `extras`: “словарь доп. данных” по id (обычно события `planner_input`) -- `base_dir`: база для резолва относительных путей ресурсов (полезно для бандлов фикстур) - -#### `REPLAY_HANDLERS` -Глобальный реестр обработчиков: -```py -REPLAY_HANDLERS: Dict[str, Callable[[dict, ReplayContext], dict]] = {} + def emit(self, event: dict) -> None: ... ``` -Ключ: `replay_point.id` -Значение: функция `handler(input_dict, ctx) -> output_dict` - ---- +### `log_replay_case` +Хелпер формирует событие v2 и валидирует: +- `id` не пустой +- `input` — dict +- XOR `observed` / `observed_error` +- `requires` — список `{kind,id}` -### `snapshots.py` -Снапшоты данных о провайдерах, чтобы реплей был более стабильным: - -- `snapshot_provider_info(info: ProviderInfo) -> Dict[str, object]` -- `snapshot_provider_catalog(provider_catalog: Mapping[str, object]) -> Dict[str, object]` - -Смысл: вместо того чтобы зависеть от “живых” объектов, сохраняем стабильный JSON-слепок. - ---- - -### Пример обработчика: `plan_normalize.spec_v1` (`plan_normalize.py`) -Функция: -```py -def replay_plan_normalize_spec_v1(inp: dict, ctx: ReplayContext) -> dict: ... -REPLAY_HANDLERS["plan_normalize.spec_v1"] = replay_plan_normalize_spec_v1 -``` - -Что делает (по коду): -1) Берёт `inp["spec"]` и `inp["options"]` -2) Строит `PlanNormalizerOptions` -3) Вытаскивает правила нормализации из `normalizer_rules` или `normalizer_registry` -4) Пытается восстановить `ProviderInfo` из `ctx.extras["planner_input_v1"]["input"]["provider_catalog"][provider]` - - если не получилось — создаёт `ProviderInfo(name=provider, capabilities=[])` -5) Собирает `PlanNormalizer` и нормализует один `ContextFetchSpec` -6) Возвращает: - - `out_spec` (provider/mode/selectors) - - `notes_last` (последняя заметка нормализатора) - ---- - -### Экспорт фикстур: `export.py` - -#### Что делает `export.py` -1) Читает `events.jsonl` построчно (`iter_events`) -2) Находит `replay_point` с нужным `id` (+ фильтры `spec_idx`/`provider`) -3) Опционально подтягивает зависимости из `requires`: - - `replay_resource` события → `ctx.resources` - - `planner_input` события → `ctx.extras` -4) Пишет фикстуру в `out_dir`: - - **простая фикстура** (`write_fixture`) — только replay_point + `source` - - **bundle** (`write_bundle`) — root replay_point + resources + extras + `source`, - и при необходимости копирует файлы ресурсов в структуру `resources//...` - -#### Имена фикстур -Имя вычисляется стабильно: +Пример: ```py -fixture_name = f"{event_id}__{sha256(event_id + canonical_json(input))[:8]}.json" -``` +from fetchgraph.tracer import log_replay_case ---- - -## 4) Как этим пользоваться - -### 4.1 В рантайме: как логировать replay-point’ы - -1) Подготовить объект, реализующий `EventLoggerLike.emit(...)`. - -Пример (упрощённый) — JSONL writer: -```py -from pathlib import Path -import json, datetime - -class JsonlEventLog: - def __init__(self, path: Path): - self.path = path - self.path.parent.mkdir(parents=True, exist_ok=True) - - def emit(self, event: dict) -> None: - payload = {"timestamp": datetime.datetime.utcnow().isoformat() + "Z", **event} - with self.path.open("a", encoding="utf-8") as f: - f.write(json.dumps(payload, ensure_ascii=False) + "\n") -``` - -2) В точке, где хотите получать реплейный тест, вызвать `log_replay_point(...)`: - -```py -from fetchgraph.tracer import log_replay_point # путь зависит от того, где пакет лежит у вас - -log_replay_point( +log_replay_case( logger=event_log, id="plan_normalize.spec_v1", meta={"spec_idx": i, "provider": spec.provider}, - input={"spec": spec.model_dump(), "options": options.model_dump(), "normalizer_rules": rules}, - expected={"out_spec": out_spec, "notes_last": notes_last}, - requires=["planner_input_v1"], # если обработчику нужен контекст + input={"spec": spec.model_dump(), "options": options.model_dump()}, + observed={"out_spec": out_spec}, + requires=[{"kind": "extra", "id": "planner_input_v1"}], ) ``` -3) Если есть внешние файлы/ресурсы, которые нужно будет копировать в bundle — записывайте отдельные события `replay_resource` -(формат в коде не зафиксирован типами, но `export.py` ожидает хотя бы `type="replay_resource"`, `id=str`, и опционально `data_ref.file`): +Если есть файлы, логируйте отдельные события: ```py event_log.emit({ "type": "replay_resource", "id": "catalog_csv", - "data_ref": {"file": "relative/path/to/catalog.csv"} + "data_ref": {"file": "artifacts/catalog.csv"} }) ``` --- -### 4.2 Реплей: как выполнить фикстуру +## 4) Replay runtime -1) Импортировать модули обработчиков, чтобы они зарегистрировались в `REPLAY_HANDLERS` -(сейчас регистрация — через side-effect: модуль при импорте пишет в dict). - -Например: +### Регистрация обработчиков +Обработчики регистрируются через side-effect импорта. +Рекомендуемый способ: ```py -from fetchgraph.tracer.runtime import REPLAY_HANDLERS, ReplayContext -import fetchgraph.tracer.plan_normalize # важно: импорт модуля регистрирует обработчик +import fetchgraph.tracer.handlers # noqa: F401 ``` -2) Загрузить фикстуру (или bundle) и построить `ReplayContext`. +### `ReplayContext` +```py +@dataclass(frozen=True) +class ReplayContext: + resources: dict[str, dict] + extras: dict[str, dict] + base_dir: Path | None = None -- Для простой фикстуры: + def resolve_resource_path(self, resource_path: str | Path) -> Path: ... +``` + +### `run_case` ```py -fixture = json.load(open(path, "r", encoding="utf-8")) -ctx = ReplayContext(resources={}, extras={}, base_dir=path.parent) -handler = REPLAY_HANDLERS[fixture["id"]] -out = handler(fixture["input"], ctx) -assert out == fixture["expected"] +from fetchgraph.tracer.runtime import run_case + +out = run_case(root_case, ctx) ``` -- Для bundle: +### `load_case_bundle` ```py -bundle = json.load(open(path, "r", encoding="utf-8")) -root = bundle["root"] -ctx = ReplayContext( - resources=bundle.get("resources", {}), - extras=bundle.get("extras", {}), - base_dir=path.parent, -) -handler = REPLAY_HANDLERS[root["id"]] -out = handler(root["input"], ctx) -assert out == root["expected"] +from fetchgraph.tracer.runtime import load_case_bundle + +root, ctx = load_case_bundle(Path(".../case.case.json")) ``` --- -### 4.3 Экспорт фикстур из events.jsonl +## 5) Export case bundles -Python API (как в `export.py`): - -- **Одна фикстура**: +### Python API ```py -from fetchgraph.tracer.export import export_replay_fixture -export_replay_fixture( - events_path=Path("_demo_data/.../events.jsonl"), - out_dir=Path("tests/fixtures/replay"), +from fetchgraph.tracer.export import export_replay_case_bundle, export_replay_case_bundles + +path = export_replay_case_bundle( + events_path=Path(".../events.jsonl"), + out_dir=Path("tests/fixtures/replay_cases"), replay_id="plan_normalize.spec_v1", spec_idx=0, provider="sql", - with_requires=True, - run_dir=Path("_demo_data/.../run_dir"), + run_dir=Path(".../run_dir"), + allow_bad_json=False, + overwrite=False, ) ``` -- **Все совпадения (если один `id` встречается много раз)**: +Все совпадения: ```py -from fetchgraph.tracer.export import export_replay_fixtures -paths = export_replay_fixtures( - events_path=..., - out_dir=..., +paths = export_replay_case_bundles( + events_path=Path(".../events.jsonl"), + out_dir=Path("tests/fixtures/replay_cases"), replay_id="plan_normalize.spec_v1", - with_requires=False, + allow_bad_json=True, + overwrite=True, ) ``` +### Layout ресурсов +Файлы копируются в: +``` +resources/// +``` + --- -## 5) Рекомендации по pytest-фикстурам +## 6) CLI + +### tracer CLI +```bash +fetchgraph-tracer export-case-bundle \ + --events path/to/events.jsonl \ + --out tests/fixtures/replay_cases \ + --id plan_normalize.spec_v1 \ + --spec-idx 0 \ + --run-dir path/to/run_dir \ + --allow-bad-json \ + --overwrite +``` -Минимальный удобный слой (совет): +### Makefile +```bash +make tracer-export REPLAY_ID=plan_normalize.spec_v1 EVENTS=path/to/events.jsonl \ + TRACER_OUT_DIR=tests/fixtures/replay_cases RUN_DIR=path/to/run_dir OVERWRITE=1 +``` -- `load_replay_fixture(path) -> (root_event, ctx)` -- `run_replay(root_event, ctx) -> out` +--- -Чтобы каждый тест был “одной строкой”. +## 7) Тестовый workflow -Пример (псевдокод): +### known_bad ```py -@pytest.mark.parametrize("fixture_path", glob("tests/fixtures/replay/*.json")) -def test_replay_fixture(fixture_path): - root, ctx = load_fixture_and_ctx(fixture_path) - handler = REPLAY_HANDLERS[root["id"]] - assert handler(root["input"], ctx) == root["expected"] +import pytest +from pydantic import ValidationError + +import fetchgraph.tracer.handlers # noqa: F401 +from fetchgraph.tracer.runtime import load_case_bundle, run_case +from fetchgraph.tracer.validators import validate_plan_normalize_spec_v1 + +@pytest.mark.known_bad +@pytest.mark.parametrize("case_path", [...]) +def test_known_bad(case_path): + root, ctx = load_case_bundle(case_path) + out = run_case(root, ctx) + with pytest.raises((AssertionError, ValidationError)): + validate_plan_normalize_spec_v1(out) ``` +### Green (explicit expected) +Если рядом лежит `*.expected.json`, сравниваем его с результатом; иначе можно свериться с `root["observed"]`. + --- -## 6) “Короткая памятка” для разработчика +## 8) Короткая памятка -1) В коде, где хотите регрессию, используйте `log_replay_point(...)`. -2) Если реплей требует контекст/файлы: - - логируйте `planner_input` (extras) и/или `replay_resource` - - добавляйте их `id` в `requires` -3) Для тестов: - - экспортируйте фикстуры из `events.jsonl` через `export.py` - - импортируйте модули обработчиков (чтобы заполнить `REPLAY_HANDLERS`) - - грузите фикстуру, строите `ReplayContext`, вызывайте handler, сравнивайте с `expected` +1) Логируйте `replay_case` через `log_replay_case` (observed-first). +2) Extras/Resources логируйте отдельными событиями (`planner_input`, `replay_resource`). +3) Экспортируйте bundle через `export_replay_case_bundle(s)`. +4) В тестах грузите bundle через `load_case_bundle` и запускайте `run_case`. From 0a88c7375d2616023ccc7527ab518a4eaf2c7949 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 14:01:58 +0300 Subject: [PATCH 37/79] Fix tracer export imports and expected-only tests --- src/fetchgraph/tracer/export.py | 6 ++++-- tests/test_replay_fixtures.py | 9 +++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/fetchgraph/tracer/export.py b/src/fetchgraph/tracer/export.py index 785aa6da..0164223e 100644 --- a/src/fetchgraph/tracer/export.py +++ b/src/fetchgraph/tracer/export.py @@ -2,18 +2,20 @@ from fetchgraph.replay.export import ( case_bundle_name, - collect_requires, copy_resource_files, export_replay_case_bundle, export_replay_case_bundles, + index_requires, iter_events, + resolve_requires, ) __all__ = [ "case_bundle_name", - "collect_requires", "copy_resource_files", "export_replay_case_bundle", "export_replay_case_bundles", + "index_requires", "iter_events", + "resolve_requires", ] diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index 4ff93fa4..24b54bff 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -48,14 +48,11 @@ def test_known_bad_cases(case_path: Path) -> None: @pytest.mark.parametrize("case_path", _iter_case_paths(FIXED_DIR)) def test_replay_cases_expected(case_path: Path) -> None: expected_path = _expected_path(case_path) + if not expected_path.exists(): + pytest.skip(f"Expected fixture missing: {expected_path}") root, ctx = load_case_bundle(case_path) out = run_case(root, ctx) - if expected_path.exists(): - expected = json.loads(expected_path.read_text(encoding="utf-8")) - else: - expected = root.get("observed") - if expected is None: - pytest.skip(f"Expected fixture missing and no observed data in {case_path}") + expected = json.loads(expected_path.read_text(encoding="utf-8")) assert out == expected From da12338f3fb11c5232e1a424a5382a4a06453645 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 14:14:36 +0300 Subject: [PATCH 38/79] Add tracer fixture utility helpers --- src/fetchgraph/tracer/fixture_tools.py | 278 +++++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 src/fetchgraph/tracer/fixture_tools.py diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py new file mode 100644 index 00000000..b6b74150 --- /dev/null +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import filecmp +import json +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +from .runtime import load_case_bundle, run_case + + +@dataclass(frozen=True) +class FixturePaths: + case_path: Path + expected_path: Path + resources_dir: Path + stem: str + bucket: str + + +def _fixture_paths(root: Path, bucket: str, stem: str) -> FixturePaths: + case_path = root / bucket / f"{stem}.case.json" + expected_path = root / bucket / f"{stem}.expected.json" + resources_dir = root / bucket / "resources" / stem + return FixturePaths(case_path, expected_path, resources_dir, stem, bucket) + + +def _iter_case_paths(root: Path, bucket: str) -> Iterable[Path]: + bucket_dir = root / bucket + if not bucket_dir.exists(): + return [] + return sorted(bucket_dir.glob("*.case.json")) + + +def _validate_bundle_schema(payload: dict, *, path: Path) -> None: + if payload.get("schema") != "fetchgraph.tracer.case_bundle" or payload.get("v") != 1: + raise ValueError(f"Unsupported case bundle schema in {path}") + + +def _safe_resource_path(path: str, *, stem: str) -> Path: + rel = Path(path) + if rel.is_absolute() or ".." in rel.parts: + raise ValueError(f"Invalid resource path in bundle {stem}: {path}") + return rel + + +def fixture_green( + *, + case_path: Path, + out_root: Path, + validate: bool = False, + overwrite_expected: bool = False, + dry_run: bool = False, +) -> None: + case_path = case_path.resolve() + out_root = out_root.resolve() + if out_root not in case_path.parents: + raise ValueError(f"Case path must be under {out_root}") + if "known_bad" not in case_path.parts: + raise ValueError("fixture-green expects a case under known_bad") + stem = case_path.stem.replace(".case", "") + known_paths = _fixture_paths(out_root, "known_bad", stem) + fixed_paths = _fixture_paths(out_root, "fixed", stem) + + payload = json.loads(case_path.read_text(encoding="utf-8")) + _validate_bundle_schema(payload, path=case_path) + root = payload.get("root") or {} + observed = root.get("observed") + if not isinstance(observed, dict): + raise ValueError(f"Bundle missing observed payload: {case_path}") + + if fixed_paths.case_path.exists() and not dry_run: + raise FileExistsError(f"Target case already exists: {fixed_paths.case_path}") + if fixed_paths.expected_path.exists() and not overwrite_expected and not dry_run: + raise FileExistsError(f"Expected already exists: {fixed_paths.expected_path}") + + resources = payload.get("resources") or {} + resource_paths = [] + if isinstance(resources, dict): + for resource in resources.values(): + if not isinstance(resource, dict): + continue + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + continue + rel = _safe_resource_path(file_name, stem=stem) + resource_paths.append(rel) + + if resource_paths: + src_dir = known_paths.resources_dir + dst_dir = fixed_paths.resources_dir + if dst_dir.exists() and not dry_run: + raise FileExistsError(f"Target resources already exist: {dst_dir}") + + if dry_run: + print(f"Would write expected: {fixed_paths.expected_path}") + print(f"Would move case: {known_paths.case_path} -> {fixed_paths.case_path}") + if resource_paths: + print(f"Would move resources: {known_paths.resources_dir} -> {fixed_paths.resources_dir}") + if validate: + print("Would validate replay output against expected") + return + + fixed_paths.expected_path.parent.mkdir(parents=True, exist_ok=True) + fixed_paths.expected_path.write_text( + json.dumps(observed, ensure_ascii=False, sort_keys=True, indent=2), + encoding="utf-8", + ) + fixed_paths.case_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(known_paths.case_path), str(fixed_paths.case_path)) + + if resource_paths: + fixed_paths.resources_dir.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(known_paths.resources_dir), str(fixed_paths.resources_dir)) + + if validate: + import fetchgraph.tracer.handlers # noqa: F401 + + root_case, ctx = load_case_bundle(fixed_paths.case_path) + out = run_case(root_case, ctx) + expected = json.loads(fixed_paths.expected_path.read_text(encoding="utf-8")) + if out != expected: + raise AssertionError("Replay output does not match expected after fixture-green") + + +def fixture_rm( + *, + root: Path, + name: str | None, + pattern: str | None, + bucket: str, + scope: str, + dry_run: bool, +) -> None: + root = root.resolve() + buckets = ["fixed", "known_bad"] if bucket == "all" else [bucket] + matched = [] + for b in buckets: + for case_path in _iter_case_paths(root, b): + stem = case_path.stem.replace(".case", "") + if name and stem != name: + continue + if pattern and not case_path.match(pattern): + continue + matched.append(_fixture_paths(root, b, stem)) + + if name and not matched: + raise FileNotFoundError(f"No fixtures found for name={name!r}") + + targets: list[Path] = [] + for paths in matched: + if scope in ("cases", "both"): + targets.extend([paths.case_path, paths.expected_path]) + if scope in ("resources", "both"): + targets.append(paths.resources_dir) + + if dry_run: + for target in targets: + print(f"Would remove: {target}") + return + + for target in targets: + if target.is_dir(): + shutil.rmtree(target, ignore_errors=True) + elif target.exists(): + target.unlink() + + +def fixture_fix( + *, + root: Path, + name: str, + new_name: str, + bucket: str, + dry_run: bool, +) -> None: + root = root.resolve() + paths = _fixture_paths(root, bucket, name) + new_paths = _fixture_paths(root, bucket, new_name) + + if not paths.case_path.exists(): + raise FileNotFoundError(f"Missing case bundle: {paths.case_path}") + if new_paths.case_path.exists(): + raise FileExistsError(f"Target case already exists: {new_paths.case_path}") + + payload = json.loads(paths.case_path.read_text(encoding="utf-8")) + _validate_bundle_schema(payload, path=paths.case_path) + resources = payload.get("resources") or {} + updated = False + if isinstance(resources, dict): + for resource in resources.values(): + if not isinstance(resource, dict): + continue + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + continue + rel = _safe_resource_path(file_name, stem=name) + old_prefix = Path("resources") / name + if rel.parts[:2] == old_prefix.parts[:2]: + new_rel = Path("resources") / new_name / Path(*rel.parts[2:]) + data_ref["file"] = new_rel.as_posix() + updated = True + + if dry_run: + print(f"Would rename case: {paths.case_path} -> {new_paths.case_path}") + if paths.expected_path.exists(): + print(f"Would rename expected: {paths.expected_path} -> {new_paths.expected_path}") + if paths.resources_dir.exists(): + print(f"Would rename resources: {paths.resources_dir} -> {new_paths.resources_dir}") + if updated: + print("Would update data_ref.file paths inside bundle") + return + + new_paths.case_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(paths.case_path), str(new_paths.case_path)) + if paths.expected_path.exists(): + shutil.move(str(paths.expected_path), str(new_paths.expected_path)) + if paths.resources_dir.exists(): + if new_paths.resources_dir.exists(): + raise FileExistsError(f"Target resources already exist: {new_paths.resources_dir}") + shutil.move(str(paths.resources_dir), str(new_paths.resources_dir)) + if updated: + new_paths.case_path.write_text( + json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2), + encoding="utf-8", + ) + + +def fixture_migrate(*, root: Path, dry_run: bool) -> None: + root = root.resolve() + for bucket in ("fixed", "known_bad"): + for case_path in _iter_case_paths(root, bucket): + stem = case_path.stem.replace(".case", "") + payload = json.loads(case_path.read_text(encoding="utf-8")) + _validate_bundle_schema(payload, path=case_path) + resources = payload.get("resources") or {} + if not isinstance(resources, dict): + continue + updated = False + for resource in resources.values(): + if not isinstance(resource, dict): + continue + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + continue + rel = _safe_resource_path(file_name, stem=stem) + target_rel = Path("resources") / stem / rel + if rel == target_rel: + continue + src_path = case_path.parent / rel + if not src_path.exists(): + raise FileNotFoundError(f"Missing resource file: {src_path}") + dest_path = case_path.parent / target_rel + if dest_path.exists() and not filecmp.cmp(src_path, dest_path, shallow=False): + raise FileExistsError(f"Resource collision at {dest_path}") + if dry_run: + print(f"Would move {src_path} -> {dest_path}") + else: + dest_path.parent.mkdir(parents=True, exist_ok=True) + if not dest_path.exists(): + shutil.move(str(src_path), str(dest_path)) + data_ref["file"] = target_rel.as_posix() + updated = True + if updated and not dry_run: + case_path.write_text( + json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2), + encoding="utf-8", + ) From ba8a2e80d92b1c1571aa747812b5a593d681edbf Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 14:21:58 +0300 Subject: [PATCH 39/79] Refine tracer fixture tooling --- Makefile | 35 ++- src/fetchgraph/replay/export.py | 15 + src/fetchgraph/tracer/cli.py | 142 +++++++-- src/fetchgraph/tracer/export.py | 2 + src/fetchgraph/tracer/fixture_layout.py | 69 +++++ src/fetchgraph/tracer/fixture_tools.py | 372 ++++++++++++++---------- tests/test_replay_fixtures.py | 6 +- 7 files changed, 456 insertions(+), 185 deletions(-) create mode 100644 src/fetchgraph/tracer/fixture_layout.py diff --git a/Makefile b/Makefile index 046017b1..8aa5f7b0 100644 --- a/Makefile +++ b/Makefile @@ -47,13 +47,15 @@ TAG ?= NOTE ?= CASE ?= NAME ?= +NEW_NAME ?= PATTERN ?= SPEC_IDX ?= PROVIDER ?= BUCKET ?= fixed REPLAY_ID ?= EVENTS ?= -TRACER_OUT_DIR ?= tests/fixtures/replay_cases/$(BUCKET) +TRACER_ROOT ?= tests/fixtures/replay_cases +TRACER_OUT_DIR ?= $(TRACER_ROOT)/$(BUCKET) RUN_DIR ?= ALLOW_BAD_JSON ?= OVERWRITE ?= @@ -83,6 +85,8 @@ PURGE_RUNS ?= 0 PRUNE_HISTORY ?= 0 PRUNE_CASE_HISTORY ?= 0 DRY ?= 0 +VALIDATE ?= 0 +OVERWRITE_EXPECTED ?= 0 MOVE_TRACES ?= 0 # ============================================================================== @@ -114,6 +118,7 @@ LIMIT_FLAG := $(if $(strip $(LIMIT)),--limit $(LIMIT),) batch-missed batch-missed-from batch-failed-tag batch-missed-tag \ batch-fail-fast batch-max-fails \ stats history-case report-tag report-tag-changes tags tag-rm case-run case-open tracer-export \ + fixture-green fixture-rm fixture-fix fixture-migrate \ compare compare-tag # ============================================================================== @@ -164,6 +169,10 @@ help: @echo " make tracer-export REPLAY_ID=... EVENTS=... TRACER_OUT_DIR=... [SPEC_IDX=...] [PROVIDER=...] [RUN_DIR=...] [ALLOW_BAD_JSON=1] [OVERWRITE=1]" @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources//..." + @echo " make fixture-green CASE=path/to/case.case.json [TRACER_ROOT=...] [VALIDATE=1] [OVERWRITE_EXPECTED=1] [DRY=1]" + @echo " make fixture-rm [BUCKET=fixed|known_bad|all] [NAME=...] [PATTERN=...] [SCOPE=cases|resources|both] [DRY=1]" + @echo " make fixture-fix BUCKET=... NAME=... NEW_NAME=... [DRY=1]" + @echo " make fixture-migrate [BUCKET=fixed|known_bad|all] [DRY=1]" @echo "" @echo "Уборка:" @echo " make tag-rm TAG=... [DRY=1] [PURGE_RUNS=1] [PRUNE_HISTORY=1] [PRUNE_CASE_HISTORY=1]" @@ -364,6 +373,30 @@ tracer-export: $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) \ $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) +fixture-green: + @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture-green CASE=tests/fixtures/replay_cases/known_bad/fixture.case.json" && exit 1) + @$(PYTHON) -m fetchgraph.tracer.cli fixture-green --case "$(CASE)" --root "$(TRACER_ROOT)" \ + $(if $(filter 1 true yes on,$(VALIDATE)),--validate,) \ + $(if $(filter 1 true yes on,$(OVERWRITE_EXPECTED)),--overwrite-expected,) \ + $(if $(filter 1 true yes on,$(DRY)),--dry-run,) + +fixture-rm: + @$(PYTHON) -m fetchgraph.tracer.cli fixture-rm --root "$(TRACER_ROOT)" --bucket "$(BUCKET)" \ + $(if $(strip $(NAME)),--name "$(NAME)",) \ + $(if $(strip $(PATTERN)),--pattern "$(PATTERN)",) \ + $(if $(strip $(SCOPE)),--scope "$(SCOPE)",) \ + $(if $(filter 1 true yes on,$(DRY)),--dry-run,) + +fixture-fix: + @test -n "$(strip $(NAME))" || (echo "NAME обязателен: make fixture-fix NAME=old_stem NEW_NAME=new_stem" && exit 1) + @test -n "$(strip $(NEW_NAME))" || (echo "NEW_NAME обязателен: make fixture-fix NAME=old_stem NEW_NAME=new_stem" && exit 1) + @$(PYTHON) -m fetchgraph.tracer.cli fixture-fix --root "$(TRACER_ROOT)" --bucket "$(BUCKET)" \ + --name "$(NAME)" --new-name "$(NEW_NAME)" \ + $(if $(filter 1 true yes on,$(DRY)),--dry-run,) + +fixture-migrate: + @$(PYTHON) -m fetchgraph.tracer.cli fixture-migrate --root "$(TRACER_ROOT)" --bucket "$(BUCKET)" \ + $(if $(filter 1 true yes on,$(DRY)),--dry-run,) diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py index 0b216d8d..32f8a8ec 100644 --- a/src/fetchgraph/replay/export.py +++ b/src/fetchgraph/replay/export.py @@ -154,6 +154,21 @@ def resolve_requires( return resolved_resources, resolved_extras +def collect_requires( + requires: list[dict] | list[str], + *, + resources: dict[str, dict], + extras: dict[str, dict], + events_path: Path, +) -> tuple[dict[str, dict], dict[str, dict]]: + return resolve_requires( + requires, + resources=resources, + extras=extras, + events_path=events_path, + ) + + def write_case_bundle( out_path: Path, *, diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index 5de7b4a8..aaee555c 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -5,6 +5,14 @@ from pathlib import Path from fetchgraph.tracer.export import export_replay_case_bundle, export_replay_case_bundles +from fetchgraph.tracer.fixture_tools import ( + fixture_fix, + fixture_green, + fixture_migrate, + fixture_rm, +) + +DEFAULT_ROOT = Path("tests/fixtures/replay_cases") def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: @@ -34,35 +42,125 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) export.add_argument("--all", action="store_true", help="Export all matching replay cases") + green = sub.add_parser("fixture-green", help="Promote known_bad case to fixed") + green.add_argument("--case", type=Path, required=True, help="Path to known_bad case bundle") + green.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") + green.add_argument("--validate", action="store_true", help="Validate replay output") + green.add_argument( + "--overwrite-expected", + action="store_true", + help="Overwrite existing expected output", + ) + green.add_argument("--dry-run", action="store_true", help="Print actions without changing files") + + rm_cmd = sub.add_parser("fixture-rm", help="Remove replay fixtures") + rm_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") + rm_cmd.add_argument( + "--bucket", + choices=["fixed", "known_bad", "all"], + default="all", + help="Fixture bucket", + ) + rm_cmd.add_argument("--name", help="Fixture stem name") + rm_cmd.add_argument("--pattern", help="Glob pattern for fixture stems or case bundles") + rm_cmd.add_argument( + "--scope", + choices=["cases", "resources", "both"], + default="both", + help="What to remove", + ) + rm_cmd.add_argument("--dry-run", action="store_true", help="Print actions without changing files") + + fix_cmd = sub.add_parser("fixture-fix", help="Rename fixture stem") + fix_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") + fix_cmd.add_argument("--bucket", choices=["fixed", "known_bad"], default="fixed") + fix_cmd.add_argument("--name", required=True, help="Old fixture stem") + fix_cmd.add_argument("--new-name", required=True, help="New fixture stem") + fix_cmd.add_argument("--dry-run", action="store_true", help="Print actions without changing files") + + migrate_cmd = sub.add_parser("fixture-migrate", help="Normalize resource layout") + migrate_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") + migrate_cmd.add_argument( + "--bucket", + choices=["fixed", "known_bad", "all"], + default="all", + help="Fixture bucket", + ) + migrate_cmd.add_argument("--dry-run", action="store_true", help="Print actions without changing files") + return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) - if args.command == "export-case-bundle": - if args.all: - export_replay_case_bundles( - events_path=args.events, - out_dir=args.out, - replay_id=args.id, - spec_idx=args.spec_idx, - provider=args.provider, - run_dir=args.run_dir, - allow_bad_json=args.allow_bad_json, - overwrite=args.overwrite, + try: + if args.command == "export-case-bundle": + if args.all: + export_replay_case_bundles( + events_path=args.events, + out_dir=args.out, + replay_id=args.id, + spec_idx=args.spec_idx, + provider=args.provider, + run_dir=args.run_dir, + allow_bad_json=args.allow_bad_json, + overwrite=args.overwrite, + ) + else: + export_replay_case_bundle( + events_path=args.events, + out_dir=args.out, + replay_id=args.id, + spec_idx=args.spec_idx, + provider=args.provider, + run_dir=args.run_dir, + allow_bad_json=args.allow_bad_json, + overwrite=args.overwrite, + ) + return 0 + if args.command == "fixture-green": + fixture_green( + case_path=args.case, + out_root=args.root, + validate=args.validate, + overwrite_expected=args.overwrite_expected, + dry_run=args.dry_run, + ) + return 0 + if args.command == "fixture-rm": + removed = fixture_rm( + root=args.root, + bucket=args.bucket, + name=args.name, + pattern=args.pattern, + scope=args.scope, + dry_run=args.dry_run, + ) + print(f"Removed {removed} paths") + return 0 + if args.command == "fixture-fix": + fixture_fix( + root=args.root, + bucket=args.bucket, + name=args.name, + new_name=args.new_name, + dry_run=args.dry_run, ) - else: - export_replay_case_bundle( - events_path=args.events, - out_dir=args.out, - replay_id=args.id, - spec_idx=args.spec_idx, - provider=args.provider, - run_dir=args.run_dir, - allow_bad_json=args.allow_bad_json, - overwrite=args.overwrite, + return 0 + if args.command == "fixture-migrate": + bundles_updated, files_moved = fixture_migrate( + root=args.root, + bucket=args.bucket, + dry_run=args.dry_run, ) - return 0 + print(f"Updated {bundles_updated} bundles; moved {files_moved} files") + return 0 + except (ValueError, FileNotFoundError, LookupError, KeyError) as exc: + print(str(exc), file=sys.stderr) + return 2 + except Exception as exc: # pragma: no cover - unexpected + print(f"Unexpected error: {exc}", file=sys.stderr) + return 1 raise SystemExit(f"Unknown command: {args.command}") diff --git a/src/fetchgraph/tracer/export.py b/src/fetchgraph/tracer/export.py index 0164223e..c37a3b2c 100644 --- a/src/fetchgraph/tracer/export.py +++ b/src/fetchgraph/tracer/export.py @@ -2,6 +2,7 @@ from fetchgraph.replay.export import ( case_bundle_name, + collect_requires, copy_resource_files, export_replay_case_bundle, export_replay_case_bundles, @@ -12,6 +13,7 @@ __all__ = [ "case_bundle_name", + "collect_requires", "copy_resource_files", "export_replay_case_bundle", "export_replay_case_bundles", diff --git a/src/fetchgraph/tracer/fixture_layout.py b/src/fetchgraph/tracer/fixture_layout.py new file mode 100644 index 00000000..08424b8f --- /dev/null +++ b/src/fetchgraph/tracer/fixture_layout.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +VALID_BUCKETS = {"fixed", "known_bad"} + + +@dataclass(frozen=True) +class FixtureLayout: + root: Path + bucket: str + + @property + def bucket_dir(self) -> Path: + return self.root / self.bucket + + def case_path(self, stem: str) -> Path: + return self.bucket_dir / f"{stem}.case.json" + + def expected_path(self, stem: str) -> Path: + return self.bucket_dir / f"{stem}.expected.json" + + def resources_dir(self, stem: str) -> Path: + return self.bucket_dir / "resources" / stem + + +def find_case_bundles( + *, + root: Path, + bucket: str | None, + name: str | None, + pattern: str | None, +) -> list[Path]: + if name and pattern: + raise ValueError("Use only one of name or pattern.") + + root = root.resolve() + if bucket in (None, "all"): + buckets = sorted(VALID_BUCKETS) + elif bucket in VALID_BUCKETS: + buckets = [bucket] + else: + raise ValueError(f"Unsupported bucket: {bucket}") + + if name: + matches: list[Path] = [] + for entry in buckets: + layout = FixtureLayout(root, entry) + case_path = layout.case_path(name) + if case_path.exists(): + matches.append(case_path) + return matches + + if pattern: + if pattern.endswith(".case.json"): + glob_pattern = pattern + else: + glob_pattern = f"{pattern}.case.json" + else: + glob_pattern = "*.case.json" + + matches = [] + for entry in buckets: + layout = FixtureLayout(root, entry) + if not layout.bucket_dir.exists(): + continue + matches.extend(layout.bucket_dir.glob(glob_pattern)) + return sorted(matches) diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py index b6b74150..6cb0084b 100644 --- a/src/fetchgraph/tracer/fixture_tools.py +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -3,39 +3,36 @@ import filecmp import json import shutil -from dataclasses import dataclass from pathlib import Path -from typing import Iterable +from .fixture_layout import FixtureLayout, find_case_bundles from .runtime import load_case_bundle, run_case -@dataclass(frozen=True) -class FixturePaths: - case_path: Path - expected_path: Path - resources_dir: Path - stem: str - bucket: str - - -def _fixture_paths(root: Path, bucket: str, stem: str) -> FixturePaths: - case_path = root / bucket / f"{stem}.case.json" - expected_path = root / bucket / f"{stem}.expected.json" - resources_dir = root / bucket / "resources" / stem - return FixturePaths(case_path, expected_path, resources_dir, stem, bucket) +def load_bundle_json(path: Path) -> dict: + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("schema") != "fetchgraph.tracer.case_bundle" or payload.get("v") != 1: + raise ValueError(f"Not a tracer case bundle: {path}") + root = payload.get("root") + if not isinstance(root, dict): + raise ValueError(f"Bundle root must be a mapping: {path}") -def _iter_case_paths(root: Path, bucket: str) -> Iterable[Path]: - bucket_dir = root / bucket - if not bucket_dir.exists(): - return [] - return sorted(bucket_dir.glob("*.case.json")) + root_type = root.get("type") + root_version = root.get("v") + if root_type is not None and root_type != "replay_case": + raise ValueError(f"Bundle root.type must be replay_case: {path}") + if root_version is not None and root_version != 2: + raise ValueError(f"Bundle root.v must be 2: {path}") + resources = payload.get("resources") + if resources is not None and not isinstance(resources, dict): + raise ValueError(f"Bundle resources must be a mapping: {path}") + extras = payload.get("extras") + if extras is not None and not isinstance(extras, dict): + raise ValueError(f"Bundle extras must be a mapping: {path}") -def _validate_bundle_schema(payload: dict, *, path: Path) -> None: - if payload.get("schema") != "fetchgraph.tracer.case_bundle" or payload.get("v") != 1: - raise ValueError(f"Unsupported case bundle schema in {path}") + return payload def _safe_resource_path(path: str, *, stem: str) -> Path: @@ -45,6 +42,15 @@ def _safe_resource_path(path: str, *, stem: str) -> Path: return rel +def _atomic_write_json(path: Path, payload: dict) -> None: + tmp_path = path.with_suffix(path.suffix + ".tmp") + tmp_path.write_text( + json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2), + encoding="utf-8", + ) + tmp_path.replace(path) + + def fixture_green( *, case_path: Path, @@ -55,77 +61,89 @@ def fixture_green( ) -> None: case_path = case_path.resolve() out_root = out_root.resolve() - if out_root not in case_path.parents: - raise ValueError(f"Case path must be under {out_root}") - if "known_bad" not in case_path.parts: - raise ValueError("fixture-green expects a case under known_bad") - stem = case_path.stem.replace(".case", "") - known_paths = _fixture_paths(out_root, "known_bad", stem) - fixed_paths = _fixture_paths(out_root, "fixed", stem) - - payload = json.loads(case_path.read_text(encoding="utf-8")) - _validate_bundle_schema(payload, path=case_path) + known_layout = FixtureLayout(out_root, "known_bad") + if not case_path.is_relative_to(known_layout.bucket_dir): + raise ValueError(f"fixture-green expects a known_bad case path, got: {case_path}") + if not case_path.name.endswith(".case.json"): + raise ValueError(f"fixture-green expects a .case.json bundle, got: {case_path}") + stem = case_path.name.replace(".case.json", "") + fixed_layout = FixtureLayout(out_root, "fixed") + + payload = load_bundle_json(case_path) root = payload.get("root") or {} observed = root.get("observed") if not isinstance(observed, dict): - raise ValueError(f"Bundle missing observed payload: {case_path}") - - if fixed_paths.case_path.exists() and not dry_run: - raise FileExistsError(f"Target case already exists: {fixed_paths.case_path}") - if fixed_paths.expected_path.exists() and not overwrite_expected and not dry_run: - raise FileExistsError(f"Expected already exists: {fixed_paths.expected_path}") - - resources = payload.get("resources") or {} - resource_paths = [] - if isinstance(resources, dict): - for resource in resources.values(): - if not isinstance(resource, dict): - continue - data_ref = resource.get("data_ref") - if not isinstance(data_ref, dict): - continue - file_name = data_ref.get("file") - if not isinstance(file_name, str) or not file_name: - continue - rel = _safe_resource_path(file_name, stem=stem) - resource_paths.append(rel) + raise ValueError( + "Cannot green fixture: root.observed is missing.\n" + f"Case: {case_path}\n" + "Hint: export observed-first replay_case bundles; green requires observed to freeze behavior." + ) - if resource_paths: - src_dir = known_paths.resources_dir - dst_dir = fixed_paths.resources_dir - if dst_dir.exists() and not dry_run: - raise FileExistsError(f"Target resources already exist: {dst_dir}") + known_case_path = known_layout.case_path(stem) + fixed_case_path = fixed_layout.case_path(stem) + fixed_expected_path = fixed_layout.expected_path(stem) + known_expected_path = known_layout.expected_path(stem) + resources_from = known_layout.resources_dir(stem) + resources_to = fixed_layout.resources_dir(stem) + + if fixed_case_path.exists() and not dry_run: + raise FileExistsError(f"Target case already exists: {fixed_case_path}") + if fixed_expected_path.exists() and not overwrite_expected and not dry_run: + raise FileExistsError(f"Expected already exists: {fixed_expected_path}") + if resources_from.exists() and resources_to.exists() and not dry_run: + raise FileExistsError( + "Resources destination already exists:\n" + f" dest: {resources_to}\n" + "Actions:\n" + " - remove destination resources directory, or\n" + " - run fixture-fix to rename stems, or\n" + " - choose a different out root." + ) if dry_run: - print(f"Would write expected: {fixed_paths.expected_path}") - print(f"Would move case: {known_paths.case_path} -> {fixed_paths.case_path}") - if resource_paths: - print(f"Would move resources: {known_paths.resources_dir} -> {fixed_paths.resources_dir}") + print("fixture-green:") + print(f" case: {known_case_path}") + print(f" move: -> {fixed_case_path}") + print(f" write: -> {fixed_expected_path} (from root.observed)") + if resources_from.exists(): + print(f" move: resources -> {resources_to}") if validate: - print("Would validate replay output against expected") + print(" validate: would run") return - fixed_paths.expected_path.parent.mkdir(parents=True, exist_ok=True) - fixed_paths.expected_path.write_text( + fixed_expected_path.parent.mkdir(parents=True, exist_ok=True) + fixed_expected_path.write_text( json.dumps(observed, ensure_ascii=False, sort_keys=True, indent=2), encoding="utf-8", ) - fixed_paths.case_path.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(known_paths.case_path), str(fixed_paths.case_path)) + fixed_case_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(known_case_path), str(fixed_case_path)) - if resource_paths: - fixed_paths.resources_dir.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(known_paths.resources_dir), str(fixed_paths.resources_dir)) + if known_expected_path.exists(): + known_expected_path.unlink() + + if resources_from.exists(): + resources_to.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(resources_from), str(resources_to)) if validate: import fetchgraph.tracer.handlers # noqa: F401 - root_case, ctx = load_case_bundle(fixed_paths.case_path) + root_case, ctx = load_case_bundle(fixed_case_path) out = run_case(root_case, ctx) - expected = json.loads(fixed_paths.expected_path.read_text(encoding="utf-8")) + expected = json.loads(fixed_expected_path.read_text(encoding="utf-8")) if out != expected: raise AssertionError("Replay output does not match expected after fixture-green") + print("fixture-green:") + print(f" case: {known_case_path}") + print(f" move: -> {fixed_case_path}") + print(f" write: -> {fixed_expected_path} (from root.observed)") + if resources_from.exists(): + print(f" move: resources -> {resources_to}") + if validate: + print(" validate: OK") + def fixture_rm( *, @@ -135,39 +153,40 @@ def fixture_rm( bucket: str, scope: str, dry_run: bool, -) -> None: +) -> int: root = root.resolve() - buckets = ["fixed", "known_bad"] if bucket == "all" else [bucket] - matched = [] - for b in buckets: - for case_path in _iter_case_paths(root, b): - stem = case_path.stem.replace(".case", "") - if name and stem != name: - continue - if pattern and not case_path.match(pattern): - continue - matched.append(_fixture_paths(root, b, stem)) + bucket = None if bucket == "all" else bucket + matched = find_case_bundles(root=root, bucket=bucket, name=name, pattern=pattern) if name and not matched: raise FileNotFoundError(f"No fixtures found for name={name!r}") targets: list[Path] = [] - for paths in matched: + for case_path in matched: + bucket_name = case_path.parent.name + stem = case_path.name.replace(".case.json", "") + layout = FixtureLayout(root, bucket_name) if scope in ("cases", "both"): - targets.extend([paths.case_path, paths.expected_path]) + targets.extend([layout.case_path(stem), layout.expected_path(stem)]) if scope in ("resources", "both"): - targets.append(paths.resources_dir) + targets.append(layout.resources_dir(stem)) + + existing_targets = [target for target in targets if target.exists()] + + if name and not existing_targets: + raise FileNotFoundError(f"No fixtures found for name={name!r} with scope={scope}") if dry_run: for target in targets: print(f"Would remove: {target}") - return + return len(existing_targets) for target in targets: if target.is_dir(): shutil.rmtree(target, ignore_errors=True) elif target.exists(): target.unlink() + return len(existing_targets) def fixture_fix( @@ -179,16 +198,27 @@ def fixture_fix( dry_run: bool, ) -> None: root = root.resolve() - paths = _fixture_paths(root, bucket, name) - new_paths = _fixture_paths(root, bucket, new_name) - - if not paths.case_path.exists(): - raise FileNotFoundError(f"Missing case bundle: {paths.case_path}") - if new_paths.case_path.exists(): - raise FileExistsError(f"Target case already exists: {new_paths.case_path}") - - payload = json.loads(paths.case_path.read_text(encoding="utf-8")) - _validate_bundle_schema(payload, path=paths.case_path) + if name == new_name: + raise ValueError("fixture-fix requires a new name different from the old name.") + + layout = FixtureLayout(root, bucket) + case_path = layout.case_path(name) + expected_path = layout.expected_path(name) + resources_dir = layout.resources_dir(name) + new_case_path = layout.case_path(new_name) + new_expected_path = layout.expected_path(new_name) + new_resources_dir = layout.resources_dir(new_name) + + if not case_path.exists(): + raise FileNotFoundError(f"Missing case bundle: {case_path}") + if new_case_path.exists(): + raise FileExistsError(f"Target case already exists: {new_case_path}") + if new_expected_path.exists(): + raise FileExistsError(f"Target expected already exists: {new_expected_path}") + if new_resources_dir.exists(): + raise FileExistsError(f"Target resources already exist: {new_resources_dir}") + + payload = load_bundle_json(case_path) resources = payload.get("resources") or {} updated = False if isinstance(resources, dict): @@ -202,77 +232,97 @@ def fixture_fix( if not isinstance(file_name, str) or not file_name: continue rel = _safe_resource_path(file_name, stem=name) - old_prefix = Path("resources") / name - if rel.parts[:2] == old_prefix.parts[:2]: - new_rel = Path("resources") / new_name / Path(*rel.parts[2:]) - data_ref["file"] = new_rel.as_posix() + old_prefix = f"resources/{name}/" + rel_posix = rel.as_posix() + if rel_posix.startswith(old_prefix): + data_ref["file"] = f"resources/{new_name}/{rel_posix[len(old_prefix):]}" updated = True if dry_run: - print(f"Would rename case: {paths.case_path} -> {new_paths.case_path}") - if paths.expected_path.exists(): - print(f"Would rename expected: {paths.expected_path} -> {new_paths.expected_path}") - if paths.resources_dir.exists(): - print(f"Would rename resources: {paths.resources_dir} -> {new_paths.resources_dir}") + print("fixture-fix:") + print(f" rename: {case_path} -> {new_case_path}") + if expected_path.exists(): + print(f" move: {expected_path} -> {new_expected_path}") + if resources_dir.exists(): + print(f" move: {resources_dir} -> {new_resources_dir}") if updated: - print("Would update data_ref.file paths inside bundle") + print(" rewrite: data_ref.file paths updated") return - new_paths.case_path.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(paths.case_path), str(new_paths.case_path)) - if paths.expected_path.exists(): - shutil.move(str(paths.expected_path), str(new_paths.expected_path)) - if paths.resources_dir.exists(): - if new_paths.resources_dir.exists(): - raise FileExistsError(f"Target resources already exist: {new_paths.resources_dir}") - shutil.move(str(paths.resources_dir), str(new_paths.resources_dir)) + new_case_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(case_path), str(new_case_path)) + if expected_path.exists(): + shutil.move(str(expected_path), str(new_expected_path)) + if resources_dir.exists(): + shutil.move(str(resources_dir), str(new_resources_dir)) if updated: - new_paths.case_path.write_text( - json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2), - encoding="utf-8", - ) + _atomic_write_json(new_case_path, payload) + + print("fixture-fix:") + print(f" rename: {case_path} -> {new_case_path}") + if expected_path.exists(): + print(f" move: {expected_path} -> {new_expected_path}") + if resources_dir.exists(): + print(f" move: {resources_dir} -> {new_resources_dir}") + if updated: + print(" rewrite: data_ref.file paths updated") -def fixture_migrate(*, root: Path, dry_run: bool) -> None: +def fixture_migrate(*, root: Path, bucket: str = "all", dry_run: bool) -> tuple[int, int]: root = root.resolve() - for bucket in ("fixed", "known_bad"): - for case_path in _iter_case_paths(root, bucket): - stem = case_path.stem.replace(".case", "") - payload = json.loads(case_path.read_text(encoding="utf-8")) - _validate_bundle_schema(payload, path=case_path) - resources = payload.get("resources") or {} - if not isinstance(resources, dict): + bucket_filter = None if bucket == "all" else bucket + matched = find_case_bundles(root=root, bucket=bucket_filter, name=None, pattern=None) + bundles_updated = 0 + files_moved = 0 + for case_path in matched: + stem = case_path.name.replace(".case.json", "") + payload = load_bundle_json(case_path) + resources = payload.get("resources") or {} + if not isinstance(resources, dict): + continue + updated = False + for resource_id, resource in resources.items(): + if not isinstance(resource, dict): continue - updated = False - for resource in resources.values(): - if not isinstance(resource, dict): - continue - data_ref = resource.get("data_ref") - if not isinstance(data_ref, dict): - continue - file_name = data_ref.get("file") - if not isinstance(file_name, str) or not file_name: - continue - rel = _safe_resource_path(file_name, stem=stem) - target_rel = Path("resources") / stem / rel - if rel == target_rel: - continue - src_path = case_path.parent / rel - if not src_path.exists(): - raise FileNotFoundError(f"Missing resource file: {src_path}") - dest_path = case_path.parent / target_rel - if dest_path.exists() and not filecmp.cmp(src_path, dest_path, shallow=False): - raise FileExistsError(f"Resource collision at {dest_path}") - if dry_run: - print(f"Would move {src_path} -> {dest_path}") - else: - dest_path.parent.mkdir(parents=True, exist_ok=True) - if not dest_path.exists(): - shutil.move(str(src_path), str(dest_path)) - data_ref["file"] = target_rel.as_posix() - updated = True - if updated and not dry_run: - case_path.write_text( - json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2), - encoding="utf-8", + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + continue + rel = _safe_resource_path(file_name, stem=stem) + if rel.parts[:3] == ("resources", stem, resource_id): + continue + if rel.parts[:1] == ("resources",) and len(rel.parts) >= 2: + rel_tail = Path(*rel.parts[2:]) + else: + rel_tail = rel + if not rel_tail.parts: + rel_tail = Path(rel.name) + target_rel = Path("resources") / stem / resource_id / rel_tail + src_path = case_path.parent / rel + if not src_path.exists(): + raise FileNotFoundError(f"Missing resource file: {src_path}") + dest_path = case_path.parent / target_rel + if dest_path.exists() and not filecmp.cmp(src_path, dest_path, shallow=False): + raise FileExistsError( + "Resource collision at destination:\n" + f" dest: {dest_path}\n" + "Hint: clean the destination or run migrate in an empty output directory." ) + if dry_run: + print(f"Would move {src_path} -> {dest_path}") + else: + dest_path.parent.mkdir(parents=True, exist_ok=True) + if not dest_path.exists(): + shutil.move(str(src_path), str(dest_path)) + files_moved += 1 + data_ref["file"] = target_rel.as_posix() + updated = True + if updated: + bundles_updated += 1 + if dry_run: + print(f"Would update bundle: {case_path}") + else: + _atomic_write_json(case_path, payload) + return bundles_updated, files_moved diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index 24b54bff..41b54649 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -49,7 +49,11 @@ def test_known_bad_cases(case_path: Path) -> None: def test_replay_cases_expected(case_path: Path) -> None: expected_path = _expected_path(case_path) if not expected_path.exists(): - pytest.skip(f"Expected fixture missing: {expected_path}") + pytest.fail( + "Expected file is required for fixed fixtures:\n" + f" missing: {expected_path}\n" + "Hint: run `fetchgraph-tracer fixture-green --case ...` or create expected json manually." + ) root, ctx = load_case_bundle(case_path) out = run_case(root, ctx) expected = json.loads(expected_path.read_text(encoding="utf-8")) From 702ca759db62ce2ab1a21e1bce1fabb4fdb1ae9e Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 14:49:21 +0300 Subject: [PATCH 40/79] Guard tracer-export bucket usage --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 8aa5f7b0..28ca46cc 100644 --- a/Makefile +++ b/Makefile @@ -365,6 +365,7 @@ case-open: check tracer-export: @test -n "$(strip $(REPLAY_ID))" || (echo "REPLAY_ID обязателен: make tracer-export REPLAY_ID=plan_normalize.spec_v1" && exit 1) @test -n "$(strip $(EVENTS))" || (echo "EVENTS обязателен: make tracer-export EVENTS=path/to/events.jsonl" && exit 1) + @case "$(BUCKET)" in fixed|known_bad) ;; *) echo "BUCKET должен быть fixed или known_bad для tracer-export" && exit 1 ;; esac @# TRACER_OUT_DIR has a default; override if needed. @$(PYTHON) -m fetchgraph.tracer.cli export-case-bundle --events "$(EVENTS)" --out "$(TRACER_OUT_DIR)" --id "$(REPLAY_ID)" \ $(if $(strip $(SPEC_IDX)),--spec-idx $(SPEC_IDX),) \ From 8decb64c663d13a48c4b1dc9cd3334a247a89988 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 14:56:20 +0300 Subject: [PATCH 41/79] Remove legacy replay fixture entrypoints --- examples/demo_qa/fixture_cli.py | 175 -------- examples/demo_qa/fixture_tools.py | 683 ------------------------------ src/fetchgraph/cli.py | 74 ---- src/fetchgraph/replay/__init__.py | 4 +- 4 files changed, 1 insertion(+), 935 deletions(-) delete mode 100644 examples/demo_qa/fixture_cli.py delete mode 100644 examples/demo_qa/fixture_tools.py delete mode 100644 src/fetchgraph/cli.py diff --git a/examples/demo_qa/fixture_cli.py b/examples/demo_qa/fixture_cli.py deleted file mode 100644 index daef31d9..00000000 --- a/examples/demo_qa/fixture_cli.py +++ /dev/null @@ -1,175 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path -from typing import Iterable, Optional - -from fetchgraph.replay.export import export_replay_fixture, export_replay_fixtures - - -def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: - p = argparse.ArgumentParser(description="DemoQA: export replay fixtures from .runs") - p.add_argument("--case", required=True, help="Case id to extract from runs") - p.add_argument("--tag", default=None, help="Optional run tag filter") - p.add_argument("--run-id", default=None, help="Exact run_id to select") - p.add_argument("--id", default="plan_normalize.spec_v1", help="Replay point id to extract") - p.add_argument("--spec-idx", type=int, default=None, help="Filter replay_point by meta.spec_idx") - p.add_argument("--provider", default=None, help="Filter replay_point by meta.provider") - p.add_argument("--with-requires", action="store_true", help="Export replay bundle with dependencies") - p.add_argument("--all", action="store_true", help="Export all matching replay points") - - p.add_argument("--data", type=Path, default=None, help="Data dir containing .runs (default: cwd)") - p.add_argument("--runs-dir", type=Path, default=None, help="Explicit .runs/runs dir (overrides --data)") - p.add_argument( - "--out-dir", - type=Path, - default=Path("tests") / "fixtures" / "replay_points", - help="Output directory for replay fixtures", - ) - return p.parse_args(argv) - - -def _load_json(path: Path) -> Optional[dict]: - if not path.exists(): - return None - try: - return json.loads(path.read_text(encoding="utf-8")) - except Exception: - return None - - -def _iter_run_folders(runs_root: Path) -> Iterable[Path]: - if not runs_root.exists(): - return [] - for entry in runs_root.iterdir(): - if entry.is_dir(): - yield entry - - -def _load_run_meta(run_folder: Path) -> dict: - return (_load_json(run_folder / "run_meta.json") or {}) or (_load_json(run_folder / "summary.json") or {}) - - -def _parse_run_id_from_name(run_folder: Path) -> str | None: - parts = run_folder.name.split("_") - return parts[-1] if parts else None - - -def _case_dirs(run_folder: Path, case_id: str) -> list[Path]: - cases_root = run_folder / "cases" - if not cases_root.exists(): - return [] - return sorted(cases_root.glob(f"{case_id}_*")) - - -def _pick_latest(paths: Iterable[Path]) -> Optional[Path]: - candidates = list(paths) - if not candidates: - return None - return max(candidates, key=lambda p: p.stat().st_mtime) - - -def _is_missed_case(case_dir: Path) -> bool: - status = _load_json(case_dir / "status.json") or {} - status_value = str(status.get("status") or "").lower() - if status_value in {"missed"}: - return True - if status_value and status_value != "skipped": - return False - reason = str(status.get("reason") or "").lower() - if "missed" in reason or "missing" in reason: - return True - return bool(status.get("missed")) - - -def _resolve_runs_root(args: argparse.Namespace) -> Path: - if args.runs_dir: - return args.runs_dir - if args.data: - return args.data / ".runs" / "runs" - return Path(".runs") / "runs" - - -def _resolve_run_folder(args: argparse.Namespace, runs_root: Path) -> Path: - if args.run_id: - for run_folder in _iter_run_folders(runs_root): - meta = _load_run_meta(run_folder) - run_id = meta.get("run_id") or _parse_run_id_from_name(run_folder) - if run_id == args.run_id: - return run_folder - raise SystemExit(f"run_id={args.run_id!r} not found in {runs_root}") - - tag = args.tag - candidates = [] - for run_folder in _iter_run_folders(runs_root): - meta = _load_run_meta(run_folder) - if tag: - if meta.get("tag") != tag: - continue - case_dirs = _case_dirs(run_folder, args.case) - if not case_dirs: - continue - latest_case = _pick_latest(case_dirs) - if latest_case and _is_missed_case(latest_case): - continue - candidates.append(run_folder) - - latest = _pick_latest(candidates) - if latest is None: - raise SystemExit(f"No runs found for case={args.case!r} (tag={tag!r}) in {runs_root}") - return latest - - -def _find_case_run_dir(run_folder: Path, case_id: str) -> Path: - latest = _pick_latest(_case_dirs(run_folder, case_id)) - if latest is None: - raise SystemExit(f"No case run dir found for case={case_id!r} in {run_folder}") - return latest - - -def main(argv: list[str] | None = None) -> int: - args = _parse_args(argv) - runs_root = _resolve_runs_root(args) - run_folder = _resolve_run_folder(args, runs_root) - case_run_dir = _find_case_run_dir(run_folder, args.case) - events_path = case_run_dir / "events.jsonl" - if not events_path.exists(): - raise SystemExit(f"events.jsonl not found at {events_path}") - - if args.all: - export_replay_fixtures( - events_path=events_path, - run_dir=case_run_dir, - out_dir=args.out_dir, - replay_id=args.id, - spec_idx=args.spec_idx, - provider=args.provider, - with_requires=args.with_requires, - source_extra={ - "case_id": args.case, - "tag": args.tag, - "picked": "run_id" if args.run_id else "latest_non_missed", - }, - ) - else: - export_replay_fixture( - events_path=events_path, - run_dir=case_run_dir, - out_dir=args.out_dir, - replay_id=args.id, - spec_idx=args.spec_idx, - provider=args.provider, - with_requires=args.with_requires, - source_extra={ - "case_id": args.case, - "tag": args.tag, - "picked": "run_id" if args.run_id else "latest_non_missed", - }, - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/demo_qa/fixture_tools.py b/examples/demo_qa/fixture_tools.py deleted file mode 100644 index f075b26e..00000000 --- a/examples/demo_qa/fixture_tools.py +++ /dev/null @@ -1,683 +0,0 @@ -from __future__ import annotations - -import argparse -import fnmatch -import json -import shutil -import subprocess -import sys -from pathlib import Path -from typing import Iterable, Optional - -REPO_ROOT = Path(__file__).resolve().parents[2] -REPLAY_ROOT = REPO_ROOT / "tests" / "fixtures" / "replay_points" -TRACE_ROOT = REPO_ROOT / "tests" / "fixtures" / "plan_traces" -BUCKETS = ("fixed", "known_bad") - - -def _normalize(value: Optional[str]) -> Optional[str]: - if value is None: - return None - value = value.strip() - return value or None - - -def _is_git_repo() -> bool: - result = subprocess.run( - ["git", "rev-parse", "--is-inside-work-tree"], - cwd=REPO_ROOT, - capture_output=True, - text=True, - check=False, - ) - return result.returncode == 0 and result.stdout.strip() == "true" - - -def _git_path(path: Path) -> str: - return str(path.relative_to(REPO_ROOT)) - - -def _git_tracked(path: Path) -> bool: - result = subprocess.run( - ["git", "ls-files", "--error-unmatch", _git_path(path)], - cwd=REPO_ROOT, - capture_output=True, - text=True, - check=False, - ) - return result.returncode == 0 - - -def _git_has_tracked(path: Path) -> bool: - result = subprocess.run( - ["git", "ls-files", "-z", "--", _git_path(path)], - cwd=REPO_ROOT, - capture_output=True, - check=False, - ) - return result.returncode == 0 and bool(result.stdout) - - -def _is_relative_to(path: Path, base: Path) -> bool: - try: - path.relative_to(base) - return True - except ValueError: - return False - - -def _remove_file(path: Path, *, use_git: bool, dry_run: bool) -> None: - if dry_run: - print(f"DRY: remove {path}") - return - if use_git and _git_tracked(path): - try: - subprocess.run(["git", "rm", "-f", _git_path(path)], cwd=REPO_ROOT, check=True) - return - except subprocess.CalledProcessError: - pass - path.unlink(missing_ok=True) - - -def _remove_tree(path: Path, *, use_git: bool, dry_run: bool) -> None: - if not path.exists(): - return - if dry_run: - if use_git and _git_has_tracked(path): - print(f"DRY: git rm -r -f {_git_path(path)}") - print(f"DRY: rmtree (cleanup untracked) {path}") - else: - print(f"DRY: remove tree {path}") - return - - # If directory contains tracked files, remove them via git rm first. - if use_git and _git_has_tracked(path): - try: - subprocess.run(["git", "rm", "-r", "-f", _git_path(path)], cwd=REPO_ROOT, check=True) - except subprocess.CalledProcessError: - # Fall back to filesystem removal below. - pass - - # git rm deletes only tracked files; remove any remaining untracked files/dirs. - if path.exists(): - shutil.rmtree(path, ignore_errors=True) - - -def _move_file(src: Path, dst: Path, *, use_git: bool, dry_run: bool) -> None: - if dry_run: - print(f"DRY: move {src} -> {dst}") - return - dst.parent.mkdir(parents=True, exist_ok=True) - if use_git and (_git_tracked(src) or _git_tracked(dst)): - try: - subprocess.run(["git", "mv", _git_path(src), _git_path(dst)], cwd=REPO_ROOT, check=True) - return - except subprocess.CalledProcessError: - pass - shutil.move(str(src), str(dst)) - - -def _move_tree(src: Path, dst: Path, *, use_git: bool, dry_run: bool) -> None: - if dry_run: - print(f"DRY: move tree {src} -> {dst}") - return - dst.parent.mkdir(parents=True, exist_ok=True) - if use_git and (_git_has_tracked(src) or _git_has_tracked(dst)): - try: - subprocess.run(["git", "mv", _git_path(src), _git_path(dst)], cwd=REPO_ROOT, check=True) - return - except subprocess.CalledProcessError: - pass - shutil.move(str(src), str(dst)) - - -def _rollback_moves(moves_done: list[tuple[Path, Path, bool]], *, use_git: bool) -> None: - """ - Best-effort rollback of already executed moves. - moves_done: list of (src, dst, is_dir) that were successfully moved src -> dst. - Rollback tries to move dst -> src in reverse order. - """ - if not moves_done: - return - print("Rolling back already moved files...", file=sys.stderr) - for src, dst, is_dir in reversed(moves_done): - try: - if not dst.exists(): - print(f"ROLLBACK: skip (missing) {dst}", file=sys.stderr) - continue - if src.exists(): - print(f"ROLLBACK: skip (src exists) {src} <- {dst}", file=sys.stderr) - continue - if is_dir: - _move_tree(dst, src, use_git=use_git, dry_run=False) - else: - _move_file(dst, src, use_git=use_git, dry_run=False) - print(f"ROLLBACK: {dst} -> {src}", file=sys.stderr) - except Exception as exc: - print(f"ROLLBACK ERROR: failed {dst} -> {src}: {exc}", file=sys.stderr) - - -def _matches_filters(path: Path, *, name: Optional[str], pattern: Optional[str]) -> bool: - if name and path.name != name and path.stem != name: - return False - if pattern and not fnmatch.fnmatch(path.name, pattern): - return False - return True - - -def _iter_replay_paths(bucket: Optional[str]) -> Iterable[Path]: - buckets = [bucket] if bucket else BUCKETS - for bkt in buckets: - root = REPLAY_ROOT / bkt - if not root.exists(): - continue - for path in root.rglob("*.json"): - if "resources" in path.parts: - continue - yield path - - -def _iter_trace_paths(bucket: Optional[str]) -> Iterable[Path]: - buckets = [bucket] if bucket else BUCKETS - for bkt in buckets: - root = TRACE_ROOT / bkt - if not root.exists(): - continue - yield from root.rglob("*_plan_trace.txt") - - -def _relative(path: Path) -> str: - try: - return str(path.relative_to(REPO_ROOT)) - except ValueError: - return str(path) - - -def _resource_key_for_fixture(fixture_path: Path) -> str: - if _is_relative_to(fixture_path, REPLAY_ROOT): - rel = fixture_path.relative_to(REPLAY_ROOT) - if len(rel.parts) >= 2: - bucket = rel.parts[0] - fixture_rel = fixture_path.relative_to(REPLAY_ROOT / bucket).with_suffix("") - return "__".join(fixture_rel.parts) - return fixture_path.stem - - -def _resources_dir_for_fixture(fixture_path: Path) -> Path: - if _is_relative_to(fixture_path, REPLAY_ROOT): - rel = fixture_path.relative_to(REPLAY_ROOT) - if len(rel.parts) >= 2: - bucket = rel.parts[0] - return REPLAY_ROOT / bucket / "resources" / _resource_key_for_fixture(fixture_path) - return fixture_path.parent / "resources" / _resource_key_for_fixture(fixture_path) - - -def _canonical_json(payload: object) -> str: - return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) - - -def _resource_destination( - fixture_path: Path, - file_name: str, - resource_key: str, - used_paths: set[Path], -) -> Path: - rel_path = Path(file_name) - if rel_path.is_absolute() or ".." in rel_path.parts: - raise ValueError(f"Invalid resource path: {file_name}") - base_dir = Path("resources") / _resource_key_for_fixture(fixture_path) - if rel_path.parent != Path("."): - dest_rel = base_dir / rel_path - else: - dest_rel = base_dir / rel_path.name - if dest_rel in used_paths: - prefix = resource_key or "resource" - dest_rel = base_dir / f"{prefix}__{rel_path.name}" - suffix = 1 - while dest_rel in used_paths: - dest_rel = base_dir / f"{prefix}_{suffix}__{rel_path.name}" - suffix += 1 - used_paths.add(dest_rel) - return dest_rel - - -def _bucket_from_path(path: Path, root: Path) -> str: - try: - return path.relative_to(root).parts[0] - except ValueError: - return "unknown" - - -def _load_case_id(path: Path) -> Optional[str]: - try: - data = json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - return None - if data.get("type") == "replay_bundle": - root = data.get("root") or {} - else: - root = data - case_id = ( - root.get("case_id") - or (root.get("meta") or {}).get("case_id") - or (data.get("meta") or {}).get("case_id") - or (data.get("input") or {}).get("case_id") - ) - return case_id if isinstance(case_id, str) else None - - -def _select_with_case(paths: list[Path], case_id: str) -> list[Path]: - name_matches = [path for path in paths if case_id in path.name] - if name_matches: - return name_matches - return [path for path in paths if _load_case_id(path) == case_id] - - -def _validate_name_or_pattern(name: Optional[str], pattern: Optional[str]) -> None: - if not name and not pattern: - raise ValueError("Нужно задать хотя бы NAME или PATTERN.") - - -def _validate_name_pattern_case(name: Optional[str], pattern: Optional[str], case_id: Optional[str]) -> None: - if not name and not pattern and not case_id: - raise ValueError("Нужно задать хотя бы NAME, PATTERN или CASE.") - - -def _collect_candidates( - *, - scope: str, - bucket: Optional[str], - name: Optional[str], - pattern: Optional[str], -) -> list[Path]: - candidates: list[Path] = [] - if scope in ("replay", "both"): - for path in _iter_replay_paths(bucket): - if _matches_filters(path, name=name, pattern=pattern): - candidates.append(path) - if scope in ("traces", "both"): - for path in _iter_trace_paths(bucket): - if _matches_filters(path, name=name, pattern=pattern): - candidates.append(path) - return sorted(candidates, key=lambda p: _relative(p)) - - -def cmd_rm(args: argparse.Namespace) -> int: - name = _normalize(args.name) - pattern = _normalize(args.pattern) - bucket = _normalize(args.bucket) - scope = _normalize(args.scope) or "both" - dry_run = bool(args.dry) - with_resources = bool(args.with_resources) - - if bucket and bucket not in BUCKETS: - print(f"Неизвестный BUCKET: {bucket}", file=sys.stderr) - return 1 - if scope not in ("replay", "traces", "both"): - print(f"Неизвестный SCOPE: {scope}", file=sys.stderr) - return 1 - - try: - _validate_name_or_pattern(name, pattern) - except ValueError as exc: - print(str(exc), file=sys.stderr) - return 1 - - candidates = _collect_candidates(scope=scope, bucket=bucket, name=name, pattern=pattern) - if not candidates: - print("Ничего не найдено.", file=sys.stderr) - return 1 - - use_git = _is_git_repo() - print("Found files to remove:") - for path in candidates: - root = REPLAY_ROOT if _is_relative_to(path, REPLAY_ROOT) else TRACE_ROOT - print(f"- {_bucket_from_path(path, root)}: {_relative(path)}") - if with_resources and _is_relative_to(path, REPLAY_ROOT): - resources_dir = _resources_dir_for_fixture(path) - if resources_dir.exists(): - print(f" - resources: {_relative(resources_dir)}") - - if dry_run: - print(f"DRY: would remove {len(candidates)} files.") - return 0 - - resource_dirs: list[Path] = [] - if with_resources and scope in ("replay", "both"): - for path in candidates: - if _is_relative_to(path, REPLAY_ROOT): - resources_dir = _resources_dir_for_fixture(path) - if resources_dir.exists(): - resource_dirs.append(resources_dir) - - for resource_dir in sorted(set(resource_dirs), key=lambda p: _relative(p)): - _remove_tree(resource_dir, use_git=use_git, dry_run=False) - for path in candidates: - _remove_file(path, use_git=use_git, dry_run=False) - - if resource_dirs: - print(f"Removed {len(candidates)} files and {len(set(resource_dirs))} resource trees.") - else: - print(f"Removed {len(candidates)} files.") - return 0 - - -def _collect_replay_known_bad( - *, - name: Optional[str], - pattern: Optional[str], - case_id: Optional[str], -) -> list[Path]: - candidates = [path for path in _iter_replay_paths("known_bad") if _matches_filters(path, name=name, pattern=pattern)] - if case_id: - candidates = _select_with_case(candidates, case_id) - return sorted(candidates, key=lambda p: _relative(p)) - - -def _collect_traces_for_case(case_id: str) -> list[Path]: - root = TRACE_ROOT / "known_bad" - if not root.exists(): - return [] - return sorted(root.glob(f"*{case_id}*plan_trace*.txt"), key=lambda p: _relative(p)) - - -def cmd_fix(args: argparse.Namespace) -> int: - name = _normalize(args.name) - pattern = _normalize(args.pattern) - case_id = _normalize(args.case) - move_traces = bool(args.move_traces) - dry_run = bool(args.dry) - - try: - _validate_name_pattern_case(name, pattern, case_id) - except ValueError as exc: - print(str(exc), file=sys.stderr) - return 1 - - candidates = _collect_replay_known_bad(name=name, pattern=pattern, case_id=case_id) - if not candidates: - print("Ничего не найдено в known_bad.", file=sys.stderr) - return 1 - - dests = [] - resource_moves: list[tuple[Path, Path]] = [] - conflicts = [] - dst_seen: dict[Path, Path] = {} - for src in candidates: - rel_path = src.relative_to(REPLAY_ROOT / "known_bad") - dst = REPLAY_ROOT / "fixed" / rel_path - if dst in dst_seen and dst_seen[dst] != src: - conflicts.append(dst) - if dst.exists(): - conflicts.append(dst) - dst_seen[dst] = src - dests.append((src, dst)) - src_resources = _resources_dir_for_fixture(src) - if src_resources.exists(): - dst_resources = _resources_dir_for_fixture(dst) - if dst_resources.exists(): - conflicts.append(dst_resources) - resource_moves.append((src_resources, dst_resources)) - if conflicts: - print("Конфликт имён в fixed:", file=sys.stderr) - for conflict in conflicts: - print(f"- {_relative(conflict)}", file=sys.stderr) - return 1 - - use_git = _is_git_repo() - print("Found replay fixtures to promote:") - for src, dst in dests: - print(f"- {_relative(src)} -> {_relative(dst)}") - for src, dst in resource_moves: - print(f" - resources: {_relative(src)} -> {_relative(dst)}") - - case_ids: set[str] = set() - promoted_traces: list[Path] = [] - trace_dests: list[tuple[Path, Path]] = [] - if move_traces: - for src in candidates: - resolved_case_id = case_id or _load_case_id(src) - if resolved_case_id: - case_ids.add(resolved_case_id) - - for cid in sorted(case_ids): - promoted_traces.extend(_collect_traces_for_case(cid)) - - for trace in promoted_traces: - trace_dests.append((trace, TRACE_ROOT / "fixed" / trace.name)) - - trace_conflicts = [dst for _, dst in trace_dests if dst.exists()] - if trace_conflicts: - print("Конфликт trace в fixed:", file=sys.stderr) - for conflict in trace_conflicts: - print(f"- {_relative(conflict)}", file=sys.stderr) - return 1 - - if dry_run: - print(f"DRY: would promote {len(candidates)} replay fixtures.") - if move_traces: - if not promoted_traces: - print("DRY: no plan traces found to promote.") - else: - print(f"DRY: would also promote {len(promoted_traces)} plan traces:") - for src, dst in trace_dests: - print(f"- {_relative(src)} -> {_relative(dst)}") - return 0 - - moves_done: list[tuple[Path, Path, bool]] = [] - try: - for src, dst in dests: - _move_file(src, dst, use_git=use_git, dry_run=False) - moves_done.append((src, dst, False)) - for src, dst in resource_moves: - _move_tree(src, dst, use_git=use_git, dry_run=False) - moves_done.append((src, dst, True)) - - print(f"Promoted {len(candidates)} replay fixtures to fixed.") - - if move_traces: - if not promoted_traces: - print("No plan traces found to promote.") - return 0 - for src, dst in trace_dests: - _move_file(src, dst, use_git=use_git, dry_run=False) - moves_done.append((src, dst, False)) - print(f"Also promoted {len(promoted_traces)} plan traces.") - - return 0 - except Exception as exc: - print("ERROR: fix failed during move. State may be partial; attempting rollback.", file=sys.stderr) - print(f"Cause: {exc}", file=sys.stderr) - _rollback_moves(moves_done, use_git=use_git) - return 1 - - -def cmd_migrate(args: argparse.Namespace) -> int: - bucket = _normalize(args.bucket) - dry_run = bool(args.dry) - if bucket and bucket not in BUCKETS: - print(f"Неизвестный BUCKET: {bucket}", file=sys.stderr) - return 1 - - buckets = [bucket] if bucket else BUCKETS - use_git = _is_git_repo() - total_moves = 0 - total_updates = 0 - - for bkt in buckets: - root = REPLAY_ROOT / bkt - if not root.exists(): - continue - fixture_candidates = [path for path in root.rglob("*.json") if "resources" not in path.parts] - for fixture_path in sorted(fixture_candidates, key=lambda p: _relative(p)): - if fixture_path.parent != root: - print( - "Skip nested fixture path; expected fixtures at bucket root: " - f"{_relative(fixture_path)}", - file=sys.stderr, - ) - continue - original_text: str | None = None - try: - original_text = fixture_path.read_text(encoding="utf-8") - data = json.loads(original_text) - except (json.JSONDecodeError, OSError) as exc: - print(f"Skip invalid json {_relative(fixture_path)}: {exc}", file=sys.stderr) - continue - if data.get("type") != "replay_bundle": - continue - resources = data.get("resources") - if not isinstance(resources, dict): - continue - used_paths: set[Path] = set() - moves: list[tuple[Path, Path]] = [] - src_to_dest_rel: dict[Path, Path] = {} - changes = False - conflicts: list[Path] = [] - for rid, resource in resources.items(): - if not isinstance(resource, dict): - continue - data_ref = resource.get("data_ref") - if not isinstance(data_ref, dict): - continue - file_name = data_ref.get("file") - if not isinstance(file_name, str) or not file_name: - continue - rel_path = Path(file_name) - if rel_path.is_absolute() or ".." in rel_path.parts: - print( - f"Skip invalid resource path {_relative(fixture_path)}: {file_name}", - file=sys.stderr, - ) - continue - if ( - len(rel_path.parts) >= 2 - and rel_path.parts[0] == "resources" - and rel_path.parts[1] == _resource_key_for_fixture(fixture_path) - ): - used_paths.add(rel_path) - continue - src_path = fixture_path.parent / file_name - if not src_path.exists(): - print( - f"Missing resource for {_relative(fixture_path)}: {_relative(src_path)}", - file=sys.stderr, - ) - continue - if src_path in src_to_dest_rel: - existing_rel = src_to_dest_rel[src_path] - updated_resource = dict(resource) - updated_data_ref = dict(data_ref) - updated_data_ref["file"] = existing_rel.as_posix() - updated_resource["data_ref"] = updated_data_ref - resources[rid] = updated_resource - changes = True - continue - - try: - dest_rel = _resource_destination(fixture_path, file_name, rid, used_paths) - except ValueError as exc: - print(f"Skip resource in {_relative(fixture_path)}: {exc}", file=sys.stderr) - continue - - dst_path = fixture_path.parent / dest_rel - if dst_path.exists(): - conflicts.append(dst_path) - continue - - # De-duplicate moves so shared resources are only relocated once. - moves.append((src_path, dst_path)) - src_to_dest_rel[src_path] = dest_rel - updated_resource = dict(resource) - updated_data_ref = dict(data_ref) - updated_data_ref["file"] = dest_rel.as_posix() - updated_resource["data_ref"] = updated_data_ref - resources[rid] = updated_resource - changes = True - - if conflicts: - print(f"Conflicts for {_relative(fixture_path)}:", file=sys.stderr) - for conflict in conflicts: - print(f"- {_relative(conflict)}", file=sys.stderr) - continue - - if not moves and not changes: - continue - - print(f"Migrate resources for {_relative(fixture_path)}:") - for src, dst in moves: - print(f"- {_relative(src)} -> {_relative(dst)}") - - if dry_run: - continue - - moves_done: list[tuple[Path, Path, bool]] = [] - try: - for src, dst in moves: - _move_file(src, dst, use_git=use_git, dry_run=False) - moves_done.append((src, dst, False)) - - if changes: - data["resources"] = resources - fixture_path.write_text(_canonical_json(data), encoding="utf-8") - total_updates += 1 - total_moves += len(moves) - except Exception as exc: - print( - f"ERROR: migrate failed for {_relative(fixture_path)}: {exc}", - file=sys.stderr, - ) - _rollback_moves(moves_done, use_git=use_git) - if original_text is not None: - try: - fixture_path.write_text(original_text, encoding="utf-8") - except Exception as rollback_exc: - print( - f"ROLLBACK ERROR: failed to restore {_relative(fixture_path)}: {rollback_exc}", - file=sys.stderr, - ) - - if dry_run: - print("DRY: migration scan complete.") - else: - print(f"Migrated {total_moves} resources across {total_updates} fixtures.") - return 0 - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Tools for managing test fixtures.") - sub = parser.add_subparsers(dest="command", required=True) - - rm_parser = sub.add_parser("rm", help="Remove fixtures by name or pattern.") - rm_parser.add_argument("--name", default="") - rm_parser.add_argument("--pattern", default="") - rm_parser.add_argument("--bucket", default="") - rm_parser.add_argument("--scope", default="both") - rm_parser.add_argument("--with-resources", type=int, default=1) - rm_parser.add_argument("--dry", type=int, default=0) - rm_parser.set_defaults(func=cmd_rm) - - fix_parser = sub.add_parser("fix", help="Promote known_bad fixtures to fixed.") - fix_parser.add_argument("--name", default="") - fix_parser.add_argument("--pattern", default="") - fix_parser.add_argument("--case", dest="case", default="") - fix_parser.add_argument("--move-traces", type=int, default=0) - fix_parser.add_argument("--dry", type=int, default=0) - fix_parser.set_defaults(func=cmd_fix) - - migrate_parser = sub.add_parser("migrate", help="Migrate replay bundle resources into resources//") - migrate_parser.add_argument("--bucket", default="") - migrate_parser.add_argument("--dry", type=int, default=0) - migrate_parser.set_defaults(func=cmd_migrate) - - return parser - - -def main() -> int: - parser = build_parser() - args = parser.parse_args() - return args.func(args) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/fetchgraph/cli.py b/src/fetchgraph/cli.py deleted file mode 100644 index a967f8bf..00000000 --- a/src/fetchgraph/cli.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -import argparse -import sys -from pathlib import Path - -from fetchgraph.replay.export import export_replay_case_bundle, export_replay_case_bundles - - -def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Fetchgraph utilities") - sub = parser.add_subparsers(dest="command", required=True) - - fixture = sub.add_parser("fixture", help="Export replay case bundle from events.jsonl") - fixture.add_argument("--events", type=Path, required=True, help="Path to events.jsonl") - fixture.add_argument("--run-dir", type=Path, default=None, help="Case run dir (needed for file resources)") - fixture.add_argument("--id", default="plan_normalize.spec_v1", help="Replay case id to extract") - fixture.add_argument("--spec-idx", type=int, default=None, help="Filter replay_case by meta.spec_idx") - fixture.add_argument( - "--provider", - default=None, - help="Filter replay_case by meta.provider (case-insensitive)", - ) - fixture.add_argument( - "--allow-bad-json", - action="store_true", - help="Skip invalid JSON lines in events.jsonl", - ) - fixture.add_argument( - "--overwrite", - action="store_true", - help="Overwrite existing bundles and resource copies", - ) - fixture.add_argument("--all", action="store_true", help="Export all matching replay cases") - fixture.add_argument( - "--out-dir", - type=Path, - default=Path("tests") / "fixtures" / "replay_cases", - help="Output directory for replay case bundles", - ) - return parser.parse_args(argv) - - -def main(argv: list[str] | None = None) -> int: - args = _parse_args(argv) - if args.command == "fixture": - if args.all: - export_replay_case_bundles( - events_path=args.events, - out_dir=args.out_dir, - replay_id=args.id, - spec_idx=args.spec_idx, - provider=args.provider, - run_dir=args.run_dir, - allow_bad_json=args.allow_bad_json, - overwrite=args.overwrite, - ) - else: - export_replay_case_bundle( - events_path=args.events, - out_dir=args.out_dir, - replay_id=args.id, - spec_idx=args.spec_idx, - provider=args.provider, - run_dir=args.run_dir, - allow_bad_json=args.allow_bad_json, - overwrite=args.overwrite, - ) - return 0 - raise SystemExit(f"Unknown command: {args.command}") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/fetchgraph/replay/__init__.py b/src/fetchgraph/replay/__init__.py index 80da066e..0f5f8b31 100644 --- a/src/fetchgraph/replay/__init__.py +++ b/src/fetchgraph/replay/__init__.py @@ -1,13 +1,11 @@ from __future__ import annotations from .log import EventLoggerLike, log_replay_case -from .runtime import REPLAY_HANDLERS, ReplayContext, load_case_bundle, run_case +from .runtime import REPLAY_HANDLERS, ReplayContext __all__ = [ "EventLoggerLike", "REPLAY_HANDLERS", "ReplayContext", - "load_case_bundle", "log_replay_case", - "run_case", ] From a92f65151d386a206e97adbb86cb661cbaf21a41 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 14:56:24 +0300 Subject: [PATCH 42/79] Allow fixture-green shorthand case names --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 28ca46cc..274afcbb 100644 --- a/Makefile +++ b/Makefile @@ -375,8 +375,12 @@ tracer-export: $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) fixture-green: - @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture-green CASE=tests/fixtures/replay_cases/known_bad/fixture.case.json" && exit 1) - @$(PYTHON) -m fetchgraph.tracer.cli fixture-green --case "$(CASE)" --root "$(TRACER_ROOT)" \ + @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture-green CASE=tests/fixtures/replay_cases/known_bad/fixture.case.json (или CASE=fixture_stem)" && exit 1) + @case_path="$(CASE)"; \ + if [ "$${case_path##*/}" = "$$case_path" ] && [ "$$case_path" != *".case.json" ]; then \ + case_path="$(TRACER_ROOT)/known_bad/$$case_path.case.json"; \ + fi; \ + $(PYTHON) -m fetchgraph.tracer.cli fixture-green --case "$$case_path" --root "$(TRACER_ROOT)" \ $(if $(filter 1 true yes on,$(VALIDATE)),--validate,) \ $(if $(filter 1 true yes on,$(OVERWRITE_EXPECTED)),--overwrite-expected,) \ $(if $(filter 1 true yes on,$(DRY)),--dry-run,) From 19a38e52dc9e73c747f9ae753ee150a94d171a08 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 14:59:07 +0300 Subject: [PATCH 43/79] Add provider snapshot to replay inputs --- .../planning/normalize/plan_normalizer.py | 15 ++++++++++++++ src/fetchgraph/replay/fetchgraph_tracer.md | 12 +++++++++-- .../replay/handlers/plan_normalize.py | 20 +++++++++++-------- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/fetchgraph/planning/normalize/plan_normalizer.py b/src/fetchgraph/planning/normalize/plan_normalizer.py index 94732780..dda4daaa 100644 --- a/src/fetchgraph/planning/normalize/plan_normalizer.py +++ b/src/fetchgraph/planning/normalize/plan_normalizer.py @@ -14,6 +14,7 @@ from ...relational.normalize import normalize_relational_selectors from ...relational.providers.base import RelationalDataProvider from ...replay.log import EventLoggerLike, log_replay_case +from ...replay.snapshots import snapshot_provider_info logger = logging.getLogger(__name__) @@ -179,6 +180,18 @@ def _normalize_specs( notes.append(note) rule_kind = rule.kind if replay_logger and rule_kind: + provider_info_snapshot = None + provider_info = self.provider_catalog.get(spec.provider) + if isinstance(provider_info, ProviderInfo): + provider_info_snapshot = snapshot_provider_info(provider_info) + elif spec.provider in self.schema_registry: + provider_info_snapshot = snapshot_provider_info( + ProviderInfo( + name=spec.provider, + capabilities=[], + selectors_schema=self.schema_registry[spec.provider], + ) + ) input_payload = { "spec": { "provider": spec.provider, @@ -188,6 +201,8 @@ def _normalize_specs( "options": asdict(self.options), "normalizer_rules": {spec.provider: rule_kind}, } + if provider_info_snapshot: + input_payload["provider_info_snapshot"] = provider_info_snapshot observed_payload = { "out_spec": { "provider": spec.provider, diff --git a/src/fetchgraph/replay/fetchgraph_tracer.md b/src/fetchgraph/replay/fetchgraph_tracer.md index d7c7494b..121c675c 100644 --- a/src/fetchgraph/replay/fetchgraph_tracer.md +++ b/src/fetchgraph/replay/fetchgraph_tracer.md @@ -35,6 +35,7 @@ - `id`: идентификатор обработчика - `meta`: опциональные метаданные (например `spec_idx`, `provider`) - `input`: вход для реплея +- `input.provider_info_snapshot`: минимальный snapshot провайдера (например `selectors_schema`), чтобы реплей был детерминированным без extras - **ровно одно** из `observed` или `observed_error` - `requires`: список зависимостей `[{"kind":"extra"|"resource","id":"..."}]` @@ -45,7 +46,7 @@ "v": 2, "id": "plan_normalize.spec_v1", "meta": {"spec_idx": 0, "provider": "sql"}, - "input": {"spec": {...}, "options": {...}}, + "input": {"spec": {...}, "options": {...}, "provider_info_snapshot": {"name": "sql", "selectors_schema": {...}}}, "observed": {"out_spec": {...}}, "requires": [{"kind": "extra", "id": "planner_input_v1"}] } @@ -81,7 +82,14 @@ log_replay_case( logger=event_log, id="plan_normalize.spec_v1", meta={"spec_idx": i, "provider": spec.provider}, - input={"spec": spec.model_dump(), "options": options.model_dump()}, + input={ + "spec": spec.model_dump(), + "options": options.model_dump(), + "provider_info_snapshot": { + "name": spec.provider, + "selectors_schema": provider_info.selectors_schema, + }, + }, observed={"out_spec": out_spec}, requires=[{"kind": "extra", "id": "planner_input_v1"}], ) diff --git a/src/fetchgraph/replay/handlers/plan_normalize.py b/src/fetchgraph/replay/handlers/plan_normalize.py index ecea6c5f..09decb0c 100644 --- a/src/fetchgraph/replay/handlers/plan_normalize.py +++ b/src/fetchgraph/replay/handlers/plan_normalize.py @@ -21,15 +21,19 @@ def replay_plan_normalize_spec_v1(inp: dict, ctx: ReplayContext) -> dict: rules = inp.get("normalizer_rules") or inp.get("normalizer_registry") or {} provider = spec_dict["provider"] provider_catalog: Dict[str, ProviderInfo] = {} - planner = ctx.extras.get("planner_input_v1") or {} - planner_input = planner.get("input") if isinstance(planner, dict) else {} - catalog_raw = {} - if isinstance(planner_input, dict): - catalog_raw = planner_input.get("provider_catalog") or {} - if provider in catalog_raw and isinstance(catalog_raw[provider], dict): - provider_catalog[provider] = ProviderInfo(**catalog_raw[provider]) + provider_snapshot = inp.get("provider_info_snapshot") + if isinstance(provider_snapshot, dict): + provider_catalog[provider] = ProviderInfo(**provider_snapshot) else: - provider_catalog[provider] = ProviderInfo(name=provider, capabilities=[]) + planner = ctx.extras.get("planner_input_v1") or {} + planner_input = planner.get("input") if isinstance(planner, dict) else {} + catalog_raw = {} + if isinstance(planner_input, dict): + catalog_raw = planner_input.get("provider_catalog") or {} + if provider in catalog_raw and isinstance(catalog_raw[provider], dict): + provider_catalog[provider] = ProviderInfo(**catalog_raw[provider]) + else: + provider_catalog[provider] = ProviderInfo(name=provider, capabilities=[]) rule_kind = rules.get(provider) normalizer_registry: Dict[str, SelectorNormalizationRule] = {} From 22f8cfce445b38246b93bd0a4c7daf8972851ff8 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 15:21:07 +0300 Subject: [PATCH 44/79] Refine tracer export and fixture behavior --- Makefile | 8 +++++- .../planning/normalize/plan_normalizer.py | 4 --- .../{replay => tracer}/fetchgraph_tracer.md | 3 +-- src/fetchgraph/tracer/fixture_tools.py | 26 ++++++++++++------- 4 files changed, 25 insertions(+), 16 deletions(-) rename src/fetchgraph/{replay => tracer}/fetchgraph_tracer.md (97%) diff --git a/Makefile b/Makefile index 274afcbb..85342c80 100644 --- a/Makefile +++ b/Makefile @@ -366,13 +366,18 @@ tracer-export: @test -n "$(strip $(REPLAY_ID))" || (echo "REPLAY_ID обязателен: make tracer-export REPLAY_ID=plan_normalize.spec_v1" && exit 1) @test -n "$(strip $(EVENTS))" || (echo "EVENTS обязателен: make tracer-export EVENTS=path/to/events.jsonl" && exit 1) @case "$(BUCKET)" in fixed|known_bad) ;; *) echo "BUCKET должен быть fixed или known_bad для tracer-export" && exit 1 ;; esac + @if [ -z "$(strip $(SPEC_IDX))" ] && ! printf "%s" "$(ALL)" | grep -Eiq '^(1|true|yes|on)$$'; then \ + echo "Для tracer-export нужно задать SPEC_IDX или ALL=1"; \ + exit 1; \ + fi @# TRACER_OUT_DIR has a default; override if needed. @$(PYTHON) -m fetchgraph.tracer.cli export-case-bundle --events "$(EVENTS)" --out "$(TRACER_OUT_DIR)" --id "$(REPLAY_ID)" \ $(if $(strip $(SPEC_IDX)),--spec-idx $(SPEC_IDX),) \ $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ $(if $(strip $(RUN_DIR)),--run-dir "$(RUN_DIR)",) \ $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) \ - $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) + $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) \ + $(if $(filter 1 true yes on,$(ALL)),--all,) fixture-green: @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture-green CASE=tests/fixtures/replay_cases/known_bad/fixture.case.json (или CASE=fixture_stem)" && exit 1) @@ -386,6 +391,7 @@ fixture-green: $(if $(filter 1 true yes on,$(DRY)),--dry-run,) fixture-rm: + @case "$(BUCKET)" in fixed|known_bad|all) ;; *) echo "BUCKET должен быть fixed, known_bad или all для fixture-rm" && exit 1 ;; esac @$(PYTHON) -m fetchgraph.tracer.cli fixture-rm --root "$(TRACER_ROOT)" --bucket "$(BUCKET)" \ $(if $(strip $(NAME)),--name "$(NAME)",) \ $(if $(strip $(PATTERN)),--pattern "$(PATTERN)",) \ diff --git a/src/fetchgraph/planning/normalize/plan_normalizer.py b/src/fetchgraph/planning/normalize/plan_normalizer.py index dda4daaa..22b2eda2 100644 --- a/src/fetchgraph/planning/normalize/plan_normalizer.py +++ b/src/fetchgraph/planning/normalize/plan_normalizer.py @@ -210,9 +210,6 @@ def _normalize_specs( "selectors": use, }, } - requires = None - if getattr(replay_logger, "case_id", None): - requires = [{"kind": "extra", "id": "planner_input_v1"}] log_replay_case( replay_logger, id="plan_normalize.spec_v1", @@ -223,7 +220,6 @@ def _normalize_specs( }, input=input_payload, observed=observed_payload, - requires=requires, diag={ "selectors_valid_before": before_ok, "selectors_valid_after": after_ok, diff --git a/src/fetchgraph/replay/fetchgraph_tracer.md b/src/fetchgraph/tracer/fetchgraph_tracer.md similarity index 97% rename from src/fetchgraph/replay/fetchgraph_tracer.md rename to src/fetchgraph/tracer/fetchgraph_tracer.md index 121c675c..a91605ef 100644 --- a/src/fetchgraph/replay/fetchgraph_tracer.md +++ b/src/fetchgraph/tracer/fetchgraph_tracer.md @@ -37,7 +37,7 @@ - `input`: вход для реплея - `input.provider_info_snapshot`: минимальный snapshot провайдера (например `selectors_schema`), чтобы реплей был детерминированным без extras - **ровно одно** из `observed` или `observed_error` -- `requires`: список зависимостей `[{"kind":"extra"|"resource","id":"..."}]` +- `requires`: опциональный список зависимостей `[{"kind":"extra"|"resource","id":"..."}]` Пример: ```json @@ -91,7 +91,6 @@ log_replay_case( }, }, observed={"out_spec": out_spec}, - requires=[{"kind": "extra", "id": "planner_input_v1"}], ) ``` diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py index 6cb0084b..95fbe047 100644 --- a/src/fetchgraph/tracer/fixture_tools.py +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -45,7 +45,7 @@ def _safe_resource_path(path: str, *, stem: str) -> Path: def _atomic_write_json(path: Path, payload: dict) -> None: tmp_path = path.with_suffix(path.suffix + ".tmp") tmp_path.write_text( - json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2), + json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")), encoding="utf-8", ) tmp_path.replace(path) @@ -112,19 +112,27 @@ def fixture_green( return fixed_expected_path.parent.mkdir(parents=True, exist_ok=True) - fixed_expected_path.write_text( + tmp_expected_path = fixed_expected_path.with_suffix(fixed_expected_path.suffix + ".tmp") + tmp_expected_path.write_text( json.dumps(observed, ensure_ascii=False, sort_keys=True, indent=2), encoding="utf-8", ) - fixed_case_path.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(known_case_path), str(fixed_case_path)) + try: + fixed_case_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(known_case_path), str(fixed_case_path)) - if known_expected_path.exists(): - known_expected_path.unlink() + if resources_from.exists(): + resources_to.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(resources_from), str(resources_to)) - if resources_from.exists(): - resources_to.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(resources_from), str(resources_to)) + if known_expected_path.exists(): + known_expected_path.unlink() + + tmp_expected_path.replace(fixed_expected_path) + except Exception: + if tmp_expected_path.exists(): + tmp_expected_path.unlink() + raise if validate: import fetchgraph.tracer.handlers # noqa: F401 From cdc91c0bdc2a19a79d046e6eb0876aaeb4193617 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 15:24:39 +0300 Subject: [PATCH 45/79] Add tracer auto-discovery for events.jsonl --- Makefile | 28 +++-- src/fetchgraph/tracer/auto_resolve.py | 150 ++++++++++++++++++++++++++ src/fetchgraph/tracer/cli.py | 53 ++++++++- tests/test_tracer_auto_resolve.py | 83 ++++++++++++++ 4 files changed, 301 insertions(+), 13 deletions(-) create mode 100644 src/fetchgraph/tracer/auto_resolve.py create mode 100644 tests/test_tracer_auto_resolve.py diff --git a/Makefile b/Makefile index 85342c80..8a596cea 100644 --- a/Makefile +++ b/Makefile @@ -167,6 +167,7 @@ help: @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" @echo " make tracer-export REPLAY_ID=... EVENTS=... TRACER_OUT_DIR=... [SPEC_IDX=...] [PROVIDER=...] [RUN_DIR=...] [ALLOW_BAD_JSON=1] [OVERWRITE=1]" + @echo " make tracer-export REPLAY_ID=... CASE=... DATA=... [TAG=...] [SPEC_IDX=...] [PROVIDER=...] [ALLOW_BAD_JSON=1] [OVERWRITE=1]" @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources//..." @echo " make fixture-green CASE=path/to/case.case.json [TRACER_ROOT=...] [VALIDATE=1] [OVERWRITE_EXPECTED=1] [DRY=1]" @@ -364,20 +365,31 @@ case-open: check tracer-export: @test -n "$(strip $(REPLAY_ID))" || (echo "REPLAY_ID обязателен: make tracer-export REPLAY_ID=plan_normalize.spec_v1" && exit 1) - @test -n "$(strip $(EVENTS))" || (echo "EVENTS обязателен: make tracer-export EVENTS=path/to/events.jsonl" && exit 1) @case "$(BUCKET)" in fixed|known_bad) ;; *) echo "BUCKET должен быть fixed или known_bad для tracer-export" && exit 1 ;; esac @if [ -z "$(strip $(SPEC_IDX))" ] && ! printf "%s" "$(ALL)" | grep -Eiq '^(1|true|yes|on)$$'; then \ echo "Для tracer-export нужно задать SPEC_IDX или ALL=1"; \ exit 1; \ fi @# TRACER_OUT_DIR has a default; override if needed. - @$(PYTHON) -m fetchgraph.tracer.cli export-case-bundle --events "$(EVENTS)" --out "$(TRACER_OUT_DIR)" --id "$(REPLAY_ID)" \ - $(if $(strip $(SPEC_IDX)),--spec-idx $(SPEC_IDX),) \ - $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ - $(if $(strip $(RUN_DIR)),--run-dir "$(RUN_DIR)",) \ - $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) \ - $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) \ - $(if $(filter 1 true yes on,$(ALL)),--all,) + @if [ -n "$(strip $(EVENTS))" ]; then \ + $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle --events "$(EVENTS)" --out "$(TRACER_OUT_DIR)" --id "$(REPLAY_ID)" \ + $(if $(strip $(SPEC_IDX)),--spec-idx $(SPEC_IDX),) \ + $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ + $(if $(strip $(RUN_DIR)),--run-dir "$(RUN_DIR)",) \ + $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) \ + $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) \ + $(if $(filter 1 true yes on,$(ALL)),--all,); \ + else \ + test -n "$(strip $(CASE))" || (echo "CASE обязателен для auto режима: make tracer-export CASE=case_id DATA=... REPLAY_ID=..." && exit 1); \ + test -n "$(strip $(DATA))" || (echo "DATA обязателен для auto режима: make tracer-export CASE=case_id DATA=... REPLAY_ID=..." && exit 1); \ + $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle --case "$(CASE)" --data "$(DATA)" --out "$(TRACER_OUT_DIR)" --id "$(REPLAY_ID)" \ + $(if $(strip $(TAG)),--tag "$(TAG)",) \ + $(if $(strip $(SPEC_IDX)),--spec-idx $(SPEC_IDX),) \ + $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ + $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) \ + $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) \ + $(if $(filter 1 true yes on,$(ALL)),--all,); \ + fi fixture-green: @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture-green CASE=tests/fixtures/replay_cases/known_bad/fixture.case.json (или CASE=fixture_stem)" && exit 1) diff --git a/src/fetchgraph/tracer/auto_resolve.py b/src/fetchgraph/tracer/auto_resolve.py new file mode 100644 index 00000000..937d27e3 --- /dev/null +++ b/src/fetchgraph/tracer/auto_resolve.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + + +@dataclass(frozen=True) +class CaseResolution: + run_dir: Path + case_dir: Path + events_path: Path + tag: str | None + + +def resolve_case_events( + *, + case_id: str, + data_dir: Path, + tag: str | None = None, + runs_subdir: str = ".runs/runs", + pick_run: str = "latest_non_missed", +) -> CaseResolution: + if not case_id: + raise ValueError("case_id is required") + if not data_dir: + raise ValueError("data_dir is required") + if pick_run != "latest_non_missed": + raise ValueError(f"Unsupported pick_run mode: {pick_run}") + + runs_root = (data_dir / runs_subdir).resolve() + if not runs_root.exists(): + raise FileNotFoundError(f"Runs directory does not exist: {runs_root}") + + run_dirs = list(_iter_run_dirs(runs_root)) + inspected_runs = 0 + missing_cases = 0 + missed_cases = 0 + tag_mismatches = 0 + + for run_dir in run_dirs: + inspected_runs += 1 + run_tag = _extract_run_tag(run_dir) + if tag and run_tag != tag: + tag_mismatches += 1 + continue + + case_dirs = _case_dirs(run_dir, case_id) + if not case_dirs: + missing_cases += 1 + continue + + for case_dir in case_dirs: + if _case_is_missed(case_dir): + missed_cases += 1 + continue + events_path = case_dir / "events.jsonl" + if not events_path.exists(): + raise FileNotFoundError(f"events.jsonl not found at {events_path}") + return CaseResolution( + run_dir=run_dir, + case_dir=case_dir, + events_path=events_path, + tag=run_tag, + ) + + details = [ + "No suitable case run found.", + f"runs_root: {runs_root}", + f"case_id: {case_id}", + f"inspected_runs: {inspected_runs}", + f"missing_cases: {missing_cases}", + f"missed_cases: {missed_cases}", + ] + if tag: + details.append(f"tag: {tag}") + details.append(f"tag_mismatches: {tag_mismatches}") + raise LookupError("\n".join(details)) + + +def _iter_run_dirs(runs_root: Path) -> Iterable[Path]: + candidates = [p for p in runs_root.iterdir() if p.is_dir()] + return sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True) + + +def _case_dirs(run_dir: Path, case_id: str) -> list[Path]: + cases_root = run_dir / "cases" + if not cases_root.exists(): + return [] + return sorted(cases_root.glob(f"{case_id}_*"), key=lambda p: p.stat().st_mtime, reverse=True) + + +def _extract_run_tag(run_dir: Path) -> str | None: + for name in ("run_meta.json", "meta.json", "summary.json"): + path = run_dir / name + if not path.exists(): + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + continue + tag = _extract_tag_value(payload) + if tag: + return tag + return None + + +def _extract_tag_value(payload: dict) -> str | None: + for key in ("tag", "TAG"): + value = payload.get(key) + if isinstance(value, str) and value: + return value + tags_value = payload.get("tags") + if isinstance(tags_value, list) and tags_value: + for entry in tags_value: + if isinstance(entry, str) and entry: + return entry + return None + + +def _case_is_missed(case_dir: Path) -> bool: + for name in ("status.json", "result.json"): + path = case_dir / name + if not path.exists(): + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + continue + if _payload_is_missed(payload): + return True + if _payload_is_non_missed(payload): + return False + return False + + +def _payload_is_missed(payload: dict) -> bool: + if payload.get("missed") is True: + return True + status = str(payload.get("status") or payload.get("result") or "").lower() + if status in {"missed", "missing"}: + return True + reason = str(payload.get("reason") or "").lower() + return "missed" in reason or "missing" in reason + + +def _payload_is_non_missed(payload: dict) -> bool: + status = str(payload.get("status") or payload.get("result") or "").lower() + return status in {"ok", "pass", "passed", "success"} diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index aaee555c..d43e60b4 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -4,6 +4,7 @@ import sys from pathlib import Path +from fetchgraph.tracer.auto_resolve import resolve_case_events from fetchgraph.tracer.export import export_replay_case_bundle, export_replay_case_bundles from fetchgraph.tracer.fixture_tools import ( fixture_fix, @@ -20,7 +21,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: sub = parser.add_subparsers(dest="command", required=True) export = sub.add_parser("export-case-bundle", help="Export replay case bundle from events.jsonl") - export.add_argument("--events", type=Path, required=True, help="Path to events.jsonl") + export.add_argument("--events", type=Path, help="Path to events.jsonl") export.add_argument("--out", type=Path, required=True, help="Output directory for bundle") export.add_argument("--id", required=True, help="Replay case id to export") export.add_argument("--spec-idx", type=int, default=None, help="Filter replay_case by meta.spec_idx") @@ -41,6 +42,24 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="Overwrite existing bundles and resource copies", ) export.add_argument("--all", action="store_true", help="Export all matching replay cases") + export.add_argument("--case", help="Case id for auto-resolving events.jsonl") + export.add_argument("--data", type=Path, help="Data directory containing .runs") + export.add_argument("--tag", default=None, help="Run tag filter for auto-resolve") + export.add_argument( + "--runs-subdir", + default=".runs/runs", + help="Runs subdir relative to data dir (default: .runs/runs)", + ) + export.add_argument( + "--pick-run", + default="latest_non_missed", + help="Run selection strategy (default: latest_non_missed)", + ) + export.add_argument( + "--print-resolve", + action="store_true", + help="Print resolved run_dir/events.jsonl", + ) green = sub.add_parser("fixture-green", help="Promote known_bad case to fixed") green.add_argument("--case", type=Path, required=True, help="Path to known_bad case bundle") @@ -95,25 +114,49 @@ def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) try: if args.command == "export-case-bundle": + if args.events: + if args.case or args.data or args.tag: + raise ValueError("Do not combine --events with --case/--data/--tag.") + events_path = args.events + run_dir = args.run_dir + else: + if args.run_dir: + raise ValueError("Do not combine --run-dir with auto-resolve.") + if not args.case or not args.data: + raise ValueError("--case and --data are required when --events is not provided.") + resolution = resolve_case_events( + case_id=args.case, + data_dir=args.data, + tag=args.tag, + runs_subdir=args.runs_subdir, + pick_run=args.pick_run, + ) + events_path = resolution.events_path + run_dir = resolution.case_dir + if args.print_resolve: + print(f"Resolved run_dir: {resolution.case_dir}") + print(f"Resolved events.jsonl: {resolution.events_path}") + if resolution.tag: + print(f"Resolved tag: {resolution.tag}") if args.all: export_replay_case_bundles( - events_path=args.events, + events_path=events_path, out_dir=args.out, replay_id=args.id, spec_idx=args.spec_idx, provider=args.provider, - run_dir=args.run_dir, + run_dir=run_dir, allow_bad_json=args.allow_bad_json, overwrite=args.overwrite, ) else: export_replay_case_bundle( - events_path=args.events, + events_path=events_path, out_dir=args.out, replay_id=args.id, spec_idx=args.spec_idx, provider=args.provider, - run_dir=args.run_dir, + run_dir=run_dir, allow_bad_json=args.allow_bad_json, overwrite=args.overwrite, ) diff --git a/tests/test_tracer_auto_resolve.py b/tests/test_tracer_auto_resolve.py new file mode 100644 index 00000000..528f0d86 --- /dev/null +++ b/tests/test_tracer_auto_resolve.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from fetchgraph.tracer.auto_resolve import resolve_case_events + + +def _write_json(path: Path, payload: dict) -> None: + path.write_text(json.dumps(payload), encoding="utf-8") + + +def _touch(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}", encoding="utf-8") + + +def _set_mtime(path: Path, ts: float) -> None: + os.utime(path, (ts, ts)) + + +def _make_case_dir(run_dir: Path, case_id: str, suffix: str, *, status: str) -> Path: + case_dir = run_dir / "cases" / f"{case_id}_{suffix}" + case_dir.mkdir(parents=True, exist_ok=True) + _touch(case_dir / "events.jsonl") + _write_json(case_dir / "status.json", {"status": status}) + return case_dir + + +def test_resolve_latest_non_missed(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + run_old = runs_root / "run_old" + run_old.mkdir() + _make_case_dir(run_old, "case_1", "abc", status="ok") + _set_mtime(run_old, 100) + + run_new = runs_root / "run_new" + run_new.mkdir() + _make_case_dir(run_new, "case_1", "def", status="missed") + _set_mtime(run_new, 200) + + resolution = resolve_case_events(case_id="case_1", data_dir=data_dir) + assert resolution.case_dir.parent.parent == run_old + + +def test_resolve_with_tag(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + run_a = runs_root / "run_a" + run_a.mkdir() + _write_json(run_a / "run_meta.json", {"tag": "alpha"}) + _make_case_dir(run_a, "case_2", "aaa", status="ok") + _set_mtime(run_a, 100) + + run_b = runs_root / "run_b" + run_b.mkdir() + _write_json(run_b / "run_meta.json", {"tag": "beta"}) + _make_case_dir(run_b, "case_2", "bbb", status="ok") + _set_mtime(run_b, 200) + + resolution = resolve_case_events(case_id="case_2", data_dir=data_dir, tag="alpha") + assert resolution.run_dir == run_a + + +def test_resolve_not_found(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + run_dir = runs_root / "run_only" + run_dir.mkdir() + _make_case_dir(run_dir, "case_3", "ccc", status="missed") + + with pytest.raises(LookupError, match="No suitable case run found"): + resolve_case_events(case_id="case_3", data_dir=data_dir) From b1bf05d4071722d17014b6e8b03cf82f6ce3eed0 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 15:32:18 +0300 Subject: [PATCH 46/79] Improve replay case selection UX --- src/fetchgraph/replay/export.py | 160 +++++++++++++++++++++++++++++--- src/fetchgraph/tracer/cli.py | 39 +++++++- src/fetchgraph/tracer/export.py | 4 + 3 files changed, 191 insertions(+), 12 deletions(-) diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py index 32f8a8ec..c0ca9326 100644 --- a/src/fetchgraph/replay/export.py +++ b/src/fetchgraph/replay/export.py @@ -8,8 +8,9 @@ import shutil import textwrap from dataclasses import dataclass +from datetime import datetime from pathlib import Path -from typing import Dict, Iterable +from typing import Callable, Dict, Iterable logger = logging.getLogger(__name__) @@ -79,6 +80,126 @@ def _select_replay_cases( return selected +def find_replay_case_matches( + events_path: Path, + *, + replay_id: str, + spec_idx: int | None = None, + provider: str | None = None, + allow_bad_json: bool = False, +) -> list[ExportSelection]: + return _select_replay_cases( + events_path, + replay_id=replay_id, + spec_idx=spec_idx, + provider=provider, + allow_bad_json=allow_bad_json, + ) + + +def format_replay_case_matches(selections: list[ExportSelection], *, limit: int | None = 10) -> str: + rows = [] + for idx, selection in enumerate(selections[:limit], start=1): + event = selection.event + meta = event.get("meta") or {} + provider = meta.get("provider") + spec_idx = meta.get("spec_idx") + timestamp = event.get("timestamp") + input_payload = event.get("input") if isinstance(event.get("input"), dict) else {} + fingerprint = hashlib.sha256(canonical_json(input_payload).encode("utf-8")).hexdigest()[:8] + preview = canonical_json(input_payload)[:80] + rows.append( + " " + f"{idx}. line={selection.line} " + f"ts={timestamp!r} " + f"provider={provider!r} " + f"spec_idx={spec_idx!r} " + f"input={fingerprint} " + f"preview={preview!r}" + ) + if len(selections) > (limit or 0): + rows.append(f" ... ({len(selections) - (limit or 0)} more)") + return "\n".join(rows) + + +def _parse_timestamp(value: object) -> datetime | None: + if not isinstance(value, str) or not value: + return None + cleaned = value + if cleaned.endswith("Z"): + cleaned = cleaned[:-1] + "+00:00" + try: + return datetime.fromisoformat(cleaned) + except ValueError: + return None + + +def _select_by_timestamp(selections: list[ExportSelection]) -> ExportSelection: + with_ts = [] + for selection in selections: + ts = _parse_timestamp(selection.event.get("timestamp")) + if ts is not None: + with_ts.append((ts, selection)) + if with_ts: + return max(with_ts, key=lambda pair: pair[0])[1] + return max(selections, key=lambda sel: sel.line) + + +def _select_replay_case( + selections: list[ExportSelection], + *, + events_path: Path, + selection_policy: str, + select_index: int | None, + require_unique: bool, + allow_prompt: bool, + prompt_fn: Callable[[str], str] | None, +) -> tuple[ExportSelection, str]: + if not selections: + raise LookupError(f"No replay_case entries found in {events_path}") + if require_unique and len(selections) > 1: + details = format_replay_case_matches(selections) + raise LookupError( + "Multiple replay_case entries matched; run with --select/--select-index/--list-matches.\n" + f"{details}" + ) + if len(selections) == 1: + return selections[0], "unique" + if select_index is not None: + if select_index < 1 or select_index > len(selections): + raise ValueError(f"select_index must be between 1 and {len(selections)}") + return selections[select_index - 1], "index" + + selection_policy = selection_policy or "latest" + if allow_prompt and prompt_fn is not None: + details = format_replay_case_matches(selections) + prompt = ( + "Multiple replay_case entries matched:\n" + f"{details}\n" + "Select entry (1..N) or press Enter for latest: " + ) + response = prompt_fn(prompt).strip() + if response: + if response.isdigit(): + choice = int(response) + if 1 <= choice <= len(selections): + return selections[choice - 1], "prompt" + raise ValueError(f"Selection must be between 1 and {len(selections)}") + raise ValueError("Selection must be a number") + + if selection_policy == "latest": + return _select_by_timestamp(selections), "policy" + if selection_policy == "by-timestamp": + return _select_by_timestamp(selections), "policy" + if selection_policy == "by-line": + return max(selections, key=lambda sel: sel.line), "policy" + if selection_policy == "last": + return selections[-1], "policy" + if selection_policy == "first": + return selections[0], "policy" + raise ValueError(f"Unsupported selection policy: {selection_policy}") + + def index_requires( events_path: Path, *, @@ -298,6 +419,11 @@ def export_replay_case_bundle( run_dir: Path | None = None, allow_bad_json: bool = False, overwrite: bool = False, + selection_policy: str = "latest", + select_index: int | None = None, + require_unique: bool = False, + allow_prompt: bool = False, + prompt_fn: Callable[[str], str] | None = None, ) -> Path: selections = _select_replay_cases( events_path, @@ -314,17 +440,29 @@ def export_replay_case_bundle( details.append(f"provider={provider!r}") detail_str = f" (filters: {', '.join(details)})" if details else "" raise LookupError(f"No replay_case id={replay_id!r} found in {events_path}{detail_str}") - if len(selections) > 1: - details = "\n".join( - f"- line {sel.line} run_id={sel.event.get('run_id')!r} timestamp={sel.event.get('timestamp')!r}" - for sel in selections[:5] - ) - raise LookupError( - "Multiple replay_case entries matched; use export_replay_case_bundles to export all.\n" - f"{details}" + selection, selection_mode = _select_replay_case( + selections, + events_path=events_path, + selection_policy=selection_policy, + select_index=select_index, + require_unique=require_unique, + allow_prompt=allow_prompt, + prompt_fn=prompt_fn, + ) + if len(selections) > 1 and selection_mode == "policy": + input_hashes = { + hashlib.sha256(canonical_json(sel.event.get("input") or {}).encode("utf-8")).hexdigest() + for sel in selections + } + details = format_replay_case_matches(selections, limit=5) + suffix = "Candidates differ by input payload." if len(input_hashes) > 1 else "Candidates share input." + logger.warning( + "Multiple replay_case entries matched; selection policy=%s chose line %s.\n%s\n%s", + selection_policy, + selection.line, + details, + suffix, ) - - selection = selections[0] root_event = selection.event requires = root_event.get("requires") or [] diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index d43e60b4..c6974e8e 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -5,7 +5,12 @@ from pathlib import Path from fetchgraph.tracer.auto_resolve import resolve_case_events -from fetchgraph.tracer.export import export_replay_case_bundle, export_replay_case_bundles +from fetchgraph.tracer.export import ( + export_replay_case_bundle, + export_replay_case_bundles, + find_replay_case_matches, + format_replay_case_matches, +) from fetchgraph.tracer.fixture_tools import ( fixture_fix, fixture_green, @@ -60,6 +65,15 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: action="store_true", help="Print resolved run_dir/events.jsonl", ) + export.add_argument( + "--select", + choices=["latest", "first", "last", "by-timestamp", "by-line"], + default="latest", + help="Selection policy when multiple replay_case entries match", + ) + export.add_argument("--select-index", type=int, default=None, help="Select a specific match (1-based)") + export.add_argument("--list-matches", action="store_true", help="List matches and exit") + export.add_argument("--require-unique", action="store_true", help="Error if multiple matches exist") green = sub.add_parser("fixture-green", help="Promote known_bad case to fixed") green.add_argument("--case", type=Path, required=True, help="Path to known_bad case bundle") @@ -138,6 +152,24 @@ def main(argv: list[str] | None = None) -> int: print(f"Resolved events.jsonl: {resolution.events_path}") if resolution.tag: print(f"Resolved tag: {resolution.tag}") + if args.list_matches: + selections = find_replay_case_matches( + events_path, + replay_id=args.id, + spec_idx=args.spec_idx, + provider=args.provider, + allow_bad_json=args.allow_bad_json, + ) + if not selections: + raise LookupError(f"No replay_case id={args.id!r} found in {events_path}") + print(format_replay_case_matches(selections, limit=20)) + return 0 + allow_prompt = ( + sys.stdin.isatty() + and args.select_index is None + and not args.require_unique + and not args.list_matches + ) if args.all: export_replay_case_bundles( events_path=events_path, @@ -159,6 +191,11 @@ def main(argv: list[str] | None = None) -> int: run_dir=run_dir, allow_bad_json=args.allow_bad_json, overwrite=args.overwrite, + selection_policy=args.select, + select_index=args.select_index, + require_unique=args.require_unique, + allow_prompt=allow_prompt, + prompt_fn=input, ) return 0 if args.command == "fixture-green": diff --git a/src/fetchgraph/tracer/export.py b/src/fetchgraph/tracer/export.py index c37a3b2c..7e9cb008 100644 --- a/src/fetchgraph/tracer/export.py +++ b/src/fetchgraph/tracer/export.py @@ -6,6 +6,8 @@ copy_resource_files, export_replay_case_bundle, export_replay_case_bundles, + find_replay_case_matches, + format_replay_case_matches, index_requires, iter_events, resolve_requires, @@ -17,6 +19,8 @@ "copy_resource_files", "export_replay_case_bundle", "export_replay_case_bundles", + "find_replay_case_matches", + "format_replay_case_matches", "index_requires", "iter_events", "resolve_requires", From 93571a473e237f6c693ac842215e5b50342b04a5 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 15:44:56 +0300 Subject: [PATCH 47/79] Move tracer auto-resolve module --- src/fetchgraph/tracer/cli.py | 2 +- src/fetchgraph/tracer/{auto_resolve.py => resolve.py} | 0 tests/test_tracer_auto_resolve.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/fetchgraph/tracer/{auto_resolve.py => resolve.py} (100%) diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index c6974e8e..acfc824b 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -4,7 +4,7 @@ import sys from pathlib import Path -from fetchgraph.tracer.auto_resolve import resolve_case_events +from fetchgraph.tracer.resolve import resolve_case_events from fetchgraph.tracer.export import ( export_replay_case_bundle, export_replay_case_bundles, diff --git a/src/fetchgraph/tracer/auto_resolve.py b/src/fetchgraph/tracer/resolve.py similarity index 100% rename from src/fetchgraph/tracer/auto_resolve.py rename to src/fetchgraph/tracer/resolve.py diff --git a/tests/test_tracer_auto_resolve.py b/tests/test_tracer_auto_resolve.py index 528f0d86..40c03196 100644 --- a/tests/test_tracer_auto_resolve.py +++ b/tests/test_tracer_auto_resolve.py @@ -6,7 +6,7 @@ import pytest -from fetchgraph.tracer.auto_resolve import resolve_case_events +from fetchgraph.tracer.resolve import resolve_case_events def _write_json(path: Path, payload: dict) -> None: From 08857c5f1f8da7e6db5088865e94e11980f82e00 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 15:56:25 +0300 Subject: [PATCH 48/79] Fix fixture-rm bucket typing --- src/fetchgraph/tracer/fixture_tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py index 95fbe047..8da9bb51 100644 --- a/src/fetchgraph/tracer/fixture_tools.py +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -163,8 +163,8 @@ def fixture_rm( dry_run: bool, ) -> int: root = root.resolve() - bucket = None if bucket == "all" else bucket - matched = find_case_bundles(root=root, bucket=bucket, name=name, pattern=pattern) + bucket_filter: str | None = None if bucket == "all" else bucket + matched = find_case_bundles(root=root, bucket=bucket_filter, name=name, pattern=pattern) if name and not matched: raise FileNotFoundError(f"No fixtures found for name={name!r}") From 36eca655f130a4297ef42df90a36cc75347b007e Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 16:05:55 +0300 Subject: [PATCH 49/79] Finalize tracer workflow tests and DX --- Makefile | 15 ++- tests/test_replay_export_selection.py | 142 ++++++++++++++++++++++++++ tests/test_replay_runtime.py | 17 ++- tests/test_tracer_fixture_tools.py | 134 ++++++++++++++++++++++++ 4 files changed, 297 insertions(+), 11 deletions(-) create mode 100644 tests/test_replay_export_selection.py create mode 100644 tests/test_tracer_fixture_tools.py diff --git a/Makefile b/Makefile index 8a596cea..fbe9d6f7 100644 --- a/Makefile +++ b/Makefile @@ -169,7 +169,7 @@ help: @echo " make tracer-export REPLAY_ID=... EVENTS=... TRACER_OUT_DIR=... [SPEC_IDX=...] [PROVIDER=...] [RUN_DIR=...] [ALLOW_BAD_JSON=1] [OVERWRITE=1]" @echo " make tracer-export REPLAY_ID=... CASE=... DATA=... [TAG=...] [SPEC_IDX=...] [PROVIDER=...] [ALLOW_BAD_JSON=1] [OVERWRITE=1]" @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" - @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources//..." + @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources///..." @echo " make fixture-green CASE=path/to/case.case.json [TRACER_ROOT=...] [VALIDATE=1] [OVERWRITE_EXPECTED=1] [DRY=1]" @echo " make fixture-rm [BUCKET=fixed|known_bad|all] [NAME=...] [PATTERN=...] [SCOPE=cases|resources|both] [DRY=1]" @echo " make fixture-fix BUCKET=... NAME=... NEW_NAME=... [DRY=1]" @@ -233,9 +233,22 @@ show-config: @echo "OUT = $(OUT)" @echo "EVENTS = $(EVENTS)" @echo "REPLAY_ID = $(REPLAY_ID)" + @echo "SPEC_IDX = $(SPEC_IDX)" + @echo "PROVIDER = $(PROVIDER)" + @echo "RUN_DIR = $(RUN_DIR)" + @echo "TRACER_ROOT = $(TRACER_ROOT)" @echo "TRACER_OUT_DIR = $(TRACER_OUT_DIR)" + @echo "BUCKET = $(BUCKET)" @echo "ALLOW_BAD_JSON = $(ALLOW_BAD_JSON)" @echo "OVERWRITE = $(OVERWRITE)" + @echo "ALL = $(ALL)" + @echo "NAME = $(NAME)" + @echo "NEW_NAME = $(NEW_NAME)" + @echo "PATTERN = $(PATTERN)" + @echo "SCOPE = $(SCOPE)" + @echo "DRY = $(DRY)" + @echo "VALIDATE = $(VALIDATE)" + @echo "OVERWRITE_EXPECTED = $(OVERWRITE_EXPECTED)" @echo "LLM_TOML= $(LLM_TOML)" @echo "TAG = $(TAG)" @echo "NOTE = $(NOTE)" diff --git a/tests/test_replay_export_selection.py b/tests/test_replay_export_selection.py new file mode 100644 index 00000000..849703ed --- /dev/null +++ b/tests/test_replay_export_selection.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fetchgraph.replay.export import ( + case_bundle_name, + export_replay_case_bundle, + find_replay_case_matches, + iter_events, +) + + +def _write_events(path: Path, events: list[dict]) -> None: + lines = [json.dumps(event) for event in events] + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def test_iter_events_allow_bad_json(tmp_path: Path) -> None: + events_path = tmp_path / "events.jsonl" + events_path.write_text('{"type":"ok"}\n{bad json}\n{"type":"ok2"}\n', encoding="utf-8") + + with pytest.raises(ValueError, match="Invalid JSON on line 2"): + list(iter_events(events_path, allow_bad_json=False)) + + events = list(iter_events(events_path, allow_bad_json=True)) + assert [event["type"] for _, event in events] == ["ok", "ok2"] + + +def test_export_replay_case_selection_controls(tmp_path: Path) -> None: + events_path = tmp_path / "events.jsonl" + out_dir = tmp_path / "out" + event_base = { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}}, + "observed": {"out_spec": {"provider": "sql"}}, + } + _write_events( + events_path, + [ + {**event_base, "timestamp": "2024-01-01T00:00:00Z"}, + {**event_base, "timestamp": "2024-01-02T00:00:00Z"}, + ], + ) + + selections = find_replay_case_matches(events_path, replay_id="plan_normalize.spec_v1") + assert len(selections) == 2 + + with pytest.raises(LookupError, match="Multiple replay_case entries matched"): + export_replay_case_bundle( + events_path=events_path, + out_dir=out_dir, + replay_id="plan_normalize.spec_v1", + require_unique=True, + ) + + out_path = export_replay_case_bundle( + events_path=events_path, + out_dir=out_dir, + replay_id="plan_normalize.spec_v1", + select_index=1, + ) + payload = json.loads(out_path.read_text(encoding="utf-8")) + assert payload["source"]["line"] == 1 + + +def test_export_replay_case_requires_run_dir_for_resources(tmp_path: Path) -> None: + events_path = tmp_path / "events.jsonl" + out_dir = tmp_path / "out" + events = [ + { + "type": "replay_resource", + "id": "rid1", + "data_ref": {"file": "artifact.txt"}, + }, + { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}}, + "observed": {"out_spec": {"provider": "sql"}}, + "requires": [{"kind": "resource", "id": "rid1"}], + }, + ] + _write_events(events_path, events) + + with pytest.raises(ValueError, match="run_dir is required to export file resources"): + export_replay_case_bundle( + events_path=events_path, + out_dir=out_dir, + replay_id="plan_normalize.spec_v1", + ) + + +def test_export_replay_case_overwrite_cleans_resources(tmp_path: Path) -> None: + events_path = tmp_path / "events.jsonl" + out_dir = tmp_path / "out" + run_dir = tmp_path / "run" + run_dir.mkdir() + (run_dir / "artifact.txt").write_text("data", encoding="utf-8") + + events = [ + { + "type": "replay_resource", + "id": "rid1", + "data_ref": {"file": "artifact.txt"}, + }, + { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}}, + "observed": {"out_spec": {"provider": "sql"}}, + "requires": [{"kind": "resource", "id": "rid1"}], + }, + ] + _write_events(events_path, events) + + out_path = export_replay_case_bundle( + events_path=events_path, + out_dir=out_dir, + replay_id="plan_normalize.spec_v1", + run_dir=run_dir, + ) + fixture_stem = case_bundle_name("plan_normalize.spec_v1", events[1]["input"]).replace(".case.json", "") + extra_path = out_dir / "resources" / fixture_stem / "extra.txt" + extra_path.parent.mkdir(parents=True, exist_ok=True) + extra_path.write_text("extra", encoding="utf-8") + + export_replay_case_bundle( + events_path=events_path, + out_dir=out_dir, + replay_id="plan_normalize.spec_v1", + run_dir=run_dir, + overwrite=True, + ) + assert out_path.exists() + assert not extra_path.exists() diff --git a/tests/test_replay_runtime.py b/tests/test_replay_runtime.py index ce03ba2a..62d06b97 100644 --- a/tests/test_replay_runtime.py +++ b/tests/test_replay_runtime.py @@ -1,15 +1,12 @@ from __future__ import annotations -from pathlib import Path +import pytest -from fetchgraph.replay.runtime import ReplayContext +from fetchgraph.replay.runtime import ReplayContext, run_case -def test_resolve_resource_path(tmp_path) -> None: - ctx = ReplayContext(base_dir=tmp_path) - - rel_path = Path("resources/sample.json") - assert ctx.resolve_resource_path(rel_path) == tmp_path / rel_path - - abs_path = Path("/var/data/sample.json") - assert ctx.resolve_resource_path(abs_path) == abs_path +def test_run_case_missing_handler_message() -> None: + root = {"id": "unknown.handler", "input": {}} + ctx = ReplayContext() + with pytest.raises(KeyError, match="Did you import fetchgraph.tracer.handlers\\?"): + run_case(root, ctx) diff --git a/tests/test_tracer_fixture_tools.py b/tests/test_tracer_fixture_tools.py new file mode 100644 index 00000000..99d9363a --- /dev/null +++ b/tests/test_tracer_fixture_tools.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fetchgraph.tracer.fixture_tools import fixture_fix, fixture_green, fixture_migrate + + +def _write_bundle(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")), encoding="utf-8") + + +def _bundle_payload(root: dict, *, resources: dict | None = None, extras: dict | None = None) -> dict: + return { + "schema": "fetchgraph.tracer.case_bundle", + "v": 1, + "root": root, + "resources": resources or {}, + "extras": extras or {}, + "source": {"events_path": "events.jsonl"}, + } + + +def test_fixture_green_requires_observed(tmp_path: Path) -> None: + root = tmp_path / "fixtures" + case_path = root / "known_bad" / "case.case.json" + payload = _bundle_payload( + { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}}, + "observed_error": {"type": "ValueError", "message": "boom"}, + } + ) + _write_bundle(case_path, payload) + + with pytest.raises(ValueError, match="root.observed is missing"): + fixture_green(case_path=case_path, out_root=root) + + +def test_fixture_green_moves_case_and_resources(tmp_path: Path) -> None: + root = tmp_path / "fixtures" + case_path = root / "known_bad" / "case.case.json" + resources_dir = root / "known_bad" / "resources" / "case" + resources_dir.mkdir(parents=True, exist_ok=True) + (resources_dir / "rid1.txt").write_text("data", encoding="utf-8") + payload = _bundle_payload( + { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}}, + "observed": {"out_spec": {"provider": "sql"}}, + } + ) + _write_bundle(case_path, payload) + + fixture_green(case_path=case_path, out_root=root) + + fixed_case = root / "fixed" / "case.case.json" + expected_path = root / "fixed" / "case.expected.json" + fixed_resources = root / "fixed" / "resources" / "case" + assert fixed_case.exists() + assert expected_path.exists() + assert fixed_resources.exists() + assert not (root / "known_bad" / "case.case.json").exists() + assert not resources_dir.exists() + + +def test_fixture_fix_renames_and_updates_resource_paths(tmp_path: Path) -> None: + root = tmp_path / "fixtures" + bucket = "fixed" + case_path = root / bucket / "old.case.json" + resources_dir = root / bucket / "resources" / "old" / "rid1" + resources_dir.mkdir(parents=True, exist_ok=True) + (resources_dir / "file.txt").write_text("data", encoding="utf-8") + payload = _bundle_payload( + { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}}, + "observed": {"out_spec": {"provider": "sql"}}, + }, + resources={ + "rid1": {"data_ref": {"file": "resources/old/rid1/file.txt"}}, + }, + ) + _write_bundle(case_path, payload) + expected_path = root / bucket / "old.expected.json" + expected_path.write_text("{}", encoding="utf-8") + + fixture_fix(root=root, name="old", new_name="new", bucket=bucket, dry_run=False) + + new_case = root / bucket / "new.case.json" + new_expected = root / bucket / "new.expected.json" + new_resources = root / bucket / "resources" / "new" / "rid1" / "file.txt" + assert new_case.exists() + assert new_expected.exists() + assert new_resources.exists() + data = json.loads(new_case.read_text(encoding="utf-8")) + assert data["resources"]["rid1"]["data_ref"]["file"] == "resources/new/rid1/file.txt" + + +def test_fixture_migrate_moves_resources(tmp_path: Path) -> None: + root = tmp_path / "fixtures" + bucket = "fixed" + case_path = root / bucket / "case.case.json" + legacy_dir = root / bucket / "legacy" + legacy_dir.mkdir(parents=True, exist_ok=True) + legacy_path = legacy_dir / "file.txt" + legacy_path.write_text("data", encoding="utf-8") + payload = _bundle_payload( + { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}}, + "observed": {"out_spec": {"provider": "sql"}}, + }, + resources={"rid1": {"data_ref": {"file": "legacy/file.txt"}}}, + ) + _write_bundle(case_path, payload) + + bundles_updated, files_moved = fixture_migrate(root=root, bucket=bucket, dry_run=False) + assert bundles_updated == 1 + assert files_moved == 1 + data = json.loads(case_path.read_text(encoding="utf-8")) + assert data["resources"]["rid1"]["data_ref"]["file"] == "resources/case/rid1/legacy/file.txt" + assert (root / bucket / "resources" / "case" / "rid1" / "legacy" / "file.txt").exists() From 3c855b01f4bdef681ffb6a74da452ef26ed6cd1f Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 16:12:46 +0300 Subject: [PATCH 50/79] Simplify tracer export defaults and event resolution --- Makefile | 68 +++++------ src/fetchgraph/tracer/cli.py | 82 +++++++++---- src/fetchgraph/tracer/resolve.py | 195 +++++++++++++++++++++++++----- tests/test_tracer_auto_resolve.py | 10 +- 4 files changed, 264 insertions(+), 91 deletions(-) diff --git a/Makefile b/Makefile index fbe9d6f7..941b0964 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ SHELL := /bin/bash # ============================================================================== CONFIG ?= .demo_qa.mk -include $(CONFIG) +-include .demo_qa.mk # ============================================================================== # 2) Значения по умолчанию (для make init) @@ -35,7 +36,7 @@ CLI := $(PYTHON) -m examples.demo_qa.cli # ============================================================================== # 4) Пути demo_qa (можно переопределять через CLI или в $(CONFIG)) # ============================================================================== -DATA ?= +DATA ?= _demo_data/shop SCHEMA ?= CASES ?= OUT ?= $(DATA)/.runs/results.jsonl @@ -49,16 +50,16 @@ CASE ?= NAME ?= NEW_NAME ?= PATTERN ?= -SPEC_IDX ?= -PROVIDER ?= -BUCKET ?= fixed +SPEC_IDX ?= 0 +PROVIDER ?= demo_qa +BUCKET ?= known_bad REPLAY_ID ?= EVENTS ?= TRACER_ROOT ?= tests/fixtures/replay_cases TRACER_OUT_DIR ?= $(TRACER_ROOT)/$(BUCKET) RUN_DIR ?= ALLOW_BAD_JSON ?= -OVERWRITE ?= +OVERWRITE ?= 0 SCOPE ?= both WITH_RESOURCES ?= 1 ALL ?= @@ -117,7 +118,7 @@ LIMIT_FLAG := $(if $(strip $(LIMIT)),--limit $(LIMIT),) batch batch-tag batch-failed batch-failed-from \ batch-missed batch-missed-from batch-failed-tag batch-missed-tag \ batch-fail-fast batch-max-fails \ - stats history-case report-tag report-tag-changes tags tag-rm case-run case-open tracer-export \ + stats history-case report-tag report-tag-changes tags tag-rm case-run case-open tracer-export tracer-ls \ fixture-green fixture-rm fixture-fix fixture-migrate \ compare compare-tag @@ -166,8 +167,8 @@ help: @echo " make tags [PATTERN=*] DATA=... - показать список тегов" @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" - @echo " make tracer-export REPLAY_ID=... EVENTS=... TRACER_OUT_DIR=... [SPEC_IDX=...] [PROVIDER=...] [RUN_DIR=...] [ALLOW_BAD_JSON=1] [OVERWRITE=1]" - @echo " make tracer-export REPLAY_ID=... CASE=... DATA=... [TAG=...] [SPEC_IDX=...] [PROVIDER=...] [ALLOW_BAD_JSON=1] [OVERWRITE=1]" + @echo " make tracer-export REPLAY_ID=... CASE=... [EVENTS=...] [RUN_DIR=...] [DATA=...] [PROVIDER=...] [BUCKET=...] [SPEC_IDX=...] [OVERWRITE=1] [ALLOW_BAD_JSON=1]" + @echo " make tracer-ls CASE=... [DATA=...] [TAG=...]" @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources///..." @echo " make fixture-green CASE=path/to/case.case.json [TRACER_ROOT=...] [VALIDATE=1] [OVERWRITE_EXPECTED=1] [DRY=1]" @@ -377,32 +378,31 @@ case-open: check @$(CLI) case open "$(CASE)" --data "$(DATA)" tracer-export: - @test -n "$(strip $(REPLAY_ID))" || (echo "REPLAY_ID обязателен: make tracer-export REPLAY_ID=plan_normalize.spec_v1" && exit 1) - @case "$(BUCKET)" in fixed|known_bad) ;; *) echo "BUCKET должен быть fixed или known_bad для tracer-export" && exit 1 ;; esac - @if [ -z "$(strip $(SPEC_IDX))" ] && ! printf "%s" "$(ALL)" | grep -Eiq '^(1|true|yes|on)$$'; then \ - echo "Для tracer-export нужно задать SPEC_IDX или ALL=1"; \ - exit 1; \ - fi - @# TRACER_OUT_DIR has a default; override if needed. - @if [ -n "$(strip $(EVENTS))" ]; then \ - $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle --events "$(EVENTS)" --out "$(TRACER_OUT_DIR)" --id "$(REPLAY_ID)" \ - $(if $(strip $(SPEC_IDX)),--spec-idx $(SPEC_IDX),) \ - $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ - $(if $(strip $(RUN_DIR)),--run-dir "$(RUN_DIR)",) \ - $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) \ - $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) \ - $(if $(filter 1 true yes on,$(ALL)),--all,); \ - else \ - test -n "$(strip $(CASE))" || (echo "CASE обязателен для auto режима: make tracer-export CASE=case_id DATA=... REPLAY_ID=..." && exit 1); \ - test -n "$(strip $(DATA))" || (echo "DATA обязателен для auto режима: make tracer-export CASE=case_id DATA=... REPLAY_ID=..." && exit 1); \ - $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle --case "$(CASE)" --data "$(DATA)" --out "$(TRACER_OUT_DIR)" --id "$(REPLAY_ID)" \ - $(if $(strip $(TAG)),--tag "$(TAG)",) \ - $(if $(strip $(SPEC_IDX)),--spec-idx $(SPEC_IDX),) \ - $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ - $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) \ - $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) \ - $(if $(filter 1 true yes on,$(ALL)),--all,); \ - fi + @test -n "$(strip $(REPLAY_ID))" || (echo "REPLAY_ID обязателен: make tracer-export REPLAY_ID=plan_normalize.spec_v1" && exit 2) + @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make tracer-export CASE=agg_003" && exit 2) + @case "$(BUCKET)" in fixed|known_bad) ;; *) echo "BUCKET должен быть fixed или known_bad для tracer-export" && exit 2 ;; esac + @fetchgraph-tracer export-case-bundle \ + --id "$(REPLAY_ID)" \ + --spec-idx "$(SPEC_IDX)" \ + --out "$(TRACER_OUT_DIR)" \ + --case "$(CASE)" \ + --data "$(DATA)" \ + --provider "$(PROVIDER)" \ + $(if $(RUN_DIR),--run-dir "$(RUN_DIR)",) \ + $(if $(EVENTS),--events "$(EVENTS)",) \ + $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) \ + $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) + +tracer-ls: + @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make tracer-ls CASE=agg_003" && exit 2) + @fetchgraph-tracer export-case-bundle \ + --case "$(CASE)" \ + --data "$(DATA)" \ + --out "$(TRACER_OUT_DIR)" \ + $(if $(RUN_DIR),--run-dir "$(RUN_DIR)",) \ + $(if $(EVENTS),--events "$(EVENTS)",) \ + $(if $(strip $(TAG)),--tag "$(TAG)",) \ + --list-matches fixture-green: @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture-green CASE=tests/fixtures/replay_cases/known_bad/fixture.case.json (или CASE=fixture_stem)" && exit 1) diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index acfc824b..7562210e 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -4,7 +4,14 @@ import sys from pathlib import Path -from fetchgraph.tracer.resolve import resolve_case_events +from fetchgraph.tracer.resolve import ( + find_events_file, + format_case_runs, + format_events_search, + list_case_runs, + resolve_case_events, + select_case_run, +) from fetchgraph.tracer.export import ( export_replay_case_bundle, export_replay_case_bundles, @@ -28,7 +35,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: export = sub.add_parser("export-case-bundle", help="Export replay case bundle from events.jsonl") export.add_argument("--events", type=Path, help="Path to events.jsonl") export.add_argument("--out", type=Path, required=True, help="Output directory for bundle") - export.add_argument("--id", required=True, help="Replay case id to export") + export.add_argument("--id", help="Replay case id to export") export.add_argument("--spec-idx", type=int, default=None, help="Filter replay_case by meta.spec_idx") export.add_argument( "--provider", @@ -71,8 +78,19 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: default="latest", help="Selection policy when multiple replay_case entries match", ) - export.add_argument("--select-index", type=int, default=None, help="Select a specific match (1-based)") - export.add_argument("--list-matches", action="store_true", help="List matches and exit") + export.add_argument("--select-index", type=int, default=None, help="Select a case run (1-based)") + export.add_argument("--list-matches", action="store_true", help="List case runs and exit") + export.add_argument( + "--replay-select-index", + type=int, + default=None, + help="Select a specific replay_case match (1-based)", + ) + export.add_argument( + "--list-replay-matches", + action="store_true", + help="List replay_case matches and exit", + ) export.add_argument("--require-unique", action="store_true", help="Error if multiple matches exist") green = sub.add_parser("fixture-green", help="Promote known_bad case to fixed") @@ -135,24 +153,38 @@ def main(argv: list[str] | None = None) -> int: run_dir = args.run_dir else: if args.run_dir: - raise ValueError("Do not combine --run-dir with auto-resolve.") - if not args.case or not args.data: - raise ValueError("--case and --data are required when --events is not provided.") - resolution = resolve_case_events( - case_id=args.case, - data_dir=args.data, - tag=args.tag, - runs_subdir=args.runs_subdir, - pick_run=args.pick_run, - ) - events_path = resolution.events_path - run_dir = resolution.case_dir + run_dir = args.run_dir + else: + if not args.case or not args.data: + raise ValueError("--case and --data are required when --events is not provided.") + candidates, stats = list_case_runs( + case_id=args.case, + data_dir=args.data, + tag=args.tag, + runs_subdir=args.runs_subdir, + ) + if args.list_matches: + if not candidates: + raise LookupError( + "No suitable case run found.\n" + f"runs_root: {stats.runs_root}\n" + f"case_id: {args.case}\n" + f"inspected_runs: {stats.inspected_runs}" + ) + print(format_case_runs(candidates, limit=20)) + return 0 + selected = select_case_run(candidates, select_index=args.select_index) + run_dir = selected.case_dir + events_resolution = find_events_file(run_dir) + if not events_resolution.events_path: + raise FileNotFoundError(format_events_search(run_dir, events_resolution)) + events_path = events_resolution.events_path if args.print_resolve: - print(f"Resolved run_dir: {resolution.case_dir}") - print(f"Resolved events.jsonl: {resolution.events_path}") - if resolution.tag: - print(f"Resolved tag: {resolution.tag}") - if args.list_matches: + print(f"Resolved run_dir: {run_dir}") + print(f"Resolved events.jsonl: {events_path}") + if args.list_replay_matches: + if not args.id: + raise ValueError("--id is required to list replay_case matches.") selections = find_replay_case_matches( events_path, replay_id=args.id, @@ -166,10 +198,12 @@ def main(argv: list[str] | None = None) -> int: return 0 allow_prompt = ( sys.stdin.isatty() - and args.select_index is None + and args.replay_select_index is None and not args.require_unique - and not args.list_matches + and not args.list_replay_matches ) + if not args.id: + raise ValueError("--id is required to export replay case bundles.") if args.all: export_replay_case_bundles( events_path=events_path, @@ -192,7 +226,7 @@ def main(argv: list[str] | None = None) -> int: allow_bad_json=args.allow_bad_json, overwrite=args.overwrite, selection_policy=args.select, - select_index=args.select_index, + select_index=args.replay_select_index, require_unique=args.require_unique, allow_prompt=allow_prompt, prompt_fn=input, diff --git a/src/fetchgraph/tracer/resolve.py b/src/fetchgraph/tracer/resolve.py index 937d27e3..9362b6e0 100644 --- a/src/fetchgraph/tracer/resolve.py +++ b/src/fetchgraph/tracer/resolve.py @@ -14,6 +14,21 @@ class CaseResolution: tag: str | None +@dataclass(frozen=True) +class CaseRunCandidate: + run_dir: Path + case_dir: Path + tag: str | None + mtime: float + + +@dataclass(frozen=True) +class EventsResolution: + events_path: Path | None + searched: list[str] + found: list[Path] + + def resolve_case_events( *, case_id: str, @@ -21,6 +36,7 @@ def resolve_case_events( tag: str | None = None, runs_subdir: str = ".runs/runs", pick_run: str = "latest_non_missed", + select_index: int | None = None, ) -> CaseResolution: if not case_id: raise ValueError("case_id is required") @@ -29,66 +45,181 @@ def resolve_case_events( if pick_run != "latest_non_missed": raise ValueError(f"Unsupported pick_run mode: {pick_run}") + candidates, stats = list_case_runs( + case_id=case_id, + data_dir=data_dir, + tag=tag, + runs_subdir=runs_subdir, + ) + if not candidates: + details = [ + "No suitable case run found.", + f"runs_root: {stats.runs_root}", + f"case_id: {case_id}", + f"inspected_runs: {stats.inspected_runs}", + f"missing_cases: {stats.missing_cases}", + f"missed_cases: {stats.missed_cases}", + ] + if tag: + details.append(f"tag: {tag}") + details.append(f"tag_mismatches: {stats.tag_mismatches}") + raise LookupError("\n".join(details)) + + candidate = select_case_run(candidates, select_index=select_index) + events = find_events_file(candidate.case_dir) + return CaseResolution( + run_dir=candidate.run_dir, + case_dir=candidate.case_dir, + events_path=events.events_path, + tag=candidate.tag, + ) + + +def _iter_run_dirs(runs_root: Path) -> Iterable[Path]: + candidates = [p for p in runs_root.iterdir() if p.is_dir()] + return sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True) + + +def _case_dirs(run_dir: Path, case_id: str) -> list[Path]: + cases_root = run_dir / "cases" + if not cases_root.exists(): + return [] + return sorted(cases_root.glob(f"{case_id}_*"), key=lambda p: p.stat().st_mtime, reverse=True) + + +@dataclass(frozen=True) +class RunScanStats: + runs_root: Path + inspected_runs: int + missing_cases: int + missed_cases: int + tag_mismatches: int + + +def list_case_runs( + *, + case_id: str, + data_dir: Path, + tag: str | None = None, + runs_subdir: str = ".runs/runs", +) -> tuple[list[CaseRunCandidate], RunScanStats]: runs_root = (data_dir / runs_subdir).resolve() if not runs_root.exists(): raise FileNotFoundError(f"Runs directory does not exist: {runs_root}") - run_dirs = list(_iter_run_dirs(runs_root)) + candidates: list[CaseRunCandidate] = [] inspected_runs = 0 missing_cases = 0 missed_cases = 0 tag_mismatches = 0 - for run_dir in run_dirs: + for run_dir in _iter_run_dirs(runs_root): inspected_runs += 1 run_tag = _extract_run_tag(run_dir) if tag and run_tag != tag: tag_mismatches += 1 continue - case_dirs = _case_dirs(run_dir, case_id) if not case_dirs: missing_cases += 1 continue - for case_dir in case_dirs: if _case_is_missed(case_dir): missed_cases += 1 continue - events_path = case_dir / "events.jsonl" - if not events_path.exists(): - raise FileNotFoundError(f"events.jsonl not found at {events_path}") - return CaseResolution( - run_dir=run_dir, - case_dir=case_dir, - events_path=events_path, - tag=run_tag, + candidates.append( + CaseRunCandidate( + run_dir=run_dir, + case_dir=case_dir, + tag=run_tag, + mtime=case_dir.stat().st_mtime, + ) ) - details = [ - "No suitable case run found.", - f"runs_root: {runs_root}", - f"case_id: {case_id}", - f"inspected_runs: {inspected_runs}", - f"missing_cases: {missing_cases}", - f"missed_cases: {missed_cases}", - ] - if tag: - details.append(f"tag: {tag}") - details.append(f"tag_mismatches: {tag_mismatches}") - raise LookupError("\n".join(details)) + candidates.sort(key=lambda candidate: candidate.mtime, reverse=True) + stats = RunScanStats( + runs_root=runs_root, + inspected_runs=inspected_runs, + missing_cases=missing_cases, + missed_cases=missed_cases, + tag_mismatches=tag_mismatches, + ) + return candidates, stats -def _iter_run_dirs(runs_root: Path) -> Iterable[Path]: - candidates = [p for p in runs_root.iterdir() if p.is_dir()] - return sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True) +def select_case_run(candidates: list[CaseRunCandidate], *, select_index: int | None = None) -> CaseRunCandidate: + if not candidates: + raise LookupError("No case run candidates available.") + if select_index is None: + return candidates[0] + if select_index < 1 or select_index > len(candidates): + raise ValueError(f"select_index must be between 1 and {len(candidates)}") + return candidates[select_index - 1] -def _case_dirs(run_dir: Path, case_id: str) -> list[Path]: - cases_root = run_dir / "cases" - if not cases_root.exists(): - return [] - return sorted(cases_root.glob(f"{case_id}_*"), key=lambda p: p.stat().st_mtime, reverse=True) +def format_case_runs(candidates: list[CaseRunCandidate], *, limit: int | None = 10) -> str: + rows = [] + for idx, candidate in enumerate(candidates[:limit], start=1): + rows.append( + " " + f"{idx}. run_dir={candidate.run_dir} " + f"case_dir={candidate.case_dir.name} " + f"tag={candidate.tag!r} " + f"mtime={candidate.mtime:.0f}" + ) + if len(candidates) > (limit or 0): + rows.append(f" ... ({len(candidates) - (limit or 0)} more)") + return "\n".join(rows) + + +_EVENTS_CANDIDATES = ( + "events.jsonl", + "events.ndjson", + "trace.jsonl", + "trace.ndjson", + "traces/events.jsonl", + "traces/trace.jsonl", +) + + +def find_events_file(run_dir: Path) -> EventsResolution: + searched = list(_EVENTS_CANDIDATES) + found: list[Path] = [] + for rel in _EVENTS_CANDIDATES: + candidate = run_dir / rel + if candidate.exists(): + return EventsResolution(events_path=candidate, searched=searched, found=found) + + candidates = [] + for path in run_dir.rglob("*"): + if path.is_dir(): + continue + if path.suffix not in {".jsonl", ".ndjson"}: + continue + rel_parts = path.relative_to(run_dir).parts + if "resources" in rel_parts: + continue + if len(rel_parts) > 3: + continue + candidates.append(path) + if candidates: + candidates.sort(key=lambda p: p.stat().st_size, reverse=True) + found = candidates + return EventsResolution(events_path=candidates[0], searched=searched, found=found) + + return EventsResolution(events_path=None, searched=searched, found=found) + + +def format_events_search(run_dir: Path, resolution: EventsResolution) -> str: + found_list = [str(path) for path in resolution.found] + return "\n".join( + [ + f"events file not found in {run_dir}.", + f"Looked for: {', '.join(resolution.searched)}", + f"Found jsonl/ndjson: {found_list}", + "You can pass --events explicitly.", + ] + ) def _extract_run_tag(run_dir: Path) -> str | None: diff --git a/tests/test_tracer_auto_resolve.py b/tests/test_tracer_auto_resolve.py index 40c03196..0ded707e 100644 --- a/tests/test_tracer_auto_resolve.py +++ b/tests/test_tracer_auto_resolve.py @@ -6,7 +6,7 @@ import pytest -from fetchgraph.tracer.resolve import resolve_case_events +from fetchgraph.tracer.resolve import EventsResolution, find_events_file, resolve_case_events def _write_json(path: Path, payload: dict) -> None: @@ -81,3 +81,11 @@ def test_resolve_not_found(tmp_path: Path) -> None: with pytest.raises(LookupError, match="No suitable case run found"): resolve_case_events(case_id="case_3", data_dir=data_dir) + + +def test_find_events_file_not_found(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + run_dir.mkdir() + resolution = find_events_file(run_dir) + assert isinstance(resolution, EventsResolution) + assert resolution.events_path is None From 6dd25de48cba0da305c34e1360c2d4f58d8351c8 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 16:23:16 +0300 Subject: [PATCH 51/79] Ensure events.jsonl on runner failures --- examples/demo_qa/runner.py | 140 ++++++++++++------ examples/demo_qa/tests/test_demo_qa_runner.py | 63 ++++++++ 2 files changed, 159 insertions(+), 44 deletions(-) diff --git a/examples/demo_qa/runner.py b/examples/demo_qa/runner.py index 58fb7785..93115d8e 100644 --- a/examples/demo_qa/runner.py +++ b/examples/demo_qa/runner.py @@ -1,6 +1,7 @@ from __future__ import annotations import datetime +import traceback import hashlib import json import re @@ -10,7 +11,9 @@ import uuid from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, Iterable, List, Mapping, NotRequired, TypedDict +from typing import Dict, Iterable, List, Mapping, TypedDict + +from typing_extensions import NotRequired from fetchgraph.core import create_generic_agent from fetchgraph.core.context import BaseGraphAgent @@ -387,70 +390,119 @@ def run_one( else: run_id = run_dir.name.split("_")[-1] - case_logger = event_logger.for_case(case.id, run_dir / "events.jsonl") if event_logger else None + run_dir.mkdir(parents=True, exist_ok=True) + events_path = run_dir / "events.jsonl" + case_logger = event_logger.for_case(case.id, events_path) if event_logger else EventLogger(events_path, run_id, case.id) if case_logger: + case_logger.emit({"type": "run_started", "case_id": case.id, "run_dir": str(run_dir)}) case_logger.emit({"type": "case_started", "case_id": case.id, "run_dir": str(run_dir)}) schema_ref: str | None = None - if case_logger and schema_path and schema_path.exists(): - schema_ref = _emit_schema_snapshot(case_logger, run_dir, schema_path) - if case.skip: - run_dir.mkdir(parents=True, exist_ok=True) - _save_text(run_dir / "skipped.txt", "Skipped by request") + ok = False + result: RunResult | None = None + try: + if case_logger and schema_path and schema_path.exists(): + schema_ref = _emit_schema_snapshot(case_logger, run_dir, schema_path) + if case.skip: + _save_text(run_dir / "skipped.txt", "Skipped by request") + result = RunResult( + id=case.id, + question=case.question, + status="skipped", + checked=False, + reason="skipped", + details=None, + artifacts_dir=str(run_dir), + duration_ms=0, + tags=list(case.tags), + answer=None, + error=None, + plan_path=None, + timings=RunTimings(), + expected_check=None, + ) + save_status(result) + ok = True + if case_logger: + case_logger.emit({"type": "case_finished", "case_id": case.id, "status": "skipped"}) + return result + + artifacts = runner.run_question( + case, + run_id, + run_dir, + plan_only=plan_only, + event_logger=case_logger, + schema_ref=schema_ref, + ) + save_artifacts(artifacts) + + expected_check = None if plan_only else _match_expected(case, artifacts.answer) + result = _build_result(case, artifacts, run_dir, expected_check) + save_status(result) + ok = result.status != "error" + if case_logger: + if result.status == "error": + case_logger.emit( + { + "type": "case_failed", + "case_id": case.id, + "status": result.status, + "reason": result.reason, + "artifacts_dir": result.artifacts_dir, + } + ) + case_logger.emit( + { + "type": "case_finished", + "case_id": case.id, + "status": result.status, + "duration_ms": result.duration_ms, + "artifacts_dir": result.artifacts_dir, + } + ) + return result + except BaseException as exc: + tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) + error_message = str(exc) or repr(exc) + if case_logger: + case_logger.emit( + { + "type": "run_failed", + "case_id": case.id, + "exc_type": type(exc).__name__, + "exc_message": error_message, + "traceback": tb, + "artifacts_dir": str(run_dir), + } + ) result = RunResult( id=case.id, question=case.question, - status="skipped", - checked=False, - reason="skipped", - details=None, + status="error", + checked=case.has_asserts, + reason=error_message, + details={"error": error_message, "traceback": tb, "events_path": str(events_path)}, artifacts_dir=str(run_dir), duration_ms=0, tags=list(case.tags), answer=None, - error=None, + error=error_message, plan_path=None, timings=RunTimings(), expected_check=None, ) save_status(result) + raise + finally: if case_logger: - case_logger.emit({"type": "case_finished", "case_id": case.id, "status": "skipped"}) - return result - - artifacts = runner.run_question( - case, - run_id, - run_dir, - plan_only=plan_only, - event_logger=case_logger, - schema_ref=schema_ref, - ) - save_artifacts(artifacts) - - expected_check = None if plan_only else _match_expected(case, artifacts.answer) - result = _build_result(case, artifacts, run_dir, expected_check) - save_status(result) - if case_logger: - if result.status == "error": case_logger.emit( { - "type": "case_failed", + "type": "run_finished", "case_id": case.id, - "status": result.status, - "reason": result.reason, - "artifacts_dir": result.artifacts_dir, + "ok": ok, + "artifacts_dir": str(run_dir), } ) - case_logger.emit( - { - "type": "case_finished", - "case_id": case.id, - "status": result.status, - "duration_ms": result.duration_ms, - "artifacts_dir": result.artifacts_dir, - } - ) - return result def summarize(results: Iterable[RunResult]) -> Dict[str, object]: diff --git a/examples/demo_qa/tests/test_demo_qa_runner.py b/examples/demo_qa/tests/test_demo_qa_runner.py index f9723dd2..545f0550 100644 --- a/examples/demo_qa/tests/test_demo_qa_runner.py +++ b/examples/demo_qa/tests/test_demo_qa_runner.py @@ -2,15 +2,78 @@ from typing import cast +import pytest + from examples.demo_qa.runner import ( Case, + RunArtifacts, RunResult, + RunTimings, _match_expected, diff_runs, + run_one, summarize, ) +class _FailingRunner: + def run_question(self, *args, **kwargs): + raise RuntimeError("boom") + + +class _ArtifactRunner: + def run_question(self, case, run_id, run_dir, **kwargs): + return RunArtifacts( + run_id=run_id, + run_dir=run_dir, + question=case.question, + answer="ok", + timings=RunTimings(total_s=0.01), + ) + + +def _read_events(path): + return [line for line in path.read_text(encoding="utf-8").splitlines() if line] + + +def test_run_one_writes_events_on_early_failure(tmp_path) -> None: + case = Case(id="case_fail", question="Q") + artifacts_root = tmp_path / "runs" + + with pytest.raises(RuntimeError): + run_one(case, _FailingRunner(), artifacts_root) + + case_dirs = list(artifacts_root.iterdir()) + assert case_dirs + events_path = case_dirs[0] / "events.jsonl" + assert events_path.exists() + events = _read_events(events_path) + assert any("run_started" in line for line in events) + assert any("run_failed" in line for line in events) + assert any("run_finished" in line for line in events) + + +def test_run_one_writes_events_on_late_failure(tmp_path, monkeypatch) -> None: + case = Case(id="case_late", question="Q", expected="ok") + artifacts_root = tmp_path / "runs" + + def _boom(*args, **kwargs): + raise ValueError("late boom") + + monkeypatch.setattr("examples.demo_qa.runner._match_expected", _boom) + + with pytest.raises(ValueError): + run_one(case, _ArtifactRunner(), artifacts_root) + + case_dirs = list(artifacts_root.iterdir()) + assert case_dirs + events_path = case_dirs[0] / "events.jsonl" + assert events_path.exists() + events = _read_events(events_path) + assert any("run_started" in line for line in events) + assert any("run_failed" in line for line in events) + + def test_match_expected_unchecked_when_no_expectations() -> None: case = Case(id="c1", question="What is foo?") assert _match_expected(case, "anything") is None From 57b9f8b0be99691eabf0338202bdcc5a71ce8a08 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 16:46:37 +0300 Subject: [PATCH 52/79] Improve tracer export run selection --- Makefile | 16 ++- src/fetchgraph/tracer/cli.py | 126 ++++++++++++++-- src/fetchgraph/tracer/resolve.py | 229 +++++++++++++++++++++--------- tests/test_tracer_auto_resolve.py | 30 ++-- 4 files changed, 305 insertions(+), 96 deletions(-) diff --git a/Makefile b/Makefile index 941b0964..7aa27098 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,7 @@ CLI := $(PYTHON) -m examples.demo_qa.cli # 4) Пути demo_qa (можно переопределять через CLI или в $(CONFIG)) # ============================================================================== DATA ?= _demo_data/shop +REPLAY_IDATA ?= $(DATA) SCHEMA ?= CASES ?= OUT ?= $(DATA)/.runs/results.jsonl @@ -55,6 +56,8 @@ PROVIDER ?= demo_qa BUCKET ?= known_bad REPLAY_ID ?= EVENTS ?= +RUN_ID ?= +CASE_DIR ?= TRACER_ROOT ?= tests/fixtures/replay_cases TRACER_OUT_DIR ?= $(TRACER_ROOT)/$(BUCKET) RUN_DIR ?= @@ -167,8 +170,8 @@ help: @echo " make tags [PATTERN=*] DATA=... - показать список тегов" @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" - @echo " make tracer-export REPLAY_ID=... CASE=... [EVENTS=...] [RUN_DIR=...] [DATA=...] [PROVIDER=...] [BUCKET=...] [SPEC_IDX=...] [OVERWRITE=1] [ALLOW_BAD_JSON=1]" - @echo " make tracer-ls CASE=... [DATA=...] [TAG=...]" + @echo " make tracer-export REPLAY_ID=... CASE=... [EVENTS=...] [RUN_ID=...] [CASE_DIR=...] [DATA=...] [PROVIDER=...] [BUCKET=...] [SPEC_IDX=...] [OVERWRITE=1] [ALLOW_BAD_JSON=1]" + @echo " make tracer-ls CASE=... [DATA=...] [TAG=...] [RUN_ID=...] [CASE_DIR=...]" @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources///..." @echo " make fixture-green CASE=path/to/case.case.json [TRACER_ROOT=...] [VALIDATE=1] [OVERWRITE_EXPECTED=1] [DRY=1]" @@ -386,10 +389,13 @@ tracer-export: --spec-idx "$(SPEC_IDX)" \ --out "$(TRACER_OUT_DIR)" \ --case "$(CASE)" \ - --data "$(DATA)" \ + --data "$(REPLAY_IDATA)" \ --provider "$(PROVIDER)" \ + $(if $(RUN_ID),--run-id "$(RUN_ID)",) \ + $(if $(CASE_DIR),--case-dir "$(CASE_DIR)",) \ $(if $(RUN_DIR),--run-dir "$(RUN_DIR)",) \ $(if $(EVENTS),--events "$(EVENTS)",) \ + $(if $(TAG),--tag "$(TAG)",) \ $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) \ $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) @@ -397,8 +403,10 @@ tracer-ls: @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make tracer-ls CASE=agg_003" && exit 2) @fetchgraph-tracer export-case-bundle \ --case "$(CASE)" \ - --data "$(DATA)" \ + --data "$(REPLAY_IDATA)" \ --out "$(TRACER_OUT_DIR)" \ + $(if $(RUN_ID),--run-id "$(RUN_ID)",) \ + $(if $(CASE_DIR),--case-dir "$(CASE_DIR)",) \ $(if $(RUN_DIR),--run-dir "$(RUN_DIR)",) \ $(if $(EVENTS),--events "$(EVENTS)",) \ $(if $(strip $(TAG)),--tag "$(TAG)",) \ diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index 7562210e..f7f67039 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -1,15 +1,17 @@ from __future__ import annotations import argparse +import os import sys from pathlib import Path from fetchgraph.tracer.resolve import ( find_events_file, format_case_runs, + format_case_run_debug, format_events_search, list_case_runs, - resolve_case_events, + scan_case_runs, select_case_run, ) from fetchgraph.tracer.export import ( @@ -43,6 +45,8 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="Filter replay_case by meta.provider (case-insensitive)", ) export.add_argument("--run-dir", type=Path, default=None, help="Run dir (required for file resources)") + export.add_argument("--run-id", default=None, help="Run id (select case dir within runs root)") + export.add_argument("--case-dir", type=Path, default=None, help="Case dir (explicit path to case)") export.add_argument( "--allow-bad-json", action="store_true", @@ -72,6 +76,11 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: action="store_true", help="Print resolved run_dir/events.jsonl", ) + export.add_argument( + "--debug", + action="store_true", + help="Print debug information about candidate runs", + ) export.add_argument( "--select", choices=["latest", "first", "last", "by-timestamp", "by-line"], @@ -146,17 +155,39 @@ def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) try: if args.command == "export-case-bundle": + debug_enabled = args.debug or bool(os.getenv("DEBUG")) if args.events: - if args.case or args.data or args.tag: - raise ValueError("Do not combine --events with --case/--data/--tag.") + if args.case or args.data or args.tag or args.run_id: + raise ValueError("Do not combine --events with --case/--data/--tag/--run-id.") events_path = args.events - run_dir = args.run_dir + run_dir = args.case_dir or args.run_dir else: - if args.run_dir: - run_dir = args.run_dir + if args.case_dir and args.run_id: + raise ValueError("Do not combine --case-dir with --run-id.") + if args.case_dir or args.run_dir: + run_dir = args.case_dir or args.run_dir + selection_rule = "explicit CASE_DIR" if args.case_dir else "explicit RUN_DIR" + elif args.run_id: + if not args.case or not args.data: + raise ValueError("--case and --data are required when --run-id is provided.") + run_dir = _resolve_case_dir_from_run_id( + data_dir=args.data, + runs_subdir=args.runs_subdir, + run_id=args.run_id, + case_id=args.case, + ) + selection_rule = f"explicit RUN_ID={args.run_id}" else: if not args.case or not args.data: raise ValueError("--case and --data are required when --events is not provided.") + infos, stats = scan_case_runs( + case_id=args.case, + data_dir=args.data, + runs_subdir=args.runs_subdir, + ) + if debug_enabled: + print("Debug: case run candidates (most recent first):") + print(format_case_run_debug(infos, limit=10)) candidates, stats = list_case_runs( case_id=args.case, data_dir=args.data, @@ -166,19 +197,31 @@ def main(argv: list[str] | None = None) -> int: if args.list_matches: if not candidates: raise LookupError( - "No suitable case run found.\n" - f"runs_root: {stats.runs_root}\n" - f"case_id: {args.case}\n" - f"inspected_runs: {stats.inspected_runs}" + _format_case_run_error( + stats, + case_id=args.case, + tag=args.tag, + ) ) print(format_case_runs(candidates, limit=20)) return 0 selected = select_case_run(candidates, select_index=args.select_index) run_dir = selected.case_dir - events_resolution = find_events_file(run_dir) - if not events_resolution.events_path: - raise FileNotFoundError(format_events_search(run_dir, events_resolution)) - events_path = events_resolution.events_path + selection_rule = "latest with events" + if args.tag: + selection_rule = f"latest with events filtered by TAG={args.tag!r}" + events_path = selected.events_path + if "events_path" not in locals(): + events_resolution = find_events_file(run_dir) + if not events_resolution.events_path: + raise FileNotFoundError( + _format_events_error( + run_dir, + events_resolution, + selection_rule=selection_rule, + ) + ) + events_path = events_resolution.events_path if args.print_resolve: print(f"Resolved run_dir: {run_dir}") print(f"Resolved events.jsonl: {events_path}") @@ -278,5 +321,60 @@ def main(argv: list[str] | None = None) -> int: raise SystemExit(f"Unknown command: {args.command}") +def _resolve_case_dir_from_run_id(*, data_dir: Path, runs_subdir: str, run_id: str, case_id: str) -> Path: + runs_root = data_dir / runs_subdir / run_id / "cases" + if not runs_root.exists(): + raise FileNotFoundError(f"Run directory does not exist: {runs_root}") + case_dirs = sorted(runs_root.glob(f"{case_id}_*"), key=lambda p: p.stat().st_mtime, reverse=True) + if not case_dirs: + raise LookupError( + "No case directories found under run.\n" + f"run_id: {run_id}\n" + f"case_id: {case_id}\n" + f"runs_root: {runs_root}" + ) + return case_dirs[0] + + +def _format_case_run_error(stats, *, case_id: str, tag: str | None) -> str: + lines = [ + "No suitable case run found.", + f"selection_rule: {'latest with events' if not tag else f'latest with events filtered by TAG={tag!r}'}", + f"runs_root: {stats.runs_root}", + f"case_id: {case_id}", + f"inspected_runs: {stats.inspected_runs}", + f"inspected_cases: {stats.inspected_cases}", + f"missing_cases: {stats.missing_cases}", + f"missing_events: {stats.missing_events}", + ] + if tag: + lines.append(f"tag: {tag}") + lines.append(f"tag_mismatches: {stats.tag_mismatches}") + if stats.recent: + lines.append("recent_cases:") + for info in stats.recent[:5]: + lines.append( + " " + f"case_dir={info.case_dir} " + f"tag={info.tag!r} " + f"events={bool(info.events.events_path)}" + ) + lines.append("Tip: verify TAG or pass RUN_ID/CASE_DIR/EVENTS.") + else: + lines.append("Tip: pass TAG/RUN_ID/CASE_DIR/EVENTS for a narrower selection.") + return "\n".join(lines) + + +def _format_events_error(run_dir: Path, resolution, *, selection_rule: str) -> str: + return "\n".join( + [ + f"Selected case_dir: {run_dir}", + f"selection_rule: {selection_rule}", + format_events_search(run_dir, resolution), + "Tip: rerun the case or pass EVENTS=... explicitly.", + ] + ) + + if __name__ == "__main__": sys.exit(main()) diff --git a/src/fetchgraph/tracer/resolve.py b/src/fetchgraph/tracer/resolve.py index 9362b6e0..88748f8b 100644 --- a/src/fetchgraph/tracer/resolve.py +++ b/src/fetchgraph/tracer/resolve.py @@ -18,8 +18,20 @@ class CaseResolution: class CaseRunCandidate: run_dir: Path case_dir: Path + events_path: Path + tag: str | None + run_mtime: float + case_mtime: float + + +@dataclass(frozen=True) +class CaseRunInfo: + run_dir: Path + case_dir: Path + events: "EventsResolution" tag: str | None - mtime: float + run_mtime: float + case_mtime: float @dataclass(frozen=True) @@ -52,25 +64,19 @@ def resolve_case_events( runs_subdir=runs_subdir, ) if not candidates: - details = [ - "No suitable case run found.", - f"runs_root: {stats.runs_root}", - f"case_id: {case_id}", - f"inspected_runs: {stats.inspected_runs}", - f"missing_cases: {stats.missing_cases}", - f"missed_cases: {stats.missed_cases}", - ] - if tag: - details.append(f"tag: {tag}") - details.append(f"tag_mismatches: {stats.tag_mismatches}") - raise LookupError("\n".join(details)) + details = _format_missing_case_runs( + stats, + case_id=case_id, + tag=tag, + rule=_resolve_rule(tag=tag), + ) + raise LookupError(details) candidate = select_case_run(candidates, select_index=select_index) - events = find_events_file(candidate.case_dir) return CaseResolution( run_dir=candidate.run_dir, case_dir=candidate.case_dir, - events_path=events.events_path, + events_path=candidate.events_path, tag=candidate.tag, ) @@ -91,9 +97,11 @@ def _case_dirs(run_dir: Path, case_id: str) -> list[Path]: class RunScanStats: runs_root: Path inspected_runs: int + inspected_cases: int missing_cases: int - missed_cases: int + missing_events: int tag_mismatches: int + recent: list[CaseRunInfo] def list_case_runs( @@ -103,48 +111,99 @@ def list_case_runs( tag: str | None = None, runs_subdir: str = ".runs/runs", ) -> tuple[list[CaseRunCandidate], RunScanStats]: + infos, stats = scan_case_runs( + case_id=case_id, + data_dir=data_dir, + tag=tag, + runs_subdir=runs_subdir, + ) + candidates = _filter_case_run_infos(infos, tag=tag) + return candidates, stats + + +def scan_case_runs( + *, + case_id: str, + data_dir: Path, + tag: str | None = None, + runs_subdir: str = ".runs/runs", +) -> tuple[list[CaseRunInfo], RunScanStats]: runs_root = (data_dir / runs_subdir).resolve() if not runs_root.exists(): raise FileNotFoundError(f"Runs directory does not exist: {runs_root}") - candidates: list[CaseRunCandidate] = [] + infos: list[CaseRunInfo] = [] inspected_runs = 0 + inspected_cases = 0 missing_cases = 0 - missed_cases = 0 - tag_mismatches = 0 + missing_events = 0 for run_dir in _iter_run_dirs(runs_root): inspected_runs += 1 - run_tag = _extract_run_tag(run_dir) - if tag and run_tag != tag: - tag_mismatches += 1 - continue case_dirs = _case_dirs(run_dir, case_id) if not case_dirs: missing_cases += 1 continue + run_mtime = run_dir.stat().st_mtime for case_dir in case_dirs: - if _case_is_missed(case_dir): - missed_cases += 1 - continue - candidates.append( - CaseRunCandidate( + inspected_cases += 1 + events = find_events_file(case_dir) + if events.events_path is None: + missing_events += 1 + case_mtime = case_dir.stat().st_mtime + tag_value = _extract_case_tag(case_dir) + infos.append( + CaseRunInfo( run_dir=run_dir, case_dir=case_dir, - tag=run_tag, - mtime=case_dir.stat().st_mtime, + events=events, + tag=tag_value, + run_mtime=run_mtime, + case_mtime=case_mtime, ) ) - candidates.sort(key=lambda candidate: candidate.mtime, reverse=True) + infos.sort(key=lambda info: (info.run_mtime, info.case_mtime), reverse=True) stats = RunScanStats( runs_root=runs_root, inspected_runs=inspected_runs, + inspected_cases=inspected_cases, missing_cases=missing_cases, - missed_cases=missed_cases, - tag_mismatches=tag_mismatches, + missing_events=missing_events, + tag_mismatches=_count_tag_mismatches(infos, tag=tag), + recent=infos[:10], ) - return candidates, stats + return infos, stats + + +def _filter_case_run_infos(infos: list[CaseRunInfo], *, tag: str | None = None) -> list[CaseRunCandidate]: + candidates: list[CaseRunCandidate] = [] + for info in infos: + if info.events.events_path is None: + continue + if tag and info.tag != tag: + continue + candidates.append( + CaseRunCandidate( + run_dir=info.run_dir, + case_dir=info.case_dir, + events_path=info.events.events_path, + tag=info.tag, + run_mtime=info.run_mtime, + case_mtime=info.case_mtime, + ) + ) + return candidates + + +def _count_tag_mismatches(infos: list[CaseRunInfo], tag: str | None) -> int: + if not tag: + return 0 + mismatches = 0 + for info in infos: + if info.tag != tag: + mismatches += 1 + return mismatches def select_case_run(candidates: list[CaseRunCandidate], *, select_index: int | None = None) -> CaseRunCandidate: @@ -165,13 +224,31 @@ def format_case_runs(candidates: list[CaseRunCandidate], *, limit: int | None = f"{idx}. run_dir={candidate.run_dir} " f"case_dir={candidate.case_dir.name} " f"tag={candidate.tag!r} " - f"mtime={candidate.mtime:.0f}" + f"run_mtime={candidate.run_mtime:.0f} " + f"case_mtime={candidate.case_mtime:.0f}" ) if len(candidates) > (limit or 0): rows.append(f" ... ({len(candidates) - (limit or 0)} more)") return "\n".join(rows) +def format_case_run_debug(infos: list[CaseRunInfo], *, limit: int = 10) -> str: + rows = [] + for idx, info in enumerate(infos[:limit], start=1): + rows.append( + " " + f"{idx}. run_dir={info.run_dir} " + f"case_dir={info.case_dir.name} " + f"tag={info.tag!r} " + f"events={bool(info.events.events_path)} " + f"run_mtime={info.run_mtime:.0f} " + f"case_mtime={info.case_mtime:.0f}" + ) + if len(infos) > limit: + rows.append(f" ... ({len(infos) - limit} more)") + return "\n".join(rows) + + _EVENTS_CANDIDATES = ( "events.jsonl", "events.ndjson", @@ -222,9 +299,9 @@ def format_events_search(run_dir: Path, resolution: EventsResolution) -> str: ) -def _extract_run_tag(run_dir: Path) -> str | None: - for name in ("run_meta.json", "meta.json", "summary.json"): - path = run_dir / name +def _extract_case_tag(case_dir: Path) -> str | None: + for name in ("status.json", "result.json"): + path = case_dir / name if not path.exists(): continue try: @@ -234,11 +311,17 @@ def _extract_run_tag(run_dir: Path) -> str | None: tag = _extract_tag_value(payload) if tag: return tag + for meta_key in ("run_meta", "meta"): + nested = payload.get(meta_key) + if isinstance(nested, dict): + tag = _extract_tag_value(nested) + if tag: + return tag return None def _extract_tag_value(payload: dict) -> str | None: - for key in ("tag", "TAG"): + for key in ("tag", "TAG", "bucket", "batch_tag", "bucket_tag"): value = payload.get(key) if isinstance(value, str) and value: return value @@ -250,32 +333,42 @@ def _extract_tag_value(payload: dict) -> str | None: return None -def _case_is_missed(case_dir: Path) -> bool: - for name in ("status.json", "result.json"): - path = case_dir / name - if not path.exists(): - continue - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - continue - if _payload_is_missed(payload): - return True - if _payload_is_non_missed(payload): - return False - return False - - -def _payload_is_missed(payload: dict) -> bool: - if payload.get("missed") is True: - return True - status = str(payload.get("status") or payload.get("result") or "").lower() - if status in {"missed", "missing"}: - return True - reason = str(payload.get("reason") or "").lower() - return "missed" in reason or "missing" in reason - - -def _payload_is_non_missed(payload: dict) -> bool: - status = str(payload.get("status") or payload.get("result") or "").lower() - return status in {"ok", "pass", "passed", "success"} +def _resolve_rule(*, tag: str | None) -> str: + if tag: + return f"latest with events filtered by TAG={tag!r}" + return "latest with events" + + +def _format_missing_case_runs( + stats: RunScanStats, + *, + case_id: str, + tag: str | None, + rule: str, +) -> str: + details = [ + "No suitable case run found.", + f"selection_rule: {rule}", + f"runs_root: {stats.runs_root}", + f"case_id: {case_id}", + f"inspected_runs: {stats.inspected_runs}", + f"inspected_cases: {stats.inspected_cases}", + f"missing_cases: {stats.missing_cases}", + f"missing_events: {stats.missing_events}", + ] + if tag: + details.append(f"tag: {tag}") + details.append(f"tag_mismatches: {stats.tag_mismatches}") + if stats.recent: + details.append("recent_cases:") + for info in stats.recent[:5]: + details.append( + " " + f"case_dir={info.case_dir} " + f"tag={info.tag!r} " + f"events={bool(info.events.events_path)}" + ) + details.append("Tip: verify TAG or pass RUN_ID/CASE_DIR/EVENTS.") + else: + details.append("Tip: pass TAG/RUN_ID/CASE_DIR/EVENTS for a narrower selection.") + return "\n".join(details) diff --git a/tests/test_tracer_auto_resolve.py b/tests/test_tracer_auto_resolve.py index 0ded707e..65639986 100644 --- a/tests/test_tracer_auto_resolve.py +++ b/tests/test_tracer_auto_resolve.py @@ -22,15 +22,27 @@ def _set_mtime(path: Path, ts: float) -> None: os.utime(path, (ts, ts)) -def _make_case_dir(run_dir: Path, case_id: str, suffix: str, *, status: str) -> Path: +def _make_case_dir( + run_dir: Path, + case_id: str, + suffix: str, + *, + status: str, + tag: str | None = None, + with_events: bool = True, +) -> Path: case_dir = run_dir / "cases" / f"{case_id}_{suffix}" case_dir.mkdir(parents=True, exist_ok=True) - _touch(case_dir / "events.jsonl") - _write_json(case_dir / "status.json", {"status": status}) + if with_events: + _touch(case_dir / "events.jsonl") + payload = {"status": status} + if tag: + payload["tag"] = tag + _write_json(case_dir / "status.json", payload) return case_dir -def test_resolve_latest_non_missed(tmp_path: Path) -> None: +def test_resolve_latest_with_events(tmp_path: Path) -> None: data_dir = tmp_path / "data" runs_root = data_dir / ".runs" / "runs" runs_root.mkdir(parents=True, exist_ok=True) @@ -42,7 +54,7 @@ def test_resolve_latest_non_missed(tmp_path: Path) -> None: run_new = runs_root / "run_new" run_new.mkdir() - _make_case_dir(run_new, "case_1", "def", status="missed") + _make_case_dir(run_new, "case_1", "def", status="ok", with_events=False) _set_mtime(run_new, 200) resolution = resolve_case_events(case_id="case_1", data_dir=data_dir) @@ -56,14 +68,12 @@ def test_resolve_with_tag(tmp_path: Path) -> None: run_a = runs_root / "run_a" run_a.mkdir() - _write_json(run_a / "run_meta.json", {"tag": "alpha"}) - _make_case_dir(run_a, "case_2", "aaa", status="ok") + _make_case_dir(run_a, "case_2", "aaa", status="ok", tag="alpha") _set_mtime(run_a, 100) run_b = runs_root / "run_b" run_b.mkdir() - _write_json(run_b / "run_meta.json", {"tag": "beta"}) - _make_case_dir(run_b, "case_2", "bbb", status="ok") + _make_case_dir(run_b, "case_2", "bbb", status="ok", tag="beta") _set_mtime(run_b, 200) resolution = resolve_case_events(case_id="case_2", data_dir=data_dir, tag="alpha") @@ -77,7 +87,7 @@ def test_resolve_not_found(tmp_path: Path) -> None: run_dir = runs_root / "run_only" run_dir.mkdir() - _make_case_dir(run_dir, "case_3", "ccc", status="missed") + _make_case_dir(run_dir, "case_3", "ccc", status="missed", with_events=False) with pytest.raises(LookupError, match="No suitable case run found"): resolve_case_events(case_id="case_3", data_dir=data_dir) From 7b462abc427a81cd3fa63dbd7f00c544eab775fe Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 17:28:17 +0300 Subject: [PATCH 53/79] Fix tracer selection typing --- src/fetchgraph/tracer/cli.py | 3 ++- src/fetchgraph/tracer/resolve.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index f7f67039..63027a27 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -156,6 +156,7 @@ def main(argv: list[str] | None = None) -> int: try: if args.command == "export-case-bundle": debug_enabled = args.debug or bool(os.getenv("DEBUG")) + events_path: Path | None = None if args.events: if args.case or args.data or args.tag or args.run_id: raise ValueError("Do not combine --events with --case/--data/--tag/--run-id.") @@ -211,7 +212,7 @@ def main(argv: list[str] | None = None) -> int: if args.tag: selection_rule = f"latest with events filtered by TAG={args.tag!r}" events_path = selected.events_path - if "events_path" not in locals(): + if events_path is None: events_resolution = find_events_file(run_dir) if not events_resolution.events_path: raise FileNotFoundError( diff --git a/src/fetchgraph/tracer/resolve.py b/src/fetchgraph/tracer/resolve.py index 88748f8b..bf0bd094 100644 --- a/src/fetchgraph/tracer/resolve.py +++ b/src/fetchgraph/tracer/resolve.py @@ -260,7 +260,7 @@ def format_case_run_debug(infos: list[CaseRunInfo], *, limit: int = 10) -> str: def find_events_file(run_dir: Path) -> EventsResolution: - searched = list(_EVENTS_CANDIDATES) + searched = [str(entry) for entry in _EVENTS_CANDIDATES] found: list[Path] = [] for rel in _EVENTS_CANDIDATES: candidate = run_dir / rel From 80dc7f56cdca46831315f4c4a932104391d2370e Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 18:38:27 +0300 Subject: [PATCH 54/79] Improve replay fixture pytest diagnostics --- tests/test_replay_fixtures.py | 122 +++++++++++++++++++++++++++++++--- 1 file changed, 112 insertions(+), 10 deletions(-) diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index 41b54649..32f384c2 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import traceback from pathlib import Path import pytest @@ -10,15 +11,26 @@ from fetchgraph.tracer.runtime import load_case_bundle, run_case from fetchgraph.tracer.validators import validate_plan_normalize_spec_v1 -FIXTURES_ROOT = Path(__file__).parent / "fixtures" / "replay_cases" -KNOWN_BAD_DIR = FIXTURES_ROOT / "known_bad" -FIXED_DIR = FIXTURES_ROOT / "fixed" +REPLAY_CASES_ROOT = Path(__file__).parent / "fixtures" / "replay_cases" +KNOWN_BAD_DIR = REPLAY_CASES_ROOT / "known_bad" +FIXED_DIR = REPLAY_CASES_ROOT / "fixed" def _iter_case_paths(directory: Path) -> list[Path]: if not directory.exists(): return [] - return sorted(directory.glob("*.case.json")) + return sorted(directory.rglob("*.case.json")) + + +def _case_id(case_path: Path, base: Path) -> str: + try: + return str(case_path.relative_to(base)) + except ValueError: + return case_path.stem + + +def _iter_case_params(directory: Path, base: Path) -> list[pytest.ParamSpec]: + return [pytest.param(path, id=_case_id(path, base)) for path in _iter_case_paths(directory)] def _iter_all_case_paths() -> list[Path]: @@ -36,16 +48,106 @@ def _expected_path(case_path: Path) -> Path: return case_path.with_name(case_path.name.replace(".case.json", ".expected.json")) +def format_json(obj: object, *, max_chars: int = 10_000, max_depth: int = 6) -> str: + def _prune(value: object, depth: int) -> object: + if depth <= 0: + return "...(max depth reached)" + if isinstance(value, dict): + return {str(key): _prune(val, depth - 1) for key, val in value.items()} + if isinstance(value, list): + return [_prune(item, depth - 1) for item in value] + if isinstance(value, tuple): + return tuple(_prune(item, depth - 1) for item in value) + return value + + text = json.dumps( + _prune(obj, max_depth), + ensure_ascii=False, + indent=2, + sort_keys=True, + default=str, + ) + if len(text) <= max_chars: + return text + return f"{text[:max_chars]}...(truncated)" + + +def _format_case_debug( + *, + case_path: Path, + case_id: str, + root: object, + out: object | None = None, + exc: BaseException | None = None, + tb: str | None = None, +) -> str: + root_id = root.get("id") if isinstance(root, dict) else None + root_meta = root.get("meta") if isinstance(root, dict) else None + root_input = root.get("input") if isinstance(root, dict) else None + lines = [ + "Replay fixture diagnostics:", + f"case_id: {case_id}", + f"case_path: {case_path}", + f"root.id: {root_id!r}", + f"root.meta: {format_json(root_meta)}", + f"root.input: {format_json(root_input)}", + ] + if out is not None: + lines.append(f"handler.out: {format_json(out)}") + if exc is not None: + lines.append(f"handler.exc: {type(exc).__name__}: {exc}") + if tb: + lines.append(f"handler.traceback:\n{tb}") + return "\n".join(lines) + + @pytest.mark.known_bad -@pytest.mark.parametrize("case_path", _iter_case_paths(KNOWN_BAD_DIR)) +@pytest.mark.parametrize("case_path", _iter_case_params(KNOWN_BAD_DIR, REPLAY_CASES_ROOT)) def test_known_bad_cases(case_path: Path) -> None: root, ctx = load_case_bundle(case_path) - out = run_case(root, ctx) - with pytest.raises((AssertionError, ValidationError)): + case_id = _case_id(case_path, REPLAY_CASES_ROOT) + try: + out = run_case(root, ctx) + except Exception as exc: + pytest.fail( + _format_case_debug( + case_path=case_path, + case_id=case_id, + root=root, + exc=exc, + tb=traceback.format_exc(), + ), + pytrace=False, + ) + try: validate_plan_normalize_spec_v1(out) - - -@pytest.mark.parametrize("case_path", _iter_case_paths(FIXED_DIR)) + except (AssertionError, ValidationError): + return + except Exception as exc: + pytest.fail( + _format_case_debug( + case_path=case_path, + case_id=case_id, + root=root, + out=out, + exc=exc, + tb=traceback.format_exc(), + ), + pytrace=False, + ) + pytest.fail( + "DID NOT RAISE: expected validate_plan_normalize_spec_v1 to fail.\n" + + _format_case_debug( + case_path=case_path, + case_id=case_id, + root=root, + out=out, + ), + pytrace=False, + ) + + +@pytest.mark.parametrize("case_path", _iter_case_params(FIXED_DIR, REPLAY_CASES_ROOT)) def test_replay_cases_expected(case_path: Path) -> None: expected_path = _expected_path(case_path) if not expected_path.exists(): From 672129e7dc1251bf186fe39f5db3b8f079e51a46 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 18:45:54 +0300 Subject: [PATCH 55/79] Refine tracer fixture diagnostics --- src/fetchgraph/tracer/cli.py | 4 +++ src/fetchgraph/tracer/fetchgraph_tracer.md | 39 ++++++++++++++++++++++ tests/test_replay_fixtures.py | 2 +- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index 63027a27..0758abfe 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -227,6 +227,8 @@ def main(argv: list[str] | None = None) -> int: print(f"Resolved run_dir: {run_dir}") print(f"Resolved events.jsonl: {events_path}") if args.list_replay_matches: + if events_path is None: + raise ValueError("events_path was not resolved.") if not args.id: raise ValueError("--id is required to list replay_case matches.") selections = find_replay_case_matches( @@ -246,6 +248,8 @@ def main(argv: list[str] | None = None) -> int: and not args.require_unique and not args.list_replay_matches ) + if events_path is None: + raise ValueError("events_path was not resolved.") if not args.id: raise ValueError("--id is required to export replay case bundles.") if args.all: diff --git a/src/fetchgraph/tracer/fetchgraph_tracer.md b/src/fetchgraph/tracer/fetchgraph_tracer.md index a91605ef..18fdd4a1 100644 --- a/src/fetchgraph/tracer/fetchgraph_tracer.md +++ b/src/fetchgraph/tracer/fetchgraph_tracer.md @@ -192,10 +192,49 @@ fetchgraph-tracer export-case-bundle \ --overwrite ``` +Разрешение источника events: +- `--events` — явный путь к events/trace файлу (самый высокий приоритет). +- `--run-id` или `--case-dir` — выбрать конкретный запуск/кейс. +- `--tag` — выбрать самый свежий кейс с events, совпадающий с тегом. +- по умолчанию — самый свежий кейс с events. + +Доступные форматы events: `events.jsonl`, `events.ndjson`, `trace.jsonl`, `trace.ndjson`, `traces/events.jsonl`, `traces/trace.jsonl`. + +Примеры: +```bash +# По RUN_ID +fetchgraph-tracer export-case-bundle \ + --case agg_003 \ + --data _demo_data/shop \ + --run-id 20260125_160601_retail_cases \ + --id plan_normalize.spec_v1 \ + --out tests/fixtures/replay_cases + +# По TAG +fetchgraph-tracer export-case-bundle \ + --case agg_003 \ + --data _demo_data/shop \ + --tag known_bad \ + --id plan_normalize.spec_v1 \ + --out tests/fixtures/replay_cases +``` + ### Makefile ```bash +make tracer-export REPLAY_ID=plan_normalize.spec_v1 CASE=agg_003 \ + REPLAY_IDATA=_demo_data/shop TRACER_OUT_DIR=tests/fixtures/replay_cases + +# Явный events make tracer-export REPLAY_ID=plan_normalize.spec_v1 EVENTS=path/to/events.jsonl \ TRACER_OUT_DIR=tests/fixtures/replay_cases RUN_DIR=path/to/run_dir OVERWRITE=1 + +# Фильтр по TAG +make tracer-export REPLAY_ID=plan_normalize.spec_v1 CASE=agg_003 TAG=known_bad \ + TRACER_OUT_DIR=tests/fixtures/replay_cases + +# Явный run/case +make tracer-export REPLAY_ID=plan_normalize.spec_v1 CASE=agg_003 RUN_ID=20260125_160601_retail_cases \ + TRACER_OUT_DIR=tests/fixtures/replay_cases ``` --- diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index 32f384c2..dfc4c867 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -29,7 +29,7 @@ def _case_id(case_path: Path, base: Path) -> str: return case_path.stem -def _iter_case_params(directory: Path, base: Path) -> list[pytest.ParamSpec]: +def _iter_case_params(directory: Path, base: Path) -> list[pytest.ParameterSet]: return [pytest.param(path, id=_case_id(path, base)) for path in _iter_case_paths(directory)] From a1673ea796eb5692872c3bef04237daa8169fd2b Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 18:52:10 +0300 Subject: [PATCH 56/79] Fix pytest parameter typing --- tests/test_replay_fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py index dfc4c867..3cbfc9a8 100644 --- a/tests/test_replay_fixtures.py +++ b/tests/test_replay_fixtures.py @@ -29,7 +29,7 @@ def _case_id(case_path: Path, base: Path) -> str: return case_path.stem -def _iter_case_params(directory: Path, base: Path) -> list[pytest.ParameterSet]: +def _iter_case_params(directory: Path, base: Path) -> list[object]: return [pytest.param(path, id=_case_id(path, base)) for path in _iter_case_paths(directory)] From 170e94e71f28c2db9ddab1e7e8077a3b2b9081f4 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 19:23:05 +0300 Subject: [PATCH 57/79] Add known_bad backlog suite --- Makefile | 11 +- tests/helpers/replay_dx.py | 85 +++++++++++ tests/test_replay_fixed.py | 90 ++++++++++++ tests/test_replay_fixtures.py | 189 ------------------------- tests/test_replay_known_bad_backlog.py | 154 ++++++++++++++++++++ 5 files changed, 339 insertions(+), 190 deletions(-) create mode 100644 tests/helpers/replay_dx.py create mode 100644 tests/test_replay_fixed.py delete mode 100644 tests/test_replay_fixtures.py create mode 100644 tests/test_replay_known_bad_backlog.py diff --git a/Makefile b/Makefile index 7aa27098..4ec1a433 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,7 @@ LIMIT_FLAG := $(if $(strip $(LIMIT)),--limit $(LIMIT),) batch batch-tag batch-failed batch-failed-from \ batch-missed batch-missed-from batch-failed-tag batch-missed-tag \ batch-fail-fast batch-max-fails \ - stats history-case report-tag report-tag-changes tags tag-rm case-run case-open tracer-export tracer-ls \ + stats history-case report-tag report-tag-changes tags tag-rm case-run case-open tracer-export tracer-ls known-bad known-bad-one \ fixture-green fixture-rm fixture-fix fixture-migrate \ compare compare-tag @@ -172,6 +172,8 @@ help: @echo " make case-open CASE=case_42 - открыть артефакты кейса" @echo " make tracer-export REPLAY_ID=... CASE=... [EVENTS=...] [RUN_ID=...] [CASE_DIR=...] [DATA=...] [PROVIDER=...] [BUCKET=...] [SPEC_IDX=...] [OVERWRITE=1] [ALLOW_BAD_JSON=1]" @echo " make tracer-ls CASE=... [DATA=...] [TAG=...] [RUN_ID=...] [CASE_DIR=...]" + @echo " make known-bad - запустить backlog-suite для known_bad (ожидаемо красный)" + @echo " make known-bad-one NAME=fixture_stem - запустить один known_bad кейс" @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources///..." @echo " make fixture-green CASE=path/to/case.case.json [TRACER_ROOT=...] [VALIDATE=1] [OVERWRITE_EXPECTED=1] [DRY=1]" @@ -412,6 +414,13 @@ tracer-ls: $(if $(strip $(TAG)),--tag "$(TAG)",) \ --list-matches +known-bad: + @pytest -m known_bad -vv + +known-bad-one: + @test -n "$(strip $(NAME))" || (echo "NAME обязателен: make known-bad-one NAME=fixture_stem" && exit 2) + @pytest -m known_bad -k "$(NAME)" -vv + fixture-green: @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture-green CASE=tests/fixtures/replay_cases/known_bad/fixture.case.json (или CASE=fixture_stem)" && exit 1) @case_path="$(CASE)"; \ diff --git a/tests/helpers/replay_dx.py b/tests/helpers/replay_dx.py new file mode 100644 index 00000000..d2e4a5e7 --- /dev/null +++ b/tests/helpers/replay_dx.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path + + +def _int_env(name: str, default: int) -> int: + value = os.getenv(name) + if not value: + return default + try: + return int(value) + except ValueError: + return default + + +def truncate(text: str, *, limit: int = 8000) -> str: + if len(text) <= limit: + return text + remaining = len(text) - limit + return f"{text[:limit]}...[truncated {remaining} chars]" + + +def format_json(obj: object, *, max_chars: int = 12_000, max_depth: int = 6) -> str: + def _prune(value: object, depth: int) -> object: + if depth <= 0: + return "...(max depth reached)" + if isinstance(value, dict): + return {str(key): _prune(val, depth - 1) for key, val in value.items()} + if isinstance(value, list): + return [_prune(item, depth - 1) for item in value] + if isinstance(value, tuple): + return tuple(_prune(item, depth - 1) for item in value) + return value + + text = json.dumps( + _prune(obj, max_depth), + ensure_ascii=False, + indent=2, + sort_keys=True, + default=str, + ) + return truncate(text, limit=max_chars) + + +def format_rule_trace(diag: dict | None, *, tail: int = 30) -> str: + if not diag: + return "" + rule_trace = diag.get("rule_trace") + if not isinstance(rule_trace, list) or not rule_trace: + return "" + tail_items = rule_trace[-tail:] if tail > 0 else rule_trace + return truncate(format_json(tail_items), limit=_int_env("FETCHGRAPH_REPLAY_TRUNCATE", 8000)) + + +def build_rerun_hints(bundle_path: Path) -> list[str]: + stem = bundle_path.stem + return [ + f"pytest -k {stem} -m known_bad -vv", + f"fetchgraph-tracer fixture-green --case {bundle_path} --validate", + f"fetchgraph-tracer replay --case {bundle_path} --debug", + ] + + +def ids_from_path(path: Path, base: Path) -> str: + try: + return str(path.relative_to(base)) + except ValueError: + return path.stem + + +def truncate_limits() -> tuple[int, int]: + return ( + _int_env("FETCHGRAPH_REPLAY_TRUNCATE", 12_000), + _int_env("FETCHGRAPH_REPLAY_META_TRUNCATE", 8000), + ) + + +def rule_trace_tail() -> int: + return _int_env("FETCHGRAPH_RULE_TRACE_TAIL", 30) + + +def debug_enabled() -> bool: + return os.getenv("FETCHGRAPH_REPLAY_DEBUG") not in (None, "", "0", "false", "False") diff --git a/tests/test_replay_fixed.py b/tests/test_replay_fixed.py new file mode 100644 index 00000000..edd0bc7b --- /dev/null +++ b/tests/test_replay_fixed.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +import fetchgraph.tracer.handlers # noqa: F401 +from fetchgraph.tracer.runtime import load_case_bundle, run_case +from tests.helpers.replay_dx import ( + format_json, + ids_from_path, + truncate, + truncate_limits, +) + +REPLAY_CASES_ROOT = Path(__file__).parent / "fixtures" / "replay_cases" +FIXED_DIR = REPLAY_CASES_ROOT / "fixed" + + +def _iter_fixed_paths() -> list[Path]: + if not FIXED_DIR.exists(): + return [] + return sorted(FIXED_DIR.rglob("*.case.json")) + + +def _iter_case_params() -> list[object]: + return [pytest.param(path, id=ids_from_path(path, REPLAY_CASES_ROOT)) for path in _iter_fixed_paths()] + + +def _expected_path(case_path: Path) -> Path: + if not case_path.name.endswith(".case.json"): + raise ValueError(f"Unexpected case filename: {case_path}") + return case_path.with_name(case_path.name.replace(".case.json", ".expected.json")) + + +@pytest.mark.parametrize("case_path", _iter_case_params()) +def test_replay_fixed_cases(case_path: Path) -> None: + expected_path = _expected_path(case_path) + if not expected_path.exists(): + pytest.fail( + "Expected file is required for fixed fixtures:\n" + f" missing: {expected_path}\n" + "Hint: run `fetchgraph-tracer fixture-green --case ...` or create expected json manually." + ) + root, ctx = load_case_bundle(case_path) + out = run_case(root, ctx) + expected = json.loads(expected_path.read_text(encoding="utf-8")) + if out != expected: + input_limit, meta_limit = truncate_limits() + meta = root.get("meta") if isinstance(root, dict) else None + note = root.get("note") if isinstance(root, dict) else None + message = "\n".join( + [ + "Fixed fixture mismatch.", + f"fixture: {case_path}", + f"meta: {truncate(format_json(meta, max_chars=meta_limit), limit=meta_limit)}", + f"note: {truncate(format_json(note, max_chars=meta_limit), limit=meta_limit)}", + f"out: {truncate(format_json(out, max_chars=input_limit), limit=input_limit)}", + ] + ) + pytest.fail(message, pytrace=False) + assert out == expected + + +def test_replay_case_resources_exist() -> None: + case_paths = _iter_fixed_paths() + if not case_paths: + pytest.skip("No replay case bundles found.") + missing: list[tuple[Path, Path]] = [] + for case_path in case_paths: + raw = json.loads(case_path.read_text(encoding="utf-8")) + resources = raw.get("resources") or {} + if not isinstance(resources, dict): + continue + for resource in resources.values(): + if not isinstance(resource, dict): + continue + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + continue + resolved = case_path.parent / file_name + if not resolved.exists(): + missing.append((case_path, resolved)) + if missing: + details = "\n".join(f"- {fixture}: {resource}" for fixture, resource in missing) + pytest.fail(f"Missing replay resources:\n{details}") diff --git a/tests/test_replay_fixtures.py b/tests/test_replay_fixtures.py deleted file mode 100644 index 3cbfc9a8..00000000 --- a/tests/test_replay_fixtures.py +++ /dev/null @@ -1,189 +0,0 @@ -from __future__ import annotations - -import json -import traceback -from pathlib import Path - -import pytest -from pydantic import ValidationError - -import fetchgraph.tracer.handlers # noqa: F401 -from fetchgraph.tracer.runtime import load_case_bundle, run_case -from fetchgraph.tracer.validators import validate_plan_normalize_spec_v1 - -REPLAY_CASES_ROOT = Path(__file__).parent / "fixtures" / "replay_cases" -KNOWN_BAD_DIR = REPLAY_CASES_ROOT / "known_bad" -FIXED_DIR = REPLAY_CASES_ROOT / "fixed" - - -def _iter_case_paths(directory: Path) -> list[Path]: - if not directory.exists(): - return [] - return sorted(directory.rglob("*.case.json")) - - -def _case_id(case_path: Path, base: Path) -> str: - try: - return str(case_path.relative_to(base)) - except ValueError: - return case_path.stem - - -def _iter_case_params(directory: Path, base: Path) -> list[object]: - return [pytest.param(path, id=_case_id(path, base)) for path in _iter_case_paths(directory)] - - -def _iter_all_case_paths() -> list[Path]: - return _iter_case_paths(FIXED_DIR) + _iter_case_paths(KNOWN_BAD_DIR) - - -def test_replay_cases_present() -> None: - if not _iter_all_case_paths(): - pytest.skip("No replay case bundles found under tests/fixtures/replay_cases") - - -def _expected_path(case_path: Path) -> Path: - if not case_path.name.endswith(".case.json"): - raise ValueError(f"Unexpected case filename: {case_path}") - return case_path.with_name(case_path.name.replace(".case.json", ".expected.json")) - - -def format_json(obj: object, *, max_chars: int = 10_000, max_depth: int = 6) -> str: - def _prune(value: object, depth: int) -> object: - if depth <= 0: - return "...(max depth reached)" - if isinstance(value, dict): - return {str(key): _prune(val, depth - 1) for key, val in value.items()} - if isinstance(value, list): - return [_prune(item, depth - 1) for item in value] - if isinstance(value, tuple): - return tuple(_prune(item, depth - 1) for item in value) - return value - - text = json.dumps( - _prune(obj, max_depth), - ensure_ascii=False, - indent=2, - sort_keys=True, - default=str, - ) - if len(text) <= max_chars: - return text - return f"{text[:max_chars]}...(truncated)" - - -def _format_case_debug( - *, - case_path: Path, - case_id: str, - root: object, - out: object | None = None, - exc: BaseException | None = None, - tb: str | None = None, -) -> str: - root_id = root.get("id") if isinstance(root, dict) else None - root_meta = root.get("meta") if isinstance(root, dict) else None - root_input = root.get("input") if isinstance(root, dict) else None - lines = [ - "Replay fixture diagnostics:", - f"case_id: {case_id}", - f"case_path: {case_path}", - f"root.id: {root_id!r}", - f"root.meta: {format_json(root_meta)}", - f"root.input: {format_json(root_input)}", - ] - if out is not None: - lines.append(f"handler.out: {format_json(out)}") - if exc is not None: - lines.append(f"handler.exc: {type(exc).__name__}: {exc}") - if tb: - lines.append(f"handler.traceback:\n{tb}") - return "\n".join(lines) - - -@pytest.mark.known_bad -@pytest.mark.parametrize("case_path", _iter_case_params(KNOWN_BAD_DIR, REPLAY_CASES_ROOT)) -def test_known_bad_cases(case_path: Path) -> None: - root, ctx = load_case_bundle(case_path) - case_id = _case_id(case_path, REPLAY_CASES_ROOT) - try: - out = run_case(root, ctx) - except Exception as exc: - pytest.fail( - _format_case_debug( - case_path=case_path, - case_id=case_id, - root=root, - exc=exc, - tb=traceback.format_exc(), - ), - pytrace=False, - ) - try: - validate_plan_normalize_spec_v1(out) - except (AssertionError, ValidationError): - return - except Exception as exc: - pytest.fail( - _format_case_debug( - case_path=case_path, - case_id=case_id, - root=root, - out=out, - exc=exc, - tb=traceback.format_exc(), - ), - pytrace=False, - ) - pytest.fail( - "DID NOT RAISE: expected validate_plan_normalize_spec_v1 to fail.\n" - + _format_case_debug( - case_path=case_path, - case_id=case_id, - root=root, - out=out, - ), - pytrace=False, - ) - - -@pytest.mark.parametrize("case_path", _iter_case_params(FIXED_DIR, REPLAY_CASES_ROOT)) -def test_replay_cases_expected(case_path: Path) -> None: - expected_path = _expected_path(case_path) - if not expected_path.exists(): - pytest.fail( - "Expected file is required for fixed fixtures:\n" - f" missing: {expected_path}\n" - "Hint: run `fetchgraph-tracer fixture-green --case ...` or create expected json manually." - ) - root, ctx = load_case_bundle(case_path) - out = run_case(root, ctx) - expected = json.loads(expected_path.read_text(encoding="utf-8")) - assert out == expected - - -def test_replay_case_resources_exist() -> None: - case_paths = _iter_all_case_paths() - if not case_paths: - pytest.skip("No replay case bundles found.") - missing: list[tuple[Path, Path]] = [] - for case_path in case_paths: - raw = json.loads(case_path.read_text(encoding="utf-8")) - resources = raw.get("resources") or {} - if not isinstance(resources, dict): - continue - for resource in resources.values(): - if not isinstance(resource, dict): - continue - data_ref = resource.get("data_ref") - if not isinstance(data_ref, dict): - continue - file_name = data_ref.get("file") - if not isinstance(file_name, str) or not file_name: - continue - resolved = case_path.parent / file_name - if not resolved.exists(): - missing.append((case_path, resolved)) - if missing: - details = "\n".join(f"- {fixture}: {resource}" for fixture, resource in missing) - pytest.fail(f"Missing replay resources:\n{details}") diff --git a/tests/test_replay_known_bad_backlog.py b/tests/test_replay_known_bad_backlog.py new file mode 100644 index 00000000..cdfa27f6 --- /dev/null +++ b/tests/test_replay_known_bad_backlog.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import json +import traceback +from pathlib import Path +from typing import Callable + +import pytest +from pydantic import ValidationError + +import fetchgraph.tracer.handlers # noqa: F401 +from fetchgraph.tracer.runtime import load_case_bundle, run_case +from fetchgraph.tracer.validators import validate_plan_normalize_spec_v1 +from tests.helpers.replay_dx import ( + build_rerun_hints, + debug_enabled, + format_json, + format_rule_trace, + ids_from_path, + rule_trace_tail, + truncate, + truncate_limits, +) + +REPLAY_CASES_ROOT = Path(__file__).parent / "fixtures" / "replay_cases" +KNOWN_BAD_DIR = REPLAY_CASES_ROOT / "known_bad" + +VALIDATORS: dict[str, Callable[[dict], None]] = { + "plan_normalize.spec_v1": validate_plan_normalize_spec_v1, +} + + +def _iter_known_bad_paths() -> list[Path]: + if not KNOWN_BAD_DIR.exists(): + return [] + return sorted(KNOWN_BAD_DIR.rglob("*.case.json")) + + +def _iter_case_params() -> list[object]: + return [pytest.param(path, id=ids_from_path(path, REPLAY_CASES_ROOT)) for path in _iter_known_bad_paths()] + + +def _format_common_block( + *, + bundle_path: Path, + root: dict, + input_limit: int, + meta_limit: int, +) -> list[str]: + note = root.get("note") + requires = root.get("requires") + meta = root.get("meta") + lines = [ + f"fixture: {bundle_path}", + "bucket: known_bad", + f"id: {root.get('id')!r}", + "rerun:", + ] + lines.extend([f" - {hint}" for hint in build_rerun_hints(bundle_path)]) + lines.append(f"meta: {truncate(format_json(meta, max_chars=meta_limit), limit=meta_limit)}") + if note: + lines.append(f"note: {truncate(format_json(note, max_chars=meta_limit), limit=meta_limit)}") + lines.append(f"requires: {truncate(format_json(requires, max_chars=meta_limit), limit=meta_limit)}") + lines.append(f"input: {truncate(format_json(root.get('input'), max_chars=input_limit), limit=input_limit)}") + return lines + + +def _validate_root_schema(root: dict) -> None: + if root.get("schema") != "fetchgraph.tracer.case_bundle": + raise ValueError(f"Unexpected schema: {root.get('schema')!r}") + if root.get("v") != 1: + raise ValueError(f"Unexpected bundle version: {root.get('v')!r}") + root_case = root.get("root") + if not isinstance(root_case, dict): + raise ValueError("Missing root replay_case entry.") + if root_case.get("type") != "replay_case": + raise ValueError(f"Unexpected root type: {root_case.get('type')!r}") + if root_case.get("v") != 2: + raise ValueError(f"Unexpected replay_case version: {root_case.get('v')!r}") + + +@pytest.mark.known_bad +@pytest.mark.parametrize("bundle_path", _iter_case_params()) +def test_known_bad_backlog(bundle_path: Path) -> None: + root, ctx = load_case_bundle(bundle_path) + if not isinstance(root, dict): + pytest.fail(f"Unexpected bundle payload type: {type(root)}", pytrace=False) + _validate_root_schema(root) + replay_id = root.get("id") + validator = VALIDATORS.get(replay_id) + if validator is None: + pytest.fail( + f"No validator registered for replay id={replay_id!r}. Add it to VALIDATORS.", + pytrace=False, + ) + input_limit, meta_limit = truncate_limits() + common_block = _format_common_block( + bundle_path=bundle_path, + root=root, + input_limit=input_limit, + meta_limit=meta_limit, + ) + if debug_enabled(): + print("\n".join(["KNOWN_BAD debug summary:"] + common_block)) + + try: + out = run_case(root, ctx) + except Exception as exc: + message = "\n".join( + [ + "KNOWN_BAD (backlog): handler raised exception", + *common_block, + f"exception: {exc!r}", + f"traceback:\n{truncate(traceback.format_exc(), limit=meta_limit)}", + ] + ) + pytest.fail(message, pytrace=False) + + try: + validator(out) + except (AssertionError, ValidationError) as exc: + diag = out.get("diag") if isinstance(out, dict) else None + rule_trace = format_rule_trace(diag, tail=rule_trace_tail()) + message = "\n".join( + [ + "KNOWN_BAD (backlog): output is invalid", + *common_block, + f"validator_error: {exc!r}", + f"rule_trace (tail): {rule_trace or 'N/A'}", + f"out: {truncate(format_json(out, max_chars=input_limit), limit=input_limit)}", + ] + ) + pytest.fail(message, pytrace=False) + except Exception as exc: + message = "\n".join( + [ + "KNOWN_BAD (backlog): validator raised unexpected exception", + *common_block, + f"validator_error: {exc!r}", + f"traceback:\n{truncate(traceback.format_exc(), limit=meta_limit)}", + ] + ) + pytest.fail(message, pytrace=False) + + message = "\n".join( + [ + "KNOWN_BAD is now PASSING. Promote to fixed", + *common_block, + "Promote this fixture to fixed (freeze expected) and remove it from known_bad.", + f"Command: fetchgraph-tracer fixture-green --case {bundle_path} --validate", + "expected will be created from root.observed", + ] + ) + pytest.fail(message, pytrace=False) From 124a32e1e7061da52a6777d6dfaea4ada01f735d Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 20:18:02 +0300 Subject: [PATCH 58/79] Close tracer v2 spec gaps --- .../planning/normalize/plan_normalizer.py | 30 ++++ src/fetchgraph/replay/__init__.py | 3 +- src/fetchgraph/replay/export.py | 9 +- src/fetchgraph/replay/log.py | 21 +++ src/fetchgraph/tracer/cli.py | 29 ++++ src/fetchgraph/tracer/fetchgraph_tracer.md | 2 + src/fetchgraph/tracer/fixture_tools.py | 124 ++++++++++++-- src/fetchgraph/tracer/resolve.py | 158 +++++++++++++++--- .../fixed/resource_read.v1__sample.case.json | 23 +++ .../resource_read.v1__sample.expected.json | 3 + .../sample/sample.txt | 1 + tests/helpers/handlers_resource_read.py | 24 +++ tests/test_replay_fixed.py | 22 +++ tests/test_replay_iter_events_bad_json.py | 36 ++++ tests/test_replay_known_bad_backlog.py | 19 ++- tests/test_replay_log_trace_truncate.py | 29 ++++ tests/test_tracer_auto_resolve.py | 69 ++++++++ 17 files changed, 560 insertions(+), 42 deletions(-) create mode 100644 tests/fixtures/replay_cases/fixed/resource_read.v1__sample.case.json create mode 100644 tests/fixtures/replay_cases/fixed/resource_read.v1__sample.expected.json create mode 100644 tests/fixtures/replay_cases/fixed/resources/resource_read.v1__sample/sample/sample.txt create mode 100644 tests/helpers/handlers_resource_read.py create mode 100644 tests/test_replay_iter_events_bad_json.py create mode 100644 tests/test_replay_log_trace_truncate.py diff --git a/src/fetchgraph/planning/normalize/plan_normalizer.py b/src/fetchgraph/planning/normalize/plan_normalizer.py index 22b2eda2..a4a9c199 100644 --- a/src/fetchgraph/planning/normalize/plan_normalizer.py +++ b/src/fetchgraph/planning/normalize/plan_normalizer.py @@ -160,6 +160,13 @@ def _normalize_specs( decision = "keep_original_valid" if before_ok else "keep_original_still_invalid" use = selectors_before after_ok = before_ok + rule_trace = [ + { + "stage": "select_rule", + "provider": spec.provider, + "rule_kind": rule.kind, + } + ] if not before_ok: candidate = rule.normalize_selectors(copy.deepcopy(selectors_before)) after_ok = self._validate_selectors(rule.validator, candidate) @@ -169,6 +176,28 @@ def _normalize_specs( elif candidate != selectors_before: decision = "use_normalized_unvalidated" use = candidate + rule_trace.append( + { + "stage": "normalize", + "decision": decision, + "changed": candidate != selectors_before, + "validators": { + "before_ok": before_ok, + "after_ok": after_ok, + }, + } + ) + else: + rule_trace.append( + { + "stage": "validate", + "decision": decision, + "validators": { + "before_ok": before_ok, + "after_ok": after_ok, + }, + } + ) note = self._format_selectors_note( spec.provider, before_ok, @@ -223,6 +252,7 @@ def _normalize_specs( diag={ "selectors_valid_before": before_ok, "selectors_valid_after": after_ok, + "rule_trace": rule_trace, }, note=note, ) diff --git a/src/fetchgraph/replay/__init__.py b/src/fetchgraph/replay/__init__.py index 0f5f8b31..ad08ecf5 100644 --- a/src/fetchgraph/replay/__init__.py +++ b/src/fetchgraph/replay/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations -from .log import EventLoggerLike, log_replay_case +from .log import EventLoggerLike, log_replay_case, log_replay_point from .runtime import REPLAY_HANDLERS, ReplayContext __all__ = [ @@ -8,4 +8,5 @@ "REPLAY_HANDLERS", "ReplayContext", "log_replay_case", + "log_replay_point", ] diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py index c0ca9326..2786c02a 100644 --- a/src/fetchgraph/replay/export.py +++ b/src/fetchgraph/replay/export.py @@ -16,6 +16,7 @@ def iter_events(path: Path, *, allow_bad_json: bool = False) -> Iterable[tuple[int, dict]]: + skipped_count = 0 with path.open("r", encoding="utf-8") as handle: for idx, line in enumerate(handle, start=1): line = line.strip() @@ -24,9 +25,15 @@ def iter_events(path: Path, *, allow_bad_json: bool = False) -> Iterable[tuple[i try: yield idx, json.loads(line) except json.JSONDecodeError as exc: + excerpt = line.replace("\t", " ").replace("\n", " ") + if len(excerpt) > 120: + excerpt = f"{excerpt[:120]}…" if allow_bad_json: + skipped_count += 1 continue - raise ValueError(f"Invalid JSON on line {idx} in {path}: {exc.msg}") from exc + raise ValueError(f"Invalid JSON on line {idx}: {excerpt} in {path}") from exc + if skipped_count: + logger.warning("Skipped %d invalid JSON lines in %s", skipped_count, path) def canonical_json(payload: object) -> str: diff --git a/src/fetchgraph/replay/log.py b/src/fetchgraph/replay/log.py index 2cda3f35..3f57b2e9 100644 --- a/src/fetchgraph/replay/log.py +++ b/src/fetchgraph/replay/log.py @@ -1,7 +1,10 @@ from __future__ import annotations +import warnings from typing import Dict, Protocol +TRACE_LIMIT = 20_000 + class EventLoggerLike(Protocol): def emit(self, event: Dict[str, object]) -> None: ... @@ -34,6 +37,14 @@ def log_replay_case( raise ValueError("observed_error.type must be a non-empty string") if not isinstance(observed_error.get("message"), str) or not observed_error.get("message"): raise ValueError("observed_error.message must be a non-empty string") + trace_value = observed_error.get("trace") + if not isinstance(trace_value, str) or not trace_value: + raise ValueError("observed_error.trace must be a non-empty string") + if len(trace_value) > TRACE_LIMIT: + observed_error = { + **observed_error, + "trace": f"{trace_value[:TRACE_LIMIT]}...(truncated {len(trace_value) - TRACE_LIMIT} chars)", + } if meta is not None and not isinstance(meta, dict): raise ValueError("meta must be a dict when provided") if note is not None and not isinstance(note, str): @@ -73,3 +84,13 @@ def log_replay_case( event["diag"] = diag logger.emit(event) + +def log_replay_point(logger: EventLoggerLike, **kwargs: object) -> None: + warnings.warn( + "log_replay_point is deprecated; use log_replay_case", + DeprecationWarning, + stacklevel=2, + ) + if "observed" not in kwargs and "expected" in kwargs: + kwargs = {**kwargs, "observed": kwargs.pop("expected")} + log_replay_case(logger, **kwargs) diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index 0758abfe..71e7c23e 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -112,6 +112,12 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="Overwrite existing expected output", ) green.add_argument("--dry-run", action="store_true", help="Print actions without changing files") + green.add_argument( + "--git", + choices=["auto", "on", "off"], + default="auto", + help="Use git operations when moving/removing fixtures", + ) rm_cmd = sub.add_parser("fixture-rm", help="Remove replay fixtures") rm_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") @@ -130,6 +136,12 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="What to remove", ) rm_cmd.add_argument("--dry-run", action="store_true", help="Print actions without changing files") + rm_cmd.add_argument( + "--git", + choices=["auto", "on", "off"], + default="auto", + help="Use git operations when removing fixtures", + ) fix_cmd = sub.add_parser("fixture-fix", help="Rename fixture stem") fix_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") @@ -137,6 +149,12 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: fix_cmd.add_argument("--name", required=True, help="Old fixture stem") fix_cmd.add_argument("--new-name", required=True, help="New fixture stem") fix_cmd.add_argument("--dry-run", action="store_true", help="Print actions without changing files") + fix_cmd.add_argument( + "--git", + choices=["auto", "on", "off"], + default="auto", + help="Use git operations when moving fixtures", + ) migrate_cmd = sub.add_parser("fixture-migrate", help="Normalize resource layout") migrate_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") @@ -147,6 +165,12 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="Fixture bucket", ) migrate_cmd.add_argument("--dry-run", action="store_true", help="Print actions without changing files") + migrate_cmd.add_argument( + "--git", + choices=["auto", "on", "off"], + default="auto", + help="Use git operations when moving fixtures", + ) return parser.parse_args(argv) @@ -287,6 +311,7 @@ def main(argv: list[str] | None = None) -> int: validate=args.validate, overwrite_expected=args.overwrite_expected, dry_run=args.dry_run, + git_mode=args.git, ) return 0 if args.command == "fixture-rm": @@ -297,6 +322,7 @@ def main(argv: list[str] | None = None) -> int: pattern=args.pattern, scope=args.scope, dry_run=args.dry_run, + git_mode=args.git, ) print(f"Removed {removed} paths") return 0 @@ -307,6 +333,7 @@ def main(argv: list[str] | None = None) -> int: name=args.name, new_name=args.new_name, dry_run=args.dry_run, + git_mode=args.git, ) return 0 if args.command == "fixture-migrate": @@ -314,6 +341,7 @@ def main(argv: list[str] | None = None) -> int: root=args.root, bucket=args.bucket, dry_run=args.dry_run, + git_mode=args.git, ) print(f"Updated {bundles_updated} bundles; moved {files_moved} files") return 0 @@ -351,6 +379,7 @@ def _format_case_run_error(stats, *, case_id: str, tag: str | None) -> str: f"inspected_cases: {stats.inspected_cases}", f"missing_cases: {stats.missing_cases}", f"missing_events: {stats.missing_events}", + f"missed_cases: {stats.missed_cases}", ] if tag: lines.append(f"tag: {tag}") diff --git a/src/fetchgraph/tracer/fetchgraph_tracer.md b/src/fetchgraph/tracer/fetchgraph_tracer.md index 18fdd4a1..895c1799 100644 --- a/src/fetchgraph/tracer/fetchgraph_tracer.md +++ b/src/fetchgraph/tracer/fetchgraph_tracer.md @@ -270,3 +270,5 @@ def test_known_bad(case_path): 2) Extras/Resources логируйте отдельными событиями (`planner_input`, `replay_resource`). 3) Экспортируйте bundle через `export_replay_case_bundle(s)`. 4) В тестах грузите bundle через `load_case_bundle` и запускайте `run_case`. + +Примечание: `log_replay_point` оставлен как deprecated alias, используйте `log_replay_case`. diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py index 8da9bb51..92841e7f 100644 --- a/src/fetchgraph/tracer/fixture_tools.py +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -3,6 +3,7 @@ import filecmp import json import shutil +import subprocess from pathlib import Path from .fixture_layout import FixtureLayout, find_case_bundles @@ -51,6 +52,101 @@ def _atomic_write_json(path: Path, payload: dict) -> None: tmp_path.replace(path) +def _git_available() -> bool: + try: + subprocess.run(["git", "--version"], check=True, capture_output=True, text=True) + except (OSError, subprocess.CalledProcessError): + return False + return True + + +def _is_git_repo(root: Path) -> bool: + try: + subprocess.run( + ["git", "-C", str(root), "rev-parse", "--is-inside-work-tree"], + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.CalledProcessError): + return False + return True + + +def _git_tracked(root: Path, path: Path) -> bool: + try: + subprocess.run( + ["git", "-C", str(root), "ls-files", "--error-unmatch", str(path)], + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.CalledProcessError): + return False + return True + + +def _git_dir_tracked(root: Path, path: Path) -> bool: + try: + result = subprocess.run( + ["git", "-C", str(root), "ls-files", str(path)], + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.CalledProcessError): + return False + return bool(result.stdout.strip()) + + +def _git_mv(root: Path, src: Path, dest: Path) -> None: + subprocess.run(["git", "-C", str(root), "mv", str(src), str(dest)], check=True) + + +def _git_rm(root: Path, path: Path) -> None: + subprocess.run(["git", "-C", str(root), "rm", "-r", "--", str(path)], check=True) + + +class _GitOps: + def __init__(self, root: Path, mode: str) -> None: + self.root = root + self.mode = mode + self.use_git = False + if mode not in {"auto", "on", "off"}: + raise ValueError(f"Unsupported git mode: {mode}") + if mode == "off": + return + git_ok = _git_available() and _is_git_repo(root) + if mode == "on" and not git_ok: + raise ValueError("git mode is on but no git repository is available") + self.use_git = git_ok + + def move(self, src: Path, dest: Path) -> None: + if self.use_git and self._should_git_move(src): + _git_mv(self.root, src, dest) + else: + shutil.move(str(src), str(dest)) + + def remove(self, path: Path) -> None: + if self.use_git and self._should_git_remove(path): + _git_rm(self.root, path) + else: + if path.is_dir(): + shutil.rmtree(path, ignore_errors=True) + elif path.exists(): + path.unlink() + + def _should_git_move(self, src: Path) -> bool: + if src.is_dir(): + return _git_dir_tracked(self.root, src) + return _git_tracked(self.root, src) + + def _should_git_remove(self, path: Path) -> bool: + if path.is_dir(): + return _git_dir_tracked(self.root, path) + return _git_tracked(self.root, path) + + def fixture_green( *, case_path: Path, @@ -58,9 +154,11 @@ def fixture_green( validate: bool = False, overwrite_expected: bool = False, dry_run: bool = False, + git_mode: str = "auto", ) -> None: case_path = case_path.resolve() out_root = out_root.resolve() + git_ops = _GitOps(out_root, git_mode) known_layout = FixtureLayout(out_root, "known_bad") if not case_path.is_relative_to(known_layout.bucket_dir): raise ValueError(f"fixture-green expects a known_bad case path, got: {case_path}") @@ -119,14 +217,14 @@ def fixture_green( ) try: fixed_case_path.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(known_case_path), str(fixed_case_path)) + git_ops.move(known_case_path, fixed_case_path) if resources_from.exists(): resources_to.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(resources_from), str(resources_to)) + git_ops.move(resources_from, resources_to) if known_expected_path.exists(): - known_expected_path.unlink() + git_ops.remove(known_expected_path) tmp_expected_path.replace(fixed_expected_path) except Exception: @@ -161,8 +259,10 @@ def fixture_rm( bucket: str, scope: str, dry_run: bool, + git_mode: str = "auto", ) -> int: root = root.resolve() + git_ops = _GitOps(root, git_mode) bucket_filter: str | None = None if bucket == "all" else bucket matched = find_case_bundles(root=root, bucket=bucket_filter, name=name, pattern=pattern) @@ -190,10 +290,7 @@ def fixture_rm( return len(existing_targets) for target in targets: - if target.is_dir(): - shutil.rmtree(target, ignore_errors=True) - elif target.exists(): - target.unlink() + git_ops.remove(target) return len(existing_targets) @@ -204,8 +301,10 @@ def fixture_fix( new_name: str, bucket: str, dry_run: bool, + git_mode: str = "auto", ) -> None: root = root.resolve() + git_ops = _GitOps(root, git_mode) if name == new_name: raise ValueError("fixture-fix requires a new name different from the old name.") @@ -258,11 +357,11 @@ def fixture_fix( return new_case_path.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(case_path), str(new_case_path)) + git_ops.move(case_path, new_case_path) if expected_path.exists(): - shutil.move(str(expected_path), str(new_expected_path)) + git_ops.move(expected_path, new_expected_path) if resources_dir.exists(): - shutil.move(str(resources_dir), str(new_resources_dir)) + git_ops.move(resources_dir, new_resources_dir) if updated: _atomic_write_json(new_case_path, payload) @@ -276,8 +375,9 @@ def fixture_fix( print(" rewrite: data_ref.file paths updated") -def fixture_migrate(*, root: Path, bucket: str = "all", dry_run: bool) -> tuple[int, int]: +def fixture_migrate(*, root: Path, bucket: str = "all", dry_run: bool, git_mode: str = "auto") -> tuple[int, int]: root = root.resolve() + git_ops = _GitOps(root, git_mode) bucket_filter = None if bucket == "all" else bucket matched = find_case_bundles(root=root, bucket=bucket_filter, name=None, pattern=None) bundles_updated = 0 @@ -323,7 +423,7 @@ def fixture_migrate(*, root: Path, bucket: str = "all", dry_run: bool) -> tuple[ else: dest_path.parent.mkdir(parents=True, exist_ok=True) if not dest_path.exists(): - shutil.move(str(src_path), str(dest_path)) + git_ops.move(src_path, dest_path) files_moved += 1 data_ref["file"] = target_rel.as_posix() updated = True diff --git a/src/fetchgraph/tracer/resolve.py b/src/fetchgraph/tracer/resolve.py index bf0bd094..38f58ca0 100644 --- a/src/fetchgraph/tracer/resolve.py +++ b/src/fetchgraph/tracer/resolve.py @@ -20,6 +20,9 @@ class CaseRunCandidate: case_dir: Path events_path: Path tag: str | None + tag_source: str | None + status: str | None + is_missed: bool run_mtime: float case_mtime: float @@ -30,6 +33,9 @@ class CaseRunInfo: case_dir: Path events: "EventsResolution" tag: str | None + tag_source: str | None + status: str | None + is_missed: bool run_mtime: float case_mtime: float @@ -61,6 +67,7 @@ def resolve_case_events( case_id=case_id, data_dir=data_dir, tag=tag, + pick_run=pick_run, runs_subdir=runs_subdir, ) if not candidates: @@ -100,6 +107,7 @@ class RunScanStats: inspected_cases: int missing_cases: int missing_events: int + missed_cases: int tag_mismatches: int recent: list[CaseRunInfo] @@ -109,6 +117,7 @@ def list_case_runs( case_id: str, data_dir: Path, tag: str | None = None, + pick_run: str = "latest_non_missed", runs_subdir: str = ".runs/runs", ) -> tuple[list[CaseRunCandidate], RunScanStats]: infos, stats = scan_case_runs( @@ -117,7 +126,7 @@ def list_case_runs( tag=tag, runs_subdir=runs_subdir, ) - candidates = _filter_case_run_infos(infos, tag=tag) + candidates = _filter_case_run_infos(infos, tag=tag, pick_run=pick_run) return candidates, stats @@ -137,6 +146,7 @@ def scan_case_runs( inspected_cases = 0 missing_cases = 0 missing_events = 0 + missed_cases = 0 for run_dir in _iter_run_dirs(runs_root): inspected_runs += 1 @@ -151,13 +161,19 @@ def scan_case_runs( if events.events_path is None: missing_events += 1 case_mtime = case_dir.stat().st_mtime - tag_value = _extract_case_tag(case_dir) + status_value, is_missed = _case_status(case_dir) + if is_missed: + missed_cases += 1 + tag_value, tag_source = _extract_case_tag(case_dir) infos.append( CaseRunInfo( run_dir=run_dir, case_dir=case_dir, events=events, tag=tag_value, + tag_source=tag_source, + status=status_value, + is_missed=is_missed, run_mtime=run_mtime, case_mtime=case_mtime, ) @@ -170,17 +186,25 @@ def scan_case_runs( inspected_cases=inspected_cases, missing_cases=missing_cases, missing_events=missing_events, + missed_cases=missed_cases, tag_mismatches=_count_tag_mismatches(infos, tag=tag), recent=infos[:10], ) return infos, stats -def _filter_case_run_infos(infos: list[CaseRunInfo], *, tag: str | None = None) -> list[CaseRunCandidate]: +def _filter_case_run_infos( + infos: list[CaseRunInfo], + *, + tag: str | None = None, + pick_run: str = "latest_non_missed", +) -> list[CaseRunCandidate]: candidates: list[CaseRunCandidate] = [] for info in infos: if info.events.events_path is None: continue + if pick_run == "latest_non_missed" and info.is_missed: + continue if tag and info.tag != tag: continue candidates.append( @@ -189,6 +213,9 @@ def _filter_case_run_infos(infos: list[CaseRunInfo], *, tag: str | None = None) case_dir=info.case_dir, events_path=info.events.events_path, tag=info.tag, + tag_source=info.tag_source, + status=info.status, + is_missed=info.is_missed, run_mtime=info.run_mtime, case_mtime=info.case_mtime, ) @@ -224,6 +251,8 @@ def format_case_runs(candidates: list[CaseRunCandidate], *, limit: int | None = f"{idx}. run_dir={candidate.run_dir} " f"case_dir={candidate.case_dir.name} " f"tag={candidate.tag!r} " + f"status={candidate.status!r} " + f"missed={candidate.is_missed} " f"run_mtime={candidate.run_mtime:.0f} " f"case_mtime={candidate.case_mtime:.0f}" ) @@ -240,6 +269,9 @@ def format_case_run_debug(infos: list[CaseRunInfo], *, limit: int = 10) -> str: f"{idx}. run_dir={info.run_dir} " f"case_dir={info.case_dir.name} " f"tag={info.tag!r} " + f"tag_source={info.tag_source!r} " + f"status={info.status!r} " + f"missed={info.is_missed} " f"events={bool(info.events.events_path)} " f"run_mtime={info.run_mtime:.0f} " f"case_mtime={info.case_mtime:.0f}" @@ -299,24 +331,81 @@ def format_events_search(run_dir: Path, resolution: EventsResolution) -> str: ) -def _extract_case_tag(case_dir: Path) -> str | None: +def _extract_case_tag(case_dir: Path) -> tuple[str | None, str | None]: + run_dir = case_dir.parent.parent + for name in ("run_meta.json", "meta.json"): + tag = _extract_tag_from_json(run_dir / name) + if tag: + return tag, name + tag = _extract_tag_from_json(run_dir / "summary.json") + if tag: + return tag, "summary.json" for name in ("status.json", "result.json"): - path = case_dir / name - if not path.exists(): - continue - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - continue - tag = _extract_tag_value(payload) + tag = _extract_tag_from_json(case_dir / name) if tag: - return tag - for meta_key in ("run_meta", "meta"): - nested = payload.get(meta_key) - if isinstance(nested, dict): - tag = _extract_tag_value(nested) - if tag: - return tag + return tag, name + tag = _tag_from_run_dir_name(run_dir.name) + if tag: + return tag, "run_dir" + events = find_events_file(case_dir) + if events.events_path: + tag = _extract_tag_from_events(events.events_path) + if tag: + return tag, "events" + return None, None + + +def _extract_tag_from_json(path: Path) -> str | None: + if not path.exists(): + return None + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return None + if not isinstance(payload, dict): + return None + tag = _extract_tag_value(payload) + if tag: + return tag + for meta_key in ("run_meta", "meta"): + nested = payload.get(meta_key) + if isinstance(nested, dict): + tag = _extract_tag_value(nested) + if tag: + return tag + return None + + +def _extract_tag_from_events(path: Path, *, max_lines: int = 200) -> str | None: + try: + with path.open("r", encoding="utf-8") as handle: + for idx, line in enumerate(handle, start=1): + if idx > max_lines: + break + line = line.strip() + if not line: + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + tag = _extract_tag_value(payload) + if tag: + return tag + except OSError: + return None + return None + + +def _tag_from_run_dir_name(name: str) -> str | None: + if not name: + return None + lowered = name.lower() + for prefix in ("tag=", "tag-", "bucket=", "bucket-"): + if prefix in lowered: + tail = lowered.split(prefix, 1)[1].strip() + return tail or None return None @@ -333,6 +422,32 @@ def _extract_tag_value(payload: dict) -> str | None: return None +def _case_status(case_dir: Path) -> tuple[str | None, bool]: + for name in ("status.json", "result.json"): + path = case_dir / name + if not path.exists(): + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + continue + status_value = payload.get("status") or payload.get("result") + status_str = str(status_value) if status_value is not None else None + is_missed = _payload_is_missed(payload) + return status_str, is_missed + return None, False + + +def _payload_is_missed(payload: dict) -> bool: + if payload.get("missed") is True: + return True + status = str(payload.get("status") or payload.get("result") or "").lower() + if status in {"missed", "missing"}: + return True + reason = str(payload.get("reason") or "").lower() + return "missed" in reason or "missing" in reason + + def _resolve_rule(*, tag: str | None) -> str: if tag: return f"latest with events filtered by TAG={tag!r}" @@ -355,6 +470,7 @@ def _format_missing_case_runs( f"inspected_cases: {stats.inspected_cases}", f"missing_cases: {stats.missing_cases}", f"missing_events: {stats.missing_events}", + f"missed_cases: {stats.missed_cases}", ] if tag: details.append(f"tag: {tag}") @@ -366,7 +482,9 @@ def _format_missing_case_runs( " " f"case_dir={info.case_dir} " f"tag={info.tag!r} " - f"events={bool(info.events.events_path)}" + f"events={bool(info.events.events_path)} " + f"status={info.status!r} " + f"missed={info.is_missed}" ) details.append("Tip: verify TAG or pass RUN_ID/CASE_DIR/EVENTS.") else: diff --git a/tests/fixtures/replay_cases/fixed/resource_read.v1__sample.case.json b/tests/fixtures/replay_cases/fixed/resource_read.v1__sample.case.json new file mode 100644 index 00000000..064b0042 --- /dev/null +++ b/tests/fixtures/replay_cases/fixed/resource_read.v1__sample.case.json @@ -0,0 +1,23 @@ +{ + "schema": "fetchgraph.tracer.case_bundle", + "v": 1, + "root": { + "type": "replay_case", + "v": 2, + "id": "resource_read.v1", + "input": { + "resource_id": "sample" + }, + "observed": { + "text": "hello" + } + }, + "resources": { + "sample": { + "data_ref": { + "file": "resources/resource_read.v1__sample/sample/sample.txt" + } + } + }, + "extras": {} +} diff --git a/tests/fixtures/replay_cases/fixed/resource_read.v1__sample.expected.json b/tests/fixtures/replay_cases/fixed/resource_read.v1__sample.expected.json new file mode 100644 index 00000000..ca87294c --- /dev/null +++ b/tests/fixtures/replay_cases/fixed/resource_read.v1__sample.expected.json @@ -0,0 +1,3 @@ +{ + "text": "hello\n" +} diff --git a/tests/fixtures/replay_cases/fixed/resources/resource_read.v1__sample/sample/sample.txt b/tests/fixtures/replay_cases/fixed/resources/resource_read.v1__sample/sample/sample.txt new file mode 100644 index 00000000..ce013625 --- /dev/null +++ b/tests/fixtures/replay_cases/fixed/resources/resource_read.v1__sample/sample/sample.txt @@ -0,0 +1 @@ +hello diff --git a/tests/helpers/handlers_resource_read.py b/tests/helpers/handlers_resource_read.py new file mode 100644 index 00000000..e99f06a7 --- /dev/null +++ b/tests/helpers/handlers_resource_read.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from fetchgraph.replay.runtime import REPLAY_HANDLERS, ReplayContext + + +def _resource_read_handler(payload: dict, ctx: ReplayContext) -> dict: + resource_id = payload.get("resource_id") + if not isinstance(resource_id, str) or not resource_id: + raise ValueError("resource_id is required") + resource = ctx.resources.get(resource_id) + if not isinstance(resource, dict): + raise KeyError(f"Missing resource {resource_id!r}") + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + raise ValueError("resource.data_ref must be a dict") + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + raise ValueError("resource.data_ref.file must be a string") + path = ctx.resolve_resource_path(file_name) + text = path.read_text(encoding="utf-8") + return {"text": text} + + +REPLAY_HANDLERS.setdefault("resource_read.v1", _resource_read_handler) diff --git a/tests/test_replay_fixed.py b/tests/test_replay_fixed.py index edd0bc7b..37f7e42b 100644 --- a/tests/test_replay_fixed.py +++ b/tests/test_replay_fixed.py @@ -1,11 +1,13 @@ from __future__ import annotations import json +import shutil from pathlib import Path import pytest import fetchgraph.tracer.handlers # noqa: F401 +import tests.helpers.handlers_resource_read # noqa: F401 from fetchgraph.tracer.runtime import load_case_bundle, run_case from tests.helpers.replay_dx import ( format_json, @@ -88,3 +90,23 @@ def test_replay_case_resources_exist() -> None: if missing: details = "\n".join(f"- {fixture}: {resource}" for fixture, resource in missing) pytest.fail(f"Missing replay resources:\n{details}") + + +def test_resource_read_missing_file(tmp_path: Path) -> None: + bundle_path = FIXED_DIR / "resource_read.v1__sample.case.json" + if not bundle_path.exists(): + pytest.skip("Resource read fixture not available.") + target_bundle = tmp_path / bundle_path.name + resources_dir = FIXED_DIR / "resources" / "resource_read.v1__sample" + target_resources = tmp_path / "resources" / "resource_read.v1__sample" + target_resources.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(bundle_path, target_bundle) + shutil.copytree(resources_dir, target_resources) + + missing_path = target_resources / "sample" / "sample.txt" + if missing_path.exists(): + missing_path.unlink() + + root, ctx = load_case_bundle(target_bundle) + with pytest.raises(FileNotFoundError): + run_case(root, ctx) diff --git a/tests/test_replay_iter_events_bad_json.py b/tests/test_replay_iter_events_bad_json.py new file mode 100644 index 00000000..ae68b0a9 --- /dev/null +++ b/tests/test_replay_iter_events_bad_json.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fetchgraph.replay.export import iter_events + + +def test_iter_events_bad_json_excerpt(tmp_path: Path) -> None: + events_path = tmp_path / "events.jsonl" + events_path.write_text( + '{"type":"ok"}\n' + '{"type":"bad"\n' + '{"type":"ok2"}\n', + encoding="utf-8", + ) + + with pytest.raises(ValueError, match=r"Invalid JSON on line 2: .* in .*events.jsonl"): + list(iter_events(events_path, allow_bad_json=False)) + + +def test_iter_events_allow_bad_json_warns(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + events_path = tmp_path / "events.jsonl" + events_path.write_text( + json.dumps({"type": "ok"}) + "\n" + + "{bad json}\n" + + json.dumps({"type": "ok2"}) + "\n", + encoding="utf-8", + ) + + caplog.set_level("WARNING") + events = list(iter_events(events_path, allow_bad_json=True)) + assert [event["type"] for _, event in events] == ["ok", "ok2"] + assert any("Skipped 1 invalid JSON lines" in record.message for record in caplog.records) diff --git a/tests/test_replay_known_bad_backlog.py b/tests/test_replay_known_bad_backlog.py index cdfa27f6..01b14e55 100644 --- a/tests/test_replay_known_bad_backlog.py +++ b/tests/test_replay_known_bad_backlog.py @@ -65,12 +65,12 @@ def _format_common_block( return lines -def _validate_root_schema(root: dict) -> None: - if root.get("schema") != "fetchgraph.tracer.case_bundle": - raise ValueError(f"Unexpected schema: {root.get('schema')!r}") - if root.get("v") != 1: - raise ValueError(f"Unexpected bundle version: {root.get('v')!r}") - root_case = root.get("root") +def _validate_root_schema(bundle: dict) -> None: + if bundle.get("schema") != "fetchgraph.tracer.case_bundle": + raise ValueError(f"Unexpected schema: {bundle.get('schema')!r}") + if bundle.get("v") != 1: + raise ValueError(f"Unexpected bundle version: {bundle.get('v')!r}") + root_case = bundle.get("root") if not isinstance(root_case, dict): raise ValueError("Missing root replay_case entry.") if root_case.get("type") != "replay_case": @@ -82,10 +82,13 @@ def _validate_root_schema(root: dict) -> None: @pytest.mark.known_bad @pytest.mark.parametrize("bundle_path", _iter_case_params()) def test_known_bad_backlog(bundle_path: Path) -> None: + bundle_payload = json.loads(bundle_path.read_text(encoding="utf-8")) + if not isinstance(bundle_payload, dict): + pytest.fail(f"Unexpected bundle payload type: {type(bundle_payload)}", pytrace=False) + _validate_root_schema(bundle_payload) root, ctx = load_case_bundle(bundle_path) if not isinstance(root, dict): - pytest.fail(f"Unexpected bundle payload type: {type(root)}", pytrace=False) - _validate_root_schema(root) + pytest.fail(f"Unexpected replay_case payload type: {type(root)}", pytrace=False) replay_id = root.get("id") validator = VALIDATORS.get(replay_id) if validator is None: diff --git a/tests/test_replay_log_trace_truncate.py b/tests/test_replay_log_trace_truncate.py new file mode 100644 index 00000000..5f9bd1d5 --- /dev/null +++ b/tests/test_replay_log_trace_truncate.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from fetchgraph.replay.log import TRACE_LIMIT, log_replay_case + + +class _Recorder: + def __init__(self) -> None: + self.events: list[dict] = [] + + def emit(self, event: dict) -> None: + self.events.append(event) + + +def test_log_replay_case_truncates_trace() -> None: + logger = _Recorder() + trace = "A" * (TRACE_LIMIT + 500) + log_replay_case( + logger, + id="plan_normalize.spec_v1", + input={"spec": {"provider": "sql"}}, + observed_error={"type": "Boom", "message": "bad", "trace": trace}, + ) + assert logger.events + observed_error = logger.events[0]["observed_error"] + assert isinstance(observed_error, dict) + truncated = observed_error["trace"] + assert isinstance(truncated, str) + assert len(truncated) < len(trace) + assert "truncated" in truncated diff --git a/tests/test_tracer_auto_resolve.py b/tests/test_tracer_auto_resolve.py index 65639986..69c8fd53 100644 --- a/tests/test_tracer_auto_resolve.py +++ b/tests/test_tracer_auto_resolve.py @@ -61,6 +61,25 @@ def test_resolve_latest_with_events(tmp_path: Path) -> None: assert resolution.case_dir.parent.parent == run_old +def test_resolve_latest_non_missed(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + run_old = runs_root / "run_old" + run_old.mkdir() + _make_case_dir(run_old, "case_4", "aaa", status="ok") + _set_mtime(run_old, 100) + + run_new = runs_root / "run_new" + run_new.mkdir() + _make_case_dir(run_new, "case_4", "bbb", status="missed") + _set_mtime(run_new, 200) + + resolution = resolve_case_events(case_id="case_4", data_dir=data_dir) + assert resolution.run_dir == run_old + + def test_resolve_with_tag(tmp_path: Path) -> None: data_dir = tmp_path / "data" runs_root = data_dir / ".runs" / "runs" @@ -80,6 +99,56 @@ def test_resolve_with_tag(tmp_path: Path) -> None: assert resolution.run_dir == run_a +def test_resolve_tag_from_run_meta(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + run_meta = runs_root / "run_meta" + run_meta.mkdir() + _write_json(run_meta / "run_meta.json", {"tag": "meta_tag"}) + _make_case_dir(run_meta, "case_5", "aaa", status="ok") + _set_mtime(run_meta, 100) + + run_other = runs_root / "run_other" + run_other.mkdir() + _make_case_dir(run_other, "case_5", "bbb", status="ok") + _set_mtime(run_other, 200) + + resolution = resolve_case_events(case_id="case_5", data_dir=data_dir, tag="meta_tag") + assert resolution.run_dir == run_meta + + +def test_resolve_tag_from_run_dir(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + run_dir = runs_root / "tag-known_bad" + run_dir.mkdir() + _make_case_dir(run_dir, "case_6", "aaa", status="ok") + _set_mtime(run_dir, 100) + + resolution = resolve_case_events(case_id="case_6", data_dir=data_dir, tag="known_bad") + assert resolution.run_dir == run_dir + + +def test_resolve_tag_from_events(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + run_dir = runs_root / "run_events" + run_dir.mkdir() + case_dir = _make_case_dir(run_dir, "case_7", "aaa", status="ok") + events_path = case_dir / "events.jsonl" + events_path.write_text('{"tag": "from_events"}\n', encoding="utf-8") + _set_mtime(run_dir, 100) + + resolution = resolve_case_events(case_id="case_7", data_dir=data_dir, tag="from_events") + assert resolution.run_dir == run_dir + + def test_resolve_not_found(tmp_path: Path) -> None: data_dir = tmp_path / "data" runs_root = data_dir / ".runs" / "runs" From 410b84e3f0969454aa88460be22c96a34fe9590e Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 20:45:42 +0300 Subject: [PATCH 59/79] Fix replay log and known_bad typing --- src/fetchgraph/replay/log.py | 24 ++++++++++++++++++++---- tests/test_replay_known_bad_backlog.py | 2 ++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/fetchgraph/replay/log.py b/src/fetchgraph/replay/log.py index 3f57b2e9..a169cac2 100644 --- a/src/fetchgraph/replay/log.py +++ b/src/fetchgraph/replay/log.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import Dict, Protocol +from typing import Any, Dict, Protocol, TypedDict, cast TRACE_LIMIT = 20_000 @@ -91,6 +91,22 @@ def log_replay_point(logger: EventLoggerLike, **kwargs: object) -> None: DeprecationWarning, stacklevel=2, ) - if "observed" not in kwargs and "expected" in kwargs: - kwargs = {**kwargs, "observed": kwargs.pop("expected")} - log_replay_case(logger, **kwargs) + payload = dict(kwargs) + if "observed" not in payload and "expected" in payload: + payload["observed"] = payload.pop("expected") + + class _ReplayPointArgs(TypedDict, total=False): + id: str + input: dict + meta: dict | None + observed: dict | None + observed_error: dict | None + requires: list[dict] | None + note: str | None + diag: dict | None + + typed_payload: _ReplayPointArgs = {} + for key in _ReplayPointArgs.__annotations__: + if key in payload: + typed_payload[key] = cast(Any, payload[key]) + log_replay_case(logger, **typed_payload) diff --git a/tests/test_replay_known_bad_backlog.py b/tests/test_replay_known_bad_backlog.py index 01b14e55..111140d2 100644 --- a/tests/test_replay_known_bad_backlog.py +++ b/tests/test_replay_known_bad_backlog.py @@ -90,6 +90,8 @@ def test_known_bad_backlog(bundle_path: Path) -> None: if not isinstance(root, dict): pytest.fail(f"Unexpected replay_case payload type: {type(root)}", pytrace=False) replay_id = root.get("id") + if not isinstance(replay_id, str): + pytest.fail("Replay id missing or invalid in bundle root.", pytrace=False) validator = VALIDATORS.get(replay_id) if validator is None: pytest.fail( From fff42793ee0cb9ac40939fdebb61a74f2a2c5a87 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 21:02:36 +0300 Subject: [PATCH 60/79] Tighten deprecated replay log typing --- src/fetchgraph/replay/log.py | 38 ++++++++++++++++---------- tests/test_replay_known_bad_backlog.py | 5 ++-- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/fetchgraph/replay/log.py b/src/fetchgraph/replay/log.py index a169cac2..365de977 100644 --- a/src/fetchgraph/replay/log.py +++ b/src/fetchgraph/replay/log.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import Any, Dict, Protocol, TypedDict, cast +from typing import Any, Dict, Protocol, cast TRACE_LIMIT = 20_000 @@ -95,18 +95,26 @@ def log_replay_point(logger: EventLoggerLike, **kwargs: object) -> None: if "observed" not in payload and "expected" in payload: payload["observed"] = payload.pop("expected") - class _ReplayPointArgs(TypedDict, total=False): - id: str - input: dict - meta: dict | None - observed: dict | None - observed_error: dict | None - requires: list[dict] | None - note: str | None - diag: dict | None + id_val = payload.get("id") + input_val = payload.get("input") + if not isinstance(id_val, str) or not isinstance(input_val, dict): + raise ValueError("log_replay_point requires id (str) and input (dict)") - typed_payload: _ReplayPointArgs = {} - for key in _ReplayPointArgs.__annotations__: - if key in payload: - typed_payload[key] = cast(Any, payload[key]) - log_replay_case(logger, **typed_payload) + meta_val = payload.get("meta") + observed_val = payload.get("observed") + observed_error_val = payload.get("observed_error") + requires_val = payload.get("requires") + note_val = payload.get("note") + diag_val = payload.get("diag") + + log_replay_case( + logger, + id=cast(str, id_val), + input=cast(dict, input_val), + meta=cast(dict | None, meta_val), + observed=cast(dict | None, observed_val), + observed_error=cast(dict | None, observed_error_val), + requires=cast(list[dict] | None, requires_val), + note=cast(str | None, note_val), + diag=cast(dict | None, diag_val), + ) diff --git a/tests/test_replay_known_bad_backlog.py b/tests/test_replay_known_bad_backlog.py index 111140d2..f8500bbe 100644 --- a/tests/test_replay_known_bad_backlog.py +++ b/tests/test_replay_known_bad_backlog.py @@ -90,9 +90,10 @@ def test_known_bad_backlog(bundle_path: Path) -> None: if not isinstance(root, dict): pytest.fail(f"Unexpected replay_case payload type: {type(root)}", pytrace=False) replay_id = root.get("id") - if not isinstance(replay_id, str): + if not isinstance(replay_id, str) or not replay_id: pytest.fail("Replay id missing or invalid in bundle root.", pytrace=False) - validator = VALIDATORS.get(replay_id) + replay_id_str: str = replay_id + validator = VALIDATORS.get(replay_id_str) if validator is None: pytest.fail( f"No validator registered for replay id={replay_id!r}. Add it to VALIDATORS.", From 35d5af120511266be54ea17ce5bbef32eb841295 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 21:56:49 +0300 Subject: [PATCH 61/79] Improve tracer export filters and fixture selection --- Makefile | 36 ++++-- src/fetchgraph/replay/export.py | 21 ++++ src/fetchgraph/tracer/cli.py | 77 ++++++++++++- src/fetchgraph/tracer/fixture_tools.py | 123 ++++++++++++++++++++- tests/test_make_tracer_export.py | 21 ++++ tests/test_tracer_fixture_green_resolve.py | 43 +++++++ 6 files changed, 308 insertions(+), 13 deletions(-) create mode 100644 tests/test_make_tracer_export.py create mode 100644 tests/test_tracer_fixture_green_resolve.py diff --git a/Makefile b/Makefile index 4ec1a433..0c177e69 100644 --- a/Makefile +++ b/Makefile @@ -51,8 +51,8 @@ CASE ?= NAME ?= NEW_NAME ?= PATTERN ?= -SPEC_IDX ?= 0 -PROVIDER ?= demo_qa +SPEC_IDX ?= +PROVIDER ?= BUCKET ?= known_bad REPLAY_ID ?= EVENTS ?= @@ -71,6 +71,9 @@ CHANGES ?= 10 NEW_TAG ?= TAGS_FORMAT ?= table TAGS_COLOR ?= auto +SELECT ?= latest +SELECT_INDEX ?= +REQUIRE_UNIQUE ?= 0 ONLY_FAILED_FROM ?= ONLY_MISSED_FROM ?= @@ -122,7 +125,7 @@ LIMIT_FLAG := $(if $(strip $(LIMIT)),--limit $(LIMIT),) batch-missed batch-missed-from batch-failed-tag batch-missed-tag \ batch-fail-fast batch-max-fails \ stats history-case report-tag report-tag-changes tags tag-rm case-run case-open tracer-export tracer-ls known-bad known-bad-one \ - fixture-green fixture-rm fixture-fix fixture-migrate \ + fixture-green fixture-ls fixture-rm fixture-fix fixture-migrate \ compare compare-tag # ============================================================================== @@ -171,12 +174,14 @@ help: @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" @echo " make tracer-export REPLAY_ID=... CASE=... [EVENTS=...] [RUN_ID=...] [CASE_DIR=...] [DATA=...] [PROVIDER=...] [BUCKET=...] [SPEC_IDX=...] [OVERWRITE=1] [ALLOW_BAD_JSON=1]" + @echo " PROVIDER фильтрует replay_case.meta.provider (обычно spec.provider: sql, relational, ...)" @echo " make tracer-ls CASE=... [DATA=...] [TAG=...] [RUN_ID=...] [CASE_DIR=...]" @echo " make known-bad - запустить backlog-suite для known_bad (ожидаемо красный)" @echo " make known-bad-one NAME=fixture_stem - запустить один known_bad кейс" @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources///..." @echo " make fixture-green CASE=path/to/case.case.json [TRACER_ROOT=...] [VALIDATE=1] [OVERWRITE_EXPECTED=1] [DRY=1]" + @echo " make fixture-ls CASE=agg_003 [TRACER_ROOT=...] [BUCKET=known_bad]" @echo " make fixture-rm [BUCKET=fixed|known_bad|all] [NAME=...] [PATTERN=...] [SCOPE=cases|resources|both] [DRY=1]" @echo " make fixture-fix BUCKET=... NAME=... NEW_NAME=... [DRY=1]" @echo " make fixture-migrate [BUCKET=fixed|known_bad|all] [DRY=1]" @@ -388,11 +393,11 @@ tracer-export: @case "$(BUCKET)" in fixed|known_bad) ;; *) echo "BUCKET должен быть fixed или known_bad для tracer-export" && exit 2 ;; esac @fetchgraph-tracer export-case-bundle \ --id "$(REPLAY_ID)" \ - --spec-idx "$(SPEC_IDX)" \ --out "$(TRACER_OUT_DIR)" \ --case "$(CASE)" \ --data "$(REPLAY_IDATA)" \ - --provider "$(PROVIDER)" \ + $(if $(strip $(SPEC_IDX)),--spec-idx "$(SPEC_IDX)",) \ + $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ $(if $(RUN_ID),--run-id "$(RUN_ID)",) \ $(if $(CASE_DIR),--case-dir "$(CASE_DIR)",) \ $(if $(RUN_DIR),--run-dir "$(RUN_DIR)",) \ @@ -422,16 +427,27 @@ known-bad-one: @pytest -m known_bad -k "$(NAME)" -vv fixture-green: - @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture-green CASE=tests/fixtures/replay_cases/known_bad/fixture.case.json (или CASE=fixture_stem)" && exit 1) - @case_path="$(CASE)"; \ - if [ "$${case_path##*/}" = "$$case_path" ] && [ "$$case_path" != *".case.json" ]; then \ - case_path="$(TRACER_ROOT)/known_bad/$$case_path.case.json"; \ + @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture-green CASE=agg_003 или CASE=path/to/fixture.case.json" && exit 1) + @case_value="$(CASE)"; \ + if [ -f "$$case_value" ]; then \ + case_args="--case $$case_value"; \ + elif [[ "$$case_value" == *".case.json" || "$$case_value" == *"/"* ]]; then \ + case_args="--case $(TRACER_ROOT)/known_bad/$$case_value"; \ + else \ + case_args="--case-id $$case_value"; \ fi; \ - $(PYTHON) -m fetchgraph.tracer.cli fixture-green --case "$$case_path" --root "$(TRACER_ROOT)" \ + $(PYTHON) -m fetchgraph.tracer.cli fixture-green $$case_args --root "$(TRACER_ROOT)" \ + $(if $(strip $(SELECT)),--select "$(SELECT)",) \ + $(if $(strip $(SELECT_INDEX)),--select-index "$(SELECT_INDEX)",) \ + $(if $(filter 1 true yes on,$(REQUIRE_UNIQUE)),--require-unique,) \ $(if $(filter 1 true yes on,$(VALIDATE)),--validate,) \ $(if $(filter 1 true yes on,$(OVERWRITE_EXPECTED)),--overwrite-expected,) \ $(if $(filter 1 true yes on,$(DRY)),--dry-run,) +fixture-ls: + @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture-ls CASE=agg_003" && exit 1) + @$(PYTHON) -m fetchgraph.tracer.cli fixture-ls --root "$(TRACER_ROOT)" --bucket "$(BUCKET)" --case-id "$(CASE)" + fixture-rm: @case "$(BUCKET)" in fixed|known_bad|all) ;; *) echo "BUCKET должен быть fixed, known_bad или all для fixture-rm" && exit 1 ;; esac @$(PYTHON) -m fetchgraph.tracer.cli fixture-rm --root "$(TRACER_ROOT)" --bucket "$(BUCKET)" \ diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py index 2786c02a..1217ea39 100644 --- a/src/fetchgraph/replay/export.py +++ b/src/fetchgraph/replay/export.py @@ -446,6 +446,27 @@ def export_replay_case_bundle( if provider is not None: details.append(f"provider={provider!r}") detail_str = f" (filters: {', '.join(details)})" if details else "" + unfiltered = _select_replay_cases( + events_path, + replay_id=replay_id, + spec_idx=None, + provider=None, + allow_bad_json=allow_bad_json, + ) + if unfiltered and details: + providers = sorted( + {str(sel.event.get("meta", {}).get("provider")) for sel in unfiltered if sel.event.get("meta")} + ) + spec_idxs = sorted( + {str(sel.event.get("meta", {}).get("spec_idx")) for sel in unfiltered if sel.event.get("meta")} + ) + hint_lines = [ + f"No replay_case id={replay_id!r} matched filters in {events_path}{detail_str}.", + f"Available providers: {providers}", + f"Available spec_idx: {spec_idxs}", + "Tip: rerun without --provider/--spec-idx or choose matching values.", + ] + raise LookupError("\n".join(hint_lines)) raise LookupError(f"No replay_case id={replay_id!r} found in {events_path}{detail_str}") selection, selection_mode = _select_replay_case( selections, diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index 71e7c23e..edd5c770 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -103,7 +103,9 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: export.add_argument("--require-unique", action="store_true", help="Error if multiple matches exist") green = sub.add_parser("fixture-green", help="Promote known_bad case to fixed") - green.add_argument("--case", type=Path, required=True, help="Path to known_bad case bundle") + green.add_argument("--case", type=Path, help="Path to known_bad case bundle") + green.add_argument("--case-id", help="Case id to select fixture from known_bad") + green.add_argument("--name", help="Fixture stem name to select") green.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") green.add_argument("--validate", action="store_true", help="Validate replay output") green.add_argument( @@ -118,6 +120,14 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: default="auto", help="Use git operations when moving/removing fixtures", ) + green.add_argument( + "--select", + choices=["latest", "first", "last"], + default="latest", + help="Selection policy when multiple fixtures match", + ) + green.add_argument("--select-index", type=int, default=None, help="Select fixture index (1-based)") + green.add_argument("--require-unique", action="store_true", help="Error if multiple fixtures match") rm_cmd = sub.add_parser("fixture-rm", help="Remove replay fixtures") rm_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") @@ -172,6 +182,17 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="Use git operations when moving fixtures", ) + ls_cmd = sub.add_parser("fixture-ls", help="List fixture candidates") + ls_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") + ls_cmd.add_argument( + "--bucket", + choices=["fixed", "known_bad", "all"], + default="known_bad", + help="Fixture bucket", + ) + ls_cmd.add_argument("--case-id", help="Case id to filter") + ls_cmd.add_argument("--pattern", help="Glob pattern for fixtures") + return parser.parse_args(argv) @@ -263,6 +284,34 @@ def main(argv: list[str] | None = None) -> int: allow_bad_json=args.allow_bad_json, ) if not selections: + unfiltered = find_replay_case_matches( + events_path, + replay_id=args.id, + spec_idx=None, + provider=None, + allow_bad_json=args.allow_bad_json, + ) + if unfiltered and (args.spec_idx is not None or args.provider is not None): + providers = sorted( + { + str(sel.event.get("meta", {}).get("provider")) + for sel in unfiltered + if sel.event.get("meta") + } + ) + spec_idxs = sorted( + { + str(sel.event.get("meta", {}).get("spec_idx")) + for sel in unfiltered + if sel.event.get("meta") + } + ) + raise LookupError( + "No replay_case matched filters.\n" + f"Available providers: {providers}\n" + f"Available spec_idx: {spec_idxs}\n" + "Tip: rerun without --provider/--spec-idx or choose matching values." + ) raise LookupError(f"No replay_case id={args.id!r} found in {events_path}") print(format_replay_case_matches(selections, limit=20)) return 0 @@ -307,11 +356,16 @@ def main(argv: list[str] | None = None) -> int: if args.command == "fixture-green": fixture_green( case_path=args.case, + case_id=args.case_id, + name=args.name, out_root=args.root, validate=args.validate, overwrite_expected=args.overwrite_expected, dry_run=args.dry_run, git_mode=args.git, + select=args.select, + select_index=args.select_index, + require_unique=args.require_unique, ) return 0 if args.command == "fixture-rm": @@ -345,6 +399,27 @@ def main(argv: list[str] | None = None) -> int: ) print(f"Updated {bundles_updated} bundles; moved {files_moved} files") return 0 + if args.command == "fixture-ls": + from fetchgraph.tracer.fixture_tools import fixture_ls + + fixtures = fixture_ls( + root=args.root, + bucket=args.bucket, + case_id=args.case_id, + pattern=args.pattern, + ) + if not fixtures: + print("No fixtures matched.") + return 0 + for idx, candidate in enumerate(fixtures, start=1): + source = candidate.source or {} + print( + f"{idx}. stem={candidate.stem} " + f"path={candidate.path} " + f"run_id={source.get('run_id')} " + f"timestamp={source.get('timestamp')}" + ) + return 0 except (ValueError, FileNotFoundError, LookupError, KeyError) as exc: print(str(exc), file=sys.stderr) return 2 diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py index 92841e7f..0c9f0288 100644 --- a/src/fetchgraph/tracer/fixture_tools.py +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -4,12 +4,20 @@ import json import shutil import subprocess +from dataclasses import dataclass from pathlib import Path from .fixture_layout import FixtureLayout, find_case_bundles from .runtime import load_case_bundle, run_case +@dataclass(frozen=True) +class FixtureCandidate: + path: Path + stem: str + source: dict + + def load_bundle_json(path: Path) -> dict: payload = json.loads(path.read_text(encoding="utf-8")) if payload.get("schema") != "fetchgraph.tracer.case_bundle" or payload.get("v") != 1: @@ -52,6 +60,64 @@ def _atomic_write_json(path: Path, payload: dict) -> None: tmp_path.replace(path) +def resolve_fixture_candidates( + *, + root: Path, + bucket: str, + case_id: str | None, + name: str | None, +) -> list[FixtureCandidate]: + if case_id and name: + raise ValueError("Only one of case_id or name can be used.") + candidates = find_case_bundles(root=root, bucket=bucket, name=name, pattern=None) + results: list[FixtureCandidate] = [] + for path in candidates: + payload = load_bundle_json(path) + stem = path.name.replace(".case.json", "") + source = payload.get("source") if isinstance(payload.get("source"), dict) else {} + if case_id: + source_case = source.get("case") or source.get("case_id") + if source_case == case_id or stem.startswith(case_id): + results.append(FixtureCandidate(path=path, stem=stem, source=source)) + else: + results.append(FixtureCandidate(path=path, stem=stem, source=source)) + return results + + +def select_fixture_candidate( + candidates: list[FixtureCandidate], + *, + select: str = "latest", + select_index: int | None = None, + require_unique: bool = False, +) -> FixtureCandidate: + if not candidates: + raise FileNotFoundError("No fixtures matched the selection.") + if require_unique and len(candidates) > 1: + raise FileExistsError("Multiple fixtures matched selection; use --select-index or --select.") + if select_index is not None: + if select_index < 1 or select_index > len(candidates): + raise ValueError(f"select_index must be between 1 and {len(candidates)}") + return candidates[select_index - 1] + + def _sort_key(candidate: FixtureCandidate) -> tuple[str, float]: + timestamp = candidate.source.get("timestamp") + run_id = candidate.source.get("run_id") + time_key = f"{timestamp or ''}{run_id or ''}" + try: + mtime = candidate.path.stat().st_mtime + except OSError: + mtime = 0.0 + return (time_key, mtime) + + ordered = sorted(candidates, key=_sort_key) + if select in {"latest", "last"}: + return ordered[-1] + if select == "first": + return ordered[0] + raise ValueError(f"Unsupported select policy: {select}") + + def _git_available() -> bool: try: subprocess.run(["git", "--version"], check=True, capture_output=True, text=True) @@ -149,17 +215,48 @@ def _should_git_remove(self, path: Path) -> bool: def fixture_green( *, - case_path: Path, + case_path: Path | None = None, + case_id: str | None = None, + name: str | None = None, out_root: Path, validate: bool = False, overwrite_expected: bool = False, dry_run: bool = False, git_mode: str = "auto", + select: str = "latest", + select_index: int | None = None, + require_unique: bool = False, ) -> None: - case_path = case_path.resolve() + if case_path is None and not case_id and not name: + raise ValueError("fixture-green requires --case, --case-id, or --name.") + if case_path is not None and (case_id or name): + raise ValueError("fixture-green accepts only one of --case, --case-id, or --name.") out_root = out_root.resolve() git_ops = _GitOps(out_root, git_mode) known_layout = FixtureLayout(out_root, "known_bad") + if case_path is None: + candidates = resolve_fixture_candidates( + root=out_root, + bucket="known_bad", + case_id=case_id, + name=name, + ) + if not candidates: + selector = case_id or name + raise FileNotFoundError(f"No fixtures found for selector={selector!r} in known_bad") + selected = select_fixture_candidate( + candidates, + select=select, + select_index=select_index, + require_unique=require_unique, + ) + case_path = selected.path + if len(candidates) > 1 and not require_unique: + print("fixture-green: multiple candidates found, selected:") + for idx, candidate in enumerate(candidates, start=1): + marker = " (selected)" if candidate.path == selected.path else "" + print(f" {idx}. {candidate.path}{marker}") + case_path = case_path.resolve() if not case_path.is_relative_to(known_layout.bucket_dir): raise ValueError(f"fixture-green expects a known_bad case path, got: {case_path}") if not case_path.name.endswith(".case.json"): @@ -434,3 +531,25 @@ def fixture_migrate(*, root: Path, bucket: str = "all", dry_run: bool, git_mode: else: _atomic_write_json(case_path, payload) return bundles_updated, files_moved + + +def fixture_ls( + *, + root: Path, + bucket: str, + case_id: str | None = None, + pattern: str | None = None, +) -> list[FixtureCandidate]: + bucket_filter = None if bucket == "all" else bucket + candidates = find_case_bundles(root=root, bucket=bucket_filter, name=None, pattern=pattern) + results: list[FixtureCandidate] = [] + for path in candidates: + payload = load_bundle_json(path) + stem = path.name.replace(".case.json", "") + source = payload.get("source") if isinstance(payload.get("source"), dict) else {} + if case_id: + source_case = source.get("case") or source.get("case_id") + if source_case != case_id and not stem.startswith(case_id): + continue + results.append(FixtureCandidate(path=path, stem=stem, source=source)) + return results diff --git a/tests/test_make_tracer_export.py b/tests/test_make_tracer_export.py new file mode 100644 index 00000000..c432b475 --- /dev/null +++ b/tests/test_make_tracer_export.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import subprocess + + +def test_make_tracer_export_no_default_filters() -> None: + result = subprocess.run( + [ + "make", + "-n", + "tracer-export", + "REPLAY_ID=plan_normalize.spec_v1", + "CASE=agg_003", + ], + capture_output=True, + text=True, + check=True, + ) + output = result.stdout + result.stderr + assert "--provider" not in output + assert "--spec-idx" not in output diff --git a/tests/test_tracer_fixture_green_resolve.py b/tests/test_tracer_fixture_green_resolve.py new file mode 100644 index 00000000..4b5a7985 --- /dev/null +++ b/tests/test_tracer_fixture_green_resolve.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from fetchgraph.tracer.fixture_tools import resolve_fixture_candidates, select_fixture_candidate + + +def _write_bundle(path: Path, *, case_id: str, timestamp: str) -> None: + payload = { + "schema": "fetchgraph.tracer.case_bundle", + "v": 1, + "root": {"type": "replay_case", "v": 2, "id": "plan_normalize.spec_v1", "input": {}}, + "source": {"case_id": case_id, "timestamp": timestamp}, + "resources": {}, + "extras": {}, + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + + +def test_resolve_fixture_by_case_id(tmp_path: Path) -> None: + root = tmp_path / "fixtures" + known_bad = root / "known_bad" + case_old = known_bad / "agg_003__a.case.json" + case_new = known_bad / "agg_003__b.case.json" + _write_bundle(case_old, case_id="agg_003", timestamp="2024-01-01T00:00:00Z") + _write_bundle(case_new, case_id="agg_003", timestamp="2024-01-02T00:00:00Z") + os.utime(case_old, (1, 1)) + os.utime(case_new, (2, 2)) + + candidates = resolve_fixture_candidates(root=root, bucket="known_bad", case_id="agg_003", name=None) + selected = select_fixture_candidate(candidates, select="latest") + assert selected.path == case_new + + selected_first = select_fixture_candidate(candidates, select_index=1) + assert selected_first.path == candidates[0].path + + with pytest.raises(FileExistsError): + select_fixture_candidate(candidates, require_unique=True) From 4052bba276eb5b76d3ff3b2170a39d2d05753750 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 22:07:52 +0300 Subject: [PATCH 62/79] Fix fixture tools source typing --- src/fetchgraph/tracer/fixture_tools.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py index 0c9f0288..4d9335ad 100644 --- a/src/fetchgraph/tracer/fixture_tools.py +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -74,13 +74,14 @@ def resolve_fixture_candidates( for path in candidates: payload = load_bundle_json(path) stem = path.name.replace(".case.json", "") - source = payload.get("source") if isinstance(payload.get("source"), dict) else {} + source = payload.get("source") + source_dict = source if isinstance(source, dict) else {} if case_id: - source_case = source.get("case") or source.get("case_id") + source_case = source_dict.get("case") or source_dict.get("case_id") if source_case == case_id or stem.startswith(case_id): - results.append(FixtureCandidate(path=path, stem=stem, source=source)) + results.append(FixtureCandidate(path=path, stem=stem, source=source_dict)) else: - results.append(FixtureCandidate(path=path, stem=stem, source=source)) + results.append(FixtureCandidate(path=path, stem=stem, source=source_dict)) return results @@ -546,10 +547,11 @@ def fixture_ls( for path in candidates: payload = load_bundle_json(path) stem = path.name.replace(".case.json", "") - source = payload.get("source") if isinstance(payload.get("source"), dict) else {} + source = payload.get("source") + source_dict = source if isinstance(source, dict) else {} if case_id: - source_case = source.get("case") or source.get("case_id") + source_case = source_dict.get("case") or source_dict.get("case_id") if source_case != case_id and not stem.startswith(case_id): continue - results.append(FixtureCandidate(path=path, stem=stem, source=source)) + results.append(FixtureCandidate(path=path, stem=stem, source=source_dict)) return results From c3daa91a38fc9d70de8cb9e8343d810c25824d24 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 23:01:02 +0300 Subject: [PATCH 63/79] Extend fixture selectors and add demote --- Makefile | 56 +++++- src/fetchgraph/tracer/cli.py | 80 ++++++++ src/fetchgraph/tracer/fixture_tools.py | 187 ++++++++++++++++-- tests/test_tracer_fixture_demote.py | 55 ++++++ ..._tracer_fixture_selector_for_rm_migrate.py | 66 +++++++ 5 files changed, 430 insertions(+), 14 deletions(-) create mode 100644 tests/test_tracer_fixture_demote.py create mode 100644 tests/test_tracer_fixture_selector_for_rm_migrate.py diff --git a/Makefile b/Makefile index 0c177e69..362ca4c7 100644 --- a/Makefile +++ b/Makefile @@ -125,7 +125,7 @@ LIMIT_FLAG := $(if $(strip $(LIMIT)),--limit $(LIMIT),) batch-missed batch-missed-from batch-failed-tag batch-missed-tag \ batch-fail-fast batch-max-fails \ stats history-case report-tag report-tag-changes tags tag-rm case-run case-open tracer-export tracer-ls known-bad known-bad-one \ - fixture-green fixture-ls fixture-rm fixture-fix fixture-migrate \ + fixture-green fixture-demote fixture-ls fixture-rm fixture-fix fixture-migrate \ compare compare-tag # ============================================================================== @@ -182,6 +182,9 @@ help: @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources///..." @echo " make fixture-green CASE=path/to/case.case.json [TRACER_ROOT=...] [VALIDATE=1] [OVERWRITE_EXPECTED=1] [DRY=1]" @echo " make fixture-ls CASE=agg_003 [TRACER_ROOT=...] [BUCKET=known_bad]" + @echo " make fixture-rm CASE=agg_003 [SELECT=latest|first|last] [SELECT_INDEX=N] [REQUIRE_UNIQUE=1] [ALL=1]" + @echo " make fixture-migrate CASE=agg_003 [SELECT=latest|first|last] [SELECT_INDEX=N] [REQUIRE_UNIQUE=1] [ALL=1]" + @echo " make fixture-demote CASE=agg_003 [SELECT=latest|first|last] [SELECT_INDEX=N] [REQUIRE_UNIQUE=1] [ALL=1]" @echo " make fixture-rm [BUCKET=fixed|known_bad|all] [NAME=...] [PATTERN=...] [SCOPE=cases|resources|both] [DRY=1]" @echo " make fixture-fix BUCKET=... NAME=... NEW_NAME=... [DRY=1]" @echo " make fixture-migrate [BUCKET=fixed|known_bad|all] [DRY=1]" @@ -450,10 +453,24 @@ fixture-ls: fixture-rm: @case "$(BUCKET)" in fixed|known_bad|all) ;; *) echo "BUCKET должен быть fixed, known_bad или all для fixture-rm" && exit 1 ;; esac - @$(PYTHON) -m fetchgraph.tracer.cli fixture-rm --root "$(TRACER_ROOT)" --bucket "$(BUCKET)" \ + @case_value="$(CASE)"; \ + if [ -n "$$case_value" ]; then \ + if [ -f "$$case_value" ]; then \ + case_args="--case $$case_value"; \ + elif [[ "$$case_value" == *".case.json" || "$$case_value" == *"/"* ]]; then \ + case_args="--case $(TRACER_ROOT)/$(BUCKET)/$$case_value"; \ + else \ + case_args="--case-id $$case_value"; \ + fi; \ + fi; \ + $(PYTHON) -m fetchgraph.tracer.cli fixture-rm $$case_args --root "$(TRACER_ROOT)" --bucket "$(BUCKET)" \ $(if $(strip $(NAME)),--name "$(NAME)",) \ $(if $(strip $(PATTERN)),--pattern "$(PATTERN)",) \ $(if $(strip $(SCOPE)),--scope "$(SCOPE)",) \ + $(if $(strip $(SELECT)),--select "$(SELECT)",) \ + $(if $(strip $(SELECT_INDEX)),--select-index "$(SELECT_INDEX)",) \ + $(if $(filter 1 true yes on,$(REQUIRE_UNIQUE)),--require-unique,) \ + $(if $(filter 1 true yes on,$(ALL)),--all,) \ $(if $(filter 1 true yes on,$(DRY)),--dry-run,) fixture-fix: @@ -464,7 +481,40 @@ fixture-fix: $(if $(filter 1 true yes on,$(DRY)),--dry-run,) fixture-migrate: - @$(PYTHON) -m fetchgraph.tracer.cli fixture-migrate --root "$(TRACER_ROOT)" --bucket "$(BUCKET)" \ + @case_value="$(CASE)"; \ + if [ -n "$$case_value" ]; then \ + if [ -f "$$case_value" ]; then \ + case_args="--case $$case_value"; \ + elif [[ "$$case_value" == *".case.json" || "$$case_value" == *"/"* ]]; then \ + case_args="--case $(TRACER_ROOT)/$(BUCKET)/$$case_value"; \ + else \ + case_args="--case-id $$case_value"; \ + fi; \ + fi; \ + $(PYTHON) -m fetchgraph.tracer.cli fixture-migrate $$case_args --root "$(TRACER_ROOT)" --bucket "$(BUCKET)" \ + $(if $(strip $(NAME)),--name "$(NAME)",) \ + $(if $(strip $(SELECT)),--select "$(SELECT)",) \ + $(if $(strip $(SELECT_INDEX)),--select-index "$(SELECT_INDEX)",) \ + $(if $(filter 1 true yes on,$(REQUIRE_UNIQUE)),--require-unique,) \ + $(if $(filter 1 true yes on,$(ALL)),--all,) \ + $(if $(filter 1 true yes on,$(DRY)),--dry-run,) + +fixture-demote: + @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture-demote CASE=agg_003" && exit 1) + @case_value="$(CASE)"; \ + if [ -f "$$case_value" ]; then \ + case_args="--case $$case_value"; \ + elif [[ "$$case_value" == *".case.json" || "$$case_value" == *"/"* ]]; then \ + case_args="--case $(TRACER_ROOT)/fixed/$$case_value"; \ + else \ + case_args="--case-id $$case_value"; \ + fi; \ + $(PYTHON) -m fetchgraph.tracer.cli fixture-demote $$case_args --root "$(TRACER_ROOT)" \ + $(if $(strip $(SELECT)),--select "$(SELECT)",) \ + $(if $(strip $(SELECT_INDEX)),--select-index "$(SELECT_INDEX)",) \ + $(if $(filter 1 true yes on,$(REQUIRE_UNIQUE)),--require-unique,) \ + $(if $(filter 1 true yes on,$(ALL)),--all,) \ + $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) \ $(if $(filter 1 true yes on,$(DRY)),--dry-run,) diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index edd5c770..cdccd2fc 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -139,6 +139,8 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) rm_cmd.add_argument("--name", help="Fixture stem name") rm_cmd.add_argument("--pattern", help="Glob pattern for fixture stems or case bundles") + rm_cmd.add_argument("--case", type=Path, help="Path to case bundle") + rm_cmd.add_argument("--case-id", help="Case id to select fixtures") rm_cmd.add_argument( "--scope", choices=["cases", "resources", "both"], @@ -152,6 +154,15 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: default="auto", help="Use git operations when removing fixtures", ) + rm_cmd.add_argument( + "--select", + choices=["latest", "first", "last"], + default="latest", + help="Selection policy when multiple fixtures match", + ) + rm_cmd.add_argument("--select-index", type=int, default=None, help="Select fixture index (1-based)") + rm_cmd.add_argument("--require-unique", action="store_true", help="Error if multiple fixtures match") + rm_cmd.add_argument("--all", action="store_true", help="Apply to all matching fixtures") fix_cmd = sub.add_parser("fixture-fix", help="Rename fixture stem") fix_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") @@ -174,6 +185,9 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: default="all", help="Fixture bucket", ) + migrate_cmd.add_argument("--case", type=Path, help="Path to case bundle") + migrate_cmd.add_argument("--case-id", help="Case id to select fixtures") + migrate_cmd.add_argument("--name", help="Fixture stem name") migrate_cmd.add_argument("--dry-run", action="store_true", help="Print actions without changing files") migrate_cmd.add_argument( "--git", @@ -181,6 +195,15 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: default="auto", help="Use git operations when moving fixtures", ) + migrate_cmd.add_argument( + "--select", + choices=["latest", "first", "last"], + default="latest", + help="Selection policy when multiple fixtures match", + ) + migrate_cmd.add_argument("--select-index", type=int, default=None, help="Select fixture index (1-based)") + migrate_cmd.add_argument("--require-unique", action="store_true", help="Error if multiple fixtures match") + migrate_cmd.add_argument("--all", action="store_true", help="Apply to all matching fixtures") ls_cmd = sub.add_parser("fixture-ls", help="List fixture candidates") ls_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") @@ -193,6 +216,31 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: ls_cmd.add_argument("--case-id", help="Case id to filter") ls_cmd.add_argument("--pattern", help="Glob pattern for fixtures") + demote_cmd = sub.add_parser("fixture-demote", help="Move fixed fixture back to known_bad") + demote_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") + demote_cmd.add_argument("--from-bucket", default="fixed", help="Source bucket (default: fixed)") + demote_cmd.add_argument("--to-bucket", default="known_bad", help="Destination bucket (default: known_bad)") + demote_cmd.add_argument("--case", type=Path, help="Path to case bundle") + demote_cmd.add_argument("--case-id", help="Case id to select fixtures") + demote_cmd.add_argument("--name", help="Fixture stem name") + demote_cmd.add_argument("--dry-run", action="store_true", help="Print actions without changing files") + demote_cmd.add_argument("--overwrite", action="store_true", help="Overwrite existing target fixtures") + demote_cmd.add_argument( + "--git", + choices=["auto", "on", "off"], + default="auto", + help="Use git operations when moving fixtures", + ) + demote_cmd.add_argument( + "--select", + choices=["latest", "first", "last"], + default="latest", + help="Selection policy when multiple fixtures match", + ) + demote_cmd.add_argument("--select-index", type=int, default=None, help="Select fixture index (1-based)") + demote_cmd.add_argument("--require-unique", action="store_true", help="Error if multiple fixtures match") + demote_cmd.add_argument("--all", action="store_true", help="Apply to all matching fixtures") + return parser.parse_args(argv) @@ -377,6 +425,12 @@ def main(argv: list[str] | None = None) -> int: scope=args.scope, dry_run=args.dry_run, git_mode=args.git, + case_path=args.case, + case_id=args.case_id, + select=args.select, + select_index=args.select_index, + require_unique=args.require_unique, + all_matches=args.all, ) print(f"Removed {removed} paths") return 0 @@ -396,6 +450,13 @@ def main(argv: list[str] | None = None) -> int: bucket=args.bucket, dry_run=args.dry_run, git_mode=args.git, + case_path=args.case, + case_id=args.case_id, + name=args.name, + select=args.select, + select_index=args.select_index, + require_unique=args.require_unique, + all_matches=args.all, ) print(f"Updated {bundles_updated} bundles; moved {files_moved} files") return 0 @@ -420,6 +481,25 @@ def main(argv: list[str] | None = None) -> int: f"timestamp={source.get('timestamp')}" ) return 0 + if args.command == "fixture-demote": + from fetchgraph.tracer.fixture_tools import fixture_demote + + fixture_demote( + root=args.root, + from_bucket=args.from_bucket, + to_bucket=args.to_bucket, + case_path=args.case, + case_id=args.case_id, + name=args.name, + dry_run=args.dry_run, + git_mode=args.git, + overwrite=args.overwrite, + select=args.select, + select_index=args.select_index, + require_unique=args.require_unique, + all_matches=args.all, + ) + return 0 except (ValueError, FileNotFoundError, LookupError, KeyError) as exc: print(str(exc), file=sys.stderr) return 2 diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py index 4d9335ad..8640b3ca 100644 --- a/src/fetchgraph/tracer/fixture_tools.py +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -85,6 +85,48 @@ def resolve_fixture_candidates( return results +def resolve_fixture_selector( + *, + root: Path, + bucket: str, + case_path: Path | None = None, + case_id: str | None = None, + name: str | None = None, + select: str = "latest", + select_index: int | None = None, + require_unique: bool = False, + all_matches: bool = False, +) -> list[Path]: + if case_path is not None and (case_id or name): + raise ValueError("Only one of case_path, case_id, or name can be used.") + if case_path is not None: + return [case_path] + candidates = resolve_fixture_candidates(root=root, bucket=bucket, case_id=case_id, name=name) + if not candidates: + selector = case_id or name + raise FileNotFoundError(f"No fixtures found for selector={selector!r} in {bucket}") + if require_unique and len(candidates) > 1: + raise FileExistsError("Multiple fixtures matched selection; use --select-index or --select.") + if all_matches: + print("fixture selector: bulk operation on all matches") + for idx, candidate in enumerate(candidates, start=1): + print(f" {idx}. {candidate.path}") + return [candidate.path for candidate in candidates] + + selected = select_fixture_candidate( + candidates, + select=select, + select_index=select_index, + require_unique=require_unique, + ) + if len(candidates) > 1 and not require_unique: + print("fixture selector: multiple candidates found, selected:") + for idx, candidate in enumerate(candidates, start=1): + marker = " (selected)" if candidate.path == selected.path else "" + print(f" {idx}. {candidate.path}{marker}") + return [selected.path] + + def select_fixture_candidate( candidates: list[FixtureCandidate], *, @@ -352,25 +394,42 @@ def fixture_green( def fixture_rm( *, root: Path, - name: str | None, - pattern: str | None, bucket: str, scope: str, dry_run: bool, git_mode: str = "auto", + case_path: Path | None = None, + case_id: str | None = None, + name: str | None = None, + pattern: str | None = None, + select: str = "latest", + select_index: int | None = None, + require_unique: bool = False, + all_matches: bool = False, ) -> int: root = root.resolve() git_ops = _GitOps(root, git_mode) bucket_filter: str | None = None if bucket == "all" else bucket - matched = find_case_bundles(root=root, bucket=bucket_filter, name=name, pattern=pattern) - - if name and not matched: - raise FileNotFoundError(f"No fixtures found for name={name!r}") + if case_path or case_id or name: + selector_paths = resolve_fixture_selector( + root=root, + bucket=bucket_filter or bucket, + case_path=case_path, + case_id=case_id, + name=name, + select=select, + select_index=select_index, + require_unique=require_unique, + all_matches=all_matches, + ) + matched = selector_paths + else: + matched = find_case_bundles(root=root, bucket=bucket_filter, name=name, pattern=pattern) targets: list[Path] = [] - for case_path in matched: - bucket_name = case_path.parent.name - stem = case_path.name.replace(".case.json", "") + for case_path_item in matched: + bucket_name = case_path_item.parent.name + stem = case_path_item.name.replace(".case.json", "") layout = FixtureLayout(root, bucket_name) if scope in ("cases", "both"): targets.extend([layout.case_path(stem), layout.expected_path(stem)]) @@ -473,11 +532,38 @@ def fixture_fix( print(" rewrite: data_ref.file paths updated") -def fixture_migrate(*, root: Path, bucket: str = "all", dry_run: bool, git_mode: str = "auto") -> tuple[int, int]: +def fixture_migrate( + *, + root: Path, + bucket: str = "all", + dry_run: bool, + git_mode: str = "auto", + case_path: Path | None = None, + case_id: str | None = None, + name: str | None = None, + select: str = "latest", + select_index: int | None = None, + require_unique: bool = False, + all_matches: bool = False, +) -> tuple[int, int]: root = root.resolve() git_ops = _GitOps(root, git_mode) bucket_filter = None if bucket == "all" else bucket - matched = find_case_bundles(root=root, bucket=bucket_filter, name=None, pattern=None) + if case_path or case_id or name: + selector_paths = resolve_fixture_selector( + root=root, + bucket=bucket_filter or bucket, + case_path=case_path, + case_id=case_id, + name=name, + select=select, + select_index=select_index, + require_unique=require_unique, + all_matches=all_matches, + ) + matched = selector_paths + else: + matched = find_case_bundles(root=root, bucket=bucket_filter, name=None, pattern=None) bundles_updated = 0 files_moved = 0 for case_path in matched: @@ -555,3 +641,82 @@ def fixture_ls( continue results.append(FixtureCandidate(path=path, stem=stem, source=source_dict)) return results + + +def fixture_demote( + *, + root: Path, + from_bucket: str = "fixed", + to_bucket: str = "known_bad", + case_path: Path | None = None, + case_id: str | None = None, + name: str | None = None, + dry_run: bool = False, + git_mode: str = "auto", + overwrite: bool = False, + select: str = "latest", + select_index: int | None = None, + require_unique: bool = False, + all_matches: bool = False, +) -> None: + root = root.resolve() + if from_bucket == to_bucket: + raise ValueError("from_bucket and to_bucket must be different") + git_ops = _GitOps(root, git_mode) + selected_paths = resolve_fixture_selector( + root=root, + bucket=from_bucket, + case_path=case_path, + case_id=case_id, + name=name, + select=select, + select_index=select_index, + require_unique=require_unique, + all_matches=all_matches, + ) + for case_path_item in selected_paths: + stem = case_path_item.name.replace(".case.json", "") + from_layout = FixtureLayout(root, from_bucket) + to_layout = FixtureLayout(root, to_bucket) + from_case = from_layout.case_path(stem) + from_expected = from_layout.expected_path(stem) + from_resources = from_layout.resources_dir(stem) + to_case = to_layout.case_path(stem) + to_expected = to_layout.expected_path(stem) + to_resources = to_layout.resources_dir(stem) + + if not from_case.exists(): + raise FileNotFoundError(f"Missing fixture: {from_case}") + if to_case.exists() and not overwrite and not dry_run: + raise FileExistsError(f"Target case already exists: {to_case}") + if to_expected.exists() and from_expected.exists() and not overwrite and not dry_run: + raise FileExistsError(f"Target expected already exists: {to_expected}") + if to_resources.exists() and from_resources.exists() and not overwrite and not dry_run: + raise FileExistsError(f"Target resources already exists: {to_resources}") + + if dry_run: + print("fixture-demote:") + print(f" case: {from_case}") + print(f" move: -> {to_case}") + if from_expected.exists(): + print(f" move: {from_expected} -> {to_expected}") + if from_resources.exists(): + print(f" move: {from_resources} -> {to_resources}") + continue + + to_case.parent.mkdir(parents=True, exist_ok=True) + git_ops.move(from_case, to_case) + if from_expected.exists(): + to_expected.parent.mkdir(parents=True, exist_ok=True) + git_ops.move(from_expected, to_expected) + if from_resources.exists(): + to_resources.parent.mkdir(parents=True, exist_ok=True) + git_ops.move(from_resources, to_resources) + + print("fixture-demote:") + print(f" case: {from_case}") + print(f" move: -> {to_case}") + if from_expected.exists(): + print(f" move: {from_expected} -> {to_expected}") + if from_resources.exists(): + print(f" move: {from_resources} -> {to_resources}") diff --git a/tests/test_tracer_fixture_demote.py b/tests/test_tracer_fixture_demote.py new file mode 100644 index 00000000..ed99a10c --- /dev/null +++ b/tests/test_tracer_fixture_demote.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fetchgraph.tracer.fixture_tools import fixture_demote + + +def _write_bundle(path: Path, *, case_id: str) -> None: + payload = { + "schema": "fetchgraph.tracer.case_bundle", + "v": 1, + "root": {"type": "replay_case", "v": 2, "id": "plan_normalize.spec_v1", "input": {}}, + "source": {"case_id": case_id}, + "resources": {}, + "extras": {}, + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + + +def test_fixture_demote_moves_files(tmp_path: Path) -> None: + root = tmp_path / "fixtures" + fixed = root / "fixed" + known_bad = root / "known_bad" + case_path = fixed / "agg_003__a.case.json" + expected_path = fixed / "agg_003__a.expected.json" + resources_dir = fixed / "resources" / "agg_003__a" + _write_bundle(case_path, case_id="agg_003") + expected_path.write_text('{"ok": true}', encoding="utf-8") + resources_dir.mkdir(parents=True, exist_ok=True) + (resources_dir / "sample.txt").write_text("data", encoding="utf-8") + + fixture_demote( + root=root, + case_id="agg_003", + dry_run=True, + ) + assert case_path.exists() + assert expected_path.exists() + assert (resources_dir / "sample.txt").exists() + + fixture_demote( + root=root, + case_id="agg_003", + dry_run=False, + ) + assert not case_path.exists() + assert not expected_path.exists() + assert not resources_dir.exists() + assert (known_bad / "agg_003__a.case.json").exists() + assert (known_bad / "agg_003__a.expected.json").exists() + assert (known_bad / "resources" / "agg_003__a" / "sample.txt").exists() diff --git a/tests/test_tracer_fixture_selector_for_rm_migrate.py b/tests/test_tracer_fixture_selector_for_rm_migrate.py new file mode 100644 index 00000000..170b659f --- /dev/null +++ b/tests/test_tracer_fixture_selector_for_rm_migrate.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from fetchgraph.tracer.fixture_tools import fixture_migrate, fixture_rm + + +def _write_bundle(path: Path, *, case_id: str) -> None: + payload = { + "schema": "fetchgraph.tracer.case_bundle", + "v": 1, + "root": {"type": "replay_case", "v": 2, "id": "plan_normalize.spec_v1", "input": {}}, + "source": {"case_id": case_id}, + "resources": {}, + "extras": {}, + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + + +def test_fixture_selector_for_rm_migrate(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + root = tmp_path / "fixtures" + known_bad = root / "known_bad" + case_old = known_bad / "agg_003__a.case.json" + case_new = known_bad / "agg_003__b.case.json" + _write_bundle(case_old, case_id="agg_003") + _write_bundle(case_new, case_id="agg_003") + os.utime(case_old, (1, 1)) + os.utime(case_new, (2, 2)) + + removed = fixture_rm( + root=root, + bucket="known_bad", + scope="cases", + dry_run=True, + case_id="agg_003", + ) + assert removed == 1 + assert case_old.exists() + assert case_new.exists() + + with pytest.raises(FileExistsError): + fixture_rm( + root=root, + bucket="known_bad", + scope="cases", + dry_run=True, + case_id="agg_003", + require_unique=True, + ) + + bundles_updated, files_moved = fixture_migrate( + root=root, + bucket="known_bad", + dry_run=True, + case_id="agg_003", + all_matches=True, + ) + assert bundles_updated == 0 + assert files_moved == 0 + output = capsys.readouterr().out + assert "bulk operation" in output From 1b3ab9a7f545d39922c408f694d9323b05b9877b Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Sun, 25 Jan 2026 23:10:54 +0300 Subject: [PATCH 64/79] Prefer runs with replay events --- Makefile | 8 +-- examples/demo_qa/batch.py | 5 +- src/fetchgraph/tracer/cli.py | 44 ++++++++++-- src/fetchgraph/tracer/resolve.py | 71 +++++++++++++++++-- ..._export_resolve_prefers_run_with_replay.py | 44 ++++++++++++ 5 files changed, 154 insertions(+), 18 deletions(-) create mode 100644 tests/test_tracer_export_resolve_prefers_run_with_replay.py diff --git a/Makefile b/Makefile index 362ca4c7..157b8489 100644 --- a/Makefile +++ b/Makefile @@ -180,11 +180,11 @@ help: @echo " make known-bad-one NAME=fixture_stem - запустить один known_bad кейс" @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources///..." - @echo " make fixture-green CASE=path/to/case.case.json [TRACER_ROOT=...] [VALIDATE=1] [OVERWRITE_EXPECTED=1] [DRY=1]" + @echo " make fixture-green CASE=agg_003|fixture_stem|path/to/case.case.json [TRACER_ROOT=...] [VALIDATE=1] [OVERWRITE_EXPECTED=1] [DRY=1]" @echo " make fixture-ls CASE=agg_003 [TRACER_ROOT=...] [BUCKET=known_bad]" - @echo " make fixture-rm CASE=agg_003 [SELECT=latest|first|last] [SELECT_INDEX=N] [REQUIRE_UNIQUE=1] [ALL=1]" - @echo " make fixture-migrate CASE=agg_003 [SELECT=latest|first|last] [SELECT_INDEX=N] [REQUIRE_UNIQUE=1] [ALL=1]" - @echo " make fixture-demote CASE=agg_003 [SELECT=latest|first|last] [SELECT_INDEX=N] [REQUIRE_UNIQUE=1] [ALL=1]" + @echo " make fixture-rm CASE=agg_003|fixture_stem|path [SELECT=latest|first|last] [SELECT_INDEX=N] [REQUIRE_UNIQUE=1] [ALL=1]" + @echo " make fixture-migrate CASE=agg_003|fixture_stem|path [SELECT=latest|first|last] [SELECT_INDEX=N] [REQUIRE_UNIQUE=1] [ALL=1]" + @echo " make fixture-demote CASE=agg_003|fixture_stem|path [SELECT=latest|first|last] [SELECT_INDEX=N] [REQUIRE_UNIQUE=1] [ALL=1]" @echo " make fixture-rm [BUCKET=fixed|known_bad|all] [NAME=...] [PATTERN=...] [SCOPE=cases|resources|both] [DRY=1]" @echo " make fixture-fix BUCKET=... NAME=... NEW_NAME=... [DRY=1]" @echo " make fixture-migrate [BUCKET=fixed|known_bad|all] [DRY=1]" diff --git a/examples/demo_qa/batch.py b/examples/demo_qa/batch.py index 2df3b830..41eef835 100644 --- a/examples/demo_qa/batch.py +++ b/examples/demo_qa/batch.py @@ -1450,7 +1450,10 @@ def handle_case_open(args) -> int: plan = case_dir / "plan.json" answer = case_dir / "answer.txt" status = case_dir / "status.json" - for path in [plan, answer, status]: + events = case_dir / "events.jsonl" + error = case_dir / "error.txt" + schema_snapshot = case_dir / "schema_snapshot.yaml" + for path in [plan, answer, status, events, error, schema_snapshot]: if path.exists(): print(f"- {path}") return 0 diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index cdccd2fc..3289e327 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -6,6 +6,7 @@ from pathlib import Path from fetchgraph.tracer.resolve import ( + collect_rejections, find_events_file, format_case_runs, format_case_run_debug, @@ -68,8 +69,8 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) export.add_argument( "--pick-run", - default="latest_non_missed", - help="Run selection strategy (default: latest_non_missed)", + default="latest_with_replay", + help="Run selection strategy (default: latest_with_replay)", ) export.add_argument( "--print-resolve", @@ -274,6 +275,8 @@ def main(argv: list[str] | None = None) -> int: else: if not args.case or not args.data: raise ValueError("--case and --data are required when --events is not provided.") + if args.pick_run == "latest_with_replay" and not args.id: + raise ValueError("--id is required when pick_run=latest_with_replay.") infos, stats = scan_case_runs( case_id=args.case, data_dir=args.data, @@ -286,6 +289,8 @@ def main(argv: list[str] | None = None) -> int: case_id=args.case, data_dir=args.data, tag=args.tag, + pick_run=args.pick_run, + replay_id=args.id if args.pick_run == "latest_with_replay" else None, runs_subdir=args.runs_subdir, ) if args.list_matches: @@ -295,15 +300,14 @@ def main(argv: list[str] | None = None) -> int: stats, case_id=args.case, tag=args.tag, + pick_run=args.pick_run, ) ) print(format_case_runs(candidates, limit=20)) return 0 selected = select_case_run(candidates, select_index=args.select_index) run_dir = selected.case_dir - selection_rule = "latest with events" - if args.tag: - selection_rule = f"latest with events filtered by TAG={args.tag!r}" + selection_rule = _format_selection_rule(tag=args.tag, pick_run=args.pick_run) events_path = selected.events_path if events_path is None: events_resolution = find_events_file(run_dir) @@ -318,7 +322,24 @@ def main(argv: list[str] | None = None) -> int: events_path = events_resolution.events_path if args.print_resolve: print(f"Resolved run_dir: {run_dir}") + print(f"Resolved case_dir: {run_dir}") print(f"Resolved events.jsonl: {events_path}") + if not args.events and args.case and args.data: + infos, _ = scan_case_runs( + case_id=args.case, + data_dir=args.data, + runs_subdir=args.runs_subdir, + ) + rejections = collect_rejections( + infos, + tag=args.tag, + pick_run=args.pick_run, + replay_id=args.id, + ) + if rejections: + print("Rejected candidates:") + for reject in rejections: + print(f"- {reject.case_dir} ({reject.reason})") if args.list_replay_matches: if events_path is None: raise ValueError("events_path was not resolved.") @@ -524,10 +545,10 @@ def _resolve_case_dir_from_run_id(*, data_dir: Path, runs_subdir: str, run_id: s return case_dirs[0] -def _format_case_run_error(stats, *, case_id: str, tag: str | None) -> str: +def _format_case_run_error(stats, *, case_id: str, tag: str | None, pick_run: str = "latest_non_missed") -> str: lines = [ "No suitable case run found.", - f"selection_rule: {'latest with events' if not tag else f'latest with events filtered by TAG={tag!r}'}", + f"selection_rule: {_format_selection_rule(tag=tag, pick_run=pick_run)}", f"runs_root: {stats.runs_root}", f"case_id: {case_id}", f"inspected_runs: {stats.inspected_runs}", @@ -565,5 +586,14 @@ def _format_events_error(run_dir: Path, resolution, *, selection_rule: str) -> s ) +def _format_selection_rule(*, tag: str | None, pick_run: str) -> str: + base = "latest with events" + if pick_run == "latest_with_replay": + base = "latest with replay_case" + if tag: + return f"{base} filtered by TAG={tag!r}" + return base + + if __name__ == "__main__": sys.exit(main()) diff --git a/src/fetchgraph/tracer/resolve.py b/src/fetchgraph/tracer/resolve.py index 38f58ca0..2e236323 100644 --- a/src/fetchgraph/tracer/resolve.py +++ b/src/fetchgraph/tracer/resolve.py @@ -5,6 +5,8 @@ from pathlib import Path from typing import Iterable +from fetchgraph.replay.export import iter_events + @dataclass(frozen=True) class CaseResolution: @@ -47,6 +49,13 @@ class EventsResolution: found: list[Path] +@dataclass(frozen=True) +class RejectionReason: + run_dir: Path + case_dir: Path + reason: str + + def resolve_case_events( *, case_id: str, @@ -55,19 +64,23 @@ def resolve_case_events( runs_subdir: str = ".runs/runs", pick_run: str = "latest_non_missed", select_index: int | None = None, + replay_id: str | None = None, ) -> CaseResolution: if not case_id: raise ValueError("case_id is required") if not data_dir: raise ValueError("data_dir is required") - if pick_run != "latest_non_missed": + if pick_run not in {"latest_non_missed", "latest_with_replay"}: raise ValueError(f"Unsupported pick_run mode: {pick_run}") + if pick_run == "latest_with_replay" and not replay_id: + raise ValueError("replay_id is required for latest_with_replay selection") candidates, stats = list_case_runs( case_id=case_id, data_dir=data_dir, tag=tag, pick_run=pick_run, + replay_id=replay_id, runs_subdir=runs_subdir, ) if not candidates: @@ -75,7 +88,7 @@ def resolve_case_events( stats, case_id=case_id, tag=tag, - rule=_resolve_rule(tag=tag), + rule=_resolve_rule(tag=tag, pick_run=pick_run), ) raise LookupError(details) @@ -118,6 +131,7 @@ def list_case_runs( data_dir: Path, tag: str | None = None, pick_run: str = "latest_non_missed", + replay_id: str | None = None, runs_subdir: str = ".runs/runs", ) -> tuple[list[CaseRunCandidate], RunScanStats]: infos, stats = scan_case_runs( @@ -126,7 +140,7 @@ def list_case_runs( tag=tag, runs_subdir=runs_subdir, ) - candidates = _filter_case_run_infos(infos, tag=tag, pick_run=pick_run) + candidates = _filter_case_run_infos(infos, tag=tag, pick_run=pick_run, replay_id=replay_id) return candidates, stats @@ -198,6 +212,7 @@ def _filter_case_run_infos( *, tag: str | None = None, pick_run: str = "latest_non_missed", + replay_id: str | None = None, ) -> list[CaseRunCandidate]: candidates: list[CaseRunCandidate] = [] for info in infos: @@ -205,6 +220,11 @@ def _filter_case_run_infos( continue if pick_run == "latest_non_missed" and info.is_missed: continue + if pick_run == "latest_with_replay": + if replay_id is None: + raise ValueError("replay_id is required for latest_with_replay selection") + if not _has_replay_case_id(info.events.events_path, replay_id): + continue if tag and info.tag != tag: continue candidates.append( @@ -448,10 +468,49 @@ def _payload_is_missed(payload: dict) -> bool: return "missed" in reason or "missing" in reason -def _resolve_rule(*, tag: str | None) -> str: +def _has_replay_case_id(events_path: Path, replay_id: str) -> bool: + try: + for _, event in iter_events(events_path, allow_bad_json=True): + if event.get("type") == "replay_case" and event.get("id") == replay_id: + return True + except FileNotFoundError: + return False + return False + + +def collect_rejections( + infos: list[CaseRunInfo], + *, + tag: str | None, + pick_run: str, + replay_id: str | None, +) -> list[RejectionReason]: + rejections: list[RejectionReason] = [] + for info in infos: + if tag and info.tag != tag: + rejections.append(RejectionReason(info.run_dir, info.case_dir, "tag_mismatch")) + continue + if info.events.events_path is None: + rejections.append(RejectionReason(info.run_dir, info.case_dir, "no_events")) + continue + if pick_run == "latest_non_missed" and info.is_missed: + rejections.append(RejectionReason(info.run_dir, info.case_dir, "missed")) + continue + if pick_run == "latest_with_replay": + if replay_id is None or not _has_replay_case_id(info.events.events_path, replay_id): + rejections.append(RejectionReason(info.run_dir, info.case_dir, "no_replay_id")) + continue + rejections.append(RejectionReason(info.run_dir, info.case_dir, "filtered")) + return rejections + + +def _resolve_rule(*, tag: str | None, pick_run: str = "latest_non_missed") -> str: + base = "latest with events" + if pick_run == "latest_with_replay": + base = "latest with replay_case" if tag: - return f"latest with events filtered by TAG={tag!r}" - return "latest with events" + return f"{base} filtered by TAG={tag!r}" + return base def _format_missing_case_runs( diff --git a/tests/test_tracer_export_resolve_prefers_run_with_replay.py b/tests/test_tracer_export_resolve_prefers_run_with_replay.py new file mode 100644 index 00000000..690c05c3 --- /dev/null +++ b/tests/test_tracer_export_resolve_prefers_run_with_replay.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path + +from fetchgraph.tracer.resolve import resolve_case_events + + +def _write_events(path: Path, events: list[dict]) -> None: + path.write_text("\n".join(json.dumps(event) for event in events) + "\n", encoding="utf-8") + + +def _make_case_dir(run_dir: Path, case_id: str, suffix: str, *, status: str) -> Path: + case_dir = run_dir / "cases" / f"{case_id}_{suffix}" + case_dir.mkdir(parents=True, exist_ok=True) + (case_dir / "status.json").write_text(json.dumps({"status": status}), encoding="utf-8") + return case_dir + + +def test_resolve_prefers_run_with_replay(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + old_run = runs_root / "run_old" + old_run.mkdir() + old_case = _make_case_dir(old_run, "agg_003", "old", status="ok") + _write_events(old_case / "events.jsonl", [{"type": "replay_case", "id": "other_case"}]) + os.utime(old_run, (1, 1)) + + new_run = runs_root / "run_new" + new_run.mkdir() + new_case = _make_case_dir(new_run, "agg_003", "new", status="error") + _write_events(new_case / "events.jsonl", [{"type": "replay_case", "id": "plan_normalize.spec_v1"}]) + os.utime(new_run, (2, 2)) + + resolution = resolve_case_events( + case_id="agg_003", + data_dir=data_dir, + pick_run="latest_with_replay", + replay_id="plan_normalize.spec_v1", + ) + assert resolution.case_dir == new_case From 9d66581448d2acef88a4f4a3efe330fb79b6121b Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Mon, 26 Jan 2026 07:16:13 +0300 Subject: [PATCH 65/79] Add rollback for fixture moves --- src/fetchgraph/tracer/cli.py | 2 +- src/fetchgraph/tracer/fixture_tools.py | 98 ++++++++++++++++++++------ 2 files changed, 78 insertions(+), 22 deletions(-) diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index 3289e327..1957617b 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -521,7 +521,7 @@ def main(argv: list[str] | None = None) -> int: all_matches=args.all, ) return 0 - except (ValueError, FileNotFoundError, LookupError, KeyError) as exc: + except (ValueError, FileNotFoundError, LookupError, KeyError, FileExistsError) as exc: print(str(exc), file=sys.stderr) return 2 except Exception as exc: # pragma: no cover - unexpected diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py index 8640b3ca..83339bb0 100644 --- a/src/fetchgraph/tracer/fixture_tools.py +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -256,6 +256,48 @@ def _should_git_remove(self, path: Path) -> bool: return _git_tracked(self.root, path) +def _unique_backup_path(path: Path) -> Path: + for idx in range(1, 1000): + suffix = "" if idx == 1 else f".{idx}" + candidate = path.with_name(f".{path.name}.rollback{suffix}") + if not candidate.exists(): + return candidate + raise FileExistsError(f"Unable to create rollback path for {path}") + + +class _MoveTransaction: + def __init__(self, git_ops: _GitOps) -> None: + self.git_ops = git_ops + self.moved: list[tuple[Path, Path]] = [] + self.removed: list[tuple[Path, Path]] = [] + + def move(self, src: Path, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + self.git_ops.move(src, dest) + self.moved.append((src, dest)) + + def remove(self, path: Path) -> None: + if not path.exists(): + return + backup = _unique_backup_path(path) + backup.parent.mkdir(parents=True, exist_ok=True) + self.git_ops.move(path, backup) + self.removed.append((path, backup)) + + def rollback(self) -> None: + for src, dest in reversed(self.moved): + if dest.exists(): + self.git_ops.move(dest, src) + for original, backup in reversed(self.removed): + if backup.exists(): + self.git_ops.move(backup, original) + + def commit(self) -> None: + for _, backup in self.removed: + if backup.exists(): + self.git_ops.remove(backup) + + def fixture_green( *, case_path: Path | None = None, @@ -355,21 +397,24 @@ def fixture_green( json.dumps(observed, ensure_ascii=False, sort_keys=True, indent=2), encoding="utf-8", ) + tx = _MoveTransaction(git_ops) try: - fixed_case_path.parent.mkdir(parents=True, exist_ok=True) - git_ops.move(known_case_path, fixed_case_path) + tx.move(known_case_path, fixed_case_path) if resources_from.exists(): - resources_to.parent.mkdir(parents=True, exist_ok=True) - git_ops.move(resources_from, resources_to) + tx.move(resources_from, resources_to) if known_expected_path.exists(): - git_ops.remove(known_expected_path) + tx.remove(known_expected_path) tmp_expected_path.replace(fixed_expected_path) + tx.commit() except Exception: + if fixed_expected_path.exists(): + fixed_expected_path.unlink() if tmp_expected_path.exists(): tmp_expected_path.unlink() + tx.rollback() raise if validate: @@ -513,14 +558,22 @@ def fixture_fix( print(" rewrite: data_ref.file paths updated") return - new_case_path.parent.mkdir(parents=True, exist_ok=True) - git_ops.move(case_path, new_case_path) - if expected_path.exists(): - git_ops.move(expected_path, new_expected_path) - if resources_dir.exists(): - git_ops.move(resources_dir, new_resources_dir) - if updated: - _atomic_write_json(new_case_path, payload) + original_text = case_path.read_text(encoding="utf-8") if updated else None + tx = _MoveTransaction(git_ops) + try: + tx.move(case_path, new_case_path) + if expected_path.exists(): + tx.move(expected_path, new_expected_path) + if resources_dir.exists(): + tx.move(resources_dir, new_resources_dir) + if updated: + _atomic_write_json(new_case_path, payload) + tx.commit() + except Exception: + if updated and original_text is not None and new_case_path.exists(): + new_case_path.write_text(original_text, encoding="utf-8") + tx.rollback() + raise print("fixture-fix:") print(f" rename: {case_path} -> {new_case_path}") @@ -704,14 +757,17 @@ def fixture_demote( print(f" move: {from_resources} -> {to_resources}") continue - to_case.parent.mkdir(parents=True, exist_ok=True) - git_ops.move(from_case, to_case) - if from_expected.exists(): - to_expected.parent.mkdir(parents=True, exist_ok=True) - git_ops.move(from_expected, to_expected) - if from_resources.exists(): - to_resources.parent.mkdir(parents=True, exist_ok=True) - git_ops.move(from_resources, to_resources) + tx = _MoveTransaction(git_ops) + try: + tx.move(from_case, to_case) + if from_expected.exists(): + tx.move(from_expected, to_expected) + if from_resources.exists(): + tx.move(from_resources, to_resources) + tx.commit() + except Exception: + tx.rollback() + raise print("fixture-demote:") print(f" case: {from_case}") From fd1c59c7f20f55614eba0f0a04e38b75c4ee8e86 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Mon, 26 Jan 2026 07:27:41 +0300 Subject: [PATCH 66/79] Handle overwrite in fixture demote --- src/fetchgraph/tracer/fixture_tools.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py index 83339bb0..e73d11e6 100644 --- a/src/fetchgraph/tracer/fixture_tools.py +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -759,10 +759,16 @@ def fixture_demote( tx = _MoveTransaction(git_ops) try: + if overwrite: + tx.remove(to_case) tx.move(from_case, to_case) if from_expected.exists(): + if overwrite: + tx.remove(to_expected) tx.move(from_expected, to_expected) if from_resources.exists(): + if overwrite: + tx.remove(to_resources) tx.move(from_resources, to_resources) tx.commit() except Exception: From c91a472824b79b574280a8633ed55a3473b52b44 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Mon, 26 Jan 2026 08:17:02 +0300 Subject: [PATCH 67/79] Respect events flag for case logging --- examples/demo_qa/runner.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/demo_qa/runner.py b/examples/demo_qa/runner.py index 93115d8e..30bc4a18 100644 --- a/examples/demo_qa/runner.py +++ b/examples/demo_qa/runner.py @@ -392,7 +392,7 @@ def run_one( run_dir.mkdir(parents=True, exist_ok=True) events_path = run_dir / "events.jsonl" - case_logger = event_logger.for_case(case.id, events_path) if event_logger else EventLogger(events_path, run_id, case.id) + case_logger = event_logger.for_case(case.id, events_path) if event_logger else None if case_logger: case_logger.emit({"type": "run_started", "case_id": case.id, "run_dir": str(run_dir)}) case_logger.emit({"type": "case_started", "case_id": case.id, "run_dir": str(run_dir)}) @@ -481,7 +481,11 @@ def run_one( status="error", checked=case.has_asserts, reason=error_message, - details={"error": error_message, "traceback": tb, "events_path": str(events_path)}, + details={ + "error": error_message, + "traceback": tb, + **({"events_path": str(events_path)} if case_logger else {}), + }, artifacts_dir=str(run_dir), duration_ms=0, tags=list(case.tags), From bc96facab4250dcedad167e29416e4b42f4332ba Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Mon, 26 Jan 2026 09:13:16 +0300 Subject: [PATCH 68/79] Update fixture-green validation and diagnostics --- Makefile | 5 +- src/fetchgraph/tracer/cli.py | 11 ++- src/fetchgraph/tracer/fixture_tools.py | 112 ++++++++++++++++++++++--- tests/helpers/replay_dx.py | 2 +- tests/test_replay_fixed.py | 45 ++++++++++ tests/test_replay_known_bad_backlog.py | 2 +- tests/test_tracer_fixture_tools.py | 28 ++++++- 7 files changed, 184 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 157b8489..0d3c3736 100644 --- a/Makefile +++ b/Makefile @@ -180,7 +180,7 @@ help: @echo " make known-bad-one NAME=fixture_stem - запустить один known_bad кейс" @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources///..." - @echo " make fixture-green CASE=agg_003|fixture_stem|path/to/case.case.json [TRACER_ROOT=...] [VALIDATE=1] [OVERWRITE_EXPECTED=1] [DRY=1]" + @echo " make fixture-green CASE=agg_003|fixture_stem|path/to/case.case.json [TRACER_ROOT=...] [NO_VALIDATE=1] [EXPECTED_FROM=replay|observed] [OVERWRITE_EXPECTED=1] [DRY=1]" @echo " make fixture-ls CASE=agg_003 [TRACER_ROOT=...] [BUCKET=known_bad]" @echo " make fixture-rm CASE=agg_003|fixture_stem|path [SELECT=latest|first|last] [SELECT_INDEX=N] [REQUIRE_UNIQUE=1] [ALL=1]" @echo " make fixture-migrate CASE=agg_003|fixture_stem|path [SELECT=latest|first|last] [SELECT_INDEX=N] [REQUIRE_UNIQUE=1] [ALL=1]" @@ -443,7 +443,8 @@ fixture-green: $(if $(strip $(SELECT)),--select "$(SELECT)",) \ $(if $(strip $(SELECT_INDEX)),--select-index "$(SELECT_INDEX)",) \ $(if $(filter 1 true yes on,$(REQUIRE_UNIQUE)),--require-unique,) \ - $(if $(filter 1 true yes on,$(VALIDATE)),--validate,) \ + $(if $(filter 1 true yes on,$(NO_VALIDATE)),--no-validate,) \ + $(if $(strip $(EXPECTED_FROM)),--expected-from "$(EXPECTED_FROM)",) \ $(if $(filter 1 true yes on,$(OVERWRITE_EXPECTED)),--overwrite-expected,) \ $(if $(filter 1 true yes on,$(DRY)),--dry-run,) diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py index 1957617b..4f042c1b 100644 --- a/src/fetchgraph/tracer/cli.py +++ b/src/fetchgraph/tracer/cli.py @@ -108,7 +108,13 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: green.add_argument("--case-id", help="Case id to select fixture from known_bad") green.add_argument("--name", help="Fixture stem name to select") green.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") - green.add_argument("--validate", action="store_true", help="Validate replay output") + green.add_argument("--no-validate", action="store_true", help="Disable validation after write") + green.add_argument( + "--expected-from", + choices=["replay", "observed"], + default="replay", + help="Source for expected output (default: replay)", + ) green.add_argument( "--overwrite-expected", action="store_true", @@ -428,7 +434,8 @@ def main(argv: list[str] | None = None) -> int: case_id=args.case_id, name=args.name, out_root=args.root, - validate=args.validate, + validate=not args.no_validate, + expected_from=args.expected_from, overwrite_expected=args.overwrite_expected, dry_run=args.dry_run, git_mode=args.git, diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py index e73d11e6..37aee81e 100644 --- a/src/fetchgraph/tracer/fixture_tools.py +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -60,6 +60,71 @@ def _atomic_write_json(path: Path, payload: dict) -> None: tmp_path.replace(path) +def _format_json(value: object, *, max_chars: int = 12_000) -> str: + text = json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True, default=str) + if len(text) <= max_chars: + return text + remaining = len(text) - max_chars + return f"{text[:max_chars]}...[truncated {remaining} chars]" + + +def _top_level_key_diff(output: object, expected: object) -> tuple[list[str], list[str]]: + if not isinstance(output, dict) or not isinstance(expected, dict): + return ([], []) + output_keys = set(output.keys()) + expected_keys = set(expected.keys()) + missing = sorted(expected_keys - output_keys) + extra = sorted(output_keys - expected_keys) + return (missing, extra) + + +def _first_diff_path(left: object, right: object, *, prefix: str = "") -> str | None: + if type(left) is not type(right): + return prefix or "" + if isinstance(left, dict): + left_keys = set(left.keys()) + right_keys = set(right.keys()) + for key in sorted(left_keys | right_keys): + next_prefix = f"{prefix}.{key}" if prefix else str(key) + if key not in left or key not in right: + return next_prefix + diff = _first_diff_path(left[key], right[key], prefix=next_prefix) + if diff is not None: + return diff + return None + if isinstance(left, list): + if len(left) != len(right): + return f"{prefix}.length" if prefix else "length" + for idx, (l_item, r_item) in enumerate(zip(left, right), start=1): + next_prefix = f"{prefix}[{idx}]" if prefix else f"[{idx}]" + diff = _first_diff_path(l_item, r_item, prefix=next_prefix) + if diff is not None: + return diff + return None + if left != right: + return prefix or "" + return None + + +def _format_fixture_diff(output: object, expected: object) -> str: + missing, extra = _top_level_key_diff(output, expected) + lines = [ + "Fixture output mismatch.", + f"missing_keys: {missing}", + f"extra_keys: {extra}", + ] + if isinstance(output, dict) and isinstance(expected, dict): + out_spec = output.get("out_spec") + exp_spec = expected.get("out_spec") + if out_spec != exp_spec: + diff_path = _first_diff_path(out_spec, exp_spec, prefix="out_spec") + if diff_path: + lines.append(f"out_spec diff path: {diff_path}") + lines.append(f"expected: {_format_json(expected)}") + lines.append(f"output: {_format_json(output)}") + return "\n".join(lines) + + def resolve_fixture_candidates( *, root: Path, @@ -304,7 +369,8 @@ def fixture_green( case_id: str | None = None, name: str | None = None, out_root: Path, - validate: bool = False, + validate: bool = True, + expected_from: str = "replay", overwrite_expected: bool = False, dry_run: bool = False, git_mode: str = "auto", @@ -316,6 +382,8 @@ def fixture_green( raise ValueError("fixture-green requires --case, --case-id, or --name.") if case_path is not None and (case_id or name): raise ValueError("fixture-green accepts only one of --case, --case-id, or --name.") + if expected_from not in {"replay", "observed"}: + raise ValueError(f"Unsupported expected_from: {expected_from}") out_root = out_root.resolve() git_ops = _GitOps(out_root, git_mode) known_layout = FixtureLayout(out_root, "known_bad") @@ -352,7 +420,7 @@ def fixture_green( payload = load_bundle_json(case_path) root = payload.get("root") or {} observed = root.get("observed") - if not isinstance(observed, dict): + if expected_from == "observed" and not isinstance(observed, dict): raise ValueError( "Cannot green fixture: root.observed is missing.\n" f"Case: {case_path}\n" @@ -384,17 +452,27 @@ def fixture_green( print("fixture-green:") print(f" case: {known_case_path}") print(f" move: -> {fixed_case_path}") - print(f" write: -> {fixed_expected_path} (from root.observed)") + source_label = "replay output" if expected_from == "replay" else "root.observed" + print(f" write: -> {fixed_expected_path} (from {source_label})") if resources_from.exists(): print(f" move: resources -> {resources_to}") if validate: print(" validate: would run") return + if expected_from == "replay" or validate: + import fetchgraph.tracer.handlers # noqa: F401 + + root_case, ctx = load_case_bundle(case_path) + replay_out = run_case(root_case, ctx) + else: + replay_out = None + + expected_payload = replay_out if expected_from == "replay" else observed fixed_expected_path.parent.mkdir(parents=True, exist_ok=True) tmp_expected_path = fixed_expected_path.with_suffix(fixed_expected_path.suffix + ".tmp") tmp_expected_path.write_text( - json.dumps(observed, ensure_ascii=False, sort_keys=True, indent=2), + json.dumps(expected_payload, ensure_ascii=False, sort_keys=True, indent=2), encoding="utf-8", ) tx = _MoveTransaction(git_ops) @@ -408,7 +486,6 @@ def fixture_green( tx.remove(known_expected_path) tmp_expected_path.replace(fixed_expected_path) - tx.commit() except Exception: if fixed_expected_path.exists(): fixed_expected_path.unlink() @@ -418,18 +495,29 @@ def fixture_green( raise if validate: - import fetchgraph.tracer.handlers # noqa: F401 + try: + root_case, ctx = load_case_bundle(fixed_case_path) + out = run_case(root_case, ctx) + expected = json.loads(fixed_expected_path.read_text(encoding="utf-8")) + if out != expected: + raise AssertionError(_format_fixture_diff(out, expected)) + except Exception as exc: + tx.rollback() + if fixed_expected_path.exists(): + fixed_expected_path.unlink() + raise AssertionError( + "fixture-green validation failed; rollback completed.\n" + f"fixture: {fixed_case_path}\n" + f"{exc}" + ) from exc - root_case, ctx = load_case_bundle(fixed_case_path) - out = run_case(root_case, ctx) - expected = json.loads(fixed_expected_path.read_text(encoding="utf-8")) - if out != expected: - raise AssertionError("Replay output does not match expected after fixture-green") + tx.commit() print("fixture-green:") print(f" case: {known_case_path}") print(f" move: -> {fixed_case_path}") - print(f" write: -> {fixed_expected_path} (from root.observed)") + source_label = "replay output" if expected_from == "replay" else "root.observed" + print(f" write: -> {fixed_expected_path} (from {source_label})") if resources_from.exists(): print(f" move: resources -> {resources_to}") if validate: diff --git a/tests/helpers/replay_dx.py b/tests/helpers/replay_dx.py index d2e4a5e7..d56df3c2 100644 --- a/tests/helpers/replay_dx.py +++ b/tests/helpers/replay_dx.py @@ -58,7 +58,7 @@ def build_rerun_hints(bundle_path: Path) -> list[str]: stem = bundle_path.stem return [ f"pytest -k {stem} -m known_bad -vv", - f"fetchgraph-tracer fixture-green --case {bundle_path} --validate", + f"fetchgraph-tracer fixture-green --case {bundle_path}", f"fetchgraph-tracer replay --case {bundle_path} --debug", ] diff --git a/tests/test_replay_fixed.py b/tests/test_replay_fixed.py index 37f7e42b..28f26371 100644 --- a/tests/test_replay_fixed.py +++ b/tests/test_replay_fixed.py @@ -36,6 +36,34 @@ def _expected_path(case_path: Path) -> Path: return case_path.with_name(case_path.name.replace(".case.json", ".expected.json")) +def _first_diff_path(left: object, right: object, *, prefix: str = "") -> str | None: + if type(left) is not type(right): + return prefix or "" + if isinstance(left, dict): + left_keys = set(left.keys()) + right_keys = set(right.keys()) + for key in sorted(left_keys | right_keys): + next_prefix = f"{prefix}.{key}" if prefix else str(key) + if key not in left or key not in right: + return next_prefix + diff = _first_diff_path(left[key], right[key], prefix=next_prefix) + if diff is not None: + return diff + return None + if isinstance(left, list): + if len(left) != len(right): + return f"{prefix}.length" if prefix else "length" + for idx, (l_item, r_item) in enumerate(zip(left, right), start=1): + next_prefix = f"{prefix}[{idx}]" if prefix else f"[{idx}]" + diff = _first_diff_path(l_item, r_item, prefix=next_prefix) + if diff is not None: + return diff + return None + if left != right: + return prefix or "" + return None + + @pytest.mark.parametrize("case_path", _iter_case_params()) def test_replay_fixed_cases(case_path: Path) -> None: expected_path = _expected_path(case_path) @@ -52,6 +80,19 @@ def test_replay_fixed_cases(case_path: Path) -> None: input_limit, meta_limit = truncate_limits() meta = root.get("meta") if isinstance(root, dict) else None note = root.get("note") if isinstance(root, dict) else None + missing_keys = [] + extra_keys = [] + if isinstance(out, dict) and isinstance(expected, dict): + out_keys = set(out.keys()) + expected_keys = set(expected.keys()) + missing_keys = sorted(expected_keys - out_keys) + extra_keys = sorted(out_keys - expected_keys) + out_spec_path = None + if isinstance(out, dict) and isinstance(expected, dict): + out_spec = out.get("out_spec") + expected_spec = expected.get("out_spec") + if out_spec != expected_spec: + out_spec_path = _first_diff_path(out_spec, expected_spec, prefix="out_spec") message = "\n".join( [ "Fixed fixture mismatch.", @@ -59,6 +100,10 @@ def test_replay_fixed_cases(case_path: Path) -> None: f"meta: {truncate(format_json(meta, max_chars=meta_limit), limit=meta_limit)}", f"note: {truncate(format_json(note, max_chars=meta_limit), limit=meta_limit)}", f"out: {truncate(format_json(out, max_chars=input_limit), limit=input_limit)}", + f"expected: {truncate(format_json(expected, max_chars=input_limit), limit=input_limit)}", + f"missing_keys: {missing_keys}", + f"extra_keys: {extra_keys}", + f"out_spec_path: {out_spec_path}", ] ) pytest.fail(message, pytrace=False) diff --git a/tests/test_replay_known_bad_backlog.py b/tests/test_replay_known_bad_backlog.py index f8500bbe..58af4769 100644 --- a/tests/test_replay_known_bad_backlog.py +++ b/tests/test_replay_known_bad_backlog.py @@ -153,7 +153,7 @@ def test_known_bad_backlog(bundle_path: Path) -> None: "KNOWN_BAD is now PASSING. Promote to fixed", *common_block, "Promote this fixture to fixed (freeze expected) and remove it from known_bad.", - f"Command: fetchgraph-tracer fixture-green --case {bundle_path} --validate", + f"Command: fetchgraph-tracer fixture-green --case {bundle_path}", "expected will be created from root.observed", ] ) diff --git a/tests/test_tracer_fixture_tools.py b/tests/test_tracer_fixture_tools.py index 99d9363a..7de0842d 100644 --- a/tests/test_tracer_fixture_tools.py +++ b/tests/test_tracer_fixture_tools.py @@ -39,7 +39,7 @@ def test_fixture_green_requires_observed(tmp_path: Path) -> None: _write_bundle(case_path, payload) with pytest.raises(ValueError, match="root.observed is missing"): - fixture_green(case_path=case_path, out_root=root) + fixture_green(case_path=case_path, out_root=root, expected_from="observed") def test_fixture_green_moves_case_and_resources(tmp_path: Path) -> None: @@ -53,13 +53,13 @@ def test_fixture_green_moves_case_and_resources(tmp_path: Path) -> None: "type": "replay_case", "v": 2, "id": "plan_normalize.spec_v1", - "input": {"spec": {"provider": "sql"}}, + "input": {"spec": {"provider": "sql"}, "options": {}}, "observed": {"out_spec": {"provider": "sql"}}, } ) _write_bundle(case_path, payload) - fixture_green(case_path=case_path, out_root=root) + fixture_green(case_path=case_path, out_root=root, expected_from="replay") fixed_case = root / "fixed" / "case.case.json" expected_path = root / "fixed" / "case.expected.json" @@ -71,6 +71,28 @@ def test_fixture_green_moves_case_and_resources(tmp_path: Path) -> None: assert not resources_dir.exists() +def test_fixture_green_rolls_back_on_validation_failure(tmp_path: Path) -> None: + root = tmp_path / "fixtures" + case_path = root / "known_bad" / "case.case.json" + payload = _bundle_payload( + { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}, "options": {}}, + "observed": {"out_spec": {"provider": "other"}}, + } + ) + _write_bundle(case_path, payload) + + with pytest.raises(AssertionError, match="rollback completed"): + fixture_green(case_path=case_path, out_root=root, expected_from="observed") + + assert case_path.exists() + assert not (root / "fixed" / "case.case.json").exists() + assert not (root / "fixed" / "case.expected.json").exists() + + def test_fixture_fix_renames_and_updates_resource_paths(tmp_path: Path) -> None: root = tmp_path / "fixtures" bucket = "fixed" From 0ac1dc7b19ac338dbdf6c383f94bf3bd95b3b338 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Mon, 26 Jan 2026 09:18:19 +0300 Subject: [PATCH 69/79] Fix fixture-green observed hints --- src/fetchgraph/tracer/fixture_tools.py | 2 +- tests/test_replay_known_bad_backlog.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py index 37aee81e..95c4c778 100644 --- a/src/fetchgraph/tracer/fixture_tools.py +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -424,7 +424,7 @@ def fixture_green( raise ValueError( "Cannot green fixture: root.observed is missing.\n" f"Case: {case_path}\n" - "Hint: export observed-first replay_case bundles; green requires observed to freeze behavior." + "Hint: re-export observed-first replay_case bundles or use --expected-from replay." ) known_case_path = known_layout.case_path(stem) diff --git a/tests/test_replay_known_bad_backlog.py b/tests/test_replay_known_bad_backlog.py index 58af4769..c48f70e3 100644 --- a/tests/test_replay_known_bad_backlog.py +++ b/tests/test_replay_known_bad_backlog.py @@ -154,7 +154,7 @@ def test_known_bad_backlog(bundle_path: Path) -> None: *common_block, "Promote this fixture to fixed (freeze expected) and remove it from known_bad.", f"Command: fetchgraph-tracer fixture-green --case {bundle_path}", - "expected will be created from root.observed", + "expected will be created from replay output (default)", ] ) pytest.fail(message, pytrace=False) From e248500368c8a9f46b8add2fae58ff0aee40fc47 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Mon, 26 Jan 2026 09:19:26 +0300 Subject: [PATCH 70/79] Deduplicate diff path helper --- src/fetchgraph/tracer/diff_utils.py | 29 ++++++++++++++++++++ src/fetchgraph/tracer/fixture_tools.py | 31 ++------------------- tests/test_replay_fixed.py | 38 ++------------------------ 3 files changed, 34 insertions(+), 64 deletions(-) create mode 100644 src/fetchgraph/tracer/diff_utils.py diff --git a/src/fetchgraph/tracer/diff_utils.py b/src/fetchgraph/tracer/diff_utils.py new file mode 100644 index 00000000..59913ff7 --- /dev/null +++ b/src/fetchgraph/tracer/diff_utils.py @@ -0,0 +1,29 @@ +from __future__ import annotations + + +def first_diff_path(left: object, right: object, *, prefix: str = "") -> str | None: + if type(left) is not type(right): + return prefix or "" + if isinstance(left, dict): + left_keys = set(left.keys()) + right_keys = set(right.keys()) + for key in sorted(left_keys | right_keys): + next_prefix = f"{prefix}.{key}" if prefix else str(key) + if key not in left or key not in right: + return next_prefix + diff = first_diff_path(left[key], right[key], prefix=next_prefix) + if diff is not None: + return diff + return None + if isinstance(left, list): + if len(left) != len(right): + return f"{prefix}.length" if prefix else "length" + for idx, (l_item, r_item) in enumerate(zip(left, right), start=1): + next_prefix = f"{prefix}[{idx}]" if prefix else f"[{idx}]" + diff = first_diff_path(l_item, r_item, prefix=next_prefix) + if diff is not None: + return diff + return None + if left != right: + return prefix or "" + return None diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py index 95c4c778..f9b00615 100644 --- a/src/fetchgraph/tracer/fixture_tools.py +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from pathlib import Path +from .diff_utils import first_diff_path from .fixture_layout import FixtureLayout, find_case_bundles from .runtime import load_case_bundle, run_case @@ -78,34 +79,6 @@ def _top_level_key_diff(output: object, expected: object) -> tuple[list[str], li return (missing, extra) -def _first_diff_path(left: object, right: object, *, prefix: str = "") -> str | None: - if type(left) is not type(right): - return prefix or "" - if isinstance(left, dict): - left_keys = set(left.keys()) - right_keys = set(right.keys()) - for key in sorted(left_keys | right_keys): - next_prefix = f"{prefix}.{key}" if prefix else str(key) - if key not in left or key not in right: - return next_prefix - diff = _first_diff_path(left[key], right[key], prefix=next_prefix) - if diff is not None: - return diff - return None - if isinstance(left, list): - if len(left) != len(right): - return f"{prefix}.length" if prefix else "length" - for idx, (l_item, r_item) in enumerate(zip(left, right), start=1): - next_prefix = f"{prefix}[{idx}]" if prefix else f"[{idx}]" - diff = _first_diff_path(l_item, r_item, prefix=next_prefix) - if diff is not None: - return diff - return None - if left != right: - return prefix or "" - return None - - def _format_fixture_diff(output: object, expected: object) -> str: missing, extra = _top_level_key_diff(output, expected) lines = [ @@ -117,7 +90,7 @@ def _format_fixture_diff(output: object, expected: object) -> str: out_spec = output.get("out_spec") exp_spec = expected.get("out_spec") if out_spec != exp_spec: - diff_path = _first_diff_path(out_spec, exp_spec, prefix="out_spec") + diff_path = first_diff_path(out_spec, exp_spec, prefix="out_spec") if diff_path: lines.append(f"out_spec diff path: {diff_path}") lines.append(f"expected: {_format_json(expected)}") diff --git a/tests/test_replay_fixed.py b/tests/test_replay_fixed.py index 28f26371..fd2a7767 100644 --- a/tests/test_replay_fixed.py +++ b/tests/test_replay_fixed.py @@ -9,12 +9,8 @@ import fetchgraph.tracer.handlers # noqa: F401 import tests.helpers.handlers_resource_read # noqa: F401 from fetchgraph.tracer.runtime import load_case_bundle, run_case -from tests.helpers.replay_dx import ( - format_json, - ids_from_path, - truncate, - truncate_limits, -) +from fetchgraph.tracer.diff_utils import first_diff_path +from tests.helpers.replay_dx import format_json, ids_from_path, truncate, truncate_limits REPLAY_CASES_ROOT = Path(__file__).parent / "fixtures" / "replay_cases" FIXED_DIR = REPLAY_CASES_ROOT / "fixed" @@ -36,34 +32,6 @@ def _expected_path(case_path: Path) -> Path: return case_path.with_name(case_path.name.replace(".case.json", ".expected.json")) -def _first_diff_path(left: object, right: object, *, prefix: str = "") -> str | None: - if type(left) is not type(right): - return prefix or "" - if isinstance(left, dict): - left_keys = set(left.keys()) - right_keys = set(right.keys()) - for key in sorted(left_keys | right_keys): - next_prefix = f"{prefix}.{key}" if prefix else str(key) - if key not in left or key not in right: - return next_prefix - diff = _first_diff_path(left[key], right[key], prefix=next_prefix) - if diff is not None: - return diff - return None - if isinstance(left, list): - if len(left) != len(right): - return f"{prefix}.length" if prefix else "length" - for idx, (l_item, r_item) in enumerate(zip(left, right), start=1): - next_prefix = f"{prefix}[{idx}]" if prefix else f"[{idx}]" - diff = _first_diff_path(l_item, r_item, prefix=next_prefix) - if diff is not None: - return diff - return None - if left != right: - return prefix or "" - return None - - @pytest.mark.parametrize("case_path", _iter_case_params()) def test_replay_fixed_cases(case_path: Path) -> None: expected_path = _expected_path(case_path) @@ -92,7 +60,7 @@ def test_replay_fixed_cases(case_path: Path) -> None: out_spec = out.get("out_spec") expected_spec = expected.get("out_spec") if out_spec != expected_spec: - out_spec_path = _first_diff_path(out_spec, expected_spec, prefix="out_spec") + out_spec_path = first_diff_path(out_spec, expected_spec, prefix="out_spec") message = "\n".join( [ "Fixed fixture mismatch.", From 4ed2a7b8d9d7d50abf8bd2a34de3500126d14067 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Mon, 26 Jan 2026 09:23:48 +0300 Subject: [PATCH 71/79] Guard fixture-rm path usage for BUCKET=all --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 0d3c3736..97d4ee75 100644 --- a/Makefile +++ b/Makefile @@ -459,6 +459,9 @@ fixture-rm: if [ -f "$$case_value" ]; then \ case_args="--case $$case_value"; \ elif [[ "$$case_value" == *".case.json" || "$$case_value" == *"/"* ]]; then \ + if [ "$(BUCKET)" = "all" ]; then \ + echo "CASE с относительным путем нельзя использовать при BUCKET=all; укажите --name/--case-id/--pattern." && exit 1; \ + fi; \ case_args="--case $(TRACER_ROOT)/$(BUCKET)/$$case_value"; \ else \ case_args="--case-id $$case_value"; \ From 0ba8f5ba3a6c1e31c6467b5d5fd7a75e18f86b51 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Mon, 26 Jan 2026 09:29:44 +0300 Subject: [PATCH 72/79] Revert "Guard fixture-rm path usage for BUCKET=all" This reverts commit 2685c7dc10e1586321b13073d8a586b631aab156. --- Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile b/Makefile index 97d4ee75..0d3c3736 100644 --- a/Makefile +++ b/Makefile @@ -459,9 +459,6 @@ fixture-rm: if [ -f "$$case_value" ]; then \ case_args="--case $$case_value"; \ elif [[ "$$case_value" == *".case.json" || "$$case_value" == *"/"* ]]; then \ - if [ "$(BUCKET)" = "all" ]; then \ - echo "CASE с относительным путем нельзя использовать при BUCKET=all; укажите --name/--case-id/--pattern." && exit 1; \ - fi; \ case_args="--case $(TRACER_ROOT)/$(BUCKET)/$$case_value"; \ else \ case_args="--case-id $$case_value"; \ From bd409aff8a6087bb7295fbd43e661cf84082070a Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Mon, 26 Jan 2026 09:39:17 +0300 Subject: [PATCH 73/79] Expose diff utils and adjust fixture-rm heuristic --- Makefile | 2 ++ src/fetchgraph/tracer/__init__.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 0d3c3736..ba3ee912 100644 --- a/Makefile +++ b/Makefile @@ -460,6 +460,8 @@ fixture-rm: case_args="--case $$case_value"; \ elif [[ "$$case_value" == *".case.json" || "$$case_value" == *"/"* ]]; then \ case_args="--case $(TRACER_ROOT)/$(BUCKET)/$$case_value"; \ + elif [[ "$$case_value" == *"__"* ]]; then \ + case_args="--name $$case_value"; \ else \ case_args="--case-id $$case_value"; \ fi; \ diff --git a/src/fetchgraph/tracer/__init__.py b/src/fetchgraph/tracer/__init__.py index a0d19f20..6d5e552f 100644 --- a/src/fetchgraph/tracer/__init__.py +++ b/src/fetchgraph/tracer/__init__.py @@ -2,11 +2,13 @@ from fetchgraph.replay.log import EventLoggerLike, log_replay_case from fetchgraph.replay.runtime import REPLAY_HANDLERS, ReplayContext, load_case_bundle, run_case +from fetchgraph.tracer.diff_utils import first_diff_path __all__ = [ "EventLoggerLike", "REPLAY_HANDLERS", "ReplayContext", + "first_diff_path", "load_case_bundle", "log_replay_case", "run_case", From 86c2b3bc0923ba8f040306695c270fbfd5956ba0 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Mon, 26 Jan 2026 09:42:29 +0300 Subject: [PATCH 74/79] Add fixture-green name heuristic --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index ba3ee912..e3a08e6d 100644 --- a/Makefile +++ b/Makefile @@ -436,6 +436,8 @@ fixture-green: case_args="--case $$case_value"; \ elif [[ "$$case_value" == *".case.json" || "$$case_value" == *"/"* ]]; then \ case_args="--case $(TRACER_ROOT)/known_bad/$$case_value"; \ + elif [[ "$$case_value" == *"__"* ]]; then \ + case_args="--name $$case_value"; \ else \ case_args="--case-id $$case_value"; \ fi; \ From 8129cfde8a35a69674c382d16f632ef6496ec418 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Mon, 26 Jan 2026 10:41:02 +0300 Subject: [PATCH 75/79] Default-enable event logging; allow explicit disable via None (#126) * Allow explicit disabling of event logging * Refine event logger sentinel typing --- examples/demo_qa/runner.py | 21 ++++++++++++++++--- examples/demo_qa/tests/test_demo_qa_runner.py | 12 +++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/examples/demo_qa/runner.py b/examples/demo_qa/runner.py index 30bc4a18..805f4b44 100644 --- a/examples/demo_qa/runner.py +++ b/examples/demo_qa/runner.py @@ -11,7 +11,7 @@ import uuid from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, Iterable, List, Mapping, TypedDict +from typing import Dict, Iterable, List, Mapping, Protocol, TypedDict from typing_extensions import NotRequired @@ -21,6 +21,16 @@ from fetchgraph.replay.snapshots import snapshot_provider_catalog from fetchgraph.utils import set_run_id +class CaseEventLoggerFactory(Protocol): + def for_case(self, case_id: str, events_path: Path) -> "EventLogger": ... + + +class _DefaultEventLoggerSentinel: + pass + + +_DEFAULT_EVENT_LOGGER = _DefaultEventLoggerSentinel() + @dataclass class RunTimings: @@ -380,7 +390,7 @@ def run_one( artifacts_root: Path, *, plan_only: bool = False, - event_logger: EventLogger | None = None, + event_logger: CaseEventLoggerFactory | None | _DefaultEventLoggerSentinel = _DEFAULT_EVENT_LOGGER, run_dir: Path | None = None, schema_path: Path | None = None, ) -> RunResult: @@ -392,7 +402,12 @@ def run_one( run_dir.mkdir(parents=True, exist_ok=True) events_path = run_dir / "events.jsonl" - case_logger = event_logger.for_case(case.id, events_path) if event_logger else None + if event_logger is None: + case_logger = None + elif event_logger is _DEFAULT_EVENT_LOGGER: + case_logger = EventLogger(events_path, run_id, case.id) + else: + case_logger = event_logger.for_case(case.id, events_path) if case_logger: case_logger.emit({"type": "run_started", "case_id": case.id, "run_dir": str(run_dir)}) case_logger.emit({"type": "case_started", "case_id": case.id, "run_dir": str(run_dir)}) diff --git a/examples/demo_qa/tests/test_demo_qa_runner.py b/examples/demo_qa/tests/test_demo_qa_runner.py index 545f0550..09645f81 100644 --- a/examples/demo_qa/tests/test_demo_qa_runner.py +++ b/examples/demo_qa/tests/test_demo_qa_runner.py @@ -74,6 +74,18 @@ def _boom(*args, **kwargs): assert any("run_failed" in line for line in events) +def test_run_one_with_events_disabled_does_not_write_file(tmp_path) -> None: + case = Case(id="case_no_events", question="Q") + artifacts_root = tmp_path / "runs" + + run_one(case, _ArtifactRunner(), artifacts_root, event_logger=None) + + case_dirs = list(artifacts_root.iterdir()) + assert case_dirs + events_path = case_dirs[0] / "events.jsonl" + assert not events_path.exists() + + def test_match_expected_unchecked_when_no_expectations() -> None: case = Case(id="c1", question="What is foo?") assert _match_expected(case, "anything") is None From 28a483d8478bc29f1a44f6e2829b7de216c2099f Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Mon, 26 Jan 2026 10:51:13 +0300 Subject: [PATCH 76/79] Fix relational selector validation (#125) * Fix relational selector validation * Add replay validator checks for fixtures * Use shared replay validators in known_bad tests * Require replay validators in fixture-green and fixed tests --- src/fetchgraph/tracer/fixture_tools.py | 6 ++++++ src/fetchgraph/tracer/validators.py | 22 +++++++++++++++++++++- tests/test_replay_fixed.py | 9 +++++++++ tests/test_replay_known_bad_backlog.py | 13 +++---------- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py index f9b00615..fe572844 100644 --- a/src/fetchgraph/tracer/fixture_tools.py +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -10,6 +10,7 @@ from .diff_utils import first_diff_path from .fixture_layout import FixtureLayout, find_case_bundles from .runtime import load_case_bundle, run_case +from .validators import REPLAY_VALIDATORS @dataclass(frozen=True) @@ -474,6 +475,11 @@ def fixture_green( expected = json.loads(fixed_expected_path.read_text(encoding="utf-8")) if out != expected: raise AssertionError(_format_fixture_diff(out, expected)) + replay_id = root_case.get("id") if isinstance(root_case, dict) else None + validator = REPLAY_VALIDATORS.get(replay_id) + if validator is None: + raise AssertionError(f"No validator registered for replay id={replay_id!r}") + validator(out) except Exception as exc: tx.rollback() if fixed_expected_path.exists(): diff --git a/src/fetchgraph/tracer/validators.py b/src/fetchgraph/tracer/validators.py index 01c32bc5..ee1e1d6e 100644 --- a/src/fetchgraph/tracer/validators.py +++ b/src/fetchgraph/tracer/validators.py @@ -4,6 +4,8 @@ from fetchgraph.relational.models import RelationalRequest +RELATIONAL_PROVIDERS = {"demo_qa", "relational"} + def validate_plan_normalize_spec_v1(out: dict) -> None: if not isinstance(out, dict): @@ -11,11 +13,29 @@ def validate_plan_normalize_spec_v1(out: dict) -> None: out_spec = out.get("out_spec") if not isinstance(out_spec, dict): raise AssertionError("Output must contain out_spec dict") + provider = out_spec.get("provider") selectors = out_spec.get("selectors") if selectors is None: raise AssertionError("out_spec.selectors is required") if not isinstance(selectors, dict): raise AssertionError("out_spec.selectors must be a dict") - if "root_entity" not in selectors: + is_relational = ( + provider in RELATIONAL_PROVIDERS + or (isinstance(provider, str) and provider.startswith("relational")) + or any(key in selectors for key in ("root_entity", "relations", "entity")) + ) + if not is_relational: return + if "root_entity" not in selectors: + if "entity" in selectors: + raise AssertionError( + "Relational selectors missing required key 'root_entity' " + "(looks like you produced 'entity' instead)." + ) + raise AssertionError("Relational selectors missing required key 'root_entity'") TypeAdapter(RelationalRequest).validate_python(selectors) + + +REPLAY_VALIDATORS = { + "plan_normalize.spec_v1": validate_plan_normalize_spec_v1, +} diff --git a/tests/test_replay_fixed.py b/tests/test_replay_fixed.py index fd2a7767..b59bbc1e 100644 --- a/tests/test_replay_fixed.py +++ b/tests/test_replay_fixed.py @@ -10,6 +10,7 @@ import tests.helpers.handlers_resource_read # noqa: F401 from fetchgraph.tracer.runtime import load_case_bundle, run_case from fetchgraph.tracer.diff_utils import first_diff_path +from fetchgraph.tracer.validators import REPLAY_VALIDATORS from tests.helpers.replay_dx import format_json, ids_from_path, truncate, truncate_limits REPLAY_CASES_ROOT = Path(__file__).parent / "fixtures" / "replay_cases" @@ -75,6 +76,14 @@ def test_replay_fixed_cases(case_path: Path) -> None: ] ) pytest.fail(message, pytrace=False) + replay_id = root.get("id") if isinstance(root, dict) else None + validator = REPLAY_VALIDATORS.get(replay_id) + if validator is None: + pytest.fail( + f"No validator registered for replay id={replay_id!r}. Add it to REPLAY_VALIDATORS.", + pytrace=False, + ) + validator(out) assert out == expected diff --git a/tests/test_replay_known_bad_backlog.py b/tests/test_replay_known_bad_backlog.py index c48f70e3..fcaea965 100644 --- a/tests/test_replay_known_bad_backlog.py +++ b/tests/test_replay_known_bad_backlog.py @@ -3,14 +3,12 @@ import json import traceback from pathlib import Path -from typing import Callable - import pytest from pydantic import ValidationError import fetchgraph.tracer.handlers # noqa: F401 from fetchgraph.tracer.runtime import load_case_bundle, run_case -from fetchgraph.tracer.validators import validate_plan_normalize_spec_v1 +from fetchgraph.tracer.validators import REPLAY_VALIDATORS from tests.helpers.replay_dx import ( build_rerun_hints, debug_enabled, @@ -25,11 +23,6 @@ REPLAY_CASES_ROOT = Path(__file__).parent / "fixtures" / "replay_cases" KNOWN_BAD_DIR = REPLAY_CASES_ROOT / "known_bad" -VALIDATORS: dict[str, Callable[[dict], None]] = { - "plan_normalize.spec_v1": validate_plan_normalize_spec_v1, -} - - def _iter_known_bad_paths() -> list[Path]: if not KNOWN_BAD_DIR.exists(): return [] @@ -93,10 +86,10 @@ def test_known_bad_backlog(bundle_path: Path) -> None: if not isinstance(replay_id, str) or not replay_id: pytest.fail("Replay id missing or invalid in bundle root.", pytrace=False) replay_id_str: str = replay_id - validator = VALIDATORS.get(replay_id_str) + validator = REPLAY_VALIDATORS.get(replay_id_str) if validator is None: pytest.fail( - f"No validator registered for replay id={replay_id!r}. Add it to VALIDATORS.", + f"No validator registered for replay id={replay_id!r}. Add it to REPLAY_VALIDATORS.", pytrace=False, ) input_limit, meta_limit = truncate_limits() From c3b69353f2a8b721e3fc6b634f793c2c2df7a331 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Mon, 26 Jan 2026 14:26:29 +0300 Subject: [PATCH 77/79] Add validator for resource_read replay (#127) --- src/fetchgraph/tracer/validators.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/fetchgraph/tracer/validators.py b/src/fetchgraph/tracer/validators.py index ee1e1d6e..f921b311 100644 --- a/src/fetchgraph/tracer/validators.py +++ b/src/fetchgraph/tracer/validators.py @@ -36,6 +36,15 @@ def validate_plan_normalize_spec_v1(out: dict) -> None: TypeAdapter(RelationalRequest).validate_python(selectors) +def validate_resource_read_v1(out: dict) -> None: + if not isinstance(out, dict): + raise AssertionError("Output must be a dict") + text = out.get("text") + if not isinstance(text, str): + raise AssertionError("Output must include text string") + + REPLAY_VALIDATORS = { "plan_normalize.spec_v1": validate_plan_normalize_spec_v1, + "resource_read.v1": validate_resource_read_v1, } From 6ab3c51c3acccfc8eae85e027e12632b806fa4b8 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 26 Jan 2026 14:27:27 +0300 Subject: [PATCH 78/79] =?UTF-8?q?=D0=BE=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0=20=D0=B4=D0=BE=D0=BA=D1=83=D0=BC=D0=B5=D0=BD?= =?UTF-8?q?=D1=82=D0=B0=D1=86=D0=B8=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/fetchgraph/tracer/fetchgraph_tracer.md | 486 ++++++++++++++------- 1 file changed, 331 insertions(+), 155 deletions(-) diff --git a/src/fetchgraph/tracer/fetchgraph_tracer.md b/src/fetchgraph/tracer/fetchgraph_tracer.md index 895c1799..7fcdbfd7 100644 --- a/src/fetchgraph/tracer/fetchgraph_tracer.md +++ b/src/fetchgraph/tracer/fetchgraph_tracer.md @@ -6,9 +6,12 @@ > - `handlers/*` — обработчики (регистрация в `REPLAY_HANDLERS`) > - `export.py` — экспорт `replay_case` → case bundle (`*.case.json`) +> Актуально для консольной команды `fetchgraph-tracer` и модулей `fetchgraph.tracer/*` + `fetchgraph.replay/*`. +> Путь к фикстурам по умолчанию: `tests/fixtures/replay_cases/{fixed,known_bad}`. + --- -## 1) Зачем это нужно +## 0) Зачем это нужно Трейсер решает две задачи: @@ -20,255 +23,428 @@ --- -## 2) Ключевые понятия +## 1) Термины и базовые сущности -### Event stream (JSONL) -События пишутся как JSONL: одна строка = одно событие. +### 1.1 Event stream (JSONL) -Пример общего вида: -```json -{"timestamp":"2026-01-25T00:00:00Z","run_id":"abcd1234","type":"...","...": "..."} -``` +Трейс — это JSONL/NDJSON: одна строка = одно событие. + +События, которые важны для реплея: -### Replay case (v2) -Событие `type="replay_case"`, `v=2`: -- `id`: идентификатор обработчика -- `meta`: опциональные метаданные (например `spec_idx`, `provider`) -- `input`: вход для реплея -- `input.provider_info_snapshot`: минимальный snapshot провайдера (например `selectors_schema`), чтобы реплей был детерминированным без extras -- **ровно одно** из `observed` или `observed_error` -- `requires`: опциональный список зависимостей `[{"kind":"extra"|"resource","id":"..."}]` +- `type="replay_case"` — корневое событие “кейса реплея” (v2). +- `type="planner_input"` — “extra” (например, входы планировщика), адресуется по `id`. +- `type="replay_resource"` — “resource” (например, файл), адресуется по `id`. + +### 1.2 Replay case (v2) + +Схема корневого события: + +- `type="replay_case"`, `v=2` +- `id` — идентификатор обработчика (namespace вида `something.v1`) +- `meta` — опционально (например, `{spec_idx, provider}`) +- `input` — dict, вход для обработчика + - `input.provider_info_snapshot` — минимальный snapshot провайдера (например, `selectors_schema`), чтобы реплей был детерминированным +- **ровно одно** из: + - `observed` — dict (observed outcome) + - `observed_error` — dict (observed error), если во время “наблюдения” упало +- `requires` — опционально: список зависимостей + - v2-формат: `[{"kind":"extra"|"resource","id":"..."}]` + - legacy-формат допускается в экспорте: `["id1","id2"]` (будет нормализован при экспорте) Пример: + ```json { "type": "replay_case", "v": 2, "id": "plan_normalize.spec_v1", "meta": {"spec_idx": 0, "provider": "sql"}, - "input": {"spec": {...}, "options": {...}, "provider_info_snapshot": {"name": "sql", "selectors_schema": {...}}}, - "observed": {"out_spec": {...}}, - "requires": [{"kind": "extra", "id": "planner_input_v1"}] + "input": { + "spec": {"provider": "sql", "selectors": {"q": "..." }}, + "options": {"lowercase": true}, + "provider_info_snapshot": {"name": "sql", "selectors_schema": {"type":"object","properties":{}}} + }, + "observed": {"out_spec": {"provider": "sql", "selectors": {"q": "..."}}}, + "requires": [{"kind":"extra","id":"planner_input_v1"}] } ``` -### Extras / Resources -- **Extras**: события с `type="planner_input"`, ключуются по `id`. -- **Resources**: события с `type="replay_resource"`, ключуются по `id`. - - Файлы указываются в `data_ref.file` (относительный путь внутри `run_dir`). +### 1.3 Resources и extras + +- **Extras**: события `type="planner_input"`, индексируются по `id`. +- **Resources**: события `type="replay_resource"`, индексируются по `id`. + - Если ресурс указывает `data_ref.file`, это **относительный путь** внутри `run_dir` во время экспорта. + - При экспорте файлы копируются в fixture-layout (см. ниже) и `data_ref.file` переписывается на новый относительный путь. --- -## 3) Контракт логгера и хелпер +## 2) Контракт логирования (observed-first) + +### 2.1 EventLoggerLike + +Трейсер принимает любой logger, который умеет: -### `EventLoggerLike` ```py class EventLoggerLike(Protocol): def emit(self, event: dict) -> None: ... ``` -### `log_replay_case` -Хелпер формирует событие v2 и валидирует: -- `id` не пустой +### 2.2 log_replay_case + +`log_replay_case(logger=..., id=..., input=..., observed=.../observed_error=..., requires=..., meta=...)` + +Валидация на входе: + +- `id` — непустая строка - `input` — dict -- XOR `observed` / `observed_error` +- XOR: `observed` / `observed_error` - `requires` — список `{kind,id}` -Пример: -```py -from fetchgraph.tracer import log_replay_case - -log_replay_case( - logger=event_log, - id="plan_normalize.spec_v1", - meta={"spec_idx": i, "provider": spec.provider}, - input={ - "spec": spec.model_dump(), - "options": options.model_dump(), - "provider_info_snapshot": { - "name": spec.provider, - "selectors_schema": provider_info.selectors_schema, - }, - }, - observed={"out_spec": out_spec}, -) -``` - -Если есть файлы, логируйте отдельные события: -```py -event_log.emit({ - "type": "replay_resource", - "id": "catalog_csv", - "data_ref": {"file": "artifacts/catalog.csv"} -}) -``` +**Рекомендация:** логируйте `provider_info_snapshot` (см. `fetchgraph.replay.snapshots`), чтобы реплей не зависел от внешних данных. --- -## 4) Replay runtime +## 3) Replay runtime + +### 3.1 Регистрация обработчиков + +`REPLAY_HANDLERS` — dict `{replay_id: handler(input, ctx) -> dict}`. + +Обработчики регистрируются через side-effect импорта, рекомендуемый способ в тестах/скриптах: -### Регистрация обработчиков -Обработчики регистрируются через side-effect импорта. -Рекомендуемый способ: ```py import fetchgraph.tracer.handlers # noqa: F401 ``` -### `ReplayContext` +### 3.2 ReplayContext + ```py @dataclass(frozen=True) class ReplayContext: resources: dict[str, dict] extras: dict[str, dict] - base_dir: Path | None = None + base_dir: Path | None - def resolve_resource_path(self, resource_path: str | Path) -> Path: ... + def resolve_resource_path(self, resource_path: str | Path) -> Path: + # относительные пути резолвятся относительно base_dir ``` -### `run_case` +### 3.3 run_case и load_case_bundle + ```py -from fetchgraph.tracer.runtime import run_case +from fetchgraph.tracer.runtime import load_case_bundle, run_case -out = run_case(root_case, ctx) +root, ctx = load_case_bundle(Path(".../*.case.json")) +out = run_case(root, ctx) ``` -### `load_case_bundle` -```py -from fetchgraph.tracer.runtime import load_case_bundle +--- + +## 4) Case bundle (fixture) format + +Экспорт создает JSON файл: -root, ctx = load_case_bundle(Path(".../case.case.json")) +- `schema="fetchgraph.tracer.case_bundle"`, `v=1` +- `root` — replay_case (как в events) +- `resources` — dict ресурсов по id +- `extras` — dict extras по id +- `source` — метаданные (минимум: `events_path`, `line`, опционально `run_id`, `timestamp`, `case_id`) + +### 4.1 Layout фикстур и ресурсов + +Рекомендуемый layout: + +``` +tests/fixtures/replay_cases/ + fixed/ + .case.json + .expected.json # для green кейсов + resources///... + known_bad/ + .case.json + resources///... ``` +Где `` — имя фикстуры (обычно совпадает с именем `.case.json` без расширения). + +При экспорте файлы ресурсов копируются в: + +``` +resources/// +``` + +С коллизиями экспорт падает fail-fast (чтобы не смешивать фикстуры из разных прогонов). + --- -## 5) Export case bundles +## 5) Export API (Python) -### Python API ```py from fetchgraph.tracer.export import export_replay_case_bundle, export_replay_case_bundles +# один bundle path = export_replay_case_bundle( - events_path=Path(".../events.jsonl"), - out_dir=Path("tests/fixtures/replay_cases"), + events_path=Path("./events.jsonl"), + out_dir=Path("tests/fixtures/replay_cases/known_bad"), replay_id="plan_normalize.spec_v1", - spec_idx=0, - provider="sql", - run_dir=Path(".../run_dir"), + spec_idx=0, # фильтр по meta.spec_idx (опционально) + provider="sql", # фильтр по meta.provider (опционально) + run_dir=Path("./run_dir"), # обязателен, если есть file-resources allow_bad_json=False, overwrite=False, + selection_policy="latest", # latest|first|last|by-timestamp|by-line + select_index=None, # 1-based индекс среди replay_case матчей + require_unique=False, ) -``` -Все совпадения: -```py +# все совпадения paths = export_replay_case_bundles( - events_path=Path(".../events.jsonl"), - out_dir=Path("tests/fixtures/replay_cases"), + events_path=Path("./events.jsonl"), + out_dir=Path("tests/fixtures/replay_cases/known_bad"), replay_id="plan_normalize.spec_v1", allow_bad_json=True, overwrite=True, ) ``` -### Layout ресурсов -Файлы копируются в: -``` -resources/// +--- + +## 6) CLI: `fetchgraph-tracer` + +Команда подключена как `fetchgraph-tracer = fetchgraph.tracer.cli:main`. + +### 6.1 export-case-bundle + +> Примечание: в старом/экспериментальном синтаксисе мог встречаться флаг `--replay-id`; актуальный флаг — `--id`. + + +Экспорт одного или нескольких case bundles из events: + +```bash +fetchgraph-tracer export-case-bundle --out tests/fixtures/replay_cases/known_bad --id plan_normalize.spec_v1 --events path/to/events.jsonl --run-dir path/to/run_dir --overwrite --allow-bad-json ``` +#### 6.1.1 Как выбирается events/run_dir + +Приоритет разрешения: + +1) `--events` — явный путь к events/trace файлу. + `run_dir` берется из `--case-dir` или `--run-dir` (если нужны file-resources). + +2) Иначе (auto-resolve через `.runs`): + - обязателен `--case ` и `--data ` + - дальше выбираем конкретный run/case: + - `--case-dir ` или `--run-dir ` — явный путь + - `--run-id ` — выбрать запуск и кейс внутри него + - либо “самый свежий” по стратегии `--pick-run` (+ опционально `--tag`) + +Поддерживаемые имена файлов events (быстрый поиск): +`events.jsonl`, `events.ndjson`, `trace.jsonl`, `trace.ndjson`, `traces/events.jsonl`, `traces/trace.jsonl`. + +Если ни один из стандартных путей не найден — делается fallback-поиск по `*.jsonl/*.ndjson` в глубину до 3 уровней (кроме `resources/`) и выбирается самый “тяжелый” файл. + +#### 6.1.2 Run selection (auto-resolve) flags + +- `--case ` — id кейса (например `agg_003`) +- `--data ` — директория с `.runs` +- `--runs-subdir ` — где искать `runs` относительно `DATA_DIR` (default `.runs/runs`) +- `--tag ` — фильтр по tag (если теги ведутся) +- `--pick-run ` — стратегия выбора запуска: + - `latest_non_missed` + - `latest_with_replay` (default; требует `--id`) +- `--select-index ` — выбрать конкретный run-candidate (1-based) +- `--list-matches` — вывести список кандидатов run/case и выйти +- `--debug` — детальный вывод кандидатов (или `DEBUG=1`) + +Полезно: +- `--print-resolve` — распечатать, во что именно разрешилось (`run_dir`, `events_path`). + +#### 6.1.3 Replay-case selection flags (когда в events много replay_case) + +- `--spec-idx ` — фильтр по `meta.spec_idx` +- `--provider ` — фильтр по `meta.provider` (case-insensitive) +- `--list-replay-matches` — вывести найденные replay_case entries и выйти +- `--require-unique` — упасть, если матчей > 1 +- `--select ` — политика выбора: + - `latest` (по timestamp, fallback по line) + - `first` + - `last` + - `by-timestamp` + - `by-line` +- `--replay-select-index ` — выбрать конкретный replay_case матч (1-based) +- `--all` — экспортировать **все** матчинг replay_case (игнорируя single-selection) + --- -## 6) CLI +### 6.2 Управление фикстурами (fixture tools) + +#### 6.2.1 fixture-ls + +Показать кандидатов (по умолчанию bucket=known_bad): -### tracer CLI ```bash -fetchgraph-tracer export-case-bundle \ - --events path/to/events.jsonl \ - --out tests/fixtures/replay_cases \ - --id plan_normalize.spec_v1 \ - --spec-idx 0 \ - --run-dir path/to/run_dir \ - --allow-bad-json \ - --overwrite +fetchgraph-tracer fixture-ls --case-id agg_003 +fetchgraph-tracer fixture-ls --bucket fixed --pattern "plan_normalize.*" ``` -Разрешение источника events: -- `--events` — явный путь к events/trace файлу (самый высокий приоритет). -- `--run-id` или `--case-dir` — выбрать конкретный запуск/кейс. -- `--tag` — выбрать самый свежий кейс с events, совпадающий с тегом. -- по умолчанию — самый свежий кейс с events. +#### 6.2.2 fixture-green -Доступные форматы events: `events.jsonl`, `events.ndjson`, `trace.jsonl`, `trace.ndjson`, `traces/events.jsonl`, `traces/trace.jsonl`. +“Позеленить” кейс: перенести из `known_bad` → `fixed` и записать `*.expected.json` из `root.observed`. -Примеры: ```bash -# По RUN_ID -fetchgraph-tracer export-case-bundle \ - --case agg_003 \ - --data _demo_data/shop \ - --run-id 20260125_160601_retail_cases \ - --id plan_normalize.spec_v1 \ - --out tests/fixtures/replay_cases - -# По TAG -fetchgraph-tracer export-case-bundle \ - --case agg_003 \ - --data _demo_data/shop \ - --tag known_bad \ - --id plan_normalize.spec_v1 \ - --out tests/fixtures/replay_cases +# выбрать по case_id (если несколько — используется --select/--select-index) +fetchgraph-tracer fixture-green --case-id agg_003 --validate + +# или явно указать файл +fetchgraph-tracer fixture-green --case tests/fixtures/replay_cases/known_bad/.case.json --validate ``` -### Makefile +Флаги: +- `--overwrite-expected` — перезаписать `*.expected.json`, если уже есть +- `--validate` — после перемещения прогнать `run_case()` и сравнить с expected +- `--git auto|on|off` — перемещения/удаления через git (если доступно) +- `--dry-run` — только печать действий +- `--select/--select-index/--require-unique` — выбор среди нескольких кандидатов + +> Важно: `fixture-green` требует `root.observed`. Если в кейсе только `observed_error`, сначала нужно переэкспортировать bundle после фикса (чтобы был observed). + +#### 6.2.3 fixture-demote + +Перенести `fixed` → `known_bad` (например, если баг вернулся или кейс решили держать как backlog): + ```bash -make tracer-export REPLAY_ID=plan_normalize.spec_v1 CASE=agg_003 \ - REPLAY_IDATA=_demo_data/shop TRACER_OUT_DIR=tests/fixtures/replay_cases +fetchgraph-tracer fixture-demote --case-id agg_003 +``` + +Флаги: +- `--overwrite` — перезаписать существующие target-фикстуры +- `--all` — применить ко всем матчам +- остальные — как в green (select/dry-run/git) -# Явный events -make tracer-export REPLAY_ID=plan_normalize.spec_v1 EVENTS=path/to/events.jsonl \ - TRACER_OUT_DIR=tests/fixtures/replay_cases RUN_DIR=path/to/run_dir OVERWRITE=1 +#### 6.2.4 fixture-fix -# Фильтр по TAG -make tracer-export REPLAY_ID=plan_normalize.spec_v1 CASE=agg_003 TAG=known_bad \ - TRACER_OUT_DIR=tests/fixtures/replay_cases +Переименовать stem фикстуры (case + expected + resources): -# Явный run/case -make tracer-export REPLAY_ID=plan_normalize.spec_v1 CASE=agg_003 RUN_ID=20260125_160601_retail_cases \ - TRACER_OUT_DIR=tests/fixtures/replay_cases +```bash +fetchgraph-tracer fixture-fix --bucket fixed --name old_stem --new-name new_stem ``` +#### 6.2.5 fixture-migrate + +Нормализовать layout ресурсов/путей внутри bundle (полезно при изменении схемы ресурсов): + +```bash +fetchgraph-tracer fixture-migrate --bucket all --case-id agg_003 --dry-run +``` + +#### 6.2.6 fixture-rm + +Удалить фикстуры (cases/resources): + +```bash +# удалить конкретный stem в known_bad +fetchgraph-tracer fixture-rm --bucket known_bad --name "" + +# удалить по case_id (опционально --all, иначе выберет один по --select) +fetchgraph-tracer fixture-rm --bucket all --case-id agg_003 --scope both --all +``` + +Флаги: +- `--scope cases|resources|both` +- `--pattern ` — матчить по stem/пути +- `--all` — применить ко всем матчам +- `--git auto|on|off`, `--dry-run`, `--select*` + --- -## 7) Тестовый workflow +## 7) Make targets (DX) -### known_bad -```py -import pytest -from pydantic import ValidationError +Makefile предоставляет врапперы (см. `make help`): -import fetchgraph.tracer.handlers # noqa: F401 -from fetchgraph.tracer.runtime import load_case_bundle, run_case -from fetchgraph.tracer.validators import validate_plan_normalize_spec_v1 - -@pytest.mark.known_bad -@pytest.mark.parametrize("case_path", [...]) -def test_known_bad(case_path): - root, ctx = load_case_bundle(case_path) - out = run_case(root, ctx) - with pytest.raises((AssertionError, ValidationError)): - validate_plan_normalize_spec_v1(out) +### 7.1 Экспорт + +```bash +# обычный export (auto-resolve через .runs) +make tracer-export REPLAY_ID=plan_normalize.spec_v1 CASE=agg_003 DATA=_demo_data/shop TRACER_OUT_DIR=tests/fixtures/replay_cases/known_bad + +# явный events + run_dir +make tracer-export REPLAY_ID=plan_normalize.spec_v1 CASE=agg_003 EVENTS=path/to/events.jsonl RUN_DIR=path/to/run_dir TRACER_OUT_DIR=tests/fixtures/replay_cases/known_bad OVERWRITE=1 ``` -### Green (explicit expected) -Если рядом лежит `*.expected.json`, сравниваем его с результатом; иначе можно свериться с `root["observed"]`. +### 7.2 Список кандидатов (auto-resolve) + +> В Makefile `DATA` используется как основной параметр; внутри таргетов он пробрасывается как `REPLAY_IDATA` (alias). + + +```bash +make tracer-ls CASE=agg_003 DATA=_demo_data/shop TAG=known_bad +``` + +### 7.3 Запуск known_bad + +```bash +make known-bad +make known-bad-one NAME= +``` + +### 7.4 Управление фикстурами + +```bash +# promote known_bad -> fixed +make fixture-green CASE=agg_003 VALIDATE=1 + +# list +make fixture-ls CASE=agg_003 BUCKET=known_bad + +# remove +make fixture-rm BUCKET=known_bad CASE=agg_003 ALL=1 DRY=1 + +# rename stem +make fixture-fix BUCKET=fixed NAME=old NEW_NAME=new + +# migrate +make fixture-migrate BUCKET=all CASE=agg_003 DRY=1 + +# demote fixed -> known_bad +make fixture-demote CASE=agg_003 OVERWRITE=1 +``` --- -## 8) Короткая памятка +## 8) Тестовый workflow и CI + +### 8.1 Buckets: fixed vs known_bad + +- `fixed/` — **регрессионные** кейсы: должны проходить и обычно имеют `.expected.json`. +- `known_bad/` — **backlog/TDD-like** кейсы: могут падать или возвращать невалидный результат. + В pytest они должны идти под маркером `known_bad`, чтобы их легко исключать из CI. + +Рекомендуемый CI фильтр: `-m "not known_bad"` (и при необходимости `not slow`). + +### 8.2 Диагностика known_bad + +Для подробной диагностики можно включить окружение: + +- `FETCHGRAPH_REPLAY_DEBUG=1` — больше деталей в выводе +- `FETCHGRAPH_REPLAY_TRUNCATE=` — лимит тримминга больших блоков +- `FETCHGRAPH_REPLAY_META_TRUNCATE=` — лимит для meta +- `FETCHGRAPH_RULE_TRACE_TAIL=` — сколько элементов `diag.rule_trace` показывать + +--- + +## 9) Как добавить новый replay-case + +1) Выберите `replay_id` (строка) и зафиксируйте его в коде. +2) Добавьте handler в `fetchgraph/tracer/handlers/...` и зарегистрируйте в `REPLAY_HANDLERS`. +3) При “наблюдении” логируйте: + - `replay_case` через `log_replay_case(...)` + - `planner_input` / `replay_resource` отдельными событиями, и добавляйте их в `requires` +4) Добавьте валидатор (если нужен) для fixed/regression тестов. +5) Проверьте, что экспорт реплея воспроизводим (нет внешних сервисов, нет LLM). + +--- -1) Логируйте `replay_case` через `log_replay_case` (observed-first). -2) Extras/Resources логируйте отдельными событиями (`planner_input`, `replay_resource`). -3) Экспортируйте bundle через `export_replay_case_bundle(s)`. -4) В тестах грузите bundle через `load_case_bundle` и запускайте `run_case`. +## 10) Совместимость и заметки -Примечание: `log_replay_point` оставлен как deprecated alias, используйте `log_replay_case`. +- `log_replay_point` оставлен как deprecated alias; используйте `log_replay_case`. +- Экспорт поддерживает legacy `requires=["id1","id2"]`, но желательно писать v2-формат `[{kind,id}]`. From 7091ab385c65e2806e9b85e7f466bc05064e3138 Mon Sep 17 00:00:00 2001 From: AlexanderOnischenko <74920855+AlexanderOnischenko@users.noreply.github.com> Date: Mon, 26 Jan 2026 16:45:55 +0300 Subject: [PATCH 79/79] Fix type narrowing for replay utilities (#128) --- src/fetchgraph/tracer/diff_utils.py | 4 ++-- src/fetchgraph/tracer/fixture_tools.py | 2 ++ tests/test_replay_fixed.py | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/fetchgraph/tracer/diff_utils.py b/src/fetchgraph/tracer/diff_utils.py index 59913ff7..c8e04d86 100644 --- a/src/fetchgraph/tracer/diff_utils.py +++ b/src/fetchgraph/tracer/diff_utils.py @@ -4,7 +4,7 @@ def first_diff_path(left: object, right: object, *, prefix: str = "") -> str | None: if type(left) is not type(right): return prefix or "" - if isinstance(left, dict): + if isinstance(left, dict) and isinstance(right, dict): left_keys = set(left.keys()) right_keys = set(right.keys()) for key in sorted(left_keys | right_keys): @@ -15,7 +15,7 @@ def first_diff_path(left: object, right: object, *, prefix: str = "") -> str | N if diff is not None: return diff return None - if isinstance(left, list): + if isinstance(left, list) and isinstance(right, list): if len(left) != len(right): return f"{prefix}.length" if prefix else "length" for idx, (l_item, r_item) in enumerate(zip(left, right), start=1): diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py index fe572844..1ea56366 100644 --- a/src/fetchgraph/tracer/fixture_tools.py +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -476,6 +476,8 @@ def fixture_green( if out != expected: raise AssertionError(_format_fixture_diff(out, expected)) replay_id = root_case.get("id") if isinstance(root_case, dict) else None + if not isinstance(replay_id, str): + raise AssertionError("Case bundle has missing or non-string replay id.") validator = REPLAY_VALIDATORS.get(replay_id) if validator is None: raise AssertionError(f"No validator registered for replay id={replay_id!r}") diff --git a/tests/test_replay_fixed.py b/tests/test_replay_fixed.py index b59bbc1e..6772fb8c 100644 --- a/tests/test_replay_fixed.py +++ b/tests/test_replay_fixed.py @@ -77,6 +77,8 @@ def test_replay_fixed_cases(case_path: Path) -> None: ) pytest.fail(message, pytrace=False) replay_id = root.get("id") if isinstance(root, dict) else None + if not isinstance(replay_id, str): + pytest.fail("Case bundle has missing or non-string replay id.", pytrace=False) validator = REPLAY_VALIDATORS.get(replay_id) if validator is None: pytest.fail(