diff --git a/Makefile b/Makefile index 31458374..4e03d5b0 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ SHELL := /bin/bash # ============================================================================== CONFIG ?= .demo_qa.mk -include $(CONFIG) +-include .demo_qa.mk # ============================================================================== # 2) Значения по умолчанию (для make init) @@ -35,7 +36,8 @@ CLI := $(PYTHON) -m examples.demo_qa.cli # ============================================================================== # 4) Пути demo_qa (можно переопределять через CLI или в $(CONFIG)) # ============================================================================== -DATA ?= +DATA ?= _demo_data/shop +REPLAY_IDATA ?= $(DATA) SCHEMA ?= CASES ?= OUT ?= $(DATA)/.runs/results.jsonl @@ -46,12 +48,32 @@ OUT ?= $(DATA)/.runs/results.jsonl TAG ?= NOTE ?= CASE ?= +NAME ?= +NEW_NAME ?= +PATTERN ?= +SPEC_IDX ?= +PROVIDER ?= +BUCKET ?= known_bad +REPLAY_ID ?= +EVENTS ?= +RUN_ID ?= +CASE_DIR ?= +TRACER_ROOT ?= tests/fixtures/replay_cases +TRACER_OUT_DIR ?= $(TRACER_ROOT)/$(BUCKET) +RUN_DIR ?= +ALLOW_BAD_JSON ?= +OVERWRITE ?= 0 +SCOPE ?= both +WITH_RESOURCES ?= 1 +ALL ?= LIMIT ?= 50 CHANGES ?= 10 NEW_TAG ?= -PATTERN ?= TAGS_FORMAT ?= table TAGS_COLOR ?= auto +SELECT ?= latest +SELECT_INDEX ?= +REQUIRE_UNIQUE ?= 0 ONLY_FAILED_FROM ?= ONLY_MISSED_FROM ?= @@ -70,6 +92,9 @@ PURGE_RUNS ?= 0 PRUNE_HISTORY ?= 0 PRUNE_CASE_HISTORY ?= 0 DRY ?= 0 +VALIDATE ?= 0 +OVERWRITE_EXPECTED ?= 0 +MOVE_TRACES ?= 0 # ============================================================================== # 6) Настройки LLM-конфига (редактирование/просмотр) @@ -99,7 +124,9 @@ LIMIT_FLAG := $(if $(strip $(LIMIT)),--limit $(LIMIT),) batch batch-tag batch-failed batch-failed-from \ batch-missed batch-missed-from batch-failed-tag batch-missed-tag \ batch-fail-fast batch-max-fails \ - stats history-case report-tag report-tag-changes tags tag-rm case-run case-open compare compare-tag + stats history-case report-tag report-tag-changes tags tag-rm case-run case-open tracer-export tracer-ls known-bad known-bad-one \ + fixture-green fixture-demote fixture-ls fixture-rm fixture-fix fixture-migrate \ + compare compare-tag # ============================================================================== # help (на русском) @@ -146,6 +173,22 @@ help: @echo " make tags [PATTERN=*] DATA=... - показать список тегов" @echo " make case-run CASE=case_42 - прогнать один кейс" @echo " make case-open CASE=case_42 - открыть артефакты кейса" + @echo " make tracer-export REPLAY_ID=... CASE=... [EVENTS=...] [RUN_ID=...] [CASE_DIR=...] [DATA=...] [PROVIDER=...] [BUCKET=...] [SPEC_IDX=...] [OVERWRITE=1] [ALLOW_BAD_JSON=1]" + @echo " PROVIDER фильтрует replay_case.meta.provider (обычно spec.provider: sql, relational, ...)" + @echo " make tracer-ls CASE=... [DATA=...] [TAG=...] [RUN_ID=...] [CASE_DIR=...]" + @echo " RUN_ID берётся из make stats/history-case" + @echo " make known-bad - запустить backlog-suite для known_bad (ожидаемо красный)" + @echo " make known-bad-one NAME=fixture_stem - запустить один known_bad кейс" + @echo " (или напрямую: $(PYTHON) -m fetchgraph.tracer.cli export-case-bundle ...)" + @echo " fixtures layout: replay_cases//.case.json, resources: replay_cases//resources///..." + @echo " make fixture-green CASE=agg_003|fixture_stem|path/to/case.case.json [TRACER_ROOT=...] [NO_VALIDATE=1] [EXPECTED_FROM=replay|observed] [OVERWRITE_EXPECTED=1] [DRY=1]" + @echo " make fixture-ls CASE=agg_003 [TRACER_ROOT=...] [BUCKET=known_bad]" + @echo " make fixture-rm CASE=agg_003|fixture_stem|path [SELECT=latest|first|last] [SELECT_INDEX=N] [REQUIRE_UNIQUE=1] [ALL=1]" + @echo " make fixture-migrate CASE=agg_003|fixture_stem|path [SELECT=latest|first|last] [SELECT_INDEX=N] [REQUIRE_UNIQUE=1] [ALL=1]" + @echo " make fixture-demote CASE=agg_003|fixture_stem|path [SELECT=latest|first|last] [SELECT_INDEX=N] [REQUIRE_UNIQUE=1] [ALL=1]" + @echo " make fixture-rm [BUCKET=fixed|known_bad|all] [NAME=...] [PATTERN=...] [SCOPE=cases|resources|both] [DRY=1]" + @echo " make fixture-fix BUCKET=... NAME=... NEW_NAME=... [DRY=1]" + @echo " make fixture-migrate [BUCKET=fixed|known_bad|all] [DRY=1]" @echo "" @echo "Уборка:" @echo " make tag-rm TAG=... [DRY=1] [PURGE_RUNS=1] [PRUNE_HISTORY=1] [PRUNE_CASE_HISTORY=1]" @@ -203,6 +246,24 @@ show-config: @echo "SCHEMA = $(SCHEMA)" @echo "CASES = $(CASES)" @echo "OUT = $(OUT)" + @echo "EVENTS = $(EVENTS)" + @echo "REPLAY_ID = $(REPLAY_ID)" + @echo "SPEC_IDX = $(SPEC_IDX)" + @echo "PROVIDER = $(PROVIDER)" + @echo "RUN_DIR = $(RUN_DIR)" + @echo "TRACER_ROOT = $(TRACER_ROOT)" + @echo "TRACER_OUT_DIR = $(TRACER_OUT_DIR)" + @echo "BUCKET = $(BUCKET)" + @echo "ALLOW_BAD_JSON = $(ALLOW_BAD_JSON)" + @echo "OVERWRITE = $(OVERWRITE)" + @echo "ALL = $(ALL)" + @echo "NAME = $(NAME)" + @echo "NEW_NAME = $(NEW_NAME)" + @echo "PATTERN = $(PATTERN)" + @echo "SCOPE = $(SCOPE)" + @echo "DRY = $(DRY)" + @echo "VALIDATE = $(VALIDATE)" + @echo "OVERWRITE_EXPECTED = $(OVERWRITE_EXPECTED)" @echo "LLM_TOML= $(LLM_TOML)" @echo "TAG = $(TAG)" @echo "NOTE = $(NOTE)" @@ -330,6 +391,140 @@ case-open: check @test -n "$(strip $(CASE))" || (echo "Нужно задать CASE=case_42" && exit 1) @$(CLI) case open "$(CASE)" --data "$(DATA)" +tracer-export: + @test -n "$(strip $(REPLAY_ID))" || (echo "REPLAY_ID обязателен: make tracer-export REPLAY_ID=plan_normalize.spec_v1" && exit 2) + @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make tracer-export CASE=agg_003" && exit 2) + @case "$(BUCKET)" in fixed|known_bad) ;; *) echo "BUCKET должен быть fixed или known_bad для tracer-export" && exit 2 ;; esac + @fetchgraph-tracer export-case-bundle \ + --id "$(REPLAY_ID)" \ + --out "$(TRACER_OUT_DIR)" \ + --case "$(CASE)" \ + --data "$(REPLAY_IDATA)" \ + $(if $(strip $(SPEC_IDX)),--spec-idx "$(SPEC_IDX)",) \ + $(if $(strip $(PROVIDER)),--provider "$(PROVIDER)",) \ + $(if $(RUN_ID),--run-id "$(RUN_ID)",) \ + $(if $(CASE_DIR),--case-dir "$(CASE_DIR)",) \ + $(if $(RUN_DIR),--run-dir "$(RUN_DIR)",) \ + $(if $(EVENTS),--events "$(EVENTS)",) \ + $(if $(TAG),--tag "$(TAG)",) \ + $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) \ + $(if $(filter 1 true yes on,$(ALLOW_BAD_JSON)),--allow-bad-json,) + +tracer-ls: + @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make tracer-ls CASE=agg_003" && exit 2) + @fetchgraph-tracer export-case-bundle \ + --case "$(CASE)" \ + --data "$(REPLAY_IDATA)" \ + --out "$(TRACER_OUT_DIR)" \ + $(if $(RUN_ID),--run-id "$(RUN_ID)",) \ + $(if $(CASE_DIR),--case-dir "$(CASE_DIR)",) \ + $(if $(RUN_DIR),--run-dir "$(RUN_DIR)",) \ + $(if $(EVENTS),--events "$(EVENTS)",) \ + $(if $(strip $(TAG)),--tag "$(TAG)",) \ + --list-matches + +known-bad: + @pytest -m known_bad -vv + +known-bad-one: + @test -n "$(strip $(NAME))" || (echo "NAME обязателен: make known-bad-one NAME=fixture_stem" && exit 2) + @pytest -m known_bad -k "$(NAME)" -vv + +fixture-green: + @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture-green CASE=agg_003 или CASE=path/to/fixture.case.json" && exit 1) + @case_value="$(CASE)"; \ + if [ -f "$$case_value" ]; then \ + case_args="--case $$case_value"; \ + elif [[ "$$case_value" == *".case.json" || "$$case_value" == *"/"* ]]; then \ + case_args="--case $(TRACER_ROOT)/known_bad/$$case_value"; \ + elif [[ "$$case_value" == *"__"* ]]; then \ + case_args="--name $$case_value"; \ + else \ + case_args="--case-id $$case_value"; \ + fi; \ + $(PYTHON) -m fetchgraph.tracer.cli fixture-green $$case_args --root "$(TRACER_ROOT)" \ + $(if $(strip $(SELECT)),--select "$(SELECT)",) \ + $(if $(strip $(SELECT_INDEX)),--select-index "$(SELECT_INDEX)",) \ + $(if $(filter 1 true yes on,$(REQUIRE_UNIQUE)),--require-unique,) \ + $(if $(filter 1 true yes on,$(NO_VALIDATE)),--no-validate,) \ + $(if $(strip $(EXPECTED_FROM)),--expected-from "$(EXPECTED_FROM)",) \ + $(if $(filter 1 true yes on,$(OVERWRITE_EXPECTED)),--overwrite-expected,) \ + $(if $(filter 1 true yes on,$(DRY)),--dry-run,) + +fixture-ls: + @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture-ls CASE=agg_003" && exit 1) + @$(PYTHON) -m fetchgraph.tracer.cli fixture-ls --root "$(TRACER_ROOT)" --bucket "$(BUCKET)" --case-id "$(CASE)" + +fixture-rm: + @case "$(BUCKET)" in fixed|known_bad|all) ;; *) echo "BUCKET должен быть fixed, known_bad или all для fixture-rm" && exit 1 ;; esac + @case_value="$(CASE)"; \ + if [ -n "$$case_value" ]; then \ + if [ -f "$$case_value" ]; then \ + case_args="--case $$case_value"; \ + elif [[ "$$case_value" == *".case.json" || "$$case_value" == *"/"* ]]; then \ + case_args="--case $(TRACER_ROOT)/$(BUCKET)/$$case_value"; \ + elif [[ "$$case_value" == *"__"* ]]; then \ + case_args="--name $$case_value"; \ + else \ + case_args="--case-id $$case_value"; \ + fi; \ + fi; \ + $(PYTHON) -m fetchgraph.tracer.cli fixture-rm $$case_args --root "$(TRACER_ROOT)" --bucket "$(BUCKET)" \ + $(if $(strip $(NAME)),--name "$(NAME)",) \ + $(if $(strip $(PATTERN)),--pattern "$(PATTERN)",) \ + $(if $(strip $(SCOPE)),--scope "$(SCOPE)",) \ + $(if $(strip $(SELECT)),--select "$(SELECT)",) \ + $(if $(strip $(SELECT_INDEX)),--select-index "$(SELECT_INDEX)",) \ + $(if $(filter 1 true yes on,$(REQUIRE_UNIQUE)),--require-unique,) \ + $(if $(filter 1 true yes on,$(ALL)),--all,) \ + $(if $(filter 1 true yes on,$(DRY)),--dry-run,) + +fixture-fix: + @test -n "$(strip $(NAME))" || (echo "NAME обязателен: make fixture-fix NAME=old_stem NEW_NAME=new_stem" && exit 1) + @test -n "$(strip $(NEW_NAME))" || (echo "NEW_NAME обязателен: make fixture-fix NAME=old_stem NEW_NAME=new_stem" && exit 1) + @$(PYTHON) -m fetchgraph.tracer.cli fixture-fix --root "$(TRACER_ROOT)" --bucket "$(BUCKET)" \ + --name "$(NAME)" --new-name "$(NEW_NAME)" \ + $(if $(filter 1 true yes on,$(DRY)),--dry-run,) + +fixture-migrate: + @case_value="$(CASE)"; \ + if [ -n "$$case_value" ]; then \ + if [ -f "$$case_value" ]; then \ + case_args="--case $$case_value"; \ + elif [[ "$$case_value" == *".case.json" || "$$case_value" == *"/"* ]]; then \ + case_args="--case $(TRACER_ROOT)/$(BUCKET)/$$case_value"; \ + else \ + case_args="--case-id $$case_value"; \ + fi; \ + fi; \ + $(PYTHON) -m fetchgraph.tracer.cli fixture-migrate $$case_args --root "$(TRACER_ROOT)" --bucket "$(BUCKET)" \ + $(if $(strip $(NAME)),--name "$(NAME)",) \ + $(if $(strip $(SELECT)),--select "$(SELECT)",) \ + $(if $(strip $(SELECT_INDEX)),--select-index "$(SELECT_INDEX)",) \ + $(if $(filter 1 true yes on,$(REQUIRE_UNIQUE)),--require-unique,) \ + $(if $(filter 1 true yes on,$(ALL)),--all,) \ + $(if $(filter 1 true yes on,$(DRY)),--dry-run,) + +fixture-demote: + @test -n "$(strip $(CASE))" || (echo "CASE обязателен: make fixture-demote CASE=agg_003" && exit 1) + @case_value="$(CASE)"; \ + if [ -f "$$case_value" ]; then \ + case_args="--case $$case_value"; \ + elif [[ "$$case_value" == *".case.json" || "$$case_value" == *"/"* ]]; then \ + case_args="--case $(TRACER_ROOT)/fixed/$$case_value"; \ + else \ + case_args="--case-id $$case_value"; \ + fi; \ + $(PYTHON) -m fetchgraph.tracer.cli fixture-demote $$case_args --root "$(TRACER_ROOT)" \ + $(if $(strip $(SELECT)),--select "$(SELECT)",) \ + $(if $(strip $(SELECT_INDEX)),--select-index "$(SELECT_INDEX)",) \ + $(if $(filter 1 true yes on,$(REQUIRE_UNIQUE)),--require-unique,) \ + $(if $(filter 1 true yes on,$(ALL)),--all,) \ + $(if $(filter 1 true yes on,$(OVERWRITE)),--overwrite,) \ + $(if $(filter 1 true yes on,$(DRY)),--dry-run,) + + + # compare (diff.md + junit) compare: check @test -n "$(strip $(BASE))" || (echo "Нужно задать BASE=.../results_prev.jsonl" && exit 1) @@ -359,8 +554,3 @@ compare-tag: check tag-rm: @test -n "$(strip $(TAG))" || (echo "TAG обязателен: make tag-rm TAG=..." && exit 1) @TAG="$(TAG)" DATA="$(DATA)" PURGE_RUNS="$(PURGE_RUNS)" PRUNE_HISTORY="$(PRUNE_HISTORY)" PRUNE_CASE_HISTORY="$(PRUNE_CASE_HISTORY)" DRY="$(DRY)" $(PYTHON) -m scripts.tag_rm - - - - - diff --git a/examples/demo_qa/batch.py b/examples/demo_qa/batch.py index 47074173..41eef835 100644 --- a/examples/demo_qa/batch.py +++ b/examples/demo_qa/batch.py @@ -1018,7 +1018,14 @@ def handle_batch(args) -> int: for case in cases: current_case_id = case.id try: - result = run_one(case, runner, artifacts_root, plan_only=args.plan_only, event_logger=event_logger) + result = run_one( + case, + runner, + artifacts_root, + plan_only=args.plan_only, + event_logger=event_logger, + schema_path=schema_path, + ) except KeyboardInterrupt: interrupted = True interrupted_at_case_id = current_case_id @@ -1390,7 +1397,13 @@ def handle_case_run(args) -> int: llm = build_llm(settings) runner = build_agent(llm, provider) - result = run_one(cases[args.case_id], runner, artifacts_root, plan_only=args.plan_only) + result = run_one( + cases[args.case_id], + runner, + artifacts_root, + plan_only=args.plan_only, + schema_path=args.schema, + ) write_results(results_path, [result]) counts = summarize([result]) bad = bad_statuses("bad", False) @@ -1437,7 +1450,10 @@ def handle_case_open(args) -> int: plan = case_dir / "plan.json" answer = case_dir / "answer.txt" status = case_dir / "status.json" - for path in [plan, answer, status]: + events = case_dir / "events.jsonl" + error = case_dir / "error.txt" + schema_snapshot = case_dir / "schema_snapshot.yaml" + for path in [plan, answer, status, events, error, schema_snapshot]: if path.exists(): print(f"- {path}") return 0 diff --git a/examples/demo_qa/chat_repl.py b/examples/demo_qa/chat_repl.py index 5d3feb65..e33e00d9 100644 --- a/examples/demo_qa/chat_repl.py +++ b/examples/demo_qa/chat_repl.py @@ -112,7 +112,15 @@ def start_repl( artifacts: RunArtifacts | None = None try: case = Case(id=run_id, question=line, tags=[]) - result = run_one(case, runner, runs_root, plan_only=False, event_logger=event_logger, run_dir=run_dir) + result = run_one( + case, + runner, + runs_root, + plan_only=False, + event_logger=event_logger, + run_dir=run_dir, + schema_path=None, + ) plan_obj = _load_json(Path(result.artifacts_dir) / "plan.json") ctx_obj = _load_json(Path(result.artifacts_dir) / "context.json") or {} artifacts = RunArtifacts( diff --git a/examples/demo_qa/runner.py b/examples/demo_qa/runner.py index f78ce2aa..805f4b44 100644 --- a/examples/demo_qa/runner.py +++ b/examples/demo_qa/runner.py @@ -1,19 +1,36 @@ from __future__ import annotations import datetime +import traceback +import hashlib import json import re +import shutil import statistics import time import uuid from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, Iterable, List, Mapping, NotRequired, TypedDict +from typing import Dict, Iterable, List, Mapping, Protocol, TypedDict + +from typing_extensions import NotRequired from fetchgraph.core import create_generic_agent +from fetchgraph.core.context import BaseGraphAgent from fetchgraph.core.models import TaskProfile +from fetchgraph.replay.snapshots import snapshot_provider_catalog from fetchgraph.utils import set_run_id +class CaseEventLoggerFactory(Protocol): + def for_case(self, case_id: str, events_path: Path) -> "EventLogger": ... + + +class _DefaultEventLoggerSentinel: + pass + + +_DEFAULT_EVENT_LOGGER = _DefaultEventLoggerSentinel() + @dataclass class RunTimings: @@ -114,7 +131,7 @@ def saver(feature_name: str, parsed: object) -> None: ], ) - self.agent = create_generic_agent( + self.agent: BaseGraphAgent = create_generic_agent( llm_invoke=llm, providers={provider.name: provider}, saver=saver, @@ -129,6 +146,7 @@ def run_question( *, plan_only: bool = False, event_logger: EventLogger | None = None, + schema_ref: str | None = None, ) -> RunArtifacts: set_run_id(run_id) artifacts = RunArtifacts(run_id=run_id, run_dir=run_dir, question=case.question, plan_only=plan_only) @@ -137,8 +155,19 @@ def run_question( try: if event_logger: event_logger.emit({"type": "plan_started", "case_id": case.id}) + _emit_planner_input( + event_logger, + case=case, + agent=self, + schema_ref=schema_ref, + plan_only=plan_only, + ) plan_started = time.perf_counter() - plan = self.agent._plan(case.question) # type: ignore[attr-defined] + if event_logger: + self.agent.set_event_logger(event_logger) + else: + self.agent.set_event_logger(None) + plan = self.agent._plan(case.question) artifacts.timings.plan_s = time.perf_counter() - plan_started artifacts.plan = plan.model_dump() @@ -184,6 +213,70 @@ def _save_json(path: Path, payload: object) -> None: path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") +def _hash_file(path: Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(8192), b""): + hasher.update(chunk) + return f"sha256:{hasher.hexdigest()}" + + +def _provider_catalog_snapshot(agent: object) -> Dict[str, object]: + plan_normalizer = getattr(agent, "plan_normalizer", None) + provider_catalog = getattr(plan_normalizer, "provider_catalog", {}) if plan_normalizer else {} + return snapshot_provider_catalog(provider_catalog) + + +def _emit_schema_snapshot(event_logger: EventLogger, run_dir: Path, schema_path: Path) -> str: + run_dir.mkdir(parents=True, exist_ok=True) + suffix = schema_path.suffix.lstrip(".") or "txt" + snapshot_name = f"schema_snapshot.{suffix}" + snapshot_path = run_dir / snapshot_name + shutil.copy2(schema_path, snapshot_path) + file_hash = _hash_file(snapshot_path) + event_logger.emit( + { + "type": "replay_resource", + "v": 1, + "id": "schema_v1", + "meta": {"format": suffix}, + "data": None, + "data_ref": {"file": snapshot_name, "hash": file_hash}, + "hash": file_hash, + } + ) + return "schema_v1" + + +def _emit_planner_input( + event_logger: EventLogger, + *, + case: Case, + agent: AgentRunner, + schema_ref: str | None, + plan_only: bool, +) -> None: + input_payload: Dict[str, object] = { + "feature_name": case.id, + "user_query": case.question, + "options": {"plan_only": plan_only}, + } + if schema_ref: + input_payload["schema_ref"] = schema_ref + provider_catalog = _provider_catalog_snapshot(agent.agent) + if provider_catalog: + input_payload["provider_catalog"] = provider_catalog + event_logger.emit( + { + "type": "planner_input", + "v": 1, + "id": "planner_input_v1", + "case_id": case.id, + "input": input_payload, + } + ) + + def save_artifacts(artifacts: RunArtifacts) -> None: artifacts.run_dir.mkdir(parents=True, exist_ok=True) if artifacts.plan is not None: @@ -297,8 +390,9 @@ def run_one( artifacts_root: Path, *, plan_only: bool = False, - event_logger: EventLogger | None = None, + event_logger: CaseEventLoggerFactory | None | _DefaultEventLoggerSentinel = _DEFAULT_EVENT_LOGGER, run_dir: Path | None = None, + schema_path: Path | None = None, ) -> RunResult: if run_dir is None: run_id = uuid.uuid4().hex[:8] @@ -306,60 +400,128 @@ def run_one( else: run_id = run_dir.name.split("_")[-1] - case_logger = event_logger.for_case(case.id, run_dir / "events.jsonl") if event_logger else None + run_dir.mkdir(parents=True, exist_ok=True) + events_path = run_dir / "events.jsonl" + if event_logger is None: + case_logger = None + elif event_logger is _DEFAULT_EVENT_LOGGER: + case_logger = EventLogger(events_path, run_id, case.id) + else: + case_logger = event_logger.for_case(case.id, events_path) if case_logger: + case_logger.emit({"type": "run_started", "case_id": case.id, "run_dir": str(run_dir)}) case_logger.emit({"type": "case_started", "case_id": case.id, "run_dir": str(run_dir)}) - if case.skip: - run_dir.mkdir(parents=True, exist_ok=True) - _save_text(run_dir / "skipped.txt", "Skipped by request") + schema_ref: str | None = None + ok = False + result: RunResult | None = None + try: + if case_logger and schema_path and schema_path.exists(): + schema_ref = _emit_schema_snapshot(case_logger, run_dir, schema_path) + if case.skip: + _save_text(run_dir / "skipped.txt", "Skipped by request") + result = RunResult( + id=case.id, + question=case.question, + status="skipped", + checked=False, + reason="skipped", + details=None, + artifacts_dir=str(run_dir), + duration_ms=0, + tags=list(case.tags), + answer=None, + error=None, + plan_path=None, + timings=RunTimings(), + expected_check=None, + ) + save_status(result) + ok = True + if case_logger: + case_logger.emit({"type": "case_finished", "case_id": case.id, "status": "skipped"}) + return result + + artifacts = runner.run_question( + case, + run_id, + run_dir, + plan_only=plan_only, + event_logger=case_logger, + schema_ref=schema_ref, + ) + save_artifacts(artifacts) + + expected_check = None if plan_only else _match_expected(case, artifacts.answer) + result = _build_result(case, artifacts, run_dir, expected_check) + save_status(result) + ok = result.status != "error" + if case_logger: + if result.status == "error": + case_logger.emit( + { + "type": "case_failed", + "case_id": case.id, + "status": result.status, + "reason": result.reason, + "artifacts_dir": result.artifacts_dir, + } + ) + case_logger.emit( + { + "type": "case_finished", + "case_id": case.id, + "status": result.status, + "duration_ms": result.duration_ms, + "artifacts_dir": result.artifacts_dir, + } + ) + return result + except BaseException as exc: + tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) + error_message = str(exc) or repr(exc) + if case_logger: + case_logger.emit( + { + "type": "run_failed", + "case_id": case.id, + "exc_type": type(exc).__name__, + "exc_message": error_message, + "traceback": tb, + "artifacts_dir": str(run_dir), + } + ) result = RunResult( id=case.id, question=case.question, - status="skipped", - checked=False, - reason="skipped", - details=None, + status="error", + checked=case.has_asserts, + reason=error_message, + details={ + "error": error_message, + "traceback": tb, + **({"events_path": str(events_path)} if case_logger else {}), + }, artifacts_dir=str(run_dir), duration_ms=0, tags=list(case.tags), answer=None, - error=None, + error=error_message, plan_path=None, timings=RunTimings(), expected_check=None, ) save_status(result) + raise + finally: if case_logger: - case_logger.emit({"type": "case_finished", "case_id": case.id, "status": "skipped"}) - return result - - artifacts = runner.run_question(case, run_id, run_dir, plan_only=plan_only, event_logger=case_logger) - save_artifacts(artifacts) - - expected_check = None if plan_only else _match_expected(case, artifacts.answer) - result = _build_result(case, artifacts, run_dir, expected_check) - save_status(result) - if case_logger: - if result.status == "error": case_logger.emit( { - "type": "case_failed", + "type": "run_finished", "case_id": case.id, - "status": result.status, - "reason": result.reason, - "artifacts_dir": result.artifacts_dir, + "ok": ok, + "artifacts_dir": str(run_dir), } ) - case_logger.emit( - { - "type": "case_finished", - "case_id": case.id, - "status": result.status, - "duration_ms": result.duration_ms, - "artifacts_dir": result.artifacts_dir, - } - ) - return result def summarize(results: Iterable[RunResult]) -> Dict[str, object]: @@ -791,9 +953,10 @@ def format_status_line(result: RunResult) -> str: class EventLogger: - def __init__(self, path: Path | None, run_id: str): + def __init__(self, path: Path | None, run_id: str, case_id: str | None = None): self.path = path self.run_id = run_id + self.case_id = case_id if path: path.parent.mkdir(parents=True, exist_ok=True) @@ -802,13 +965,13 @@ def emit(self, event: Dict[str, object]) -> None: return now = datetime.datetime.now(datetime.timezone.utc) payload = {"timestamp": now.isoformat().replace("+00:00", "Z"), "run_id": self.run_id, **event} + if self.case_id and "case_id" not in payload: + payload["case_id"] = self.case_id with self.path.open("a", encoding="utf-8") as f: f.write(json.dumps(payload, ensure_ascii=False) + "\n") - def for_case(self, case_id: str, path: Path | None = None) -> "EventLogger": - if path is None: - return self - return EventLogger(path, self.run_id) + def for_case(self, case_id: str, path: Path) -> "EventLogger": + return EventLogger(path, self.run_id, case_id) DiffCaseChange = TypedDict( diff --git a/examples/demo_qa/tests/test_demo_qa_batch.py b/examples/demo_qa/tests/test_demo_qa_batch.py index fa163a8b..48bebfe8 100644 --- a/examples/demo_qa/tests/test_demo_qa_batch.py +++ b/examples/demo_qa/tests/test_demo_qa_batch.py @@ -441,7 +441,7 @@ def test_format_healed_explain_includes_key_lines() -> None: assert any("run_id=r2" in line and "status=ok" in line for line in lines) -def _stubbed_run_one(case, runner, artifacts_root, *, plan_only=False, event_logger=None): +def _stubbed_run_one(case, runner, artifacts_root, *, plan_only=False, event_logger=None, schema_path=None): run_dir = artifacts_root / f"{case.id}_stub" run_dir.mkdir(parents=True, exist_ok=True) return RunResult( diff --git a/examples/demo_qa/tests/test_demo_qa_runner.py b/examples/demo_qa/tests/test_demo_qa_runner.py index f9723dd2..09645f81 100644 --- a/examples/demo_qa/tests/test_demo_qa_runner.py +++ b/examples/demo_qa/tests/test_demo_qa_runner.py @@ -2,15 +2,90 @@ from typing import cast +import pytest + from examples.demo_qa.runner import ( Case, + RunArtifacts, RunResult, + RunTimings, _match_expected, diff_runs, + run_one, summarize, ) +class _FailingRunner: + def run_question(self, *args, **kwargs): + raise RuntimeError("boom") + + +class _ArtifactRunner: + def run_question(self, case, run_id, run_dir, **kwargs): + return RunArtifacts( + run_id=run_id, + run_dir=run_dir, + question=case.question, + answer="ok", + timings=RunTimings(total_s=0.01), + ) + + +def _read_events(path): + return [line for line in path.read_text(encoding="utf-8").splitlines() if line] + + +def test_run_one_writes_events_on_early_failure(tmp_path) -> None: + case = Case(id="case_fail", question="Q") + artifacts_root = tmp_path / "runs" + + with pytest.raises(RuntimeError): + run_one(case, _FailingRunner(), artifacts_root) + + case_dirs = list(artifacts_root.iterdir()) + assert case_dirs + events_path = case_dirs[0] / "events.jsonl" + assert events_path.exists() + events = _read_events(events_path) + assert any("run_started" in line for line in events) + assert any("run_failed" in line for line in events) + assert any("run_finished" in line for line in events) + + +def test_run_one_writes_events_on_late_failure(tmp_path, monkeypatch) -> None: + case = Case(id="case_late", question="Q", expected="ok") + artifacts_root = tmp_path / "runs" + + def _boom(*args, **kwargs): + raise ValueError("late boom") + + monkeypatch.setattr("examples.demo_qa.runner._match_expected", _boom) + + with pytest.raises(ValueError): + run_one(case, _ArtifactRunner(), artifacts_root) + + case_dirs = list(artifacts_root.iterdir()) + assert case_dirs + events_path = case_dirs[0] / "events.jsonl" + assert events_path.exists() + events = _read_events(events_path) + assert any("run_started" in line for line in events) + assert any("run_failed" in line for line in events) + + +def test_run_one_with_events_disabled_does_not_write_file(tmp_path) -> None: + case = Case(id="case_no_events", question="Q") + artifacts_root = tmp_path / "runs" + + run_one(case, _ArtifactRunner(), artifacts_root, event_logger=None) + + case_dirs = list(artifacts_root.iterdir()) + assert case_dirs + events_path = case_dirs[0] / "events.jsonl" + assert not events_path.exists() + + def test_match_expected_unchecked_when_no_expectations() -> None: case = Case(id="c1", question="What is foo?") assert _match_expected(case, "anything") is None diff --git a/pyproject.toml b/pyproject.toml index b3e82dec..0b2fb8a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "fetchgraph" -version = "0.2.1" +version = "0.2.2" description = "Graph-like planning → context fetching → synthesis agent (library-style)." readme = "README.md" requires-python = ">=3.11" @@ -47,3 +47,6 @@ Homepage = "https://github.com/AlexanderOnischenko/fetchgraph" Repository = "https://github.com/AlexanderOnischenko/fetchgraph" Documentation = "https://github.com/AlexanderOnischenko/fetchgraph#readme" Issues = "https://github.com/AlexanderOnischenko/fetchgraph/issues" + +[project.scripts] +fetchgraph-tracer = "fetchgraph.tracer.cli:main" diff --git a/src/fetchgraph/core/context.py b/src/fetchgraph/core/context.py index 0a0c08b8..265978e9 100644 --- a/src/fetchgraph/core/context.py +++ b/src/fetchgraph/core/context.py @@ -7,6 +7,7 @@ from ..parsing.plan_parser import PlanParser from ..planning.normalize import PlanNormalizer +from ..replay.log import EventLoggerLike from .models import ( BaselineSpec, ContextFetchSpec, @@ -334,6 +335,7 @@ def __init__( task_profile: Optional[TaskProfile] = None, llm_refetch: Optional[Callable[[str, Dict[str, str], Plan], str]] = None, max_refetch_iters: int = 1, + event_logger: EventLoggerLike | None = None, ): self.llm_plan = llm_plan self.llm_synth = llm_synth @@ -349,6 +351,9 @@ def __init__( self.plan_normalizer = plan_normalizer or PlanNormalizer.from_providers( providers ) + self.event_logger = event_logger + if self.plan_normalizer is not None: + self.plan_normalizer.event_logger = event_logger self.baseline = baseline or [] if self.plan_normalizer is not None and self.baseline: normalized_specs = self.plan_normalizer.normalize_specs( @@ -437,6 +442,11 @@ def run(self, feature_name: str) -> Any: return parsed # ---- steps ---- + def set_event_logger(self, event_logger: EventLoggerLike | None) -> None: + self.event_logger = event_logger + if self.plan_normalizer is not None: + self.plan_normalizer.event_logger = event_logger + def _plan(self, feature_name: str) -> Plan: t0 = time.perf_counter() lite_ctx = self._lite_context(feature_name) @@ -456,7 +466,7 @@ def _plan(self, feature_name: str) -> Plan: else: plan = Plan.model_validate_json(plan_raw.text) if self.plan_normalizer is not None: - plan = self.plan_normalizer.normalize(plan) + plan = self.plan_normalizer.normalize(plan, event_logger=self.event_logger) elapsed = time.perf_counter() - t0 logger.info( "Planning finished for feature_name=%r in %.3fs " diff --git a/src/fetchgraph/planning/normalize/__init__.py b/src/fetchgraph/planning/normalize/__init__.py index 1e4b3944..7706d3ea 100644 --- a/src/fetchgraph/planning/normalize/__init__.py +++ b/src/fetchgraph/planning/normalize/__init__.py @@ -1,5 +1,15 @@ """Normalization utilities for planning.""" -from .plan_normalizer import NormalizedPlan, PlanNormalizer, PlanNormalizerOptions +from .plan_normalizer import ( + NormalizedPlan, + PlanNormalizer, + PlanNormalizerOptions, + SelectorNormalizationRule, +) -__all__ = ["NormalizedPlan", "PlanNormalizer", "PlanNormalizerOptions"] \ No newline at end of file +__all__ = [ + "NormalizedPlan", + "PlanNormalizer", + "PlanNormalizerOptions", + "SelectorNormalizationRule", +] diff --git a/src/fetchgraph/planning/normalize/plan_normalizer.py b/src/fetchgraph/planning/normalize/plan_normalizer.py index 11c3fa24..74b98012 100644 --- a/src/fetchgraph/planning/normalize/plan_normalizer.py +++ b/src/fetchgraph/planning/normalize/plan_normalizer.py @@ -1,8 +1,9 @@ from __future__ import annotations +import copy import json import logging -from dataclasses import dataclass +from dataclasses import asdict, dataclass from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple from pydantic import Field, TypeAdapter, ValidationError @@ -12,6 +13,8 @@ from ...relational.models import RelationalRequest from ...relational.normalize import normalize_relational_selectors from ...relational.providers.base import RelationalDataProvider +from ...replay.log import EventLoggerLike, log_replay_case +from ...replay.snapshots import snapshot_provider_info logger = logging.getLogger(__name__) @@ -35,6 +38,7 @@ class PlanNormalizerOptions: class SelectorNormalizationRule: validator: TypeAdapter[Any] normalize_selectors: Callable[[Any], Any] + kind: str | None = None class PlanNormalizer: @@ -44,12 +48,14 @@ def __init__( schema_registry: Optional[Dict[str, Dict[str, Any]]] = None, normalizer_registry: Optional[Dict[str, SelectorNormalizationRule]] = None, options: Optional[PlanNormalizerOptions] = None, + event_logger: EventLoggerLike | None = None, ) -> None: self.provider_catalog = dict(provider_catalog) self.schema_registry = schema_registry or {} self.normalizer_registry = normalizer_registry or {} self.options = options or PlanNormalizerOptions() self._provider_aliases = self._build_provider_aliases(self.provider_catalog) + self.event_logger = event_logger @classmethod def from_providers( @@ -78,6 +84,7 @@ def from_providers( schema_registry[key] = info.selectors_schema if isinstance(prov, RelationalDataProvider): normalizer_registry[key] = SelectorNormalizationRule( + kind="relational_v1", validator=TypeAdapter(RelationalRequest), normalize_selectors=normalize_relational_selectors, ) @@ -88,7 +95,7 @@ def from_providers( options=options, ) - def normalize(self, plan: Plan) -> NormalizedPlan: + def normalize(self, plan: Plan, *, event_logger: EventLoggerLike | None = None) -> NormalizedPlan: notes: List[str] = [] required_context = self._normalize_required_context(plan.required_context, notes) context_plan = self._normalize_context_plan(plan.context_plan, notes) @@ -98,7 +105,7 @@ def normalize(self, plan: Plan) -> NormalizedPlan: # Baseline/plan merge owns "ensure required providers exist" logic, # and must do it in a baseline-safe way (never overriding baseline selectors/mode). - context_plan = self._normalize_specs(context_plan, notes) + context_plan = self._normalize_specs(context_plan, notes, event_logger=event_logger) adr_queries = self._normalize_text_list(plan.adr_queries, notes, "adr_queries") constraints = self._normalize_text_list( @@ -121,9 +128,10 @@ def normalize_specs( specs: Iterable[ContextFetchSpec], *, notes: Optional[List[str]] = None, + event_logger: EventLoggerLike | None = None, ) -> List[ContextFetchSpec]: local_notes: List[str] = [] - normalized = self._normalize_specs(specs, local_notes) + normalized = self._normalize_specs(specs, local_notes, event_logger=event_logger) if notes is not None: notes.extend(local_notes) if local_notes: @@ -134,39 +142,125 @@ def normalize_specs( return normalized def _normalize_specs( - self, specs: Iterable[ContextFetchSpec], notes: List[str] + self, + specs: Iterable[ContextFetchSpec], + notes: List[str], + *, + event_logger: EventLoggerLike | None = None, ) -> List[ContextFetchSpec]: normalized: List[ContextFetchSpec] = [] - for spec in specs: + replay_logger = event_logger or self.event_logger + for spec_idx, spec in enumerate(specs): rule = self.normalizer_registry.get(spec.provider) if rule is None: normalized.append(spec) continue - orig = spec.selectors - before_ok = self._validate_selectors(rule.validator, orig) + selectors_before = copy.deepcopy(spec.selectors) + before_ok = self._validate_selectors(rule.validator, selectors_before) decision = "keep_original_valid" if before_ok else "keep_original_still_invalid" - use = orig + use = selectors_before after_ok = before_ok + rule_trace = [ + { + "stage": "select_rule", + "provider": spec.provider, + "rule_kind": rule.kind, + } + ] if not before_ok: - candidate = rule.normalize_selectors(orig) + candidate = rule.normalize_selectors(copy.deepcopy(selectors_before)) after_ok = self._validate_selectors(rule.validator, candidate) if after_ok: decision = "use_normalized_fixed" use = candidate - elif candidate != orig: + elif candidate != selectors_before: decision = "use_normalized_unvalidated" use = candidate - notes.append( - self._format_selectors_note( - spec.provider, - before_ok, - after_ok, - decision, - selectors_before=orig, - selectors_after=use, + rule_trace.append( + { + "stage": "normalize", + "decision": decision, + "changed": candidate != selectors_before, + "validators": { + "before_ok": before_ok, + "after_ok": after_ok, + }, + } ) + else: + rule_trace.append( + { + "stage": "validate", + "decision": decision, + "validators": { + "before_ok": before_ok, + "after_ok": after_ok, + }, + } + ) + note = self._format_selectors_note( + spec.provider, + before_ok, + after_ok, + decision, + selectors_before=selectors_before, + selectors_after=use, ) - if use is orig: + notes.append(note) + rule_kind = rule.kind + if replay_logger and rule_kind: + provider_info_snapshot = None + provider_info = self.provider_catalog.get(spec.provider) + if isinstance(provider_info, ProviderInfo): + provider_info_snapshot = snapshot_provider_info(provider_info) + elif spec.provider in self.schema_registry: + provider_info_snapshot = snapshot_provider_info( + ProviderInfo( + name=spec.provider, + capabilities=[], + selectors_schema=self.schema_registry[spec.provider], + ) + ) + requires = None + if not self._provider_info_snapshot_sufficient(provider_info_snapshot): + requires = [{"kind": "extra", "id": "planner_input_v1"}] + input_payload = { + "spec": { + "provider": spec.provider, + "mode": spec.mode, + "selectors": selectors_before, + }, + "options": asdict(self.options), + "normalizer_rules": {spec.provider: rule_kind}, + } + if provider_info_snapshot: + input_payload["provider_info_snapshot"] = provider_info_snapshot + observed_payload = { + "out_spec": { + "provider": spec.provider, + "mode": spec.mode, + "selectors": use, + }, + } + log_replay_case( + replay_logger, + id="plan_normalize.spec_v1", + meta={ + "provider": spec.provider, + "mode": spec.mode, + "spec_idx": spec_idx, + }, + input=input_payload, + observed=observed_payload, + diag={ + "selectors_valid_before": before_ok, + "selectors_valid_after": after_ok, + "rule_trace": rule_trace, + }, + requires=requires, + note=note, + ) + if use == selectors_before: normalized.append(spec) continue data = spec.model_dump() @@ -203,6 +297,15 @@ def _format_selectors_note( payload["selectors_after"] = selectors_after return json.dumps(payload, ensure_ascii=False, default=str) + @staticmethod + def _provider_info_snapshot_sufficient(snapshot: Optional[Dict[str, Any]]) -> bool: + if not isinstance(snapshot, dict): + return False + selectors_schema = snapshot.get("selectors_schema") + if isinstance(selectors_schema, dict) and selectors_schema: + return True + return False + def _normalize_required_context( self, values: Iterable[str], notes: List[str] ) -> List[str]: diff --git a/src/fetchgraph/relational/normalize.py b/src/fetchgraph/relational/normalize.py index 6ef0a7d1..6d6e4d0b 100644 --- a/src/fetchgraph/relational/normalize.py +++ b/src/fetchgraph/relational/normalize.py @@ -1,8 +1,8 @@ from __future__ import annotations import re -from typing import Any, Dict, Optional from collections.abc import Callable, MutableMapping +from typing import Any, Dict, Optional from .types import SelectorsDict diff --git a/src/fetchgraph/replay/__init__.py b/src/fetchgraph/replay/__init__.py new file mode 100644 index 00000000..ad08ecf5 --- /dev/null +++ b/src/fetchgraph/replay/__init__.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from .log import EventLoggerLike, log_replay_case, log_replay_point +from .runtime import REPLAY_HANDLERS, ReplayContext + +__all__ = [ + "EventLoggerLike", + "REPLAY_HANDLERS", + "ReplayContext", + "log_replay_case", + "log_replay_point", +] diff --git a/src/fetchgraph/replay/export.py b/src/fetchgraph/replay/export.py new file mode 100644 index 00000000..3cf41f56 --- /dev/null +++ b/src/fetchgraph/replay/export.py @@ -0,0 +1,716 @@ +from __future__ import annotations + +import copy +import collections +import filecmp +import hashlib +import json +import logging +import shutil +import textwrap +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Callable, Dict, Iterable + +logger = logging.getLogger(__name__) + + +def iter_events(path: Path, *, allow_bad_json: bool = False) -> Iterable[tuple[int, dict]]: + skipped_count = 0 + with path.open("r", encoding="utf-8") as handle: + for idx, line in enumerate(handle, start=1): + line = line.strip() + if not line: + continue + try: + yield idx, json.loads(line) + except json.JSONDecodeError as exc: + excerpt = line.replace("\t", " ").replace("\n", " ") + if len(excerpt) > 120: + excerpt = f"{excerpt[:120]}…" + if allow_bad_json: + skipped_count += 1 + continue + raise ValueError(f"Invalid JSON on line {idx}: {excerpt} in {path}") from exc + if skipped_count: + logger.warning("Skipped %d invalid JSON lines in %s", skipped_count, path) + + +def canonical_json(payload: object) -> str: + return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def case_bundle_name(event_id: str, input_payload: dict) -> str: + digest = hashlib.sha256((event_id + canonical_json(input_payload)).encode("utf-8")).hexdigest() + return f"{event_id}__{digest[:8]}.case.json" + + +@dataclass(frozen=True) +class ExportSelection: + line: int + event: dict + + +def _match_meta(event: dict, *, spec_idx: int | None, provider: str | None) -> bool: + if spec_idx is None and provider is None: + return True + meta = event.get("meta") + if not isinstance(meta, dict): + return False + if spec_idx is not None and meta.get("spec_idx") != spec_idx: + return False + if provider is not None: + p = meta.get("provider") + if not isinstance(p, str): + return False + if p.lower() != provider.lower(): + return False + return True + + +def _select_replay_cases( + events_path: Path, + *, + replay_id: str, + spec_idx: int | None = None, + provider: str | None = None, + allow_bad_json: bool = False, +) -> list[ExportSelection]: + selected: list[ExportSelection] = [] + for line, event in iter_events(events_path, allow_bad_json=allow_bad_json): + if event.get("type") != "replay_case": + continue + if event.get("id") != replay_id: + continue + if _match_meta(event, spec_idx=spec_idx, provider=provider): + selected.append(ExportSelection(line=line, event=event)) + return selected + + +def find_replay_case_matches( + events_path: Path, + *, + replay_id: str, + spec_idx: int | None = None, + provider: str | None = None, + allow_bad_json: bool = False, +) -> list[ExportSelection]: + return _select_replay_cases( + events_path, + replay_id=replay_id, + spec_idx=spec_idx, + provider=provider, + allow_bad_json=allow_bad_json, + ) + + +def format_replay_case_matches(selections: list[ExportSelection], *, limit: int | None = 10) -> str: + rows = [] + for idx, selection in enumerate(selections[:limit], start=1): + event = selection.event + meta = event.get("meta") or {} + provider = meta.get("provider") + spec_idx = meta.get("spec_idx") + timestamp = event.get("timestamp") + input_payload = event.get("input") if isinstance(event.get("input"), dict) else {} + fingerprint = hashlib.sha256(canonical_json(input_payload).encode("utf-8")).hexdigest()[:8] + preview = canonical_json(input_payload)[:80] + rows.append( + " " + f"{idx}. line={selection.line} " + f"ts={timestamp!r} " + f"provider={provider!r} " + f"spec_idx={spec_idx!r} " + f"input={fingerprint} " + f"preview={preview!r}" + ) + if len(selections) > (limit or 0): + rows.append(f" ... ({len(selections) - (limit or 0)} more)") + return "\n".join(rows) + + +def _parse_timestamp(value: object) -> datetime | None: + if not isinstance(value, str) or not value: + return None + cleaned = value + if cleaned.endswith("Z"): + cleaned = cleaned[:-1] + "+00:00" + try: + return datetime.fromisoformat(cleaned) + except ValueError: + return None + + +def _select_by_timestamp(selections: list[ExportSelection]) -> ExportSelection: + with_ts = [] + for selection in selections: + ts = _parse_timestamp(selection.event.get("timestamp")) + if ts is not None: + with_ts.append((ts, selection)) + if with_ts: + return max(with_ts, key=lambda pair: pair[0])[1] + return max(selections, key=lambda sel: sel.line) + + +def _select_replay_case( + selections: list[ExportSelection], + *, + events_path: Path, + selection_policy: str, + select_index: int | None, + require_unique: bool, + allow_prompt: bool, + prompt_fn: Callable[[str], str] | None, +) -> tuple[ExportSelection, str]: + if not selections: + raise LookupError(f"No replay_case entries found in {events_path}") + if require_unique and len(selections) > 1: + details = format_replay_case_matches(selections) + raise LookupError( + "Multiple replay_case entries matched; run with --select/--select-index/--list-matches.\n" + f"{details}" + ) + if len(selections) == 1: + return selections[0], "unique" + if select_index is not None: + if select_index < 1 or select_index > len(selections): + raise ValueError(f"select_index must be between 1 and {len(selections)}") + return selections[select_index - 1], "index" + + selection_policy = selection_policy or "latest" + if allow_prompt and prompt_fn is not None: + details = format_replay_case_matches(selections) + prompt = ( + "Multiple replay_case entries matched:\n" + f"{details}\n" + "Select entry (1..N) or press Enter for latest: " + ) + response = prompt_fn(prompt).strip() + if response: + if response.isdigit(): + choice = int(response) + if 1 <= choice <= len(selections): + return selections[choice - 1], "prompt" + raise ValueError(f"Selection must be between 1 and {len(selections)}") + raise ValueError("Selection must be a number") + + if selection_policy == "latest": + return _select_by_timestamp(selections), "policy" + if selection_policy == "by-timestamp": + return _select_by_timestamp(selections), "policy" + if selection_policy == "by-line": + return max(selections, key=lambda sel: sel.line), "policy" + if selection_policy == "last": + return selections[-1], "policy" + if selection_policy == "first": + return selections[0], "policy" + raise ValueError(f"Unsupported selection policy: {selection_policy}") + + +def index_requires( + events_path: Path, + *, + allow_bad_json: bool = False, +) -> tuple[dict[str, dict], dict[str, dict]]: + extras: Dict[str, dict] = {} + resources: Dict[str, dict] = {} + + for _, event in iter_events(events_path, allow_bad_json=allow_bad_json): + event_type = event.get("type") + event_id = event.get("id") + if event_type == "planner_input" and isinstance(event_id, str): + extras[event_id] = event + elif event_type == "replay_resource" and isinstance(event_id, str): + resources[event_id] = event + + return resources, extras + + +def _format_replay_case_ref(replay_case: dict | None) -> str: + if not isinstance(replay_case, dict): + return "" + replay_case_dict: dict = replay_case + replay_id = replay_case_dict.get("id") + meta_value = replay_case_dict.get("meta") + meta = meta_value if isinstance(meta_value, dict) else {} + provider = meta.get("provider") + spec_idx = meta.get("spec_idx") + details = [] + if isinstance(replay_id, str): + details.append(f"id={replay_id!r}") + if provider is not None: + details.append(f"provider={provider!r}") + if spec_idx is not None: + details.append(f"spec_idx={spec_idx!r}") + if not details: + return "" + return " (replay_case " + ", ".join(details) + ")" + + +def _normalize_requires( + requires: list[dict] | list[str], + *, + resources: dict[str, dict], + extras: dict[str, dict], + events_path: Path, +) -> list[dict]: + normalized_requires: list[dict] = [] + if isinstance(requires, list) and all(isinstance(req, str) for req in requires): + for req_id in requires: + if req_id in resources: + normalized_requires.append({"kind": "resource", "id": req_id}) + elif req_id in extras: + normalized_requires.append({"kind": "extra", "id": req_id}) + else: + raise ValueError( + f"Unknown dependency {req_id!r} in {events_path}; requires must be updated to replay_case v2." + ) + return normalized_requires + if isinstance(requires, list): + for req in requires: + if not isinstance(req, dict): + raise ValueError("requires must be a list of objects with kind/id fields") + if req.get("kind") not in {"extra", "resource"}: + raise ValueError("requires entries must include kind='extra' or kind='resource'") + if not isinstance(req.get("id"), str) or not req.get("id"): + raise ValueError("requires entries must include a non-empty id") + normalized_requires.append({"kind": req["kind"], "id": req["id"]}) + return normalized_requires + raise ValueError("requires must be a list of objects with kind/id fields") + + +def _extract_extra_requires( + extra: dict, + *, + resources: dict[str, dict], + extras: dict[str, dict], + events_path: Path, +) -> list[dict]: + requirements: list[dict] = [] + extra_requires = extra.get("requires") + if isinstance(extra_requires, list) and extra_requires: + requirements.extend( + _normalize_requires( + extra_requires, + resources=resources, + extras=extras, + events_path=events_path, + ) + ) + schema_ref = extra.get("schema_ref") + if isinstance(schema_ref, str) and schema_ref: + requirements.append({"kind": "resource", "id": schema_ref}) + input_value = extra.get("input") + extra_input = input_value if isinstance(input_value, dict) else {} + schema_ref = extra_input.get("schema_ref") + if isinstance(schema_ref, str) and schema_ref: + requirements.append({"kind": "resource", "id": schema_ref}) + input_requires = extra_input.get("requires") + if isinstance(input_requires, list) and input_requires: + requirements.extend( + _normalize_requires( + input_requires, + resources=resources, + extras=extras, + events_path=events_path, + ) + ) + return requirements + + +def resolve_requires( + requires: list[dict] | list[str], + *, + resources: dict[str, dict], + extras: dict[str, dict], + events_path: Path, + replay_case: dict | None = None, +) -> tuple[dict[str, dict], dict[str, dict]]: + if not requires: + return {}, {} + + normalized_requires = _normalize_requires( + requires, + resources=resources, + extras=extras, + events_path=events_path, + ) + + resolved_resources: Dict[str, dict] = {} + resolved_extras: Dict[str, dict] = {} + pending = collections.deque(normalized_requires) + seen: set[tuple[str, str]] = set() + while pending: + req = pending.popleft() + kind = req.get("kind") + rid = req.get("id") + if not isinstance(kind, str) or not isinstance(rid, str): + raise KeyError(f"Invalid require entry {req!r} in {events_path}") + key = (kind, rid) + if key in seen: + continue + seen.add(key) + if kind == "extra": + if rid in extras: + extra = extras[rid] + resolved_extras[rid] = extra + if isinstance(extra, dict): + pending.extend( + _extract_extra_requires( + extra, + resources=resources, + extras=extras, + events_path=events_path, + ) + ) + else: + ref = _format_replay_case_ref(replay_case) + if rid == "planner_input_v1": + raise KeyError( + "Missing required extra 'planner_input_v1' in " + f"{events_path}{ref}. " + "Re-record the run with planner_input emit enabled, " + "or update the trace to include planner_input_v1." + ) + raise KeyError(f"Missing required extra {rid} in {events_path}{ref}") + elif kind == "resource": + if rid in resources: + resolved_resources[rid] = resources[rid] + else: + ref = _format_replay_case_ref(replay_case) + raise KeyError(f"Missing required resource {rid} in {events_path}{ref}") + else: + raise KeyError(f"Unknown require kind {kind!r} in {events_path}") + + return resolved_resources, resolved_extras + + +def collect_requires( + requires: list[dict] | list[str], + *, + resources: dict[str, dict], + extras: dict[str, dict], + events_path: Path, +) -> tuple[dict[str, dict], dict[str, dict]]: + return resolve_requires( + requires, + resources=resources, + extras=extras, + events_path=events_path, + ) + + +def write_case_bundle( + out_path: Path, + *, + root_case: dict, + resources: Dict[str, dict], + extras: Dict[str, dict], + source: dict, + overwrite: bool = False, +) -> None: + payload = { + "schema": "fetchgraph.tracer.case_bundle", + "v": 1, + "root": root_case, + "resources": resources, + "extras": extras, + "source": source, + } + new_text = canonical_json(payload) + if out_path.exists() and not overwrite: + try: + old_payload = json.loads(out_path.read_text(encoding="utf-8")) + old_text = canonical_json(old_payload) + except Exception as exc: + raise ValueError(f"Existing case bundle is unreadable: {out_path}: {exc}") from exc + if old_text == new_text: + logger.info("Fixture already up-to-date: %s", out_path) + return + raise FileExistsError( + textwrap.dedent( + f"""\ + Case bundle already exists and differs: {out_path} + This is fail-fast to avoid mixing fixtures from different runs. + Actions: + - delete the file, or + - choose a different --out directory, or + - rerun with --overwrite. + """ + ).rstrip() + ) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(new_text, encoding="utf-8") + + +def copy_resource_files( + resources: dict[str, dict], + *, + run_dir: Path, + out_dir: Path, + fixture_stem: str, +) -> None: + planned: dict[Path, tuple[str, Path]] = {} + for resource_id, resource in resources.items(): + if not isinstance(resource_id, str) or not resource_id: + raise ValueError("resource_id must be a non-empty string") + if "/" in resource_id or "\\" in resource_id or ".." in Path(resource_id).parts: + raise ValueError(f"resource_id must be a safe path segment: {resource_id!r}") + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + continue + rel_path = Path(file_name) + if rel_path.is_absolute(): + raise ValueError(f"Resource file path must be relative: {file_name}") + if ".." in rel_path.parts: + raise ValueError(f"Resource file path must not traverse parents: {file_name}") + src_path = run_dir / rel_path + if not src_path.exists(): + raise FileNotFoundError( + f"Missing resource file for rid={resource_id!r}: " + f"src={src_path} (run_dir={run_dir}, fixture={fixture_stem})" + ) + dest_rel = Path("resources") / fixture_stem / resource_id / rel_path + dest_path = out_dir / dest_rel + dest_path.parent.mkdir(parents=True, exist_ok=True) + prev = planned.get(dest_path) + if prev is not None: + prev_id, prev_src = prev + if prev_id != resource_id or prev_src != src_path: + if not filecmp.cmp(prev_src, src_path, shallow=False): + raise FileExistsError( + "Resource destination collision (different contents):\n" + f" dest: {dest_path}\n" + f" fixture: {fixture_stem}\n" + f" A: rid={prev_id!r} src={prev_src}\n" + f" B: rid={resource_id!r} src={src_path}\n" + "Hint: make resource filenames unique or adjust resource layout." + ) + else: + planned[dest_path] = (resource_id, src_path) + if dest_path.exists(): + if not filecmp.cmp(src_path, dest_path, shallow=False): + raise FileExistsError( + "Resource file collision (dest exists with different contents):\n" + f" rid: {resource_id!r}\n" + f" fixture: {fixture_stem}\n" + f" src: {src_path}\n" + f" dest: {dest_path}\n" + "Actions:\n" + " - delete destination resources directory, or\n" + " - export into a clean --out directory." + ) + else: + shutil.copy2(src_path, dest_path) + data_ref["file"] = dest_rel.as_posix() + + +def _has_resource_files(resources: dict[str, dict]) -> bool: + for resource in resources.values(): + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if isinstance(file_name, str) and file_name: + return True + return False + + +def export_replay_case_bundle( + *, + events_path: Path, + out_dir: Path, + replay_id: str, + spec_idx: int | None = None, + provider: str | None = None, + run_dir: Path | None = None, + allow_bad_json: bool = False, + overwrite: bool = False, + selection_policy: str = "latest", + select_index: int | None = None, + require_unique: bool = False, + allow_prompt: bool = False, + prompt_fn: Callable[[str], str] | None = None, +) -> Path: + selections = _select_replay_cases( + events_path, + replay_id=replay_id, + spec_idx=spec_idx, + provider=provider, + allow_bad_json=allow_bad_json, + ) + if not selections: + details = [] + if spec_idx is not None: + details.append(f"spec_idx={spec_idx}") + if provider is not None: + details.append(f"provider={provider!r}") + detail_str = f" (filters: {', '.join(details)})" if details else "" + unfiltered = _select_replay_cases( + events_path, + replay_id=replay_id, + spec_idx=None, + provider=None, + allow_bad_json=allow_bad_json, + ) + if unfiltered and details: + providers = sorted( + {str(sel.event.get("meta", {}).get("provider")) for sel in unfiltered if sel.event.get("meta")} + ) + spec_idxs = sorted( + {str(sel.event.get("meta", {}).get("spec_idx")) for sel in unfiltered if sel.event.get("meta")} + ) + hint_lines = [ + f"No replay_case id={replay_id!r} matched filters in {events_path}{detail_str}.", + f"Available providers: {providers}", + f"Available spec_idx: {spec_idxs}", + "Tip: rerun without --provider/--spec-idx or choose matching values.", + ] + raise LookupError("\n".join(hint_lines)) + raise LookupError(f"No replay_case id={replay_id!r} found in {events_path}{detail_str}") + selection, selection_mode = _select_replay_case( + selections, + events_path=events_path, + selection_policy=selection_policy, + select_index=select_index, + require_unique=require_unique, + allow_prompt=allow_prompt, + prompt_fn=prompt_fn, + ) + if len(selections) > 1 and selection_mode == "policy": + input_hashes = { + hashlib.sha256(canonical_json(sel.event.get("input") or {}).encode("utf-8")).hexdigest() + for sel in selections + } + details = format_replay_case_matches(selections, limit=5) + suffix = "Candidates differ by input payload." if len(input_hashes) > 1 else "Candidates share input." + logger.warning( + "Multiple replay_case entries matched; selection policy=%s chose line %s.\n%s\n%s", + selection_policy, + selection.line, + details, + suffix, + ) + root_event = selection.event + requires = root_event.get("requires") or [] + + resources_index, extras_index = index_requires(events_path, allow_bad_json=allow_bad_json) + resources, extras = resolve_requires( + requires, + resources=resources_index, + extras=extras_index, + events_path=events_path, + replay_case=root_event, + ) + resources = copy.deepcopy(resources) + extras = copy.deepcopy(extras) + fixture_name = case_bundle_name(replay_id, root_event["input"]) + fixture_stem = fixture_name.replace(".case.json", "") + if _has_resource_files(resources): + if run_dir is None: + raise ValueError("run_dir is required to export file resources") + if overwrite: + shutil.rmtree(out_dir / "resources" / fixture_stem, ignore_errors=True) + copy_resource_files( + resources, + run_dir=run_dir, + out_dir=out_dir, + fixture_stem=fixture_stem, + ) + + source = { + "events_path": str(events_path), + "line": selection.line, + "run_id": root_event.get("run_id"), + "timestamp": root_event.get("timestamp"), + "case_id": root_event.get("case_id"), + } + out_path = out_dir / fixture_name + write_case_bundle( + out_path, + root_case=root_event, + resources=resources, + extras=extras, + source=source, + overwrite=overwrite, + ) + return out_path + + +def export_replay_case_bundles( + *, + events_path: Path, + out_dir: Path, + replay_id: str, + spec_idx: int | None = None, + provider: str | None = None, + run_dir: Path | None = None, + allow_bad_json: bool = False, + overwrite: bool = False, +) -> list[Path]: + selections = _select_replay_cases( + events_path, + replay_id=replay_id, + spec_idx=spec_idx, + provider=provider, + allow_bad_json=allow_bad_json, + ) + if not selections: + details = [] + if spec_idx is not None: + details.append(f"spec_idx={spec_idx}") + if provider is not None: + details.append(f"provider={provider!r}") + detail_str = f" (filters: {', '.join(details)})" if details else "" + raise LookupError(f"No replay_case id={replay_id!r} found in {events_path}{detail_str}") + + resources_index, extras_index = index_requires(events_path, allow_bad_json=allow_bad_json) + paths: list[Path] = [] + for selection in selections: + root_event = selection.event + requires = root_event.get("requires") or [] + resources, extras = resolve_requires( + requires, + resources=resources_index, + extras=extras_index, + events_path=events_path, + replay_case=root_event, + ) + resources = copy.deepcopy(resources) + extras = copy.deepcopy(extras) + fixture_name = case_bundle_name(replay_id, root_event["input"]) + fixture_stem = fixture_name.replace(".case.json", "") + if _has_resource_files(resources): + if run_dir is None: + raise ValueError("run_dir is required to export file resources") + if overwrite: + shutil.rmtree(out_dir / "resources" / fixture_stem, ignore_errors=True) + copy_resource_files( + resources, + run_dir=run_dir, + out_dir=out_dir, + fixture_stem=fixture_stem, + ) + + source = { + "events_path": str(events_path), + "line": selection.line, + "run_id": root_event.get("run_id"), + "timestamp": root_event.get("timestamp"), + "case_id": root_event.get("case_id"), + } + out_path = out_dir / fixture_name + write_case_bundle( + out_path, + root_case=root_event, + resources=resources, + extras=extras, + source=source, + overwrite=overwrite, + ) + paths.append(out_path) + return paths diff --git a/src/fetchgraph/replay/handlers/__init__.py b/src/fetchgraph/replay/handlers/__init__.py new file mode 100644 index 00000000..55f3c2a5 --- /dev/null +++ b/src/fetchgraph/replay/handlers/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from .plan_normalize import replay_plan_normalize_spec_v1 + +__all__ = ["replay_plan_normalize_spec_v1"] diff --git a/src/fetchgraph/replay/handlers/plan_normalize.py b/src/fetchgraph/replay/handlers/plan_normalize.py new file mode 100644 index 00000000..ee084c0e --- /dev/null +++ b/src/fetchgraph/replay/handlers/plan_normalize.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import logging +from typing import Dict + +from pydantic import TypeAdapter + +from ...core.models import ContextFetchSpec, ProviderInfo +from ...planning.normalize import ( + PlanNormalizer, + PlanNormalizerOptions, + SelectorNormalizationRule, +) +from ...relational.models import RelationalRequest +from ...relational.normalize import normalize_relational_selectors +from ..runtime import REPLAY_HANDLERS, ReplayContext + +logger = logging.getLogger(__name__) + + +def replay_plan_normalize_spec_v1(inp: dict, ctx: ReplayContext) -> dict: + spec_dict = dict(inp["spec"]) + options = PlanNormalizerOptions(**inp["options"]) + rules = inp.get("normalizer_rules") or inp.get("normalizer_registry") or {} + provider = spec_dict["provider"] + provider_catalog: Dict[str, ProviderInfo] = {} + provider_info_source = "minimal_fallback" + provider_snapshot = inp.get("provider_info_snapshot") + provider_snapshot_present = isinstance(provider_snapshot, dict) + if isinstance(provider_snapshot, dict): + provider_catalog[provider] = ProviderInfo(**provider_snapshot) + provider_info_source = "snapshot" + else: + planner = ctx.extras.get("planner_input_v1") or {} + planner_input = planner.get("input") if isinstance(planner, dict) else {} + catalog_raw = {} + if isinstance(planner_input, dict): + catalog_raw = planner_input.get("provider_catalog") or {} + if provider in catalog_raw and isinstance(catalog_raw[provider], dict): + provider_catalog[provider] = ProviderInfo(**catalog_raw[provider]) + provider_info_source = "planner_input" + else: + provider_catalog[provider] = ProviderInfo(name=provider, capabilities=[]) + + rule_kind = rules.get(provider) + normalizer_registry: Dict[str, SelectorNormalizationRule] = {} + if rule_kind == "relational_v1": + normalizer_registry[provider] = SelectorNormalizationRule( + kind="relational_v1", + validator=TypeAdapter(RelationalRequest), + normalize_selectors=normalize_relational_selectors, + ) + + normalizer = PlanNormalizer( + provider_catalog, + normalizer_registry=normalizer_registry, + options=options, + ) + + spec = ContextFetchSpec(**spec_dict) + notes: list[str] = [] + out_specs = normalizer.normalize_specs([spec], notes=notes) + out = out_specs[0] + + out_spec = { + "provider": out.provider, + "mode": out.mode, + "selectors": out.selectors, + } + logger.info( + "replay_plan_normalize_spec_v1: replay_id=%s provider=%s provider_info_source=%s", + "plan_normalize.spec_v1", + provider, + provider_info_source, + ) + if provider_info_source == "minimal_fallback": + logger.warning( + "replay_plan_normalize_spec_v1: provider_info_source=minimal_fallback " + "replay_id=%s provider=%s", + "plan_normalize.spec_v1", + provider, + ) + out_payload = { + "out_spec": out_spec, + "notes_last": notes[-1] if notes else None, + } + if provider_info_source == "minimal_fallback": + out_payload["diag"] = { + "provider_info_source": provider_info_source, + "missing_planner_input": "planner_input_v1" not in ctx.extras, + "provider_snapshot_present": provider_snapshot_present, + } + return out_payload + + +REPLAY_HANDLERS["plan_normalize.spec_v1"] = replay_plan_normalize_spec_v1 diff --git a/src/fetchgraph/replay/log.py b/src/fetchgraph/replay/log.py new file mode 100644 index 00000000..365de977 --- /dev/null +++ b/src/fetchgraph/replay/log.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import warnings +from typing import Any, Dict, Protocol, cast + +TRACE_LIMIT = 20_000 + + +class EventLoggerLike(Protocol): + def emit(self, event: Dict[str, object]) -> None: ... + + +def log_replay_case( + logger: EventLoggerLike, + *, + id: str, + input: dict, + meta: dict | None = None, + observed: dict | None = None, + observed_error: dict | None = None, + requires: list[dict] | None = None, + note: str | None = None, + diag: dict | None = None, +) -> None: + if not isinstance(id, str) or not id.strip(): + raise ValueError("id must be a non-empty string") + if not isinstance(input, dict): + raise ValueError("input must be a dict") + if (observed is None) == (observed_error is None): + raise ValueError("exactly one of observed or observed_error must be set") + if observed is not None and not isinstance(observed, dict): + raise ValueError("observed must be a dict when provided") + if observed_error is not None: + if not isinstance(observed_error, dict): + raise ValueError("observed_error must be a dict when provided") + if not isinstance(observed_error.get("type"), str) or not observed_error.get("type"): + raise ValueError("observed_error.type must be a non-empty string") + if not isinstance(observed_error.get("message"), str) or not observed_error.get("message"): + raise ValueError("observed_error.message must be a non-empty string") + trace_value = observed_error.get("trace") + if not isinstance(trace_value, str) or not trace_value: + raise ValueError("observed_error.trace must be a non-empty string") + if len(trace_value) > TRACE_LIMIT: + observed_error = { + **observed_error, + "trace": f"{trace_value[:TRACE_LIMIT]}...(truncated {len(trace_value) - TRACE_LIMIT} chars)", + } + if meta is not None and not isinstance(meta, dict): + raise ValueError("meta must be a dict when provided") + if note is not None and not isinstance(note, str): + raise ValueError("note must be a string when provided") + if diag is not None and not isinstance(diag, dict): + raise ValueError("diag must be a dict when provided") + if requires is not None: + if not isinstance(requires, list): + raise ValueError("requires must be a list when provided") + for req in requires: + if not isinstance(req, dict): + raise ValueError("requires entries must be dicts") + kind = req.get("kind") + if kind not in {"extra", "resource"}: + raise ValueError("requires.kind must be 'extra' or 'resource'") + req_id = req.get("id") + if not isinstance(req_id, str) or not req_id: + raise ValueError("requires.id must be a non-empty string") + + event: Dict[str, object] = { + "type": "replay_case", + "v": 2, + "id": id, + "input": input, + } + if meta is not None: + event["meta"] = meta + if observed is not None: + event["observed"] = observed + if observed_error is not None: + event["observed_error"] = observed_error + if requires: + event["requires"] = requires + if note is not None: + event["note"] = note + if diag is not None: + event["diag"] = diag + logger.emit(event) + + +def log_replay_point(logger: EventLoggerLike, **kwargs: object) -> None: + warnings.warn( + "log_replay_point is deprecated; use log_replay_case", + DeprecationWarning, + stacklevel=2, + ) + payload = dict(kwargs) + if "observed" not in payload and "expected" in payload: + payload["observed"] = payload.pop("expected") + + id_val = payload.get("id") + input_val = payload.get("input") + if not isinstance(id_val, str) or not isinstance(input_val, dict): + raise ValueError("log_replay_point requires id (str) and input (dict)") + + meta_val = payload.get("meta") + observed_val = payload.get("observed") + observed_error_val = payload.get("observed_error") + requires_val = payload.get("requires") + note_val = payload.get("note") + diag_val = payload.get("diag") + + log_replay_case( + logger, + id=cast(str, id_val), + input=cast(dict, input_val), + meta=cast(dict | None, meta_val), + observed=cast(dict | None, observed_val), + observed_error=cast(dict | None, observed_error_val), + requires=cast(list[dict] | None, requires_val), + note=cast(str | None, note_val), + diag=cast(dict | None, diag_val), + ) diff --git a/src/fetchgraph/replay/runtime.py b/src/fetchgraph/replay/runtime.py new file mode 100644 index 00000000..f13ebfb0 --- /dev/null +++ b/src/fetchgraph/replay/runtime.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Dict + + +@dataclass(frozen=True) +class ReplayContext: + resources: Dict[str, dict] = field(default_factory=dict) + extras: Dict[str, dict] = field(default_factory=dict) + base_dir: Path | None = None + + def resolve_resource_path(self, resource_path: str | Path) -> Path: + path = Path(resource_path) + if path.is_absolute() or self.base_dir is None: + return path + return self.base_dir / path + + +REPLAY_HANDLERS: Dict[str, Callable[[dict, ReplayContext], dict]] = {} + + +def run_case(root: dict, ctx: ReplayContext) -> dict: + replay_id = root["id"] + if replay_id not in REPLAY_HANDLERS: + raise KeyError( + f"No handler for replay id={replay_id!r}. " + "Did you import fetchgraph.tracer.handlers?" + ) + handler = REPLAY_HANDLERS[replay_id] + return handler(root["input"], ctx) + + +def load_case_bundle(path: Path) -> tuple[dict, ReplayContext]: + data = json.loads(path.read_text(encoding="utf-8")) + if data.get("schema") != "fetchgraph.tracer.case_bundle" or data.get("v") != 1: + raise ValueError(f"Unsupported case bundle schema in {path}") + root = data["root"] + ctx = ReplayContext( + resources=data.get("resources", {}), + extras=data.get("extras", {}), + base_dir=path.parent, + ) + return root, ctx diff --git a/src/fetchgraph/replay/snapshots.py b/src/fetchgraph/replay/snapshots.py new file mode 100644 index 00000000..fe49d0c4 --- /dev/null +++ b/src/fetchgraph/replay/snapshots.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from typing import Dict, Mapping + +from ..core.models import ProviderInfo + + +def snapshot_provider_info(info: ProviderInfo) -> Dict[str, object]: + payload: Dict[str, object] = {"name": info.name} + if info.capabilities: + payload["capabilities"] = list(info.capabilities) + if info.selectors_schema: + payload["selectors_schema"] = info.selectors_schema + return payload + + +def snapshot_provider_catalog(provider_catalog: Mapping[str, object]) -> Dict[str, object]: + snapshot: Dict[str, object] = {} + for key, info in provider_catalog.items(): + if not isinstance(info, ProviderInfo): + continue + snapshot[key] = snapshot_provider_info(info) + return snapshot diff --git a/src/fetchgraph/tracer/__init__.py b/src/fetchgraph/tracer/__init__.py new file mode 100644 index 00000000..6d5e552f --- /dev/null +++ b/src/fetchgraph/tracer/__init__.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from fetchgraph.replay.log import EventLoggerLike, log_replay_case +from fetchgraph.replay.runtime import REPLAY_HANDLERS, ReplayContext, load_case_bundle, run_case +from fetchgraph.tracer.diff_utils import first_diff_path + +__all__ = [ + "EventLoggerLike", + "REPLAY_HANDLERS", + "ReplayContext", + "first_diff_path", + "load_case_bundle", + "log_replay_case", + "run_case", +] diff --git a/src/fetchgraph/tracer/cli.py b/src/fetchgraph/tracer/cli.py new file mode 100644 index 00000000..c15dba68 --- /dev/null +++ b/src/fetchgraph/tracer/cli.py @@ -0,0 +1,660 @@ +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +from fetchgraph.tracer.resolve import ( + collect_rejections, + find_events_file, + format_case_runs, + format_case_run_debug, + format_events_search, + list_case_runs, + resolve_run_dir_from_run_id, + scan_case_runs, + select_case_run, +) +from fetchgraph.tracer.export import ( + export_replay_case_bundle, + export_replay_case_bundles, + find_replay_case_matches, + format_replay_case_matches, +) +from fetchgraph.tracer.fixture_tools import ( + fixture_fix, + fixture_green, + fixture_migrate, + fixture_rm, +) + +DEFAULT_ROOT = Path("tests/fixtures/replay_cases") + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Fetchgraph tracer utilities") + sub = parser.add_subparsers(dest="command", required=True) + + export = sub.add_parser("export-case-bundle", help="Export replay case bundle from events.jsonl") + export.add_argument("--events", type=Path, help="Path to events.jsonl") + export.add_argument("--out", type=Path, required=True, help="Output directory for bundle") + export.add_argument("--id", help="Replay case id to export") + export.add_argument("--spec-idx", type=int, default=None, help="Filter replay_case by meta.spec_idx") + export.add_argument( + "--provider", + default=None, + help="Filter replay_case by meta.provider (case-insensitive)", + ) + export.add_argument("--run-dir", type=Path, default=None, help="Run dir path (required for file resources)") + export.add_argument( + "--run-id", + default=None, + help="Run id from history/run_meta (requires --data; selects case dir within run)", + ) + export.add_argument("--case-dir", type=Path, default=None, help="Case dir path (explicit)") + export.add_argument( + "--allow-bad-json", + action="store_true", + help="Skip invalid JSON lines in events.jsonl", + ) + export.add_argument( + "--overwrite", + action="store_true", + help="Overwrite existing bundles and resource copies", + ) + export.add_argument("--all", action="store_true", help="Export all matching replay cases") + export.add_argument("--case", help="Case id for auto-resolving events.jsonl") + export.add_argument("--data", type=Path, help="Data directory containing .runs") + export.add_argument("--tag", default=None, help="Run tag filter for auto-resolve") + export.add_argument( + "--runs-subdir", + default=".runs/runs", + help="Runs subdir relative to data dir (default: .runs/runs)", + ) + export.add_argument( + "--pick-run", + default="latest_with_replay", + help="Run selection strategy (default: latest_with_replay)", + ) + export.add_argument( + "--print-resolve", + action="store_true", + help="Print resolved run_dir/events.jsonl", + ) + export.add_argument( + "--debug", + action="store_true", + help="Print debug information about candidate runs", + ) + export.add_argument( + "--select", + choices=["latest", "first", "last", "by-timestamp", "by-line"], + default="latest", + help="Selection policy when multiple replay_case entries match", + ) + export.add_argument("--select-index", type=int, default=None, help="Select a case run (1-based)") + export.add_argument("--list-matches", action="store_true", help="List case runs and exit") + export.add_argument( + "--replay-select-index", + type=int, + default=None, + help="Select a specific replay_case match (1-based)", + ) + export.add_argument( + "--list-replay-matches", + action="store_true", + help="List replay_case matches and exit", + ) + export.add_argument("--require-unique", action="store_true", help="Error if multiple matches exist") + + green = sub.add_parser("fixture-green", help="Promote known_bad case to fixed") + green.add_argument("--case", type=Path, help="Path to known_bad case bundle") + green.add_argument("--case-id", help="Case id to select fixture from known_bad") + green.add_argument("--name", help="Fixture stem name to select") + green.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") + green.add_argument("--no-validate", action="store_true", help="Disable validation after write") + green.add_argument( + "--expected-from", + choices=["replay", "observed"], + default="replay", + help="Source for expected output (default: replay)", + ) + green.add_argument( + "--overwrite-expected", + action="store_true", + help="Overwrite existing expected output", + ) + green.add_argument("--dry-run", action="store_true", help="Print actions without changing files") + green.add_argument( + "--git", + choices=["auto", "on", "off"], + default="auto", + help="Use git operations when moving/removing fixtures", + ) + green.add_argument( + "--select", + choices=["latest", "first", "last"], + default="latest", + help="Selection policy when multiple fixtures match", + ) + green.add_argument("--select-index", type=int, default=None, help="Select fixture index (1-based)") + green.add_argument("--require-unique", action="store_true", help="Error if multiple fixtures match") + + rm_cmd = sub.add_parser("fixture-rm", help="Remove replay fixtures") + rm_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") + rm_cmd.add_argument( + "--bucket", + choices=["fixed", "known_bad", "all"], + default="all", + help="Fixture bucket", + ) + rm_cmd.add_argument("--name", help="Fixture stem name") + rm_cmd.add_argument("--pattern", help="Glob pattern for fixture stems or case bundles") + rm_cmd.add_argument("--case", type=Path, help="Path to case bundle") + rm_cmd.add_argument("--case-id", help="Case id to select fixtures") + rm_cmd.add_argument( + "--scope", + choices=["cases", "resources", "both"], + default="both", + help="What to remove", + ) + rm_cmd.add_argument("--dry-run", action="store_true", help="Print actions without changing files") + rm_cmd.add_argument( + "--git", + choices=["auto", "on", "off"], + default="auto", + help="Use git operations when removing fixtures", + ) + rm_cmd.add_argument( + "--select", + choices=["latest", "first", "last"], + default="latest", + help="Selection policy when multiple fixtures match", + ) + rm_cmd.add_argument("--select-index", type=int, default=None, help="Select fixture index (1-based)") + rm_cmd.add_argument("--require-unique", action="store_true", help="Error if multiple fixtures match") + rm_cmd.add_argument("--all", action="store_true", help="Apply to all matching fixtures") + + fix_cmd = sub.add_parser("fixture-fix", help="Rename fixture stem") + fix_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") + fix_cmd.add_argument("--bucket", choices=["fixed", "known_bad"], default="fixed") + fix_cmd.add_argument("--name", required=True, help="Old fixture stem") + fix_cmd.add_argument("--new-name", required=True, help="New fixture stem") + fix_cmd.add_argument("--dry-run", action="store_true", help="Print actions without changing files") + fix_cmd.add_argument( + "--git", + choices=["auto", "on", "off"], + default="auto", + help="Use git operations when moving fixtures", + ) + + migrate_cmd = sub.add_parser("fixture-migrate", help="Normalize resource layout") + migrate_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") + migrate_cmd.add_argument( + "--bucket", + choices=["fixed", "known_bad", "all"], + default="all", + help="Fixture bucket", + ) + migrate_cmd.add_argument("--case", type=Path, help="Path to case bundle") + migrate_cmd.add_argument("--case-id", help="Case id to select fixtures") + migrate_cmd.add_argument("--name", help="Fixture stem name") + migrate_cmd.add_argument("--dry-run", action="store_true", help="Print actions without changing files") + migrate_cmd.add_argument( + "--git", + choices=["auto", "on", "off"], + default="auto", + help="Use git operations when moving fixtures", + ) + migrate_cmd.add_argument( + "--select", + choices=["latest", "first", "last"], + default="latest", + help="Selection policy when multiple fixtures match", + ) + migrate_cmd.add_argument("--select-index", type=int, default=None, help="Select fixture index (1-based)") + migrate_cmd.add_argument("--require-unique", action="store_true", help="Error if multiple fixtures match") + migrate_cmd.add_argument("--all", action="store_true", help="Apply to all matching fixtures") + + ls_cmd = sub.add_parser("fixture-ls", help="List fixture candidates") + ls_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") + ls_cmd.add_argument( + "--bucket", + choices=["fixed", "known_bad", "all"], + default="known_bad", + help="Fixture bucket", + ) + ls_cmd.add_argument("--case-id", help="Case id to filter") + ls_cmd.add_argument("--pattern", help="Glob pattern for fixtures") + + demote_cmd = sub.add_parser("fixture-demote", help="Move fixed fixture back to known_bad") + demote_cmd.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="Fixture root") + demote_cmd.add_argument("--from-bucket", default="fixed", help="Source bucket (default: fixed)") + demote_cmd.add_argument("--to-bucket", default="known_bad", help="Destination bucket (default: known_bad)") + demote_cmd.add_argument("--case", type=Path, help="Path to case bundle") + demote_cmd.add_argument("--case-id", help="Case id to select fixtures") + demote_cmd.add_argument("--name", help="Fixture stem name") + demote_cmd.add_argument("--dry-run", action="store_true", help="Print actions without changing files") + demote_cmd.add_argument("--overwrite", action="store_true", help="Overwrite existing target fixtures") + demote_cmd.add_argument( + "--git", + choices=["auto", "on", "off"], + default="auto", + help="Use git operations when moving fixtures", + ) + demote_cmd.add_argument( + "--select", + choices=["latest", "first", "last"], + default="latest", + help="Selection policy when multiple fixtures match", + ) + demote_cmd.add_argument("--select-index", type=int, default=None, help="Select fixture index (1-based)") + demote_cmd.add_argument("--require-unique", action="store_true", help="Error if multiple fixtures match") + demote_cmd.add_argument("--all", action="store_true", help="Apply to all matching fixtures") + + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + try: + if args.command == "export-case-bundle": + debug_enabled = args.debug or bool(os.getenv("DEBUG")) + events_path: Path | None = None + run_dir: Path | None = None + case_dir: Path | None = None + selection_rule = "unknown" + run_dir_source = "unresolved" + auto_resolve = False + if args.events: + if args.case or args.data or args.tag or args.run_id: + raise ValueError("Do not combine --events with --case/--data/--tag/--run-id.") + if args.case_dir and args.run_dir: + raise ValueError("Do not combine --case-dir with --run-dir when --events is provided.") + events_path = args.events + case_dir = args.case_dir + run_dir = args.run_dir + run_dir_source = "explicit RUN_DIR" if run_dir else "unset" + if case_dir and run_dir is None: + run_dir = _run_dir_from_case_dir(case_dir) + run_dir_source = "derived from case_dir" + selection_rule = "explicit EVENTS" + else: + if args.run_id and (args.case_dir or args.run_dir): + raise ValueError("Do not combine --run-id with --run-dir/--case-dir.") + if args.run_id and not args.data: + raise ValueError("--data is required when --run-id is provided.") + if args.case_dir or args.run_dir: + if args.run_dir and not args.case and not args.case_dir: + raise ValueError("--case is required when --run-dir is provided.") + case_dir = args.case_dir + if case_dir: + run_dir = _run_dir_from_case_dir(case_dir) + selection_rule = "explicit CASE_DIR" + run_dir_source = "derived from case_dir" + else: + run_dir = args.run_dir + if run_dir is None: + raise ValueError("run_dir was not resolved.") + case_dir = _resolve_case_dir_from_run_dir(run_dir=run_dir, case_id=args.case) + selection_rule = "explicit RUN_DIR" + run_dir_source = "explicit RUN_DIR" + elif args.run_id: + if not args.case: + raise ValueError("--case is required when --run-id is provided.") + run_dir = resolve_run_dir_from_run_id( + data_dir=args.data, + runs_subdir=args.runs_subdir, + run_id=args.run_id, + ) + case_dir = _resolve_case_dir_from_run_dir(run_dir=run_dir, case_id=args.case) + selection_rule = f"explicit RUN_ID={args.run_id}" + run_dir_source = "resolved from run_id" + else: + if not args.case or not args.data: + raise ValueError("--case and --data are required when --events is not provided.") + if args.pick_run == "latest_with_replay" and not args.id: + raise ValueError("--id is required when pick_run=latest_with_replay.") + infos, stats = scan_case_runs( + case_id=args.case, + data_dir=args.data, + runs_subdir=args.runs_subdir, + ) + if debug_enabled: + print("Debug: case run candidates (most recent first):") + print(format_case_run_debug(infos, limit=10)) + candidates, stats = list_case_runs( + case_id=args.case, + data_dir=args.data, + tag=args.tag, + pick_run=args.pick_run, + replay_id=args.id if args.pick_run == "latest_with_replay" else None, + runs_subdir=args.runs_subdir, + ) + if args.list_matches: + if not candidates: + raise LookupError( + _format_case_run_error( + stats, + case_id=args.case, + tag=args.tag, + pick_run=args.pick_run, + ) + ) + print(format_case_runs(candidates, limit=20)) + return 0 + selected = select_case_run(candidates, select_index=args.select_index) + run_dir = selected.run_dir + case_dir = selected.case_dir + selection_rule = _format_selection_rule(tag=args.tag, pick_run=args.pick_run) + events_path = selected.events_path + auto_resolve = True + run_dir_source = "auto-resolve" + if events_path is None: + search_root = case_dir or run_dir + if search_root is None: + raise ValueError("run_dir was not resolved.") + events_resolution = find_events_file(search_root) + if not events_resolution.events_path: + raise FileNotFoundError( + _format_events_error( + search_root, + events_resolution, + selection_rule=selection_rule, + ) + ) + events_path = events_resolution.events_path + if args.print_resolve: + print("Input flags:") + print(f" run_id: {args.run_id}") + print(f" run_dir: {args.run_dir}") + print(f" case_dir: {args.case_dir}") + print(f"selection_method: {selection_rule}") + print(f"Resolved run_dir: {run_dir}") + print(f"run_dir_source: {run_dir_source}") + print(f"Resolved case_dir: {case_dir}") + print(f"Resolved events.jsonl: {events_path}") + if args.events and run_dir is None: + print("Note: run_dir not provided; file resources cannot be exported.") + if auto_resolve and args.case and args.data: + infos, _ = scan_case_runs( + case_id=args.case, + data_dir=args.data, + runs_subdir=args.runs_subdir, + ) + rejections = collect_rejections( + infos, + tag=args.tag, + pick_run=args.pick_run, + replay_id=args.id, + ) + if rejections: + print("Rejected candidates:") + for reject in rejections: + print(f"- {reject.case_dir} ({reject.reason})") + if args.list_replay_matches: + if events_path is None: + raise ValueError("events_path was not resolved.") + if not args.id: + raise ValueError("--id is required to list replay_case matches.") + selections = find_replay_case_matches( + events_path, + replay_id=args.id, + spec_idx=args.spec_idx, + provider=args.provider, + allow_bad_json=args.allow_bad_json, + ) + if not selections: + unfiltered = find_replay_case_matches( + events_path, + replay_id=args.id, + spec_idx=None, + provider=None, + allow_bad_json=args.allow_bad_json, + ) + if unfiltered and (args.spec_idx is not None or args.provider is not None): + providers = sorted( + { + str(sel.event.get("meta", {}).get("provider")) + for sel in unfiltered + if sel.event.get("meta") + } + ) + spec_idxs = sorted( + { + str(sel.event.get("meta", {}).get("spec_idx")) + for sel in unfiltered + if sel.event.get("meta") + } + ) + raise LookupError( + "No replay_case matched filters.\n" + f"Available providers: {providers}\n" + f"Available spec_idx: {spec_idxs}\n" + "Tip: rerun without --provider/--spec-idx or choose matching values." + ) + raise LookupError(f"No replay_case id={args.id!r} found in {events_path}") + print(format_replay_case_matches(selections, limit=20)) + return 0 + allow_prompt = ( + sys.stdin.isatty() + and args.replay_select_index is None + and not args.require_unique + and not args.list_replay_matches + ) + if events_path is None: + raise ValueError("events_path was not resolved.") + if not args.id: + raise ValueError("--id is required to export replay case bundles.") + if args.all: + export_replay_case_bundles( + events_path=events_path, + out_dir=args.out, + replay_id=args.id, + spec_idx=args.spec_idx, + provider=args.provider, + run_dir=run_dir, + allow_bad_json=args.allow_bad_json, + overwrite=args.overwrite, + ) + else: + export_replay_case_bundle( + events_path=events_path, + out_dir=args.out, + replay_id=args.id, + spec_idx=args.spec_idx, + provider=args.provider, + run_dir=run_dir, + allow_bad_json=args.allow_bad_json, + overwrite=args.overwrite, + selection_policy=args.select, + select_index=args.replay_select_index, + require_unique=args.require_unique, + allow_prompt=allow_prompt, + prompt_fn=input, + ) + return 0 + if args.command == "fixture-green": + fixture_green( + case_path=args.case, + case_id=args.case_id, + name=args.name, + out_root=args.root, + validate=not args.no_validate, + expected_from=args.expected_from, + overwrite_expected=args.overwrite_expected, + dry_run=args.dry_run, + git_mode=args.git, + select=args.select, + select_index=args.select_index, + require_unique=args.require_unique, + ) + return 0 + if args.command == "fixture-rm": + removed = fixture_rm( + root=args.root, + bucket=args.bucket, + name=args.name, + pattern=args.pattern, + scope=args.scope, + dry_run=args.dry_run, + git_mode=args.git, + case_path=args.case, + case_id=args.case_id, + select=args.select, + select_index=args.select_index, + require_unique=args.require_unique, + all_matches=args.all, + ) + print(f"Removed {removed} paths") + return 0 + if args.command == "fixture-fix": + fixture_fix( + root=args.root, + bucket=args.bucket, + name=args.name, + new_name=args.new_name, + dry_run=args.dry_run, + git_mode=args.git, + ) + return 0 + if args.command == "fixture-migrate": + bundles_updated, files_moved = fixture_migrate( + root=args.root, + bucket=args.bucket, + dry_run=args.dry_run, + git_mode=args.git, + case_path=args.case, + case_id=args.case_id, + name=args.name, + select=args.select, + select_index=args.select_index, + require_unique=args.require_unique, + all_matches=args.all, + ) + print(f"Updated {bundles_updated} bundles; moved {files_moved} files") + return 0 + if args.command == "fixture-ls": + from fetchgraph.tracer.fixture_tools import fixture_ls + + fixtures = fixture_ls( + root=args.root, + bucket=args.bucket, + case_id=args.case_id, + pattern=args.pattern, + ) + if not fixtures: + print("No fixtures matched.") + return 0 + for idx, candidate in enumerate(fixtures, start=1): + source = candidate.source or {} + print( + f"{idx}. stem={candidate.stem} " + f"path={candidate.path} " + f"run_id={source.get('run_id')} " + f"timestamp={source.get('timestamp')}" + ) + return 0 + if args.command == "fixture-demote": + from fetchgraph.tracer.fixture_tools import fixture_demote + + fixture_demote( + root=args.root, + from_bucket=args.from_bucket, + to_bucket=args.to_bucket, + case_path=args.case, + case_id=args.case_id, + name=args.name, + dry_run=args.dry_run, + git_mode=args.git, + overwrite=args.overwrite, + select=args.select, + select_index=args.select_index, + require_unique=args.require_unique, + all_matches=args.all, + ) + return 0 + except (ValueError, FileNotFoundError, LookupError, KeyError, FileExistsError) as exc: + print(str(exc), file=sys.stderr) + return 2 + except Exception as exc: # pragma: no cover - unexpected + print(f"Unexpected error: {exc}", file=sys.stderr) + return 1 + raise SystemExit(f"Unknown command: {args.command}") + + +def _resolve_case_dir_from_run_dir(*, run_dir: Path, case_id: str) -> Path: + runs_root = run_dir / "cases" + if not runs_root.exists(): + raise FileNotFoundError(f"Run directory does not exist: {runs_root}") + case_dirs = sorted(runs_root.glob(f"{case_id}_*"), key=lambda p: p.stat().st_mtime, reverse=True) + if not case_dirs: + raise LookupError( + "No case directories found under run.\n" + f"run_dir: {run_dir}\n" + f"case_id: {case_id}\n" + f"runs_root: {runs_root}" + ) + return case_dirs[0] + + +def _run_dir_from_case_dir(case_dir: Path) -> Path: + run_dir = case_dir.parent.parent + if not run_dir.exists(): + raise FileNotFoundError(f"Run directory does not exist: {run_dir}") + return run_dir + + +def _format_case_run_error(stats, *, case_id: str, tag: str | None, pick_run: str = "latest_non_missed") -> str: + lines = [ + "No suitable case run found.", + f"selection_rule: {_format_selection_rule(tag=tag, pick_run=pick_run)}", + f"runs_root: {stats.runs_root}", + f"case_id: {case_id}", + f"inspected_runs: {stats.inspected_runs}", + f"inspected_cases: {stats.inspected_cases}", + f"missing_cases: {stats.missing_cases}", + f"missing_events: {stats.missing_events}", + f"missed_cases: {stats.missed_cases}", + ] + if tag: + lines.append(f"tag: {tag}") + lines.append(f"tag_mismatches: {stats.tag_mismatches}") + if stats.recent: + lines.append("recent_cases:") + for info in stats.recent[:5]: + lines.append( + " " + f"case_dir={info.case_dir} " + f"tag={info.tag!r} " + f"events={bool(info.events.events_path)}" + ) + lines.append("Tip: verify TAG or pass RUN_ID/CASE_DIR/EVENTS.") + else: + lines.append("Tip: pass TAG/RUN_ID/CASE_DIR/EVENTS for a narrower selection.") + return "\n".join(lines) + + +def _format_events_error(run_dir: Path, resolution, *, selection_rule: str) -> str: + return "\n".join( + [ + f"Selected case_dir: {run_dir}", + f"selection_rule: {selection_rule}", + format_events_search(run_dir, resolution), + "Tip: rerun the case or pass EVENTS=... explicitly.", + ] + ) + + +def _format_selection_rule(*, tag: str | None, pick_run: str) -> str: + base = "latest with events" + if pick_run == "latest_with_replay": + base = "latest with replay_case" + if tag: + return f"{base} filtered by TAG={tag!r}" + return base + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/fetchgraph/tracer/diff_utils.py b/src/fetchgraph/tracer/diff_utils.py new file mode 100644 index 00000000..c8e04d86 --- /dev/null +++ b/src/fetchgraph/tracer/diff_utils.py @@ -0,0 +1,29 @@ +from __future__ import annotations + + +def first_diff_path(left: object, right: object, *, prefix: str = "") -> str | None: + if type(left) is not type(right): + return prefix or "" + if isinstance(left, dict) and isinstance(right, dict): + left_keys = set(left.keys()) + right_keys = set(right.keys()) + for key in sorted(left_keys | right_keys): + next_prefix = f"{prefix}.{key}" if prefix else str(key) + if key not in left or key not in right: + return next_prefix + diff = first_diff_path(left[key], right[key], prefix=next_prefix) + if diff is not None: + return diff + return None + if isinstance(left, list) and isinstance(right, list): + if len(left) != len(right): + return f"{prefix}.length" if prefix else "length" + for idx, (l_item, r_item) in enumerate(zip(left, right), start=1): + next_prefix = f"{prefix}[{idx}]" if prefix else f"[{idx}]" + diff = first_diff_path(l_item, r_item, prefix=next_prefix) + if diff is not None: + return diff + return None + if left != right: + return prefix or "" + return None diff --git a/src/fetchgraph/tracer/export.py b/src/fetchgraph/tracer/export.py new file mode 100644 index 00000000..7e9cb008 --- /dev/null +++ b/src/fetchgraph/tracer/export.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from fetchgraph.replay.export import ( + case_bundle_name, + collect_requires, + copy_resource_files, + export_replay_case_bundle, + export_replay_case_bundles, + find_replay_case_matches, + format_replay_case_matches, + index_requires, + iter_events, + resolve_requires, +) + +__all__ = [ + "case_bundle_name", + "collect_requires", + "copy_resource_files", + "export_replay_case_bundle", + "export_replay_case_bundles", + "find_replay_case_matches", + "format_replay_case_matches", + "index_requires", + "iter_events", + "resolve_requires", +] diff --git a/src/fetchgraph/tracer/fetchgraph_tracer.md b/src/fetchgraph/tracer/fetchgraph_tracer.md new file mode 100644 index 00000000..db20aadf --- /dev/null +++ b/src/fetchgraph/tracer/fetchgraph_tracer.md @@ -0,0 +1,451 @@ +# Fetchgraph Tracer: observed-first replay cases, bundles и реплей + +> Этот документ описывает актуальный формат трейса и реплея: +> - `log.py` — контракт логгера событий и хелпер `log_replay_case` +> - `runtime.py` — `ReplayContext`, `REPLAY_HANDLERS`, `run_case`, `load_case_bundle` +> - `handlers/*` — обработчики (регистрация в `REPLAY_HANDLERS`) +> - `export.py` — экспорт `replay_case` → case bundle (`*.case.json`) + +> Актуально для консольной команды `fetchgraph-tracer` и модулей `fetchgraph.tracer/*` + `fetchgraph.replay/*`. +> Путь к фикстурам по умолчанию: `tests/fixtures/replay_cases/{fixed,known_bad}`. + +--- + +## 0) Зачем это нужно + +Трейсер решает две задачи: + +1) **Observed-first логирование** + В рантайме пишется **input + observed outcome** (успех) **или** `observed_error` (ошибка) + зависимости для реплея. `expected` не логируется. + +2) **Реплей и регрессии** + Из `events.jsonl` экспортируются **case bundles** (root case + extras/resources + source). Реплей работает без LLM и внешних сервисов. + +--- + +## 1) Термины и базовые сущности + +### 1.1 Event stream (JSONL) + +Трейс — это JSONL/NDJSON: одна строка = одно событие. + +События, которые важны для реплея: + +- `type="replay_case"` — корневое событие “кейса реплея” (v2). +- `type="planner_input"` — “extra” (например, входы планировщика), адресуется по `id`. +- `type="replay_resource"` — “resource” (например, файл), адресуется по `id`. + +### 1.2 Replay case (v2) + +Схема корневого события: + +- `type="replay_case"`, `v=2` +- `id` — идентификатор обработчика (namespace вида `something.v1`) +- `meta` — опционально (например, `{spec_idx, provider}`) +- `input` — dict, вход для обработчика + - `input.provider_info_snapshot` — минимальный snapshot провайдера (например, `selectors_schema`), чтобы реплей был детерминированным +- **ровно одно** из: + - `observed` — dict (observed outcome) + - `observed_error` — dict (observed error), если во время “наблюдения” упало +- `requires` — опционально: список зависимостей + - v2-формат: `[{"kind":"extra"|"resource","id":"..."}]` + - legacy-формат допускается в экспорте: `["id1","id2"]` (будет нормализован при экспорте) + +Пример: + +```json +{ + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "meta": {"spec_idx": 0, "provider": "sql"}, + "input": { + "spec": {"provider": "sql", "selectors": {"q": "..." }}, + "options": {"lowercase": true}, + "provider_info_snapshot": {"name": "sql", "selectors_schema": {"type":"object","properties":{}}} + }, + "observed": {"out_spec": {"provider": "sql", "selectors": {"q": "..."}}}, + "requires": [{"kind":"extra","id":"planner_input_v1"}] +} +``` + +### 1.3 Resources и extras + +- **Extras**: события `type="planner_input"`, индексируются по `id`. +- **Resources**: события `type="replay_resource"`, индексируются по `id`. + - Если ресурс указывает `data_ref.file`, это **относительный путь** внутри `run_dir` во время экспорта. + - При экспорте файлы копируются в fixture-layout (см. ниже) и `data_ref.file` переписывается на новый относительный путь. + +--- + +## 2) Контракт логирования (observed-first) + +### 2.1 EventLoggerLike + +Трейсер принимает любой logger, который умеет: + +```py +class EventLoggerLike(Protocol): + def emit(self, event: dict) -> None: ... +``` + +### 2.2 log_replay_case + +`log_replay_case(logger=..., id=..., input=..., observed=.../observed_error=..., requires=..., meta=...)` + +Валидация на входе: + +- `id` — непустая строка +- `input` — dict +- XOR: `observed` / `observed_error` +- `requires` — список `{kind,id}` + +**Рекомендация:** логируйте `provider_info_snapshot` (см. `fetchgraph.replay.snapshots`), чтобы реплей не зависел от внешних данных. + +--- + +## 3) Replay runtime + +### 3.1 Регистрация обработчиков + +`REPLAY_HANDLERS` — dict `{replay_id: handler(input, ctx) -> dict}`. + +Обработчики регистрируются через side-effect импорта, рекомендуемый способ в тестах/скриптах: + +```py +import fetchgraph.tracer.handlers # noqa: F401 +``` + +### 3.2 ReplayContext + +```py +@dataclass(frozen=True) +class ReplayContext: + resources: dict[str, dict] + extras: dict[str, dict] + base_dir: Path | None + + def resolve_resource_path(self, resource_path: str | Path) -> Path: + # относительные пути резолвятся относительно base_dir +``` + +### 3.3 run_case и load_case_bundle + +```py +from fetchgraph.tracer.runtime import load_case_bundle, run_case + +root, ctx = load_case_bundle(Path(".../*.case.json")) +out = run_case(root, ctx) +``` + +--- + +## 4) Case bundle (fixture) format + +Экспорт создает JSON файл: + +- `schema="fetchgraph.tracer.case_bundle"`, `v=1` +- `root` — replay_case (как в events) +- `resources` — dict ресурсов по id +- `extras` — dict extras по id +- `source` — метаданные (минимум: `events_path`, `line`, опционально `run_id`, `timestamp`, `case_id`) + +### 4.1 Layout фикстур и ресурсов + +Рекомендуемый layout: + +``` +tests/fixtures/replay_cases/ + fixed/ + .case.json + .expected.json # для green кейсов + resources///... + known_bad/ + .case.json + resources///... +``` + +Где `` — имя фикстуры (обычно совпадает с именем `.case.json` без расширения). + +При экспорте файлы ресурсов копируются в: + +``` +resources/// +``` + +С коллизиями экспорт падает fail-fast (чтобы не смешивать фикстуры из разных прогонов). + +--- + +## 5) Export API (Python) + +```py +from fetchgraph.tracer.export import export_replay_case_bundle, export_replay_case_bundles + +# один bundle +path = export_replay_case_bundle( + events_path=Path("./events.jsonl"), + out_dir=Path("tests/fixtures/replay_cases/known_bad"), + replay_id="plan_normalize.spec_v1", + spec_idx=0, # фильтр по meta.spec_idx (опционально) + provider="sql", # фильтр по meta.provider (опционально) + run_dir=Path("./run_dir"), # обязателен, если есть file-resources + allow_bad_json=False, + overwrite=False, + selection_policy="latest", # latest|first|last|by-timestamp|by-line + select_index=None, # 1-based индекс среди replay_case матчей + require_unique=False, +) + +# все совпадения +paths = export_replay_case_bundles( + events_path=Path("./events.jsonl"), + out_dir=Path("tests/fixtures/replay_cases/known_bad"), + replay_id="plan_normalize.spec_v1", + allow_bad_json=True, + overwrite=True, +) +``` + +--- + +## 6) CLI: `fetchgraph-tracer` + +Команда подключена как `fetchgraph-tracer = fetchgraph.tracer.cli:main`. + +### 6.1 export-case-bundle + +> Примечание: в старом/экспериментальном синтаксисе мог встречаться флаг `--replay-id`; актуальный флаг — `--id`. + + +Экспорт одного или нескольких case bundles из events: + +```bash +fetchgraph-tracer export-case-bundle --out tests/fixtures/replay_cases/known_bad --id plan_normalize.spec_v1 --events path/to/events.jsonl --run-dir path/to/run_dir --overwrite --allow-bad-json +``` + +#### 6.1.1 Как выбирается events/run_dir + +Приоритет разрешения: + +1) `--events` — явный путь к events/trace файлу. + `run_dir` берется из `--case-dir` или `--run-dir` (если нужны file-resources); `--case-dir` и `--run-dir` вместе не допускаются. + +2) Иначе (auto-resolve через `.runs`): + - обязателен `--case ` и `--data ` + - `--run-id` требует `--data` и не комбинируется с `--run-dir/--case-dir` + - дальше выбираем конкретный run/case: + - `--case-dir ` или `--run-dir ` — явный путь + - `--run-id ` — выбрать запуск по meta run_id (берётся из stats/history) и кейс внутри него + - либо “самый свежий” по стратегии `--pick-run` (+ опционально `--tag`) + +Поддерживаемые имена файлов events (быстрый поиск): +`events.jsonl`, `events.ndjson`, `trace.jsonl`, `trace.ndjson`, `traces/events.jsonl`, `traces/trace.jsonl`. + +Если ни один из стандартных путей не найден — делается fallback-поиск по `*.jsonl/*.ndjson` в глубину до 3 уровней (кроме `resources/`) и выбирается самый “тяжелый” файл. + +#### 6.1.2 Run selection (auto-resolve) flags + +- `--case ` — id кейса (например `agg_003`) +- `--data ` — директория с `.runs` +- `--runs-subdir ` — где искать `runs` относительно `DATA_DIR` (default `.runs/runs`) +- `--tag ` — фильтр по tag (если теги ведутся) +- `--pick-run ` — стратегия выбора запуска: + - `latest_non_missed` + - `latest_with_replay` (default; требует `--id`) +- `--select-index ` — выбрать конкретный run-candidate (1-based) +- `--list-matches` — вывести список кандидатов run/case и выйти +- `--debug` — детальный вывод кандидатов (или `DEBUG=1`) + +Полезно: +- `--print-resolve` — распечатать входные флаги и резолв (`run_dir`, `case_dir`, `events_path`, `selection_method`). + +#### 6.1.3 Replay-case selection flags (когда в events много replay_case) + +- `--spec-idx ` — фильтр по `meta.spec_idx` +- `--provider ` — фильтр по `meta.provider` (case-insensitive) +- `--list-replay-matches` — вывести найденные replay_case entries и выйти +- `--require-unique` — упасть, если матчей > 1 +- `--select ` — политика выбора: + - `latest` (по timestamp, fallback по line) + - `first` + - `last` + - `by-timestamp` + - `by-line` +- `--replay-select-index ` — выбрать конкретный replay_case матч (1-based) +- `--all` — экспортировать **все** матчинг replay_case (игнорируя single-selection) + +--- + +### 6.2 Управление фикстурами (fixture tools) + +#### 6.2.1 fixture-ls + +Показать кандидатов (по умолчанию bucket=known_bad): + +```bash +fetchgraph-tracer fixture-ls --case-id agg_003 +fetchgraph-tracer fixture-ls --bucket fixed --pattern "plan_normalize.*" +``` + +#### 6.2.2 fixture-green + +“Позеленить” кейс: перенести из `known_bad` → `fixed` и записать `*.expected.json` из `root.observed`. + +```bash +# выбрать по case_id (если несколько — используется --select/--select-index) +fetchgraph-tracer fixture-green --case-id agg_003 --validate + +# или явно указать файл +fetchgraph-tracer fixture-green --case tests/fixtures/replay_cases/known_bad/.case.json --validate +``` + +Флаги: +- `--overwrite-expected` — перезаписать `*.expected.json`, если уже есть +- `--validate` — после перемещения прогнать `run_case()` и сравнить с expected +- `--git auto|on|off` — перемещения/удаления через git (если доступно) +- `--dry-run` — только печать действий +- `--select/--select-index/--require-unique` — выбор среди нескольких кандидатов + +> Важно: `fixture-green` требует `root.observed`. Если в кейсе только `observed_error`, сначала нужно переэкспортировать bundle после фикса (чтобы был observed). + +#### 6.2.3 fixture-demote + +Перенести `fixed` → `known_bad` (например, если баг вернулся или кейс решили держать как backlog): + +```bash +fetchgraph-tracer fixture-demote --case-id agg_003 +``` + +Флаги: +- `--overwrite` — перезаписать существующие target-фикстуры +- `--all` — применить ко всем матчам +- остальные — как в green (select/dry-run/git) + +#### 6.2.4 fixture-fix + +Переименовать stem фикстуры (case + expected + resources): + +```bash +fetchgraph-tracer fixture-fix --bucket fixed --name old_stem --new-name new_stem +``` + +#### 6.2.5 fixture-migrate + +Нормализовать layout ресурсов/путей внутри bundle (полезно при изменении схемы ресурсов): + +```bash +fetchgraph-tracer fixture-migrate --bucket all --case-id agg_003 --dry-run +``` + +#### 6.2.6 fixture-rm + +Удалить фикстуры (cases/resources): + +```bash +# удалить конкретный stem в known_bad +fetchgraph-tracer fixture-rm --bucket known_bad --name "" + +# удалить по case_id (опционально --all, иначе выберет один по --select) +fetchgraph-tracer fixture-rm --bucket all --case-id agg_003 --scope both --all +``` + +Флаги: +- `--scope cases|resources|both` +- `--pattern ` — матчить по stem/пути +- `--all` — применить ко всем матчам +- `--git auto|on|off`, `--dry-run`, `--select*` + +--- + +## 7) Make targets (DX) + +Makefile предоставляет врапперы (см. `make help`): + +### 7.1 Экспорт + +```bash +# обычный export (auto-resolve через .runs) +make tracer-export REPLAY_ID=plan_normalize.spec_v1 CASE=agg_003 DATA=_demo_data/shop TRACER_OUT_DIR=tests/fixtures/replay_cases/known_bad + +# явный events + run_dir +make tracer-export REPLAY_ID=plan_normalize.spec_v1 CASE=agg_003 EVENTS=path/to/events.jsonl RUN_DIR=path/to/run_dir TRACER_OUT_DIR=tests/fixtures/replay_cases/known_bad OVERWRITE=1 +``` + +### 7.2 Список кандидатов (auto-resolve) + +> В Makefile `DATA` используется как основной параметр; внутри таргетов он пробрасывается как `REPLAY_IDATA` (alias). + + +```bash +make tracer-ls CASE=agg_003 DATA=_demo_data/shop TAG=known_bad +``` + +### 7.3 Запуск known_bad + +```bash +make known-bad +make known-bad-one NAME= +``` + +### 7.4 Управление фикстурами + +```bash +# promote known_bad -> fixed +make fixture-green CASE=agg_003 VALIDATE=1 + +# list +make fixture-ls CASE=agg_003 BUCKET=known_bad + +# remove +make fixture-rm BUCKET=known_bad CASE=agg_003 ALL=1 DRY=1 + +# rename stem +make fixture-fix BUCKET=fixed NAME=old NEW_NAME=new + +# migrate +make fixture-migrate BUCKET=all CASE=agg_003 DRY=1 + +# demote fixed -> known_bad +make fixture-demote CASE=agg_003 OVERWRITE=1 +``` + +--- + +## 8) Тестовый workflow и CI + +### 8.1 Buckets: fixed vs known_bad + +- `fixed/` — **регрессионные** кейсы: должны проходить и обычно имеют `.expected.json`. +- `known_bad/` — **backlog/TDD-like** кейсы: могут падать или возвращать невалидный результат. + В pytest они должны идти под маркером `known_bad`, чтобы их легко исключать из CI. + +Рекомендуемый CI фильтр: `-m "not known_bad"` (и при необходимости `not slow`). + +### 8.2 Диагностика known_bad + +Для подробной диагностики можно включить окружение: + +- `FETCHGRAPH_REPLAY_DEBUG=1` — больше деталей в выводе +- `FETCHGRAPH_REPLAY_TRUNCATE=` — лимит тримминга больших блоков +- `FETCHGRAPH_REPLAY_META_TRUNCATE=` — лимит для meta +- `FETCHGRAPH_RULE_TRACE_TAIL=` — сколько элементов `diag.rule_trace` показывать + +--- + +## 9) Как добавить новый replay-case + +1) Выберите `replay_id` (строка) и зафиксируйте его в коде. +2) Добавьте handler в `fetchgraph/tracer/handlers/...` и зарегистрируйте в `REPLAY_HANDLERS`. +3) При “наблюдении” логируйте: + - `replay_case` через `log_replay_case(...)` + - `planner_input` / `replay_resource` отдельными событиями, и добавляйте их в `requires` +4) Добавьте валидатор (если нужен) для fixed/regression тестов. +5) Проверьте, что экспорт реплея воспроизводим (нет внешних сервисов, нет LLM). + +--- + +## 10) Совместимость и заметки + +- `log_replay_point` оставлен как deprecated alias; используйте `log_replay_case`. +- Экспорт поддерживает legacy `requires=["id1","id2"]`, но желательно писать v2-формат `[{kind,id}]`. diff --git a/src/fetchgraph/tracer/fixture_layout.py b/src/fetchgraph/tracer/fixture_layout.py new file mode 100644 index 00000000..08424b8f --- /dev/null +++ b/src/fetchgraph/tracer/fixture_layout.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +VALID_BUCKETS = {"fixed", "known_bad"} + + +@dataclass(frozen=True) +class FixtureLayout: + root: Path + bucket: str + + @property + def bucket_dir(self) -> Path: + return self.root / self.bucket + + def case_path(self, stem: str) -> Path: + return self.bucket_dir / f"{stem}.case.json" + + def expected_path(self, stem: str) -> Path: + return self.bucket_dir / f"{stem}.expected.json" + + def resources_dir(self, stem: str) -> Path: + return self.bucket_dir / "resources" / stem + + +def find_case_bundles( + *, + root: Path, + bucket: str | None, + name: str | None, + pattern: str | None, +) -> list[Path]: + if name and pattern: + raise ValueError("Use only one of name or pattern.") + + root = root.resolve() + if bucket in (None, "all"): + buckets = sorted(VALID_BUCKETS) + elif bucket in VALID_BUCKETS: + buckets = [bucket] + else: + raise ValueError(f"Unsupported bucket: {bucket}") + + if name: + matches: list[Path] = [] + for entry in buckets: + layout = FixtureLayout(root, entry) + case_path = layout.case_path(name) + if case_path.exists(): + matches.append(case_path) + return matches + + if pattern: + if pattern.endswith(".case.json"): + glob_pattern = pattern + else: + glob_pattern = f"{pattern}.case.json" + else: + glob_pattern = "*.case.json" + + matches = [] + for entry in buckets: + layout = FixtureLayout(root, entry) + if not layout.bucket_dir.exists(): + continue + matches.extend(layout.bucket_dir.glob(glob_pattern)) + return sorted(matches) diff --git a/src/fetchgraph/tracer/fixture_tools.py b/src/fetchgraph/tracer/fixture_tools.py new file mode 100644 index 00000000..1ea56366 --- /dev/null +++ b/src/fetchgraph/tracer/fixture_tools.py @@ -0,0 +1,853 @@ +from __future__ import annotations + +import filecmp +import json +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from .diff_utils import first_diff_path +from .fixture_layout import FixtureLayout, find_case_bundles +from .runtime import load_case_bundle, run_case +from .validators import REPLAY_VALIDATORS + + +@dataclass(frozen=True) +class FixtureCandidate: + path: Path + stem: str + source: dict + + +def load_bundle_json(path: Path) -> dict: + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("schema") != "fetchgraph.tracer.case_bundle" or payload.get("v") != 1: + raise ValueError(f"Not a tracer case bundle: {path}") + + root = payload.get("root") + if not isinstance(root, dict): + raise ValueError(f"Bundle root must be a mapping: {path}") + + root_type = root.get("type") + root_version = root.get("v") + if root_type is not None and root_type != "replay_case": + raise ValueError(f"Bundle root.type must be replay_case: {path}") + if root_version is not None and root_version != 2: + raise ValueError(f"Bundle root.v must be 2: {path}") + + resources = payload.get("resources") + if resources is not None and not isinstance(resources, dict): + raise ValueError(f"Bundle resources must be a mapping: {path}") + extras = payload.get("extras") + if extras is not None and not isinstance(extras, dict): + raise ValueError(f"Bundle extras must be a mapping: {path}") + + return payload + + +def _safe_resource_path(path: str, *, stem: str) -> Path: + rel = Path(path) + if rel.is_absolute() or ".." in rel.parts: + raise ValueError(f"Invalid resource path in bundle {stem}: {path}") + return rel + + +def _atomic_write_json(path: Path, payload: dict) -> None: + tmp_path = path.with_suffix(path.suffix + ".tmp") + tmp_path.write_text( + json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")), + encoding="utf-8", + ) + tmp_path.replace(path) + + +def _format_json(value: object, *, max_chars: int = 12_000) -> str: + text = json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True, default=str) + if len(text) <= max_chars: + return text + remaining = len(text) - max_chars + return f"{text[:max_chars]}...[truncated {remaining} chars]" + + +def _top_level_key_diff(output: object, expected: object) -> tuple[list[str], list[str]]: + if not isinstance(output, dict) or not isinstance(expected, dict): + return ([], []) + output_keys = set(output.keys()) + expected_keys = set(expected.keys()) + missing = sorted(expected_keys - output_keys) + extra = sorted(output_keys - expected_keys) + return (missing, extra) + + +def _format_fixture_diff(output: object, expected: object) -> str: + missing, extra = _top_level_key_diff(output, expected) + lines = [ + "Fixture output mismatch.", + f"missing_keys: {missing}", + f"extra_keys: {extra}", + ] + if isinstance(output, dict) and isinstance(expected, dict): + out_spec = output.get("out_spec") + exp_spec = expected.get("out_spec") + if out_spec != exp_spec: + diff_path = first_diff_path(out_spec, exp_spec, prefix="out_spec") + if diff_path: + lines.append(f"out_spec diff path: {diff_path}") + lines.append(f"expected: {_format_json(expected)}") + lines.append(f"output: {_format_json(output)}") + return "\n".join(lines) + + +def resolve_fixture_candidates( + *, + root: Path, + bucket: str, + case_id: str | None, + name: str | None, +) -> list[FixtureCandidate]: + if case_id and name: + raise ValueError("Only one of case_id or name can be used.") + candidates = find_case_bundles(root=root, bucket=bucket, name=name, pattern=None) + results: list[FixtureCandidate] = [] + for path in candidates: + payload = load_bundle_json(path) + stem = path.name.replace(".case.json", "") + source = payload.get("source") + source_dict = source if isinstance(source, dict) else {} + if case_id: + source_case = source_dict.get("case") or source_dict.get("case_id") + if source_case == case_id or stem.startswith(case_id): + results.append(FixtureCandidate(path=path, stem=stem, source=source_dict)) + else: + results.append(FixtureCandidate(path=path, stem=stem, source=source_dict)) + return results + + +def resolve_fixture_selector( + *, + root: Path, + bucket: str, + case_path: Path | None = None, + case_id: str | None = None, + name: str | None = None, + select: str = "latest", + select_index: int | None = None, + require_unique: bool = False, + all_matches: bool = False, +) -> list[Path]: + if case_path is not None and (case_id or name): + raise ValueError("Only one of case_path, case_id, or name can be used.") + if case_path is not None: + return [case_path] + candidates = resolve_fixture_candidates(root=root, bucket=bucket, case_id=case_id, name=name) + if not candidates: + selector = case_id or name + raise FileNotFoundError(f"No fixtures found for selector={selector!r} in {bucket}") + if require_unique and len(candidates) > 1: + raise FileExistsError("Multiple fixtures matched selection; use --select-index or --select.") + if all_matches: + print("fixture selector: bulk operation on all matches") + for idx, candidate in enumerate(candidates, start=1): + print(f" {idx}. {candidate.path}") + return [candidate.path for candidate in candidates] + + selected = select_fixture_candidate( + candidates, + select=select, + select_index=select_index, + require_unique=require_unique, + ) + if len(candidates) > 1 and not require_unique: + print("fixture selector: multiple candidates found, selected:") + for idx, candidate in enumerate(candidates, start=1): + marker = " (selected)" if candidate.path == selected.path else "" + print(f" {idx}. {candidate.path}{marker}") + return [selected.path] + + +def select_fixture_candidate( + candidates: list[FixtureCandidate], + *, + select: str = "latest", + select_index: int | None = None, + require_unique: bool = False, +) -> FixtureCandidate: + if not candidates: + raise FileNotFoundError("No fixtures matched the selection.") + if require_unique and len(candidates) > 1: + raise FileExistsError("Multiple fixtures matched selection; use --select-index or --select.") + if select_index is not None: + if select_index < 1 or select_index > len(candidates): + raise ValueError(f"select_index must be between 1 and {len(candidates)}") + return candidates[select_index - 1] + + def _sort_key(candidate: FixtureCandidate) -> tuple[str, float]: + timestamp = candidate.source.get("timestamp") + run_id = candidate.source.get("run_id") + time_key = f"{timestamp or ''}{run_id or ''}" + try: + mtime = candidate.path.stat().st_mtime + except OSError: + mtime = 0.0 + return (time_key, mtime) + + ordered = sorted(candidates, key=_sort_key) + if select in {"latest", "last"}: + return ordered[-1] + if select == "first": + return ordered[0] + raise ValueError(f"Unsupported select policy: {select}") + + +def _git_available() -> bool: + try: + subprocess.run(["git", "--version"], check=True, capture_output=True, text=True) + except (OSError, subprocess.CalledProcessError): + return False + return True + + +def _is_git_repo(root: Path) -> bool: + try: + subprocess.run( + ["git", "-C", str(root), "rev-parse", "--is-inside-work-tree"], + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.CalledProcessError): + return False + return True + + +def _git_tracked(root: Path, path: Path) -> bool: + try: + subprocess.run( + ["git", "-C", str(root), "ls-files", "--error-unmatch", str(path)], + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.CalledProcessError): + return False + return True + + +def _git_dir_tracked(root: Path, path: Path) -> bool: + try: + result = subprocess.run( + ["git", "-C", str(root), "ls-files", str(path)], + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.CalledProcessError): + return False + return bool(result.stdout.strip()) + + +def _git_mv(root: Path, src: Path, dest: Path) -> None: + subprocess.run(["git", "-C", str(root), "mv", str(src), str(dest)], check=True) + + +def _git_rm(root: Path, path: Path) -> None: + subprocess.run(["git", "-C", str(root), "rm", "-r", "--", str(path)], check=True) + + +class _GitOps: + def __init__(self, root: Path, mode: str) -> None: + self.root = root + self.mode = mode + self.use_git = False + if mode not in {"auto", "on", "off"}: + raise ValueError(f"Unsupported git mode: {mode}") + if mode == "off": + return + git_ok = _git_available() and _is_git_repo(root) + if mode == "on" and not git_ok: + raise ValueError("git mode is on but no git repository is available") + self.use_git = git_ok + + def move(self, src: Path, dest: Path) -> None: + if self.use_git and self._should_git_move(src): + _git_mv(self.root, src, dest) + else: + shutil.move(str(src), str(dest)) + + def remove(self, path: Path) -> None: + if self.use_git and self._should_git_remove(path): + _git_rm(self.root, path) + else: + if path.is_dir(): + shutil.rmtree(path, ignore_errors=True) + elif path.exists(): + path.unlink() + + def _should_git_move(self, src: Path) -> bool: + if src.is_dir(): + return _git_dir_tracked(self.root, src) + return _git_tracked(self.root, src) + + def _should_git_remove(self, path: Path) -> bool: + if path.is_dir(): + return _git_dir_tracked(self.root, path) + return _git_tracked(self.root, path) + + +def _unique_backup_path(path: Path) -> Path: + for idx in range(1, 1000): + suffix = "" if idx == 1 else f".{idx}" + candidate = path.with_name(f".{path.name}.rollback{suffix}") + if not candidate.exists(): + return candidate + raise FileExistsError(f"Unable to create rollback path for {path}") + + +class _MoveTransaction: + def __init__(self, git_ops: _GitOps) -> None: + self.git_ops = git_ops + self.moved: list[tuple[Path, Path]] = [] + self.removed: list[tuple[Path, Path]] = [] + + def move(self, src: Path, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + self.git_ops.move(src, dest) + self.moved.append((src, dest)) + + def remove(self, path: Path) -> None: + if not path.exists(): + return + backup = _unique_backup_path(path) + backup.parent.mkdir(parents=True, exist_ok=True) + self.git_ops.move(path, backup) + self.removed.append((path, backup)) + + def rollback(self) -> None: + for src, dest in reversed(self.moved): + if dest.exists(): + self.git_ops.move(dest, src) + for original, backup in reversed(self.removed): + if backup.exists(): + self.git_ops.move(backup, original) + + def commit(self) -> None: + for _, backup in self.removed: + if backup.exists(): + self.git_ops.remove(backup) + + +def fixture_green( + *, + case_path: Path | None = None, + case_id: str | None = None, + name: str | None = None, + out_root: Path, + validate: bool = True, + expected_from: str = "replay", + overwrite_expected: bool = False, + dry_run: bool = False, + git_mode: str = "auto", + select: str = "latest", + select_index: int | None = None, + require_unique: bool = False, +) -> None: + if case_path is None and not case_id and not name: + raise ValueError("fixture-green requires --case, --case-id, or --name.") + if case_path is not None and (case_id or name): + raise ValueError("fixture-green accepts only one of --case, --case-id, or --name.") + if expected_from not in {"replay", "observed"}: + raise ValueError(f"Unsupported expected_from: {expected_from}") + out_root = out_root.resolve() + git_ops = _GitOps(out_root, git_mode) + known_layout = FixtureLayout(out_root, "known_bad") + if case_path is None: + candidates = resolve_fixture_candidates( + root=out_root, + bucket="known_bad", + case_id=case_id, + name=name, + ) + if not candidates: + selector = case_id or name + raise FileNotFoundError(f"No fixtures found for selector={selector!r} in known_bad") + selected = select_fixture_candidate( + candidates, + select=select, + select_index=select_index, + require_unique=require_unique, + ) + case_path = selected.path + if len(candidates) > 1 and not require_unique: + print("fixture-green: multiple candidates found, selected:") + for idx, candidate in enumerate(candidates, start=1): + marker = " (selected)" if candidate.path == selected.path else "" + print(f" {idx}. {candidate.path}{marker}") + case_path = case_path.resolve() + if not case_path.is_relative_to(known_layout.bucket_dir): + raise ValueError(f"fixture-green expects a known_bad case path, got: {case_path}") + if not case_path.name.endswith(".case.json"): + raise ValueError(f"fixture-green expects a .case.json bundle, got: {case_path}") + stem = case_path.name.replace(".case.json", "") + fixed_layout = FixtureLayout(out_root, "fixed") + + payload = load_bundle_json(case_path) + root = payload.get("root") or {} + observed = root.get("observed") + if expected_from == "observed" and not isinstance(observed, dict): + raise ValueError( + "Cannot green fixture: root.observed is missing.\n" + f"Case: {case_path}\n" + "Hint: re-export observed-first replay_case bundles or use --expected-from replay." + ) + + known_case_path = known_layout.case_path(stem) + fixed_case_path = fixed_layout.case_path(stem) + fixed_expected_path = fixed_layout.expected_path(stem) + known_expected_path = known_layout.expected_path(stem) + resources_from = known_layout.resources_dir(stem) + resources_to = fixed_layout.resources_dir(stem) + + if fixed_case_path.exists() and not dry_run: + raise FileExistsError(f"Target case already exists: {fixed_case_path}") + if fixed_expected_path.exists() and not overwrite_expected and not dry_run: + raise FileExistsError(f"Expected already exists: {fixed_expected_path}") + if resources_from.exists() and resources_to.exists() and not dry_run: + raise FileExistsError( + "Resources destination already exists:\n" + f" dest: {resources_to}\n" + "Actions:\n" + " - remove destination resources directory, or\n" + " - run fixture-fix to rename stems, or\n" + " - choose a different out root." + ) + + if dry_run: + print("fixture-green:") + print(f" case: {known_case_path}") + print(f" move: -> {fixed_case_path}") + source_label = "replay output" if expected_from == "replay" else "root.observed" + print(f" write: -> {fixed_expected_path} (from {source_label})") + if resources_from.exists(): + print(f" move: resources -> {resources_to}") + if validate: + print(" validate: would run") + return + + if expected_from == "replay" or validate: + import fetchgraph.tracer.handlers # noqa: F401 + + root_case, ctx = load_case_bundle(case_path) + replay_out = run_case(root_case, ctx) + else: + replay_out = None + + expected_payload = replay_out if expected_from == "replay" else observed + fixed_expected_path.parent.mkdir(parents=True, exist_ok=True) + tmp_expected_path = fixed_expected_path.with_suffix(fixed_expected_path.suffix + ".tmp") + tmp_expected_path.write_text( + json.dumps(expected_payload, ensure_ascii=False, sort_keys=True, indent=2), + encoding="utf-8", + ) + tx = _MoveTransaction(git_ops) + try: + tx.move(known_case_path, fixed_case_path) + + if resources_from.exists(): + tx.move(resources_from, resources_to) + + if known_expected_path.exists(): + tx.remove(known_expected_path) + + tmp_expected_path.replace(fixed_expected_path) + except Exception: + if fixed_expected_path.exists(): + fixed_expected_path.unlink() + if tmp_expected_path.exists(): + tmp_expected_path.unlink() + tx.rollback() + raise + + if validate: + try: + root_case, ctx = load_case_bundle(fixed_case_path) + out = run_case(root_case, ctx) + expected = json.loads(fixed_expected_path.read_text(encoding="utf-8")) + if out != expected: + raise AssertionError(_format_fixture_diff(out, expected)) + replay_id = root_case.get("id") if isinstance(root_case, dict) else None + if not isinstance(replay_id, str): + raise AssertionError("Case bundle has missing or non-string replay id.") + validator = REPLAY_VALIDATORS.get(replay_id) + if validator is None: + raise AssertionError(f"No validator registered for replay id={replay_id!r}") + validator(out) + except Exception as exc: + tx.rollback() + if fixed_expected_path.exists(): + fixed_expected_path.unlink() + raise AssertionError( + "fixture-green validation failed; rollback completed.\n" + f"fixture: {fixed_case_path}\n" + f"{exc}" + ) from exc + + tx.commit() + + print("fixture-green:") + print(f" case: {known_case_path}") + print(f" move: -> {fixed_case_path}") + source_label = "replay output" if expected_from == "replay" else "root.observed" + print(f" write: -> {fixed_expected_path} (from {source_label})") + if resources_from.exists(): + print(f" move: resources -> {resources_to}") + if validate: + print(" validate: OK") + + +def fixture_rm( + *, + root: Path, + bucket: str, + scope: str, + dry_run: bool, + git_mode: str = "auto", + case_path: Path | None = None, + case_id: str | None = None, + name: str | None = None, + pattern: str | None = None, + select: str = "latest", + select_index: int | None = None, + require_unique: bool = False, + all_matches: bool = False, +) -> int: + root = root.resolve() + git_ops = _GitOps(root, git_mode) + bucket_filter: str | None = None if bucket == "all" else bucket + if case_path or case_id or name: + selector_paths = resolve_fixture_selector( + root=root, + bucket=bucket_filter or bucket, + case_path=case_path, + case_id=case_id, + name=name, + select=select, + select_index=select_index, + require_unique=require_unique, + all_matches=all_matches, + ) + matched = selector_paths + else: + matched = find_case_bundles(root=root, bucket=bucket_filter, name=name, pattern=pattern) + + targets: list[Path] = [] + for case_path_item in matched: + bucket_name = case_path_item.parent.name + stem = case_path_item.name.replace(".case.json", "") + layout = FixtureLayout(root, bucket_name) + if scope in ("cases", "both"): + targets.extend([layout.case_path(stem), layout.expected_path(stem)]) + if scope in ("resources", "both"): + targets.append(layout.resources_dir(stem)) + + existing_targets = [target for target in targets if target.exists()] + + if name and not existing_targets: + raise FileNotFoundError(f"No fixtures found for name={name!r} with scope={scope}") + + if dry_run: + for target in targets: + print(f"Would remove: {target}") + return len(existing_targets) + + for target in targets: + git_ops.remove(target) + return len(existing_targets) + + +def fixture_fix( + *, + root: Path, + name: str, + new_name: str, + bucket: str, + dry_run: bool, + git_mode: str = "auto", +) -> None: + root = root.resolve() + git_ops = _GitOps(root, git_mode) + if name == new_name: + raise ValueError("fixture-fix requires a new name different from the old name.") + + layout = FixtureLayout(root, bucket) + case_path = layout.case_path(name) + expected_path = layout.expected_path(name) + resources_dir = layout.resources_dir(name) + new_case_path = layout.case_path(new_name) + new_expected_path = layout.expected_path(new_name) + new_resources_dir = layout.resources_dir(new_name) + + if not case_path.exists(): + raise FileNotFoundError(f"Missing case bundle: {case_path}") + if new_case_path.exists(): + raise FileExistsError(f"Target case already exists: {new_case_path}") + if new_expected_path.exists(): + raise FileExistsError(f"Target expected already exists: {new_expected_path}") + if new_resources_dir.exists(): + raise FileExistsError(f"Target resources already exist: {new_resources_dir}") + + payload = load_bundle_json(case_path) + resources = payload.get("resources") or {} + updated = False + if isinstance(resources, dict): + for resource in resources.values(): + if not isinstance(resource, dict): + continue + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + continue + rel = _safe_resource_path(file_name, stem=name) + old_prefix = f"resources/{name}/" + rel_posix = rel.as_posix() + if rel_posix.startswith(old_prefix): + data_ref["file"] = f"resources/{new_name}/{rel_posix[len(old_prefix):]}" + updated = True + + if dry_run: + print("fixture-fix:") + print(f" rename: {case_path} -> {new_case_path}") + if expected_path.exists(): + print(f" move: {expected_path} -> {new_expected_path}") + if resources_dir.exists(): + print(f" move: {resources_dir} -> {new_resources_dir}") + if updated: + print(" rewrite: data_ref.file paths updated") + return + + original_text = case_path.read_text(encoding="utf-8") if updated else None + tx = _MoveTransaction(git_ops) + try: + tx.move(case_path, new_case_path) + if expected_path.exists(): + tx.move(expected_path, new_expected_path) + if resources_dir.exists(): + tx.move(resources_dir, new_resources_dir) + if updated: + _atomic_write_json(new_case_path, payload) + tx.commit() + except Exception: + if updated and original_text is not None and new_case_path.exists(): + new_case_path.write_text(original_text, encoding="utf-8") + tx.rollback() + raise + + print("fixture-fix:") + print(f" rename: {case_path} -> {new_case_path}") + if expected_path.exists(): + print(f" move: {expected_path} -> {new_expected_path}") + if resources_dir.exists(): + print(f" move: {resources_dir} -> {new_resources_dir}") + if updated: + print(" rewrite: data_ref.file paths updated") + + +def fixture_migrate( + *, + root: Path, + bucket: str = "all", + dry_run: bool, + git_mode: str = "auto", + case_path: Path | None = None, + case_id: str | None = None, + name: str | None = None, + select: str = "latest", + select_index: int | None = None, + require_unique: bool = False, + all_matches: bool = False, +) -> tuple[int, int]: + root = root.resolve() + git_ops = _GitOps(root, git_mode) + bucket_filter = None if bucket == "all" else bucket + if case_path or case_id or name: + selector_paths = resolve_fixture_selector( + root=root, + bucket=bucket_filter or bucket, + case_path=case_path, + case_id=case_id, + name=name, + select=select, + select_index=select_index, + require_unique=require_unique, + all_matches=all_matches, + ) + matched = selector_paths + else: + matched = find_case_bundles(root=root, bucket=bucket_filter, name=None, pattern=None) + bundles_updated = 0 + files_moved = 0 + for case_path in matched: + stem = case_path.name.replace(".case.json", "") + payload = load_bundle_json(case_path) + resources = payload.get("resources") or {} + if not isinstance(resources, dict): + continue + updated = False + for resource_id, resource in resources.items(): + if not isinstance(resource, dict): + continue + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + continue + rel = _safe_resource_path(file_name, stem=stem) + if rel.parts[:3] == ("resources", stem, resource_id): + continue + if rel.parts[:1] == ("resources",) and len(rel.parts) >= 2: + rel_tail = Path(*rel.parts[2:]) + else: + rel_tail = rel + if not rel_tail.parts: + rel_tail = Path(rel.name) + target_rel = Path("resources") / stem / resource_id / rel_tail + src_path = case_path.parent / rel + if not src_path.exists(): + raise FileNotFoundError(f"Missing resource file: {src_path}") + dest_path = case_path.parent / target_rel + if dest_path.exists() and not filecmp.cmp(src_path, dest_path, shallow=False): + raise FileExistsError( + "Resource collision at destination:\n" + f" dest: {dest_path}\n" + "Hint: clean the destination or run migrate in an empty output directory." + ) + if dry_run: + print(f"Would move {src_path} -> {dest_path}") + else: + dest_path.parent.mkdir(parents=True, exist_ok=True) + if not dest_path.exists(): + git_ops.move(src_path, dest_path) + files_moved += 1 + data_ref["file"] = target_rel.as_posix() + updated = True + if updated: + bundles_updated += 1 + if dry_run: + print(f"Would update bundle: {case_path}") + else: + _atomic_write_json(case_path, payload) + return bundles_updated, files_moved + + +def fixture_ls( + *, + root: Path, + bucket: str, + case_id: str | None = None, + pattern: str | None = None, +) -> list[FixtureCandidate]: + bucket_filter = None if bucket == "all" else bucket + candidates = find_case_bundles(root=root, bucket=bucket_filter, name=None, pattern=pattern) + results: list[FixtureCandidate] = [] + for path in candidates: + payload = load_bundle_json(path) + stem = path.name.replace(".case.json", "") + source = payload.get("source") + source_dict = source if isinstance(source, dict) else {} + if case_id: + source_case = source_dict.get("case") or source_dict.get("case_id") + if source_case != case_id and not stem.startswith(case_id): + continue + results.append(FixtureCandidate(path=path, stem=stem, source=source_dict)) + return results + + +def fixture_demote( + *, + root: Path, + from_bucket: str = "fixed", + to_bucket: str = "known_bad", + case_path: Path | None = None, + case_id: str | None = None, + name: str | None = None, + dry_run: bool = False, + git_mode: str = "auto", + overwrite: bool = False, + select: str = "latest", + select_index: int | None = None, + require_unique: bool = False, + all_matches: bool = False, +) -> None: + root = root.resolve() + if from_bucket == to_bucket: + raise ValueError("from_bucket and to_bucket must be different") + git_ops = _GitOps(root, git_mode) + selected_paths = resolve_fixture_selector( + root=root, + bucket=from_bucket, + case_path=case_path, + case_id=case_id, + name=name, + select=select, + select_index=select_index, + require_unique=require_unique, + all_matches=all_matches, + ) + for case_path_item in selected_paths: + stem = case_path_item.name.replace(".case.json", "") + from_layout = FixtureLayout(root, from_bucket) + to_layout = FixtureLayout(root, to_bucket) + from_case = from_layout.case_path(stem) + from_expected = from_layout.expected_path(stem) + from_resources = from_layout.resources_dir(stem) + to_case = to_layout.case_path(stem) + to_expected = to_layout.expected_path(stem) + to_resources = to_layout.resources_dir(stem) + + if not from_case.exists(): + raise FileNotFoundError(f"Missing fixture: {from_case}") + if to_case.exists() and not overwrite and not dry_run: + raise FileExistsError(f"Target case already exists: {to_case}") + if to_expected.exists() and from_expected.exists() and not overwrite and not dry_run: + raise FileExistsError(f"Target expected already exists: {to_expected}") + if to_resources.exists() and from_resources.exists() and not overwrite and not dry_run: + raise FileExistsError(f"Target resources already exists: {to_resources}") + + if dry_run: + print("fixture-demote:") + print(f" case: {from_case}") + print(f" move: -> {to_case}") + if from_expected.exists(): + print(f" move: {from_expected} -> {to_expected}") + if from_resources.exists(): + print(f" move: {from_resources} -> {to_resources}") + continue + + tx = _MoveTransaction(git_ops) + try: + if overwrite: + tx.remove(to_case) + tx.move(from_case, to_case) + if from_expected.exists(): + if overwrite: + tx.remove(to_expected) + tx.move(from_expected, to_expected) + if from_resources.exists(): + if overwrite: + tx.remove(to_resources) + tx.move(from_resources, to_resources) + tx.commit() + except Exception: + tx.rollback() + raise + + print("fixture-demote:") + print(f" case: {from_case}") + print(f" move: -> {to_case}") + if from_expected.exists(): + print(f" move: {from_expected} -> {to_expected}") + if from_resources.exists(): + print(f" move: {from_resources} -> {to_resources}") diff --git a/src/fetchgraph/tracer/handlers/__init__.py b/src/fetchgraph/tracer/handlers/__init__.py new file mode 100644 index 00000000..4c3d95cc --- /dev/null +++ b/src/fetchgraph/tracer/handlers/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from fetchgraph.replay.handlers.plan_normalize import replay_plan_normalize_spec_v1 + +__all__ = ["replay_plan_normalize_spec_v1"] diff --git a/src/fetchgraph/tracer/log.py b/src/fetchgraph/tracer/log.py new file mode 100644 index 00000000..375f1642 --- /dev/null +++ b/src/fetchgraph/tracer/log.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from fetchgraph.replay.log import EventLoggerLike, log_replay_case + +__all__ = ["EventLoggerLike", "log_replay_case"] diff --git a/src/fetchgraph/tracer/resolve.py b/src/fetchgraph/tracer/resolve.py new file mode 100644 index 00000000..fe19060a --- /dev/null +++ b/src/fetchgraph/tracer/resolve.py @@ -0,0 +1,620 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +from fetchgraph.replay.export import iter_events + + +@dataclass(frozen=True) +class CaseResolution: + run_dir: Path + case_dir: Path + events_path: Path + tag: str | None + + +@dataclass(frozen=True) +class CaseRunCandidate: + run_dir: Path + case_dir: Path + events_path: Path + tag: str | None + tag_source: str | None + status: str | None + is_missed: bool + run_mtime: float + case_mtime: float + + +@dataclass(frozen=True) +class CaseRunInfo: + run_dir: Path + case_dir: Path + events: "EventsResolution" + tag: str | None + tag_source: str | None + status: str | None + is_missed: bool + run_mtime: float + case_mtime: float + + +@dataclass(frozen=True) +class EventsResolution: + events_path: Path | None + searched: list[str] + found: list[Path] + + +@dataclass(frozen=True) +class RejectionReason: + run_dir: Path + case_dir: Path + reason: str + + +def resolve_case_events( + *, + case_id: str, + data_dir: Path, + tag: str | None = None, + runs_subdir: str = ".runs/runs", + pick_run: str = "latest_non_missed", + select_index: int | None = None, + replay_id: str | None = None, +) -> CaseResolution: + if not case_id: + raise ValueError("case_id is required") + if not data_dir: + raise ValueError("data_dir is required") + if pick_run not in {"latest_non_missed", "latest_with_replay"}: + raise ValueError(f"Unsupported pick_run mode: {pick_run}") + if pick_run == "latest_with_replay" and not replay_id: + raise ValueError("replay_id is required for latest_with_replay selection") + + candidates, stats = list_case_runs( + case_id=case_id, + data_dir=data_dir, + tag=tag, + pick_run=pick_run, + replay_id=replay_id, + runs_subdir=runs_subdir, + ) + if not candidates: + details = _format_missing_case_runs( + stats, + case_id=case_id, + tag=tag, + rule=_resolve_rule(tag=tag, pick_run=pick_run), + ) + raise LookupError(details) + + candidate = select_case_run(candidates, select_index=select_index) + return CaseResolution( + run_dir=candidate.run_dir, + case_dir=candidate.case_dir, + events_path=candidate.events_path, + tag=candidate.tag, + ) + + +def resolve_run_dir_from_run_id(*, data_dir: Path, runs_subdir: str, run_id: str) -> Path: + if not run_id: + raise ValueError("run_id is required") + if not data_dir: + raise ValueError("data_dir is required") + runs_root = (data_dir / runs_subdir).resolve() + history_path = data_dir / ".runs" / "history.jsonl" + history_candidates: list[Path] = [] + if history_path.exists(): + entries: list[dict] = [] + with history_path.open("r", encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + try: + entries.append(json.loads(line)) + except json.JSONDecodeError: + continue + for entry in reversed(entries): + if str(entry.get("run_id")) != run_id: + continue + run_dir_value = entry.get("run_dir") + run_folder_value = entry.get("run_folder") + if not run_dir_value and not run_folder_value: + continue + candidate: Path | None = None + if run_dir_value: + candidate = Path(run_dir_value) + if not candidate.is_absolute(): + candidate = runs_root / candidate + if candidate is None and run_folder_value: + candidate = runs_root / str(run_folder_value) + if candidate is None: + continue + if candidate.exists(): + return candidate + history_candidates.append(candidate) + + if not runs_root.exists(): + raise FileNotFoundError(f"Runs directory does not exist: {runs_root}") + matched: list[Path] = [] + for run_dir in _iter_run_dirs(runs_root): + meta_path = run_dir / "run_meta.json" + if not meta_path.exists(): + continue + try: + payload = json.loads(meta_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + continue + if str(payload.get("run_id")) != run_id: + continue + matched.append(run_dir) + if matched: + matched.sort(key=lambda p: p.stat().st_mtime, reverse=True) + return matched[0] + + hint = "" + if history_candidates: + hint = f"History entries pointed to: {', '.join(str(p) for p in history_candidates)}" + raise LookupError( + "Run id could not be resolved.\n" + f"run_id: {run_id}\n" + f"history_path: {history_path}\n" + f"runs_root: {runs_root}\n" + f"{hint}".rstrip() + ) + + +def _iter_run_dirs(runs_root: Path) -> Iterable[Path]: + candidates = [p for p in runs_root.iterdir() if p.is_dir()] + return sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True) + + +def _case_dirs(run_dir: Path, case_id: str) -> list[Path]: + cases_root = run_dir / "cases" + if not cases_root.exists(): + return [] + return sorted(cases_root.glob(f"{case_id}_*"), key=lambda p: p.stat().st_mtime, reverse=True) + + +@dataclass(frozen=True) +class RunScanStats: + runs_root: Path + inspected_runs: int + inspected_cases: int + missing_cases: int + missing_events: int + missed_cases: int + tag_mismatches: int + recent: list[CaseRunInfo] + + +def list_case_runs( + *, + case_id: str, + data_dir: Path, + tag: str | None = None, + pick_run: str = "latest_non_missed", + replay_id: str | None = None, + runs_subdir: str = ".runs/runs", +) -> tuple[list[CaseRunCandidate], RunScanStats]: + infos, stats = scan_case_runs( + case_id=case_id, + data_dir=data_dir, + tag=tag, + runs_subdir=runs_subdir, + ) + candidates = _filter_case_run_infos(infos, tag=tag, pick_run=pick_run, replay_id=replay_id) + return candidates, stats + + +def scan_case_runs( + *, + case_id: str, + data_dir: Path, + tag: str | None = None, + runs_subdir: str = ".runs/runs", +) -> tuple[list[CaseRunInfo], RunScanStats]: + runs_root = (data_dir / runs_subdir).resolve() + if not runs_root.exists(): + raise FileNotFoundError(f"Runs directory does not exist: {runs_root}") + + infos: list[CaseRunInfo] = [] + inspected_runs = 0 + inspected_cases = 0 + missing_cases = 0 + missing_events = 0 + missed_cases = 0 + + for run_dir in _iter_run_dirs(runs_root): + inspected_runs += 1 + case_dirs = _case_dirs(run_dir, case_id) + if not case_dirs: + missing_cases += 1 + continue + run_mtime = run_dir.stat().st_mtime + for case_dir in case_dirs: + inspected_cases += 1 + events = find_events_file(case_dir) + if events.events_path is None: + missing_events += 1 + case_mtime = case_dir.stat().st_mtime + status_value, is_missed = _case_status(case_dir) + if is_missed: + missed_cases += 1 + tag_value, tag_source = _extract_case_tag(case_dir) + infos.append( + CaseRunInfo( + run_dir=run_dir, + case_dir=case_dir, + events=events, + tag=tag_value, + tag_source=tag_source, + status=status_value, + is_missed=is_missed, + run_mtime=run_mtime, + case_mtime=case_mtime, + ) + ) + + infos.sort(key=lambda info: (info.run_mtime, info.case_mtime), reverse=True) + stats = RunScanStats( + runs_root=runs_root, + inspected_runs=inspected_runs, + inspected_cases=inspected_cases, + missing_cases=missing_cases, + missing_events=missing_events, + missed_cases=missed_cases, + tag_mismatches=_count_tag_mismatches(infos, tag=tag), + recent=infos[:10], + ) + return infos, stats + + +def _filter_case_run_infos( + infos: list[CaseRunInfo], + *, + tag: str | None = None, + pick_run: str = "latest_non_missed", + replay_id: str | None = None, +) -> list[CaseRunCandidate]: + candidates: list[CaseRunCandidate] = [] + for info in infos: + if info.events.events_path is None: + continue + if pick_run == "latest_non_missed" and info.is_missed: + continue + if pick_run == "latest_with_replay": + if replay_id is None: + raise ValueError("replay_id is required for latest_with_replay selection") + if not _has_replay_case_id(info.events.events_path, replay_id): + continue + if tag and info.tag != tag: + continue + candidates.append( + CaseRunCandidate( + run_dir=info.run_dir, + case_dir=info.case_dir, + events_path=info.events.events_path, + tag=info.tag, + tag_source=info.tag_source, + status=info.status, + is_missed=info.is_missed, + run_mtime=info.run_mtime, + case_mtime=info.case_mtime, + ) + ) + return candidates + + +def _count_tag_mismatches(infos: list[CaseRunInfo], tag: str | None) -> int: + if not tag: + return 0 + mismatches = 0 + for info in infos: + if info.tag != tag: + mismatches += 1 + return mismatches + + +def select_case_run(candidates: list[CaseRunCandidate], *, select_index: int | None = None) -> CaseRunCandidate: + if not candidates: + raise LookupError("No case run candidates available.") + if select_index is None: + return candidates[0] + if select_index < 1 or select_index > len(candidates): + raise ValueError(f"select_index must be between 1 and {len(candidates)}") + return candidates[select_index - 1] + + +def format_case_runs(candidates: list[CaseRunCandidate], *, limit: int | None = 10) -> str: + rows = [] + for idx, candidate in enumerate(candidates[:limit], start=1): + rows.append( + " " + f"{idx}. run_dir={candidate.run_dir} " + f"case_dir={candidate.case_dir.name} " + f"tag={candidate.tag!r} " + f"status={candidate.status!r} " + f"missed={candidate.is_missed} " + f"run_mtime={candidate.run_mtime:.0f} " + f"case_mtime={candidate.case_mtime:.0f}" + ) + if len(candidates) > (limit or 0): + rows.append(f" ... ({len(candidates) - (limit or 0)} more)") + return "\n".join(rows) + + +def format_case_run_debug(infos: list[CaseRunInfo], *, limit: int = 10) -> str: + rows = [] + for idx, info in enumerate(infos[:limit], start=1): + rows.append( + " " + f"{idx}. run_dir={info.run_dir} " + f"case_dir={info.case_dir.name} " + f"tag={info.tag!r} " + f"tag_source={info.tag_source!r} " + f"status={info.status!r} " + f"missed={info.is_missed} " + f"events={bool(info.events.events_path)} " + f"run_mtime={info.run_mtime:.0f} " + f"case_mtime={info.case_mtime:.0f}" + ) + if len(infos) > limit: + rows.append(f" ... ({len(infos) - limit} more)") + return "\n".join(rows) + + +_EVENTS_CANDIDATES = ( + "events.jsonl", + "events.ndjson", + "trace.jsonl", + "trace.ndjson", + "traces/events.jsonl", + "traces/trace.jsonl", +) + + +def find_events_file(run_dir: Path) -> EventsResolution: + searched = [str(entry) for entry in _EVENTS_CANDIDATES] + found: list[Path] = [] + for rel in _EVENTS_CANDIDATES: + candidate = run_dir / rel + if candidate.exists(): + return EventsResolution(events_path=candidate, searched=searched, found=found) + + candidates = [] + for path in run_dir.rglob("*"): + if path.is_dir(): + continue + if path.suffix not in {".jsonl", ".ndjson"}: + continue + rel_parts = path.relative_to(run_dir).parts + if "resources" in rel_parts: + continue + if len(rel_parts) > 3: + continue + candidates.append(path) + if candidates: + candidates.sort(key=lambda p: p.stat().st_size, reverse=True) + found = candidates + return EventsResolution(events_path=candidates[0], searched=searched, found=found) + + return EventsResolution(events_path=None, searched=searched, found=found) + + +def format_events_search(run_dir: Path, resolution: EventsResolution) -> str: + found_list = [str(path) for path in resolution.found] + return "\n".join( + [ + f"events file not found in {run_dir}.", + f"Looked for: {', '.join(resolution.searched)}", + f"Found jsonl/ndjson: {found_list}", + "You can pass --events explicitly.", + ] + ) + + +def _extract_case_tag(case_dir: Path) -> tuple[str | None, str | None]: + run_dir = case_dir.parent.parent + for name in ("run_meta.json", "meta.json"): + tag = _extract_tag_from_json(run_dir / name) + if tag: + return tag, name + tag = _extract_tag_from_json(run_dir / "summary.json") + if tag: + return tag, "summary.json" + for name in ("status.json", "result.json"): + tag = _extract_tag_from_json(case_dir / name) + if tag: + return tag, name + tag = _tag_from_run_dir_name(run_dir.name) + if tag: + return tag, "run_dir" + events = find_events_file(case_dir) + if events.events_path: + tag = _extract_tag_from_events(events.events_path) + if tag: + return tag, "events" + return None, None + + +def _extract_tag_from_json(path: Path) -> str | None: + if not path.exists(): + return None + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return None + if not isinstance(payload, dict): + return None + tag = _extract_tag_value(payload) + if tag: + return tag + for meta_key in ("run_meta", "meta"): + nested = payload.get(meta_key) + if isinstance(nested, dict): + tag = _extract_tag_value(nested) + if tag: + return tag + return None + + +def _extract_tag_from_events(path: Path, *, max_lines: int = 200) -> str | None: + try: + with path.open("r", encoding="utf-8") as handle: + for idx, line in enumerate(handle, start=1): + if idx > max_lines: + break + line = line.strip() + if not line: + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + tag = _extract_tag_value(payload) + if tag: + return tag + except OSError: + return None + return None + + +def _tag_from_run_dir_name(name: str) -> str | None: + if not name: + return None + lowered = name.lower() + for prefix in ("tag=", "tag-", "bucket=", "bucket-"): + if prefix in lowered: + tail = lowered.split(prefix, 1)[1].strip() + return tail or None + return None + + +def _extract_tag_value(payload: dict) -> str | None: + for key in ("tag", "TAG", "bucket", "batch_tag", "bucket_tag"): + value = payload.get(key) + if isinstance(value, str) and value: + return value + tags_value = payload.get("tags") + if isinstance(tags_value, list) and tags_value: + for entry in tags_value: + if isinstance(entry, str) and entry: + return entry + return None + + +def _case_status(case_dir: Path) -> tuple[str | None, bool]: + for name in ("status.json", "result.json"): + path = case_dir / name + if not path.exists(): + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + continue + status_value = payload.get("status") or payload.get("result") + status_str = str(status_value) if status_value is not None else None + is_missed = _payload_is_missed(payload) + return status_str, is_missed + return None, False + + +def _payload_is_missed(payload: dict) -> bool: + if payload.get("missed") is True: + return True + status = str(payload.get("status") or payload.get("result") or "").lower() + if status in {"missed", "missing"}: + return True + reason = str(payload.get("reason") or "").lower() + return "missed" in reason or "missing" in reason + + +def _has_replay_case_id(events_path: Path, replay_id: str) -> bool: + try: + for _, event in iter_events(events_path, allow_bad_json=True): + if event.get("type") == "replay_case" and event.get("id") == replay_id: + return True + except FileNotFoundError: + return False + return False + + +def collect_rejections( + infos: list[CaseRunInfo], + *, + tag: str | None, + pick_run: str, + replay_id: str | None, +) -> list[RejectionReason]: + rejections: list[RejectionReason] = [] + for info in infos: + if tag and info.tag != tag: + rejections.append(RejectionReason(info.run_dir, info.case_dir, "tag_mismatch")) + continue + if info.events.events_path is None: + rejections.append(RejectionReason(info.run_dir, info.case_dir, "no_events")) + continue + if pick_run == "latest_non_missed" and info.is_missed: + rejections.append(RejectionReason(info.run_dir, info.case_dir, "missed")) + continue + if pick_run == "latest_with_replay": + if replay_id is None or not _has_replay_case_id(info.events.events_path, replay_id): + rejections.append(RejectionReason(info.run_dir, info.case_dir, "no_replay_id")) + continue + rejections.append(RejectionReason(info.run_dir, info.case_dir, "filtered")) + return rejections + + +def _resolve_rule(*, tag: str | None, pick_run: str = "latest_non_missed") -> str: + base = "latest with events" + if pick_run == "latest_with_replay": + base = "latest with replay_case" + if tag: + return f"{base} filtered by TAG={tag!r}" + return base + + +def _format_missing_case_runs( + stats: RunScanStats, + *, + case_id: str, + tag: str | None, + rule: str, +) -> str: + details = [ + "No suitable case run found.", + f"selection_rule: {rule}", + f"runs_root: {stats.runs_root}", + f"case_id: {case_id}", + f"inspected_runs: {stats.inspected_runs}", + f"inspected_cases: {stats.inspected_cases}", + f"missing_cases: {stats.missing_cases}", + f"missing_events: {stats.missing_events}", + f"missed_cases: {stats.missed_cases}", + ] + if tag: + details.append(f"tag: {tag}") + details.append(f"tag_mismatches: {stats.tag_mismatches}") + if stats.recent: + details.append("recent_cases:") + for info in stats.recent[:5]: + details.append( + " " + f"case_dir={info.case_dir} " + f"tag={info.tag!r} " + f"events={bool(info.events.events_path)} " + f"status={info.status!r} " + f"missed={info.is_missed}" + ) + details.append("Tip: verify TAG or pass RUN_ID/CASE_DIR/EVENTS.") + else: + details.append("Tip: pass TAG/RUN_ID/CASE_DIR/EVENTS for a narrower selection.") + return "\n".join(details) diff --git a/src/fetchgraph/tracer/runtime.py b/src/fetchgraph/tracer/runtime.py new file mode 100644 index 00000000..9da70789 --- /dev/null +++ b/src/fetchgraph/tracer/runtime.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from fetchgraph.replay.runtime import REPLAY_HANDLERS, ReplayContext, load_case_bundle, run_case + +__all__ = ["REPLAY_HANDLERS", "ReplayContext", "load_case_bundle", "run_case"] diff --git a/src/fetchgraph/tracer/validators.py b/src/fetchgraph/tracer/validators.py new file mode 100644 index 00000000..f921b311 --- /dev/null +++ b/src/fetchgraph/tracer/validators.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from pydantic import TypeAdapter + +from fetchgraph.relational.models import RelationalRequest + +RELATIONAL_PROVIDERS = {"demo_qa", "relational"} + + +def validate_plan_normalize_spec_v1(out: dict) -> None: + if not isinstance(out, dict): + raise AssertionError("Output must be a dict") + out_spec = out.get("out_spec") + if not isinstance(out_spec, dict): + raise AssertionError("Output must contain out_spec dict") + provider = out_spec.get("provider") + selectors = out_spec.get("selectors") + if selectors is None: + raise AssertionError("out_spec.selectors is required") + if not isinstance(selectors, dict): + raise AssertionError("out_spec.selectors must be a dict") + is_relational = ( + provider in RELATIONAL_PROVIDERS + or (isinstance(provider, str) and provider.startswith("relational")) + or any(key in selectors for key in ("root_entity", "relations", "entity")) + ) + if not is_relational: + return + if "root_entity" not in selectors: + if "entity" in selectors: + raise AssertionError( + "Relational selectors missing required key 'root_entity' " + "(looks like you produced 'entity' instead)." + ) + raise AssertionError("Relational selectors missing required key 'root_entity'") + TypeAdapter(RelationalRequest).validate_python(selectors) + + +def validate_resource_read_v1(out: dict) -> None: + if not isinstance(out, dict): + raise AssertionError("Output must be a dict") + text = out.get("text") + if not isinstance(text, str): + raise AssertionError("Output must include text string") + + +REPLAY_VALIDATORS = { + "plan_normalize.spec_v1": validate_plan_normalize_spec_v1, + "resource_read.v1": validate_resource_read_v1, +} diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0001_20260119_133537_692233_orders_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0001_20260119_133537_692233_orders_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0001_20260119_133537_692233_orders_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0001_20260119_133537_692233_orders_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0002_20260119_133537_708677_customers_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0002_20260119_133537_708677_customers_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0002_20260119_133537_708677_customers_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0002_20260119_133537_708677_customers_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0004_20260119_133537_736816_order_items_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0004_20260119_133537_736816_order_items_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0004_20260119_133537_736816_order_items_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0004_20260119_133537_736816_order_items_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0006_20260119_133537_770370_min_orders.order_date_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0006_20260119_133537_770370_min_orders.order_date_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0006_20260119_133537_770370_min_orders.order_date_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0006_20260119_133537_770370_min_orders.order_date_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0007_20260119_133537_787946_max_orders.order_date_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0007_20260119_133537_787946_max_orders.order_date_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0007_20260119_133537_787946_max_orders.order_date_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0007_20260119_133537_787946_max_orders.order_date_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0008_20260119_133537_808662_sum_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0008_20260119_133537_808662_sum_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0008_20260119_133537_808662_sum_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0008_20260119_133537_808662_sum_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0009_20260119_133537_829192_avg_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0009_20260119_133537_829192_avg_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0009_20260119_133537_829192_avg_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0009_20260119_133537_829192_avg_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0010_20260119_133537_844853_median_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0010_20260119_133537_844853_median_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0010_20260119_133537_844853_median_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0010_20260119_133537_844853_median_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0011_20260119_133537_860976_min_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0011_20260119_133537_860976_min_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0011_20260119_133537_860976_min_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0011_20260119_133537_860976_min_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0012_20260119_133537_878573_max_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0012_20260119_133537_878573_max_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0012_20260119_133537_878573_max_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0012_20260119_133537_878573_max_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0013_20260119_133537_895165_order_id_order_total_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0013_20260119_133537_895165_order_id_order_total_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0013_20260119_133537_895165_order_id_order_total_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0013_20260119_133537_895165_order_id_order_total_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0014_20260119_133537_910859_order_id_order_total_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0014_20260119_133537_910859_order_id_order_total_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0014_20260119_133537_910859_order_id_order_total_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0014_20260119_133537_910859_order_id_order_total_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0015_20260119_133537_928468_cancelled_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0015_20260119_133537_928468_cancelled_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0015_20260119_133537_928468_cancelled_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0015_20260119_133537_928468_cancelled_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0016_20260119_133537_944943_delivered_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0016_20260119_133537_944943_delivered_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0016_20260119_133537_944943_delivered_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0016_20260119_133537_944943_delivered_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0017_20260119_133537_961602_pending_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0017_20260119_133537_961602_pending_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0017_20260119_133537_961602_pending_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0017_20260119_133537_961602_pending_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0018_20260119_133537_978273_processing_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0018_20260119_133537_978273_processing_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0018_20260119_133537_978273_processing_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0018_20260119_133537_978273_processing_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0019_20260119_133537_995082_shipped_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0019_20260119_133537_995082_shipped_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0019_20260119_133537_995082_shipped_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0019_20260119_133537_995082_shipped_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0020_20260119_133538_011607_online_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0020_20260119_133538_011607_online_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0020_20260119_133538_011607_online_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0020_20260119_133538_011607_online_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0021_20260119_133538_028336_partner_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0021_20260119_133538_028336_partner_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0021_20260119_133538_028336_partner_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0021_20260119_133538_028336_partner_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0022_20260119_133538_055979_phone_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0022_20260119_133538_055979_phone_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0022_20260119_133538_055979_phone_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0022_20260119_133538_055979_phone_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0023_20260119_133538_084494_retail_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0023_20260119_133538_084494_retail_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0023_20260119_133538_084494_retail_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0023_20260119_133538_084494_retail_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0024_20260119_133538_114093_2022_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0024_20260119_133538_114093_2022_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0024_20260119_133538_114093_2022_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0024_20260119_133538_114093_2022_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0025_20260119_133538_142347_2023_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0025_20260119_133538_142347_2023_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0025_20260119_133538_142347_2023_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0025_20260119_133538_142347_2023_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0026_20260119_133538_157966_2024_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0026_20260119_133538_157966_2024_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0026_20260119_133538_157966_2024_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0026_20260119_133538_157966_2024_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0027_20260119_133538_174624_YYYY-MM._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0027_20260119_133538_174624_YYYY-MM._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0027_20260119_133538_174624_YYYY-MM._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0027_20260119_133538_174624_YYYY-MM._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0028_20260119_133538_194420_YYYY-MM._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0028_20260119_133538_194420_YYYY-MM._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0028_20260119_133538_194420_YYYY-MM._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0028_20260119_133538_194420_YYYY-MM._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0029_20260119_133538_211755_2022-03_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0029_20260119_133538_211755_2022-03_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0029_20260119_133538_211755_2022-03_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0029_20260119_133538_211755_2022-03_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0030_20260119_133538_229173_2022-07_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0030_20260119_133538_229173_2022-07_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0030_20260119_133538_229173_2022-07_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0030_20260119_133538_229173_2022-07_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0031_20260119_133538_245323_order_items.line_total_category_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0031_20260119_133538_245323_order_items.line_total_category_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0031_20260119_133538_245323_order_items.line_total_category_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0031_20260119_133538_245323_order_items.line_total_category_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0032_20260119_133539_259529_sum_line_total_toys_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0032_20260119_133539_259529_sum_line_total_toys_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0032_20260119_133539_259529_sum_line_total_toys_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0032_20260119_133539_259529_sum_line_total_toys_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0033_20260119_133539_279390_sum_line_total_toys_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0033_20260119_133539_279390_sum_line_total_toys_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0033_20260119_133539_279390_sum_line_total_toys_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0033_20260119_133539_279390_sum_line_total_toys_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0034_20260119_133539_305892_sum_line_total_books_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0034_20260119_133539_305892_sum_line_total_books_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0034_20260119_133539_305892_sum_line_total_books_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0034_20260119_133539_305892_sum_line_total_books_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0035_20260119_133539_332861_sum_line_total_electronics_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0035_20260119_133539_332861_sum_line_total_electronics_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0035_20260119_133539_332861_sum_line_total_electronics_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0035_20260119_133539_332861_sum_line_total_electronics_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0036_20260119_133539_360048_sum_line_total_furniture_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0036_20260119_133539_360048_sum_line_total_furniture_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0036_20260119_133539_360048_sum_line_total_furniture_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0036_20260119_133539_360048_sum_line_total_furniture_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0037_20260119_133539_389073_sum_line_total_office_supplies_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0037_20260119_133539_389073_sum_line_total_office_supplies_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0037_20260119_133539_389073_sum_line_total_office_supplies_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0037_20260119_133539_389073_sum_line_total_office_supplies_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0038_20260119_133539_411584_sum_line_total_outdoors_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0038_20260119_133539_411584_sum_line_total_outdoors_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0038_20260119_133539_411584_sum_line_total_outdoors_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0038_20260119_133539_411584_sum_line_total_outdoors_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0039_20260119_133539_433448_product_id_max_products.price_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0039_20260119_133539_433448_product_id_max_products.price_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0039_20260119_133539_433448_product_id_max_products.price_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0039_20260119_133539_433448_product_id_max_products.price_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0040_20260119_133539_457942_max_products.price_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0040_20260119_133539_457942_max_products.price_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0040_20260119_133539_457942_max_products.price_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0040_20260119_133539_457942_max_products.price_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0041_20260119_133539_480567_product_id_min_products.price_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0041_20260119_133539_480567_product_id_min_products.price_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0041_20260119_133539_480567_product_id_min_products.price_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0041_20260119_133539_480567_product_id_min_products.price_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0042_20260119_133539_504992_min_products.price_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0042_20260119_133539_504992_min_products.price_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0042_20260119_133539_504992_min_products.price_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0042_20260119_133539_504992_min_products.price_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0043_20260119_133539_527837_product_id_sum_order_items.line_total_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0043_20260119_133539_527837_product_id_sum_order_items.line_total_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0043_20260119_133539_527837_product_id_sum_order_items.line_total_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0043_20260119_133539_527837_product_id_sum_order_items.line_total_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0044_20260119_133540_291264_product_id-_sum_line_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0044_20260119_133540_291264_product_id-_sum_line_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0044_20260119_133540_291264_product_id-_sum_line_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0044_20260119_133540_291264_product_id-_sum_line_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0045_20260119_133540_822643_sum_order_items.quantity_-_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0045_20260119_133540_822643_sum_order_items.quantity_-_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0045_20260119_133540_822643_sum_order_items.quantity_-_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0045_20260119_133540_822643_sum_order_items.quantity_-_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0046_20260119_133541_342609_sum_order_items.quantity_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0046_20260119_133541_342609_sum_order_items.quantity_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0046_20260119_133541_342609_sum_order_items.quantity_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0046_20260119_133541_342609_sum_order_items.quantity_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0048_20260119_133541_375247_1_order_items_per_order_id_1_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0048_20260119_133541_375247_1_order_items_per_order_id_1_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0048_20260119_133541_375247_1_order_items_per_order_id_1_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0048_20260119_133541_375247_1_order_items_per_order_id_1_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0049_20260119_133541_393992_San_Antonio_2022-03_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0049_20260119_133541_393992_San_Antonio_2022-03_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0049_20260119_133541_393992_San_Antonio_2022-03_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0049_20260119_133541_393992_San_Antonio_2022-03_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0050_20260119_133541_413772_shipped_San_Antonio_2022-03_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0050_20260119_133541_413772_shipped_San_Antonio_2022-03_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0050_20260119_133541_413772_shipped_San_Antonio_2022-03_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0050_20260119_133541_413772_shipped_San_Antonio_2022-03_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0051_20260119_133541_432318_Los_Angeles_2023-08_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0051_20260119_133541_432318_Los_Angeles_2023-08_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0051_20260119_133541_432318_Los_Angeles_2023-08_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0051_20260119_133541_432318_Los_Angeles_2023-08_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0052_20260119_133541_448371_customer_id_323_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0052_20260119_133541_448371_customer_id_323_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0052_20260119_133541_448371_customer_id_323_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0052_20260119_133541_448371_customer_id_323_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0053_20260119_133541_464736_customer_id_323_consumer_corporate_home_office._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0053_20260119_133541_464736_customer_id_323_consumer_corporate_home_office._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0053_20260119_133541_464736_customer_id_323_consumer_corporate_home_office._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0053_20260119_133541_464736_customer_id_323_consumer_corporate_home_office._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0054_20260119_133541_480392_customers.signup_date_customer_id_323_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0054_20260119_133541_480392_customers.signup_date_customer_id_323_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0054_20260119_133541_480392_customers.signup_date_customer_id_323_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0054_20260119_133541_480392_customers.signup_date_customer_id_323_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0055_20260119_133541_500165_customer_id_323_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0055_20260119_133541_500165_customer_id_323_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0055_20260119_133541_500165_customer_id_323_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0055_20260119_133541_500165_customer_id_323_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0056_20260119_163550_586309_customer_id_323_sum_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0056_20260119_163550_586309_customer_id_323_sum_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0056_20260119_163550_586309_customer_id_323_sum_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0056_20260119_163550_586309_customer_id_323_sum_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0057_20260119_163556_526136_customer_id_323_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0057_20260119_163556_526136_customer_id_323_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0057_20260119_163556_526136_customer_id_323_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0057_20260119_163556_526136_customer_id_323_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0058_20260119_163557_042633_customer_id_536_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0058_20260119_163557_042633_customer_id_536_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0058_20260119_163557_042633_customer_id_536_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0058_20260119_163557_042633_customer_id_536_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0059_20260119_163557_528349_customer_id_536_consumer_corporate_home_office._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0059_20260119_163557_528349_customer_id_536_consumer_corporate_home_office._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0059_20260119_163557_528349_customer_id_536_consumer_corporate_home_office._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0059_20260119_163557_528349_customer_id_536_consumer_corporate_home_office._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0060_20260119_163604_515336_customers.signup_date_customer_id_536_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0060_20260119_163604_515336_customers.signup_date_customer_id_536_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0060_20260119_163604_515336_customers.signup_date_customer_id_536_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0060_20260119_163604_515336_customers.signup_date_customer_id_536_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0061_20260119_163606_123642_customer_id_536_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0061_20260119_163606_123642_customer_id_536_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0061_20260119_163606_123642_customer_id_536_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0061_20260119_163606_123642_customer_id_536_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0062_20260119_163606_140388_customer_id_536_sum_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0062_20260119_163606_140388_customer_id_536_sum_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0062_20260119_163606_140388_customer_id_536_sum_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0062_20260119_163606_140388_customer_id_536_sum_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0063_20260119_163606_850367_customer_id_536_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0063_20260119_163606_850367_customer_id_536_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0063_20260119_163606_850367_customer_id_536_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0063_20260119_163606_850367_customer_id_536_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0064_20260119_163607_536957_customer_id_692_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0064_20260119_163607_536957_customer_id_692_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0064_20260119_163607_536957_customer_id_692_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0064_20260119_163607_536957_customer_id_692_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0065_20260119_163608_189367_customer_id_692_consumer_corporate_home_office._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0065_20260119_163608_189367_customer_id_692_consumer_corporate_home_office._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0065_20260119_163608_189367_customer_id_692_consumer_corporate_home_office._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0065_20260119_163608_189367_customer_id_692_consumer_corporate_home_office._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0066_20260119_163608_690578_customers.signup_date_customer_id_692_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0066_20260119_163608_690578_customers.signup_date_customer_id_692_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0066_20260119_163608_690578_customers.signup_date_customer_id_692_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0066_20260119_163608_690578_customers.signup_date_customer_id_692_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0067_20260119_163638_280588_customer_id_692_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0067_20260119_163638_280588_customer_id_692_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0067_20260119_163638_280588_customer_id_692_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0067_20260119_163638_280588_customer_id_692_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0068_20260119_163638_300137_customer_id_692_sum_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0068_20260119_163638_300137_customer_id_692_sum_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0068_20260119_163638_300137_customer_id_692_sum_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0068_20260119_163638_300137_customer_id_692_sum_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0069_20260119_163638_822771_customer_id_692_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0069_20260119_163638_822771_customer_id_692_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0069_20260119_163638_822771_customer_id_692_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0069_20260119_163638_822771_customer_id_692_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0070_20260119_163648_740364_customer_id_722_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0070_20260119_163648_740364_customer_id_722_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0070_20260119_163648_740364_customer_id_722_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0070_20260119_163648_740364_customer_id_722_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0071_20260119_163653_414936_customer_id_722_consumer_corporate_home_office._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0071_20260119_163653_414936_customer_id_722_consumer_corporate_home_office._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0071_20260119_163653_414936_customer_id_722_consumer_corporate_home_office._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0071_20260119_163653_414936_customer_id_722_consumer_corporate_home_office._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0072_20260119_163658_005036_customers.signup_date_customer_id_722_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0072_20260119_163658_005036_customers.signup_date_customer_id_722_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0072_20260119_163658_005036_customers.signup_date_customer_id_722_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0072_20260119_163658_005036_customers.signup_date_customer_id_722_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0073_20260119_163707_431102_customer_id_722_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0073_20260119_163707_431102_customer_id_722_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0073_20260119_163707_431102_customer_id_722_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0073_20260119_163707_431102_customer_id_722_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0074_20260119_163710_235823_customer_id_722_sum_orders.order_total_2_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0074_20260119_163710_235823_customer_id_722_sum_orders.order_total_2_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0074_20260119_163710_235823_customer_id_722_sum_orders.order_total_2_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0074_20260119_163710_235823_customer_id_722_sum_orders.order_total_2_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0075_20260119_163715_021803_customer_id_722_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0075_20260119_163715_021803_customer_id_722_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0075_20260119_163715_021803_customer_id_722_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0075_20260119_163715_021803_customer_id_722_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0076_20260119_163719_274457_customer_id_725_._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0076_20260119_163719_274457_customer_id_725_._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0076_20260119_163719_274457_customer_id_725_._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0076_20260119_163719_274457_customer_id_725_._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0077_20260119_163722_195616_customer_id_725_consumer_corporate_home_office._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0077_20260119_163722_195616_customer_id_725_consumer_corporate_home_office._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0077_20260119_163722_195616_customer_id_725_consumer_corporate_home_office._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0077_20260119_163722_195616_customer_id_725_consumer_corporate_home_office._plan_trace.txt diff --git a/tests/fixtures/regressions_fixed/fetchgraph_plans/0078_20260119_163726_540394_customers.signup_date_customer_id_725_YYYY-MM-DD._plan_trace.txt b/tests/fixtures/plan_traces/fixed/0078_20260119_163726_540394_customers.signup_date_customer_id_725_YYYY-MM-DD._plan_trace.txt similarity index 100% rename from tests/fixtures/regressions_fixed/fetchgraph_plans/0078_20260119_163726_540394_customers.signup_date_customer_id_725_YYYY-MM-DD._plan_trace.txt rename to tests/fixtures/plan_traces/fixed/0078_20260119_163726_540394_customers.signup_date_customer_id_725_YYYY-MM-DD._plan_trace.txt diff --git a/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_003_plan_trace.txt b/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_003_plan_trace.txt deleted file mode 100644 index d565b440..00000000 --- a/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_003_plan_trace.txt +++ /dev/null @@ -1,35 +0,0 @@ -{ - "stage": "before_normalize", - "plan": { - "required_context": [ - "demo_qa" - ], - "context_plan": [ - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "entity": "products", - "aggregations": [ - { - "field": "product_id", - "agg": "count", - "alias": "total_products" - } - ], - "group_by": [], - "filters": null - }, - "max_tokens": 100 - } - ], - "adr_queries": [], - "constraints": [], - "entities": [], - "dtos": [], - "normalization_notes": [ - "{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"error\", \"selectors_validate_after\": \"error\", \"selectors_normalization_decision\": \"use_normalized_unvalidated\", \"selectors_before\": {\"op\": \"aggregate\", \"entity\": \"products\", \"aggregations\": [{\"field\": \"product_id\", \"agg\": \"count\", \"alias\": \"total_products\"}]}, \"selectors_after\": {\"op\": \"query\", \"entity\": \"products\", \"aggregations\": [{\"field\": \"product_id\", \"agg\": \"count\", \"alias\": \"total_products\"}], \"group_by\": [], \"filters\": null}}" - ] - } -} diff --git a/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_005_plan_trace.txt b/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_005_plan_trace.txt deleted file mode 100644 index d9e5a796..00000000 --- a/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_005_plan_trace.txt +++ /dev/null @@ -1,49 +0,0 @@ -{ - "stage": "before_normalize", - "plan": { - "required_context": [ - "demo_qa" - ], - "context_plan": [ - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "customers", - "select": [ - { - "expr": "customers.customer_id", - "alias": "customer_id" - } - ], - "relations": [ - "orders_to_customers" - ], - "aggregations": [ - { - "field": "customer_id", - "agg": "count_distinct", - "alias": "unique_customers" - } - ], - "filters": { - "type": "comparison", - "entity": "orders", - "field": "order_id", - "op": "is_not_null" - }, - "group_by": [] - }, - "max_tokens": 1000 - } - ], - "adr_queries": [], - "constraints": [], - "entities": [], - "dtos": [], - "normalization_notes": [ - "{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"error\", \"selectors_validate_after\": \"error\", \"selectors_normalization_decision\": \"use_normalized_unvalidated\", \"selectors_before\": {\"op\": \"query\", \"root_entity\": \"customers\", \"select\": [{\"expr\": \"customers.customer_id\", \"alias\": \"customer_id\"}], \"relations\": [\"orders_to_customers\"], \"aggregations\": [{\"field\": \"customer_id\", \"agg\": \"count_distinct\", \"alias\": \"unique_customers\"}], \"filters\": {\"type\": \"comparison\", \"entity\": \"orders\", \"field\": \"order_id\", \"op\": \"is_not_null\"}}, \"selectors_after\": {\"op\": \"query\", \"root_entity\": \"customers\", \"select\": [{\"expr\": \"customers.customer_id\", \"alias\": \"customer_id\"}], \"relations\": [\"orders_to_customers\"], \"aggregations\": [{\"field\": \"customer_id\", \"agg\": \"count_distinct\", \"alias\": \"unique_customers\"}], \"filters\": {\"type\": \"comparison\", \"entity\": \"orders\", \"field\": \"order_id\", \"op\": \"is_not_null\"}, \"group_by\": []}}" - ] - } -} diff --git a/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_035_plan_trace.txt b/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_035_plan_trace.txt deleted file mode 100644 index ac3ab166..00000000 --- a/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_035_plan_trace.txt +++ /dev/null @@ -1,38 +0,0 @@ -{ - "stage": "before_normalize", - "plan": { - "required_context": [ - "demo_qa" - ], - "context_plan": [ - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "orders", - "aggregations": [ - { - "field": "order_id", - "agg": "count" - } - ], - "filters": { - "type": "comparison", - "field": "order_date", - "op": "between", - "value": [ - "2022-03-01", - "2022-03-31" - ] - } - }, - "max_tokens": 2000 - } - ], - "adr_queries": null, - "constraints": null, - "entities": [], - "dtos": [] - } -} diff --git a/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_036_plan_trace.txt b/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_036_plan_trace.txt deleted file mode 100644 index ac3ab166..00000000 --- a/tests/fixtures/regressions_known_bad/fetchgraph_plans/agg_036_plan_trace.txt +++ /dev/null @@ -1,38 +0,0 @@ -{ - "stage": "before_normalize", - "plan": { - "required_context": [ - "demo_qa" - ], - "context_plan": [ - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "orders", - "aggregations": [ - { - "field": "order_id", - "agg": "count" - } - ], - "filters": { - "type": "comparison", - "field": "order_date", - "op": "between", - "value": [ - "2022-03-01", - "2022-03-31" - ] - } - }, - "max_tokens": 2000 - } - ], - "adr_queries": null, - "constraints": null, - "entities": [], - "dtos": [] - } -} diff --git a/tests/fixtures/regressions_known_bad/fetchgraph_plans/geo_001_plan_trace.txt b/tests/fixtures/regressions_known_bad/fetchgraph_plans/geo_001_plan_trace.txt deleted file mode 100644 index 55373c32..00000000 --- a/tests/fixtures/regressions_known_bad/fetchgraph_plans/geo_001_plan_trace.txt +++ /dev/null @@ -1,55 +0,0 @@ -{ - "stage": "before_normalize", - "plan": { - "required_context": [ - "demo_qa" - ], - "context_plan": [ - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "orders", - "select": [ - { - "expr": "count(order_id)", - "alias": "order_count" - } - ], - "filters": { - "type": "logical", - "op": "and", - "clauses": [ - { - "type": "comparison", - "entity": "customers", - "field": "city", - "op": "=", - "value": "San Antonio" - }, - { - "type": "comparison", - "field": "order_date", - "op": "starts_with", - "value": "2022-03" - } - ] - }, - "relations": [ - "orders_to_customers" - ], - "case_sensitivity": false - }, - "max_tokens": 500 - } - ], - "adr_queries": [], - "constraints": [], - "entities": [], - "dtos": [], - "normalization_notes": [ - "{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}" - ] - } -} diff --git a/tests/fixtures/regressions_known_bad/fetchgraph_plans/items_002_plan_trace.txt b/tests/fixtures/regressions_known_bad/fetchgraph_plans/items_002_plan_trace.txt deleted file mode 100644 index 711aae8c..00000000 --- a/tests/fixtures/regressions_known_bad/fetchgraph_plans/items_002_plan_trace.txt +++ /dev/null @@ -1,54 +0,0 @@ - - -{ - "stage": "before_normalize", - "plan": { - "required_context": [ - "demo_qa" - ], - "context_plan": [ - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "order_items", - "group_by": [ - { - "field": "order_id", - "alias": "order_id" - } - ], - "aggregations": [ - { - "field": "order_item_id", - "agg": "count", - "alias": "item_count" - } - ], - "filters": { - "type": "logical", - "op": "and", - "clauses": [ - { - "type": "comparison", - "entity": "order_items", - "field": "order_id", - "op": "is_not_null" - } - ] - }, - "case_sensitivity": false - }, - "max_tokens": 2000 - } - ], - "adr_queries": [], - "constraints": [], - "entities": [], - "dtos": [], - "normalization_notes": [ - "{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"error\", \"selectors_validate_after\": \"error\", \"selectors_normalization_decision\": \"keep_original_still_invalid\", \"selectors_before\": {\"op\": \"query\", \"root_entity\": \"order_items\", \"group_by\": [{\"field\": \"order_id\", \"alias\": \"order_id\"}], \"aggregations\": [{\"field\": \"order_item_id\", \"agg\": \"count\", \"alias\": \"item_count\"}], \"filters\": {\"type\": \"logical\", \"op\": \"and\", \"clauses\": [{\"type\": \"comparison\", \"entity\": \"order_items\", \"field\": \"order_id\", \"op\": \"is_not_null\"}]}, \"case_sensitivity\": false}, \"selectors_after\": {\"op\": \"query\", \"root_entity\": \"order_items\", \"group_by\": [{\"field\": \"order_id\", \"alias\": \"order_id\"}], \"aggregations\": [{\"field\": \"order_item_id\", \"agg\": \"count\", \"alias\": \"item_count\"}], \"filters\": {\"type\": \"logical\", \"op\": \"and\", \"clauses\": [{\"type\": \"comparison\", \"entity\": \"order_items\", \"field\": \"order_id\", \"op\": \"is_not_null\"}]}, \"case_sensitivity\": false}}" - ] - } -} diff --git a/tests/fixtures/regressions_known_bad/fetchgraph_plans/qa_001_plan_trace.txt b/tests/fixtures/regressions_known_bad/fetchgraph_plans/qa_001_plan_trace.txt deleted file mode 100644 index 033a172e..00000000 --- a/tests/fixtures/regressions_known_bad/fetchgraph_plans/qa_001_plan_trace.txt +++ /dev/null @@ -1,134 +0,0 @@ -{ - "stage": "before_normalize", - "plan": { - "required_context": [ - "demo_qa" - ], - "context_plan": [ - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "customers", - "select": [ - { - "expr": "city", - "alias": "city" - }, - { - "expr": "segment", - "alias": "segment" - }, - { - "expr": "signup_date", - "alias": "signup_date" - } - ], - "filters": { - "type": "comparison", - "field": "customer_id", - "op": "=", - "value": 42 - }, - "relations": [ - "orders_to_customers" - ], - "group_by": [ - { - "field": "customer_id", - "alias": "customer_id" - } - ], - "aggregations": [ - { - "field": "order_id", - "agg": "count", - "alias": "order_count" - }, - { - "field": "order_total", - "agg": "sum", - "alias": "total_spent" - } - ], - "limit": 1 - }, - "max_tokens": null - }, - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "orders", - "select": [ - { - "expr": "order_date", - "alias": "last_order_date" - }, - { - "expr": "order_status", - "alias": "last_order_status" - }, - { - "expr": "order_channel", - "alias": "last_order_channel" - } - ], - "filters": { - "type": "comparison", - "field": "customer_id", - "op": "=", - "value": 42 - }, - "limit": 1, - "order_by": [ - { - "field": "order_date", - "direction": "desc" - } - ] - }, - "max_tokens": null - }, - { - "provider": "demo_qa", - "mode": "slice", - "selectors": { - "op": "query", - "root_entity": "order_items", - "select": [ - { - "expr": "line_total", - "alias": "line_total" - } - ], - "filters": { - "type": "comparison", - "field": "order_id", - "op": "in", - "value": "(SELECT order_id FROM orders WHERE customer_id = 42)" - }, - "limit": 5, - "order_by": [ - { - "field": "line_total", - "direction": "desc" - } - ] - }, - "max_tokens": null - } - ], - "adr_queries": [], - "constraints": [], - "entities": [], - "dtos": [], - "normalization_notes": [ - "{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}", - "{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}", - "{\"provider\": \"demo_qa\", \"selectors_validate_before\": \"ok\", \"selectors_validate_after\": \"ok\", \"selectors_normalization_decision\": \"keep_original_valid\"}" - ] - } -} diff --git a/tests/fixtures/replay_cases/.gitkeep b/tests/fixtures/replay_cases/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/replay_cases/README.md b/tests/fixtures/replay_cases/README.md new file mode 100644 index 00000000..1f84c95c --- /dev/null +++ b/tests/fixtures/replay_cases/README.md @@ -0,0 +1,4 @@ +# Replay case fixtures + +This directory is intentionally empty in the repo. Replay case bundles are generated +locally from events and should be recreated as needed for tests. diff --git a/tests/fixtures/replay_cases/fixed/.gitkeep b/tests/fixtures/replay_cases/fixed/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/replay_cases/fixed/resource_read.v1__sample.case.json b/tests/fixtures/replay_cases/fixed/resource_read.v1__sample.case.json new file mode 100644 index 00000000..064b0042 --- /dev/null +++ b/tests/fixtures/replay_cases/fixed/resource_read.v1__sample.case.json @@ -0,0 +1,23 @@ +{ + "schema": "fetchgraph.tracer.case_bundle", + "v": 1, + "root": { + "type": "replay_case", + "v": 2, + "id": "resource_read.v1", + "input": { + "resource_id": "sample" + }, + "observed": { + "text": "hello" + } + }, + "resources": { + "sample": { + "data_ref": { + "file": "resources/resource_read.v1__sample/sample/sample.txt" + } + } + }, + "extras": {} +} diff --git a/tests/fixtures/replay_cases/fixed/resource_read.v1__sample.expected.json b/tests/fixtures/replay_cases/fixed/resource_read.v1__sample.expected.json new file mode 100644 index 00000000..ca87294c --- /dev/null +++ b/tests/fixtures/replay_cases/fixed/resource_read.v1__sample.expected.json @@ -0,0 +1,3 @@ +{ + "text": "hello\n" +} diff --git a/tests/fixtures/replay_cases/fixed/resources/resource_read.v1__sample/sample/sample.txt b/tests/fixtures/replay_cases/fixed/resources/resource_read.v1__sample/sample/sample.txt new file mode 100644 index 00000000..ce013625 --- /dev/null +++ b/tests/fixtures/replay_cases/fixed/resources/resource_read.v1__sample/sample/sample.txt @@ -0,0 +1 @@ +hello diff --git a/tests/fixtures/replay_cases/known_bad/.gitkeep b/tests/fixtures/replay_cases/known_bad/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/helpers/handlers_resource_read.py b/tests/helpers/handlers_resource_read.py new file mode 100644 index 00000000..e99f06a7 --- /dev/null +++ b/tests/helpers/handlers_resource_read.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from fetchgraph.replay.runtime import REPLAY_HANDLERS, ReplayContext + + +def _resource_read_handler(payload: dict, ctx: ReplayContext) -> dict: + resource_id = payload.get("resource_id") + if not isinstance(resource_id, str) or not resource_id: + raise ValueError("resource_id is required") + resource = ctx.resources.get(resource_id) + if not isinstance(resource, dict): + raise KeyError(f"Missing resource {resource_id!r}") + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + raise ValueError("resource.data_ref must be a dict") + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + raise ValueError("resource.data_ref.file must be a string") + path = ctx.resolve_resource_path(file_name) + text = path.read_text(encoding="utf-8") + return {"text": text} + + +REPLAY_HANDLERS.setdefault("resource_read.v1", _resource_read_handler) diff --git a/tests/helpers/replay_dx.py b/tests/helpers/replay_dx.py new file mode 100644 index 00000000..d56df3c2 --- /dev/null +++ b/tests/helpers/replay_dx.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path + + +def _int_env(name: str, default: int) -> int: + value = os.getenv(name) + if not value: + return default + try: + return int(value) + except ValueError: + return default + + +def truncate(text: str, *, limit: int = 8000) -> str: + if len(text) <= limit: + return text + remaining = len(text) - limit + return f"{text[:limit]}...[truncated {remaining} chars]" + + +def format_json(obj: object, *, max_chars: int = 12_000, max_depth: int = 6) -> str: + def _prune(value: object, depth: int) -> object: + if depth <= 0: + return "...(max depth reached)" + if isinstance(value, dict): + return {str(key): _prune(val, depth - 1) for key, val in value.items()} + if isinstance(value, list): + return [_prune(item, depth - 1) for item in value] + if isinstance(value, tuple): + return tuple(_prune(item, depth - 1) for item in value) + return value + + text = json.dumps( + _prune(obj, max_depth), + ensure_ascii=False, + indent=2, + sort_keys=True, + default=str, + ) + return truncate(text, limit=max_chars) + + +def format_rule_trace(diag: dict | None, *, tail: int = 30) -> str: + if not diag: + return "" + rule_trace = diag.get("rule_trace") + if not isinstance(rule_trace, list) or not rule_trace: + return "" + tail_items = rule_trace[-tail:] if tail > 0 else rule_trace + return truncate(format_json(tail_items), limit=_int_env("FETCHGRAPH_REPLAY_TRUNCATE", 8000)) + + +def build_rerun_hints(bundle_path: Path) -> list[str]: + stem = bundle_path.stem + return [ + f"pytest -k {stem} -m known_bad -vv", + f"fetchgraph-tracer fixture-green --case {bundle_path}", + f"fetchgraph-tracer replay --case {bundle_path} --debug", + ] + + +def ids_from_path(path: Path, base: Path) -> str: + try: + return str(path.relative_to(base)) + except ValueError: + return path.stem + + +def truncate_limits() -> tuple[int, int]: + return ( + _int_env("FETCHGRAPH_REPLAY_TRUNCATE", 12_000), + _int_env("FETCHGRAPH_REPLAY_META_TRUNCATE", 8000), + ) + + +def rule_trace_tail() -> int: + return _int_env("FETCHGRAPH_RULE_TRACE_TAIL", 30) + + +def debug_enabled() -> bool: + return os.getenv("FETCHGRAPH_REPLAY_DEBUG") not in (None, "", "0", "false", "False") diff --git a/tests/test_make_tracer_export.py b/tests/test_make_tracer_export.py new file mode 100644 index 00000000..c432b475 --- /dev/null +++ b/tests/test_make_tracer_export.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import subprocess + + +def test_make_tracer_export_no_default_filters() -> None: + result = subprocess.run( + [ + "make", + "-n", + "tracer-export", + "REPLAY_ID=plan_normalize.spec_v1", + "CASE=agg_003", + ], + capture_output=True, + text=True, + check=True, + ) + output = result.stdout + result.stderr + assert "--provider" not in output + assert "--spec-idx" not in output diff --git a/tests/test_relational_normalizer_regression.py b/tests/test_relational_normalizer_regression.py index cec371ee..1a961adc 100644 --- a/tests/test_relational_normalizer_regression.py +++ b/tests/test_relational_normalizer_regression.py @@ -3,7 +3,6 @@ import copy import json import re -import zipfile from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Set @@ -38,17 +37,8 @@ def _iter_json_objects_from_trace_text(text: str) -> Iterable[Dict[str, Any]]: yield obj -def _find_fixtures_dir() -> Optional[Path]: - """ - Ищем папку `fixtures` вверх по дереву от текущего test-файла. - Это устойчиво к вложенности tests/... - """ - here = Path(__file__).resolve() - for parent in here.parents: - cand = parent / "fixtures" - if cand.exists(): - return cand - return None +def _fixtures_root() -> Path: + return Path(__file__).parent / "fixtures" / "plan_traces" @dataclass(frozen=True) @@ -69,57 +59,30 @@ def case_id(self) -> str: def _load_trace_cases_from_fixtures() -> List[TraceCase]: """ Ищет по бакетам: - - fixtures/regressions_fixed/fetchgraph_plans.zip - - fixtures/regressions_fixed/fetchgraph_plans/*.txt - - fixtures/regressions_known_bad/fetchgraph_plans.zip - - fixtures/regressions_known_bad/fetchgraph_plans/*.txt - + - fixtures/plan_traces/fixed/*.txt + - fixtures/plan_traces/known_bad/*.txt Достаёт спецификации из stage=before_normalize (plan.context_plan[*]). """ - fixtures_dir = _find_fixtures_dir() - if fixtures_dir is None: - pytest.skip("No fixtures dir found (expected .../fixtures).", allow_module_level=True) + fixtures_root = _fixtures_root() + if not fixtures_root.exists(): + pytest.skip( + "No plan fixtures found in tests/fixtures/plan_traces.", + allow_module_level=True, + ) return [] cases: List[TraceCase] = [] - buckets = ["regressions_fixed", "regressions_known_bad"] + buckets = ["fixed", "known_bad"] for bucket in buckets: - zip_path = fixtures_dir / bucket / "fetchgraph_plans.zip" - dir_path = fixtures_dir / bucket / "fetchgraph_plans" - - if zip_path.exists(): - with zipfile.ZipFile(zip_path) as zf: - for name in sorted(zf.namelist()): - if not name.endswith("_plan_trace.txt"): - continue - text = zf.read(name).decode("utf-8", errors="replace") - cases.extend(_extract_before_specs(trace_name=name, text=text, bucket=bucket)) - continue - - if dir_path.exists(): - for p in sorted(dir_path.glob("*_plan_trace.txt")): - text = p.read_text(encoding="utf-8", errors="replace") - cases.extend(_extract_before_specs(trace_name=p.name, text=text, bucket=bucket)) - - legacy_zip = fixtures_dir / "fetchgraph_plans.zip" - legacy_dir = fixtures_dir / "fetchgraph_plans" - if legacy_zip.exists(): - with zipfile.ZipFile(legacy_zip) as zf: - for name in sorted(zf.namelist()): - if not name.endswith("_plan_trace.txt"): - continue - text = zf.read(name).decode("utf-8", errors="replace") - cases.extend(_extract_before_specs(trace_name=name, text=text, bucket="regressions_fixed")) - if legacy_dir.exists(): - for p in sorted(legacy_dir.glob("*_plan_trace.txt")): + bucket_dir = fixtures_root / bucket + for p in sorted(bucket_dir.glob("*_plan_trace.txt")): text = p.read_text(encoding="utf-8", errors="replace") - cases.extend(_extract_before_specs(trace_name=p.name, text=text, bucket="regressions_fixed")) + cases.extend(_extract_before_specs(trace_name=p.name, text=text, bucket=bucket)) if not cases: pytest.skip( - "No plan fixtures found. Put fetchgraph_plans.zip into fixtures/regressions_fixed " - "or fixtures/regressions_known_bad, or unpack it to " - "fixtures/regressions_fixed/fetchgraph_plans or fixtures/regressions_known_bad/fetchgraph_plans.", + "No plan fixtures found. Put *_plan_trace.txt into " + "tests/fixtures/plan_traces/{fixed,known_bad}.", allow_module_level=True, ) return cases @@ -173,6 +136,7 @@ def _build_plan_normalizer(providers: Set[str]) -> PlanNormalizer: } relational_rule = SelectorNormalizationRule( + kind="relational_v1", validator=TypeAdapter(RelationalRequest), normalize_selectors=normalize_relational_selectors, ) @@ -215,15 +179,15 @@ def _parse_note(note: str) -> Dict[str, Any]: CASES = _load_trace_cases_from_fixtures() -# 1) Контрактный тест запускаем ТОЛЬКО на regressions_fixed -CASES_FIXED = [c for c in CASES if c.bucket == "regressions_fixed"] +# 1) Контрактный тест запускаем ТОЛЬКО на fixed +CASES_FIXED = [c for c in CASES if c.bucket == "fixed"] # 2) Для общего набора — автоматически проставляем marks по bucket CASES_ALL = [ pytest.param( c, id=c.case_id, - marks=(pytest.mark.known_bad,) if c.bucket == "regressions_known_bad" else (), + marks=(pytest.mark.known_bad,) if c.bucket == "known_bad" else (), ) for c in CASES ] @@ -325,4 +289,3 @@ def test_regression_fixtures_invalid_inputs_are_fixed_by_plan_normalizer(case: T f"Selectors(before): {json.dumps(spec.selectors, ensure_ascii=False)[:2000]}\n" f"Selectors(after): {json.dumps(out.selectors, ensure_ascii=False)[:2000]}" ) - diff --git a/tests/test_replay_export_selection.py b/tests/test_replay_export_selection.py new file mode 100644 index 00000000..849703ed --- /dev/null +++ b/tests/test_replay_export_selection.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fetchgraph.replay.export import ( + case_bundle_name, + export_replay_case_bundle, + find_replay_case_matches, + iter_events, +) + + +def _write_events(path: Path, events: list[dict]) -> None: + lines = [json.dumps(event) for event in events] + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def test_iter_events_allow_bad_json(tmp_path: Path) -> None: + events_path = tmp_path / "events.jsonl" + events_path.write_text('{"type":"ok"}\n{bad json}\n{"type":"ok2"}\n', encoding="utf-8") + + with pytest.raises(ValueError, match="Invalid JSON on line 2"): + list(iter_events(events_path, allow_bad_json=False)) + + events = list(iter_events(events_path, allow_bad_json=True)) + assert [event["type"] for _, event in events] == ["ok", "ok2"] + + +def test_export_replay_case_selection_controls(tmp_path: Path) -> None: + events_path = tmp_path / "events.jsonl" + out_dir = tmp_path / "out" + event_base = { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}}, + "observed": {"out_spec": {"provider": "sql"}}, + } + _write_events( + events_path, + [ + {**event_base, "timestamp": "2024-01-01T00:00:00Z"}, + {**event_base, "timestamp": "2024-01-02T00:00:00Z"}, + ], + ) + + selections = find_replay_case_matches(events_path, replay_id="plan_normalize.spec_v1") + assert len(selections) == 2 + + with pytest.raises(LookupError, match="Multiple replay_case entries matched"): + export_replay_case_bundle( + events_path=events_path, + out_dir=out_dir, + replay_id="plan_normalize.spec_v1", + require_unique=True, + ) + + out_path = export_replay_case_bundle( + events_path=events_path, + out_dir=out_dir, + replay_id="plan_normalize.spec_v1", + select_index=1, + ) + payload = json.loads(out_path.read_text(encoding="utf-8")) + assert payload["source"]["line"] == 1 + + +def test_export_replay_case_requires_run_dir_for_resources(tmp_path: Path) -> None: + events_path = tmp_path / "events.jsonl" + out_dir = tmp_path / "out" + events = [ + { + "type": "replay_resource", + "id": "rid1", + "data_ref": {"file": "artifact.txt"}, + }, + { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}}, + "observed": {"out_spec": {"provider": "sql"}}, + "requires": [{"kind": "resource", "id": "rid1"}], + }, + ] + _write_events(events_path, events) + + with pytest.raises(ValueError, match="run_dir is required to export file resources"): + export_replay_case_bundle( + events_path=events_path, + out_dir=out_dir, + replay_id="plan_normalize.spec_v1", + ) + + +def test_export_replay_case_overwrite_cleans_resources(tmp_path: Path) -> None: + events_path = tmp_path / "events.jsonl" + out_dir = tmp_path / "out" + run_dir = tmp_path / "run" + run_dir.mkdir() + (run_dir / "artifact.txt").write_text("data", encoding="utf-8") + + events = [ + { + "type": "replay_resource", + "id": "rid1", + "data_ref": {"file": "artifact.txt"}, + }, + { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}}, + "observed": {"out_spec": {"provider": "sql"}}, + "requires": [{"kind": "resource", "id": "rid1"}], + }, + ] + _write_events(events_path, events) + + out_path = export_replay_case_bundle( + events_path=events_path, + out_dir=out_dir, + replay_id="plan_normalize.spec_v1", + run_dir=run_dir, + ) + fixture_stem = case_bundle_name("plan_normalize.spec_v1", events[1]["input"]).replace(".case.json", "") + extra_path = out_dir / "resources" / fixture_stem / "extra.txt" + extra_path.parent.mkdir(parents=True, exist_ok=True) + extra_path.write_text("extra", encoding="utf-8") + + export_replay_case_bundle( + events_path=events_path, + out_dir=out_dir, + replay_id="plan_normalize.spec_v1", + run_dir=run_dir, + overwrite=True, + ) + assert out_path.exists() + assert not extra_path.exists() diff --git a/tests/test_replay_fixed.py b/tests/test_replay_fixed.py new file mode 100644 index 00000000..6772fb8c --- /dev/null +++ b/tests/test_replay_fixed.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pytest + +import fetchgraph.tracer.handlers # noqa: F401 +import tests.helpers.handlers_resource_read # noqa: F401 +from fetchgraph.tracer.runtime import load_case_bundle, run_case +from fetchgraph.tracer.diff_utils import first_diff_path +from fetchgraph.tracer.validators import REPLAY_VALIDATORS +from tests.helpers.replay_dx import format_json, ids_from_path, truncate, truncate_limits + +REPLAY_CASES_ROOT = Path(__file__).parent / "fixtures" / "replay_cases" +FIXED_DIR = REPLAY_CASES_ROOT / "fixed" + + +def _iter_fixed_paths() -> list[Path]: + if not FIXED_DIR.exists(): + return [] + return sorted(FIXED_DIR.rglob("*.case.json")) + + +def _iter_case_params() -> list[object]: + return [pytest.param(path, id=ids_from_path(path, REPLAY_CASES_ROOT)) for path in _iter_fixed_paths()] + + +def _expected_path(case_path: Path) -> Path: + if not case_path.name.endswith(".case.json"): + raise ValueError(f"Unexpected case filename: {case_path}") + return case_path.with_name(case_path.name.replace(".case.json", ".expected.json")) + + +@pytest.mark.parametrize("case_path", _iter_case_params()) +def test_replay_fixed_cases(case_path: Path) -> None: + expected_path = _expected_path(case_path) + if not expected_path.exists(): + pytest.fail( + "Expected file is required for fixed fixtures:\n" + f" missing: {expected_path}\n" + "Hint: run `fetchgraph-tracer fixture-green --case ...` or create expected json manually." + ) + root, ctx = load_case_bundle(case_path) + out = run_case(root, ctx) + expected = json.loads(expected_path.read_text(encoding="utf-8")) + if out != expected: + input_limit, meta_limit = truncate_limits() + meta = root.get("meta") if isinstance(root, dict) else None + note = root.get("note") if isinstance(root, dict) else None + missing_keys = [] + extra_keys = [] + if isinstance(out, dict) and isinstance(expected, dict): + out_keys = set(out.keys()) + expected_keys = set(expected.keys()) + missing_keys = sorted(expected_keys - out_keys) + extra_keys = sorted(out_keys - expected_keys) + out_spec_path = None + if isinstance(out, dict) and isinstance(expected, dict): + out_spec = out.get("out_spec") + expected_spec = expected.get("out_spec") + if out_spec != expected_spec: + out_spec_path = first_diff_path(out_spec, expected_spec, prefix="out_spec") + message = "\n".join( + [ + "Fixed fixture mismatch.", + f"fixture: {case_path}", + f"meta: {truncate(format_json(meta, max_chars=meta_limit), limit=meta_limit)}", + f"note: {truncate(format_json(note, max_chars=meta_limit), limit=meta_limit)}", + f"out: {truncate(format_json(out, max_chars=input_limit), limit=input_limit)}", + f"expected: {truncate(format_json(expected, max_chars=input_limit), limit=input_limit)}", + f"missing_keys: {missing_keys}", + f"extra_keys: {extra_keys}", + f"out_spec_path: {out_spec_path}", + ] + ) + pytest.fail(message, pytrace=False) + replay_id = root.get("id") if isinstance(root, dict) else None + if not isinstance(replay_id, str): + pytest.fail("Case bundle has missing or non-string replay id.", pytrace=False) + validator = REPLAY_VALIDATORS.get(replay_id) + if validator is None: + pytest.fail( + f"No validator registered for replay id={replay_id!r}. Add it to REPLAY_VALIDATORS.", + pytrace=False, + ) + validator(out) + assert out == expected + + +def test_replay_case_resources_exist() -> None: + case_paths = _iter_fixed_paths() + if not case_paths: + pytest.skip("No replay case bundles found.") + missing: list[tuple[Path, Path]] = [] + for case_path in case_paths: + raw = json.loads(case_path.read_text(encoding="utf-8")) + resources = raw.get("resources") or {} + if not isinstance(resources, dict): + continue + for resource in resources.values(): + if not isinstance(resource, dict): + continue + data_ref = resource.get("data_ref") + if not isinstance(data_ref, dict): + continue + file_name = data_ref.get("file") + if not isinstance(file_name, str) or not file_name: + continue + resolved = case_path.parent / file_name + if not resolved.exists(): + missing.append((case_path, resolved)) + if missing: + details = "\n".join(f"- {fixture}: {resource}" for fixture, resource in missing) + pytest.fail(f"Missing replay resources:\n{details}") + + +def test_resource_read_missing_file(tmp_path: Path) -> None: + bundle_path = FIXED_DIR / "resource_read.v1__sample.case.json" + if not bundle_path.exists(): + pytest.skip("Resource read fixture not available.") + target_bundle = tmp_path / bundle_path.name + resources_dir = FIXED_DIR / "resources" / "resource_read.v1__sample" + target_resources = tmp_path / "resources" / "resource_read.v1__sample" + target_resources.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(bundle_path, target_bundle) + shutil.copytree(resources_dir, target_resources) + + missing_path = target_resources / "sample" / "sample.txt" + if missing_path.exists(): + missing_path.unlink() + + root, ctx = load_case_bundle(target_bundle) + with pytest.raises(FileNotFoundError): + run_case(root, ctx) diff --git a/tests/test_replay_iter_events_bad_json.py b/tests/test_replay_iter_events_bad_json.py new file mode 100644 index 00000000..ae68b0a9 --- /dev/null +++ b/tests/test_replay_iter_events_bad_json.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fetchgraph.replay.export import iter_events + + +def test_iter_events_bad_json_excerpt(tmp_path: Path) -> None: + events_path = tmp_path / "events.jsonl" + events_path.write_text( + '{"type":"ok"}\n' + '{"type":"bad"\n' + '{"type":"ok2"}\n', + encoding="utf-8", + ) + + with pytest.raises(ValueError, match=r"Invalid JSON on line 2: .* in .*events.jsonl"): + list(iter_events(events_path, allow_bad_json=False)) + + +def test_iter_events_allow_bad_json_warns(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + events_path = tmp_path / "events.jsonl" + events_path.write_text( + json.dumps({"type": "ok"}) + "\n" + + "{bad json}\n" + + json.dumps({"type": "ok2"}) + "\n", + encoding="utf-8", + ) + + caplog.set_level("WARNING") + events = list(iter_events(events_path, allow_bad_json=True)) + assert [event["type"] for _, event in events] == ["ok", "ok2"] + assert any("Skipped 1 invalid JSON lines" in record.message for record in caplog.records) diff --git a/tests/test_replay_known_bad_backlog.py b/tests/test_replay_known_bad_backlog.py new file mode 100644 index 00000000..fcaea965 --- /dev/null +++ b/tests/test_replay_known_bad_backlog.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import json +import traceback +from pathlib import Path +import pytest +from pydantic import ValidationError + +import fetchgraph.tracer.handlers # noqa: F401 +from fetchgraph.tracer.runtime import load_case_bundle, run_case +from fetchgraph.tracer.validators import REPLAY_VALIDATORS +from tests.helpers.replay_dx import ( + build_rerun_hints, + debug_enabled, + format_json, + format_rule_trace, + ids_from_path, + rule_trace_tail, + truncate, + truncate_limits, +) + +REPLAY_CASES_ROOT = Path(__file__).parent / "fixtures" / "replay_cases" +KNOWN_BAD_DIR = REPLAY_CASES_ROOT / "known_bad" + +def _iter_known_bad_paths() -> list[Path]: + if not KNOWN_BAD_DIR.exists(): + return [] + return sorted(KNOWN_BAD_DIR.rglob("*.case.json")) + + +def _iter_case_params() -> list[object]: + return [pytest.param(path, id=ids_from_path(path, REPLAY_CASES_ROOT)) for path in _iter_known_bad_paths()] + + +def _format_common_block( + *, + bundle_path: Path, + root: dict, + input_limit: int, + meta_limit: int, +) -> list[str]: + note = root.get("note") + requires = root.get("requires") + meta = root.get("meta") + lines = [ + f"fixture: {bundle_path}", + "bucket: known_bad", + f"id: {root.get('id')!r}", + "rerun:", + ] + lines.extend([f" - {hint}" for hint in build_rerun_hints(bundle_path)]) + lines.append(f"meta: {truncate(format_json(meta, max_chars=meta_limit), limit=meta_limit)}") + if note: + lines.append(f"note: {truncate(format_json(note, max_chars=meta_limit), limit=meta_limit)}") + lines.append(f"requires: {truncate(format_json(requires, max_chars=meta_limit), limit=meta_limit)}") + lines.append(f"input: {truncate(format_json(root.get('input'), max_chars=input_limit), limit=input_limit)}") + return lines + + +def _validate_root_schema(bundle: dict) -> None: + if bundle.get("schema") != "fetchgraph.tracer.case_bundle": + raise ValueError(f"Unexpected schema: {bundle.get('schema')!r}") + if bundle.get("v") != 1: + raise ValueError(f"Unexpected bundle version: {bundle.get('v')!r}") + root_case = bundle.get("root") + if not isinstance(root_case, dict): + raise ValueError("Missing root replay_case entry.") + if root_case.get("type") != "replay_case": + raise ValueError(f"Unexpected root type: {root_case.get('type')!r}") + if root_case.get("v") != 2: + raise ValueError(f"Unexpected replay_case version: {root_case.get('v')!r}") + + +@pytest.mark.known_bad +@pytest.mark.parametrize("bundle_path", _iter_case_params()) +def test_known_bad_backlog(bundle_path: Path) -> None: + bundle_payload = json.loads(bundle_path.read_text(encoding="utf-8")) + if not isinstance(bundle_payload, dict): + pytest.fail(f"Unexpected bundle payload type: {type(bundle_payload)}", pytrace=False) + _validate_root_schema(bundle_payload) + root, ctx = load_case_bundle(bundle_path) + if not isinstance(root, dict): + pytest.fail(f"Unexpected replay_case payload type: {type(root)}", pytrace=False) + replay_id = root.get("id") + if not isinstance(replay_id, str) or not replay_id: + pytest.fail("Replay id missing or invalid in bundle root.", pytrace=False) + replay_id_str: str = replay_id + validator = REPLAY_VALIDATORS.get(replay_id_str) + if validator is None: + pytest.fail( + f"No validator registered for replay id={replay_id!r}. Add it to REPLAY_VALIDATORS.", + pytrace=False, + ) + input_limit, meta_limit = truncate_limits() + common_block = _format_common_block( + bundle_path=bundle_path, + root=root, + input_limit=input_limit, + meta_limit=meta_limit, + ) + if debug_enabled(): + print("\n".join(["KNOWN_BAD debug summary:"] + common_block)) + + try: + out = run_case(root, ctx) + except Exception as exc: + message = "\n".join( + [ + "KNOWN_BAD (backlog): handler raised exception", + *common_block, + f"exception: {exc!r}", + f"traceback:\n{truncate(traceback.format_exc(), limit=meta_limit)}", + ] + ) + pytest.fail(message, pytrace=False) + + try: + validator(out) + except (AssertionError, ValidationError) as exc: + diag = out.get("diag") if isinstance(out, dict) else None + rule_trace = format_rule_trace(diag, tail=rule_trace_tail()) + message = "\n".join( + [ + "KNOWN_BAD (backlog): output is invalid", + *common_block, + f"validator_error: {exc!r}", + f"rule_trace (tail): {rule_trace or 'N/A'}", + f"out: {truncate(format_json(out, max_chars=input_limit), limit=input_limit)}", + ] + ) + pytest.fail(message, pytrace=False) + except Exception as exc: + message = "\n".join( + [ + "KNOWN_BAD (backlog): validator raised unexpected exception", + *common_block, + f"validator_error: {exc!r}", + f"traceback:\n{truncate(traceback.format_exc(), limit=meta_limit)}", + ] + ) + pytest.fail(message, pytrace=False) + + message = "\n".join( + [ + "KNOWN_BAD is now PASSING. Promote to fixed", + *common_block, + "Promote this fixture to fixed (freeze expected) and remove it from known_bad.", + f"Command: fetchgraph-tracer fixture-green --case {bundle_path}", + "expected will be created from replay output (default)", + ] + ) + pytest.fail(message, pytrace=False) diff --git a/tests/test_replay_log_trace_truncate.py b/tests/test_replay_log_trace_truncate.py new file mode 100644 index 00000000..5f9bd1d5 --- /dev/null +++ b/tests/test_replay_log_trace_truncate.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from fetchgraph.replay.log import TRACE_LIMIT, log_replay_case + + +class _Recorder: + def __init__(self) -> None: + self.events: list[dict] = [] + + def emit(self, event: dict) -> None: + self.events.append(event) + + +def test_log_replay_case_truncates_trace() -> None: + logger = _Recorder() + trace = "A" * (TRACE_LIMIT + 500) + log_replay_case( + logger, + id="plan_normalize.spec_v1", + input={"spec": {"provider": "sql"}}, + observed_error={"type": "Boom", "message": "bad", "trace": trace}, + ) + assert logger.events + observed_error = logger.events[0]["observed_error"] + assert isinstance(observed_error, dict) + truncated = observed_error["trace"] + assert isinstance(truncated, str) + assert len(truncated) < len(trace) + assert "truncated" in truncated diff --git a/tests/test_replay_requires_plan_normalize.py b/tests/test_replay_requires_plan_normalize.py new file mode 100644 index 00000000..a6a428af --- /dev/null +++ b/tests/test_replay_requires_plan_normalize.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path + +import pytest + +from fetchgraph.replay.export import export_replay_case_bundle +from fetchgraph.replay.handlers.plan_normalize import replay_plan_normalize_spec_v1 +from fetchgraph.replay.runtime import ReplayContext + + +def _write_events(path: Path, events: list[dict]) -> None: + path.write_text("\n".join(json.dumps(event) for event in events) + "\n", encoding="utf-8") + + +def test_export_bundle_resolves_planner_input_schema_ref(tmp_path: Path) -> None: + events_path = tmp_path / "events.jsonl" + replay_case = { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "meta": {"provider": "relational", "spec_idx": 0}, + "input": { + "spec": {"provider": "relational", "mode": "full", "selectors": {"op": "query"}}, + "options": {}, + "normalizer_rules": {"relational": "relational_v1"}, + }, + "observed": { + "out_spec": {"provider": "relational", "mode": "full", "selectors": {"op": "query"}}, + }, + "requires": [{"kind": "extra", "id": "planner_input_v1"}], + } + planner_input = { + "type": "planner_input", + "id": "planner_input_v1", + "input": { + "provider_catalog": { + "relational": { + "name": "relational", + "selectors_schema": {"type": "object", "properties": {"op": {"type": "string"}}}, + } + }, + "schema_ref": "schema_v1", + }, + } + schema_resource = {"type": "replay_resource", "id": "schema_v1", "data": {"schema": {"type": "object"}}} + _write_events(events_path, [replay_case, planner_input, schema_resource]) + + out_dir = tmp_path / "out" + bundle_path = export_replay_case_bundle( + events_path=events_path, + out_dir=out_dir, + replay_id="plan_normalize.spec_v1", + ) + bundle = json.loads(bundle_path.read_text(encoding="utf-8")) + assert "planner_input_v1" in bundle["extras"] + assert "schema_v1" in bundle["resources"] + + +def test_export_requires_missing_planner_input_raises(tmp_path: Path) -> None: + events_path = tmp_path / "events.jsonl" + replay_case = { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "meta": {"provider": "relational", "spec_idx": 2}, + "input": { + "spec": {"provider": "relational", "mode": "full", "selectors": {"op": "query"}}, + "options": {}, + "normalizer_rules": {"relational": "relational_v1"}, + }, + "observed": { + "out_spec": {"provider": "relational", "mode": "full", "selectors": {"op": "query"}}, + }, + "requires": [{"kind": "extra", "id": "planner_input_v1"}], + } + _write_events(events_path, [replay_case]) + + with pytest.raises(KeyError) as excinfo: + export_replay_case_bundle( + events_path=events_path, + out_dir=tmp_path / "out", + replay_id="plan_normalize.spec_v1", + ) + message = str(excinfo.value) + assert "planner_input_v1" in message + assert "plan_normalize.spec_v1" in message + + +def test_replay_plan_normalize_uses_planner_input(caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level(logging.INFO, logger="fetchgraph.replay.handlers.plan_normalize") + inp = { + "spec": {"provider": "relational", "mode": "full", "selectors": {"op": "query"}}, + "options": {}, + "normalizer_rules": {"relational": "relational_v1"}, + } + ctx = ReplayContext( + extras={ + "planner_input_v1": { + "input": { + "provider_catalog": { + "relational": { + "name": "relational", + "selectors_schema": {"type": "object", "properties": {"op": {"type": "string"}}}, + } + } + } + } + } + ) + out = replay_plan_normalize_spec_v1(inp, ctx) + assert out["out_spec"]["provider"] == "relational" + assert "provider_info_source=planner_input" in caplog.text diff --git a/tests/test_replay_runtime.py b/tests/test_replay_runtime.py new file mode 100644 index 00000000..62d06b97 --- /dev/null +++ b/tests/test_replay_runtime.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import pytest + +from fetchgraph.replay.runtime import ReplayContext, run_case + + +def test_run_case_missing_handler_message() -> None: + root = {"id": "unknown.handler", "input": {}} + ctx = ReplayContext() + with pytest.raises(KeyError, match="Did you import fetchgraph.tracer.handlers\\?"): + run_case(root, ctx) diff --git a/tests/test_tracer_auto_resolve.py b/tests/test_tracer_auto_resolve.py new file mode 100644 index 00000000..69c8fd53 --- /dev/null +++ b/tests/test_tracer_auto_resolve.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from fetchgraph.tracer.resolve import EventsResolution, find_events_file, resolve_case_events + + +def _write_json(path: Path, payload: dict) -> None: + path.write_text(json.dumps(payload), encoding="utf-8") + + +def _touch(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}", encoding="utf-8") + + +def _set_mtime(path: Path, ts: float) -> None: + os.utime(path, (ts, ts)) + + +def _make_case_dir( + run_dir: Path, + case_id: str, + suffix: str, + *, + status: str, + tag: str | None = None, + with_events: bool = True, +) -> Path: + case_dir = run_dir / "cases" / f"{case_id}_{suffix}" + case_dir.mkdir(parents=True, exist_ok=True) + if with_events: + _touch(case_dir / "events.jsonl") + payload = {"status": status} + if tag: + payload["tag"] = tag + _write_json(case_dir / "status.json", payload) + return case_dir + + +def test_resolve_latest_with_events(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + run_old = runs_root / "run_old" + run_old.mkdir() + _make_case_dir(run_old, "case_1", "abc", status="ok") + _set_mtime(run_old, 100) + + run_new = runs_root / "run_new" + run_new.mkdir() + _make_case_dir(run_new, "case_1", "def", status="ok", with_events=False) + _set_mtime(run_new, 200) + + resolution = resolve_case_events(case_id="case_1", data_dir=data_dir) + assert resolution.case_dir.parent.parent == run_old + + +def test_resolve_latest_non_missed(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + run_old = runs_root / "run_old" + run_old.mkdir() + _make_case_dir(run_old, "case_4", "aaa", status="ok") + _set_mtime(run_old, 100) + + run_new = runs_root / "run_new" + run_new.mkdir() + _make_case_dir(run_new, "case_4", "bbb", status="missed") + _set_mtime(run_new, 200) + + resolution = resolve_case_events(case_id="case_4", data_dir=data_dir) + assert resolution.run_dir == run_old + + +def test_resolve_with_tag(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + run_a = runs_root / "run_a" + run_a.mkdir() + _make_case_dir(run_a, "case_2", "aaa", status="ok", tag="alpha") + _set_mtime(run_a, 100) + + run_b = runs_root / "run_b" + run_b.mkdir() + _make_case_dir(run_b, "case_2", "bbb", status="ok", tag="beta") + _set_mtime(run_b, 200) + + resolution = resolve_case_events(case_id="case_2", data_dir=data_dir, tag="alpha") + assert resolution.run_dir == run_a + + +def test_resolve_tag_from_run_meta(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + run_meta = runs_root / "run_meta" + run_meta.mkdir() + _write_json(run_meta / "run_meta.json", {"tag": "meta_tag"}) + _make_case_dir(run_meta, "case_5", "aaa", status="ok") + _set_mtime(run_meta, 100) + + run_other = runs_root / "run_other" + run_other.mkdir() + _make_case_dir(run_other, "case_5", "bbb", status="ok") + _set_mtime(run_other, 200) + + resolution = resolve_case_events(case_id="case_5", data_dir=data_dir, tag="meta_tag") + assert resolution.run_dir == run_meta + + +def test_resolve_tag_from_run_dir(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + run_dir = runs_root / "tag-known_bad" + run_dir.mkdir() + _make_case_dir(run_dir, "case_6", "aaa", status="ok") + _set_mtime(run_dir, 100) + + resolution = resolve_case_events(case_id="case_6", data_dir=data_dir, tag="known_bad") + assert resolution.run_dir == run_dir + + +def test_resolve_tag_from_events(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + run_dir = runs_root / "run_events" + run_dir.mkdir() + case_dir = _make_case_dir(run_dir, "case_7", "aaa", status="ok") + events_path = case_dir / "events.jsonl" + events_path.write_text('{"tag": "from_events"}\n', encoding="utf-8") + _set_mtime(run_dir, 100) + + resolution = resolve_case_events(case_id="case_7", data_dir=data_dir, tag="from_events") + assert resolution.run_dir == run_dir + + +def test_resolve_not_found(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + run_dir = runs_root / "run_only" + run_dir.mkdir() + _make_case_dir(run_dir, "case_3", "ccc", status="missed", with_events=False) + + with pytest.raises(LookupError, match="No suitable case run found"): + resolve_case_events(case_id="case_3", data_dir=data_dir) + + +def test_find_events_file_not_found(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + run_dir.mkdir() + resolution = find_events_file(run_dir) + assert isinstance(resolution, EventsResolution) + assert resolution.events_path is None diff --git a/tests/test_tracer_export_resolve_prefers_run_with_replay.py b/tests/test_tracer_export_resolve_prefers_run_with_replay.py new file mode 100644 index 00000000..690c05c3 --- /dev/null +++ b/tests/test_tracer_export_resolve_prefers_run_with_replay.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path + +from fetchgraph.tracer.resolve import resolve_case_events + + +def _write_events(path: Path, events: list[dict]) -> None: + path.write_text("\n".join(json.dumps(event) for event in events) + "\n", encoding="utf-8") + + +def _make_case_dir(run_dir: Path, case_id: str, suffix: str, *, status: str) -> Path: + case_dir = run_dir / "cases" / f"{case_id}_{suffix}" + case_dir.mkdir(parents=True, exist_ok=True) + (case_dir / "status.json").write_text(json.dumps({"status": status}), encoding="utf-8") + return case_dir + + +def test_resolve_prefers_run_with_replay(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + runs_root = data_dir / ".runs" / "runs" + runs_root.mkdir(parents=True, exist_ok=True) + + old_run = runs_root / "run_old" + old_run.mkdir() + old_case = _make_case_dir(old_run, "agg_003", "old", status="ok") + _write_events(old_case / "events.jsonl", [{"type": "replay_case", "id": "other_case"}]) + os.utime(old_run, (1, 1)) + + new_run = runs_root / "run_new" + new_run.mkdir() + new_case = _make_case_dir(new_run, "agg_003", "new", status="error") + _write_events(new_case / "events.jsonl", [{"type": "replay_case", "id": "plan_normalize.spec_v1"}]) + os.utime(new_run, (2, 2)) + + resolution = resolve_case_events( + case_id="agg_003", + data_dir=data_dir, + pick_run="latest_with_replay", + replay_id="plan_normalize.spec_v1", + ) + assert resolution.case_dir == new_case diff --git a/tests/test_tracer_fixture_demote.py b/tests/test_tracer_fixture_demote.py new file mode 100644 index 00000000..ed99a10c --- /dev/null +++ b/tests/test_tracer_fixture_demote.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fetchgraph.tracer.fixture_tools import fixture_demote + + +def _write_bundle(path: Path, *, case_id: str) -> None: + payload = { + "schema": "fetchgraph.tracer.case_bundle", + "v": 1, + "root": {"type": "replay_case", "v": 2, "id": "plan_normalize.spec_v1", "input": {}}, + "source": {"case_id": case_id}, + "resources": {}, + "extras": {}, + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + + +def test_fixture_demote_moves_files(tmp_path: Path) -> None: + root = tmp_path / "fixtures" + fixed = root / "fixed" + known_bad = root / "known_bad" + case_path = fixed / "agg_003__a.case.json" + expected_path = fixed / "agg_003__a.expected.json" + resources_dir = fixed / "resources" / "agg_003__a" + _write_bundle(case_path, case_id="agg_003") + expected_path.write_text('{"ok": true}', encoding="utf-8") + resources_dir.mkdir(parents=True, exist_ok=True) + (resources_dir / "sample.txt").write_text("data", encoding="utf-8") + + fixture_demote( + root=root, + case_id="agg_003", + dry_run=True, + ) + assert case_path.exists() + assert expected_path.exists() + assert (resources_dir / "sample.txt").exists() + + fixture_demote( + root=root, + case_id="agg_003", + dry_run=False, + ) + assert not case_path.exists() + assert not expected_path.exists() + assert not resources_dir.exists() + assert (known_bad / "agg_003__a.case.json").exists() + assert (known_bad / "agg_003__a.expected.json").exists() + assert (known_bad / "resources" / "agg_003__a" / "sample.txt").exists() diff --git a/tests/test_tracer_fixture_green_resolve.py b/tests/test_tracer_fixture_green_resolve.py new file mode 100644 index 00000000..4b5a7985 --- /dev/null +++ b/tests/test_tracer_fixture_green_resolve.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from fetchgraph.tracer.fixture_tools import resolve_fixture_candidates, select_fixture_candidate + + +def _write_bundle(path: Path, *, case_id: str, timestamp: str) -> None: + payload = { + "schema": "fetchgraph.tracer.case_bundle", + "v": 1, + "root": {"type": "replay_case", "v": 2, "id": "plan_normalize.spec_v1", "input": {}}, + "source": {"case_id": case_id, "timestamp": timestamp}, + "resources": {}, + "extras": {}, + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + + +def test_resolve_fixture_by_case_id(tmp_path: Path) -> None: + root = tmp_path / "fixtures" + known_bad = root / "known_bad" + case_old = known_bad / "agg_003__a.case.json" + case_new = known_bad / "agg_003__b.case.json" + _write_bundle(case_old, case_id="agg_003", timestamp="2024-01-01T00:00:00Z") + _write_bundle(case_new, case_id="agg_003", timestamp="2024-01-02T00:00:00Z") + os.utime(case_old, (1, 1)) + os.utime(case_new, (2, 2)) + + candidates = resolve_fixture_candidates(root=root, bucket="known_bad", case_id="agg_003", name=None) + selected = select_fixture_candidate(candidates, select="latest") + assert selected.path == case_new + + selected_first = select_fixture_candidate(candidates, select_index=1) + assert selected_first.path == candidates[0].path + + with pytest.raises(FileExistsError): + select_fixture_candidate(candidates, require_unique=True) diff --git a/tests/test_tracer_fixture_selector_for_rm_migrate.py b/tests/test_tracer_fixture_selector_for_rm_migrate.py new file mode 100644 index 00000000..170b659f --- /dev/null +++ b/tests/test_tracer_fixture_selector_for_rm_migrate.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from fetchgraph.tracer.fixture_tools import fixture_migrate, fixture_rm + + +def _write_bundle(path: Path, *, case_id: str) -> None: + payload = { + "schema": "fetchgraph.tracer.case_bundle", + "v": 1, + "root": {"type": "replay_case", "v": 2, "id": "plan_normalize.spec_v1", "input": {}}, + "source": {"case_id": case_id}, + "resources": {}, + "extras": {}, + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + + +def test_fixture_selector_for_rm_migrate(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + root = tmp_path / "fixtures" + known_bad = root / "known_bad" + case_old = known_bad / "agg_003__a.case.json" + case_new = known_bad / "agg_003__b.case.json" + _write_bundle(case_old, case_id="agg_003") + _write_bundle(case_new, case_id="agg_003") + os.utime(case_old, (1, 1)) + os.utime(case_new, (2, 2)) + + removed = fixture_rm( + root=root, + bucket="known_bad", + scope="cases", + dry_run=True, + case_id="agg_003", + ) + assert removed == 1 + assert case_old.exists() + assert case_new.exists() + + with pytest.raises(FileExistsError): + fixture_rm( + root=root, + bucket="known_bad", + scope="cases", + dry_run=True, + case_id="agg_003", + require_unique=True, + ) + + bundles_updated, files_moved = fixture_migrate( + root=root, + bucket="known_bad", + dry_run=True, + case_id="agg_003", + all_matches=True, + ) + assert bundles_updated == 0 + assert files_moved == 0 + output = capsys.readouterr().out + assert "bulk operation" in output diff --git a/tests/test_tracer_fixture_tools.py b/tests/test_tracer_fixture_tools.py new file mode 100644 index 00000000..7de0842d --- /dev/null +++ b/tests/test_tracer_fixture_tools.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fetchgraph.tracer.fixture_tools import fixture_fix, fixture_green, fixture_migrate + + +def _write_bundle(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")), encoding="utf-8") + + +def _bundle_payload(root: dict, *, resources: dict | None = None, extras: dict | None = None) -> dict: + return { + "schema": "fetchgraph.tracer.case_bundle", + "v": 1, + "root": root, + "resources": resources or {}, + "extras": extras or {}, + "source": {"events_path": "events.jsonl"}, + } + + +def test_fixture_green_requires_observed(tmp_path: Path) -> None: + root = tmp_path / "fixtures" + case_path = root / "known_bad" / "case.case.json" + payload = _bundle_payload( + { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}}, + "observed_error": {"type": "ValueError", "message": "boom"}, + } + ) + _write_bundle(case_path, payload) + + with pytest.raises(ValueError, match="root.observed is missing"): + fixture_green(case_path=case_path, out_root=root, expected_from="observed") + + +def test_fixture_green_moves_case_and_resources(tmp_path: Path) -> None: + root = tmp_path / "fixtures" + case_path = root / "known_bad" / "case.case.json" + resources_dir = root / "known_bad" / "resources" / "case" + resources_dir.mkdir(parents=True, exist_ok=True) + (resources_dir / "rid1.txt").write_text("data", encoding="utf-8") + payload = _bundle_payload( + { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}, "options": {}}, + "observed": {"out_spec": {"provider": "sql"}}, + } + ) + _write_bundle(case_path, payload) + + fixture_green(case_path=case_path, out_root=root, expected_from="replay") + + fixed_case = root / "fixed" / "case.case.json" + expected_path = root / "fixed" / "case.expected.json" + fixed_resources = root / "fixed" / "resources" / "case" + assert fixed_case.exists() + assert expected_path.exists() + assert fixed_resources.exists() + assert not (root / "known_bad" / "case.case.json").exists() + assert not resources_dir.exists() + + +def test_fixture_green_rolls_back_on_validation_failure(tmp_path: Path) -> None: + root = tmp_path / "fixtures" + case_path = root / "known_bad" / "case.case.json" + payload = _bundle_payload( + { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}, "options": {}}, + "observed": {"out_spec": {"provider": "other"}}, + } + ) + _write_bundle(case_path, payload) + + with pytest.raises(AssertionError, match="rollback completed"): + fixture_green(case_path=case_path, out_root=root, expected_from="observed") + + assert case_path.exists() + assert not (root / "fixed" / "case.case.json").exists() + assert not (root / "fixed" / "case.expected.json").exists() + + +def test_fixture_fix_renames_and_updates_resource_paths(tmp_path: Path) -> None: + root = tmp_path / "fixtures" + bucket = "fixed" + case_path = root / bucket / "old.case.json" + resources_dir = root / bucket / "resources" / "old" / "rid1" + resources_dir.mkdir(parents=True, exist_ok=True) + (resources_dir / "file.txt").write_text("data", encoding="utf-8") + payload = _bundle_payload( + { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}}, + "observed": {"out_spec": {"provider": "sql"}}, + }, + resources={ + "rid1": {"data_ref": {"file": "resources/old/rid1/file.txt"}}, + }, + ) + _write_bundle(case_path, payload) + expected_path = root / bucket / "old.expected.json" + expected_path.write_text("{}", encoding="utf-8") + + fixture_fix(root=root, name="old", new_name="new", bucket=bucket, dry_run=False) + + new_case = root / bucket / "new.case.json" + new_expected = root / bucket / "new.expected.json" + new_resources = root / bucket / "resources" / "new" / "rid1" / "file.txt" + assert new_case.exists() + assert new_expected.exists() + assert new_resources.exists() + data = json.loads(new_case.read_text(encoding="utf-8")) + assert data["resources"]["rid1"]["data_ref"]["file"] == "resources/new/rid1/file.txt" + + +def test_fixture_migrate_moves_resources(tmp_path: Path) -> None: + root = tmp_path / "fixtures" + bucket = "fixed" + case_path = root / bucket / "case.case.json" + legacy_dir = root / bucket / "legacy" + legacy_dir.mkdir(parents=True, exist_ok=True) + legacy_path = legacy_dir / "file.txt" + legacy_path.write_text("data", encoding="utf-8") + payload = _bundle_payload( + { + "type": "replay_case", + "v": 2, + "id": "plan_normalize.spec_v1", + "input": {"spec": {"provider": "sql"}}, + "observed": {"out_spec": {"provider": "sql"}}, + }, + resources={"rid1": {"data_ref": {"file": "legacy/file.txt"}}}, + ) + _write_bundle(case_path, payload) + + bundles_updated, files_moved = fixture_migrate(root=root, bucket=bucket, dry_run=False) + assert bundles_updated == 1 + assert files_moved == 1 + data = json.loads(case_path.read_text(encoding="utf-8")) + assert data["resources"]["rid1"]["data_ref"]["file"] == "resources/case/rid1/legacy/file.txt" + assert (root / bucket / "resources" / "case" / "rid1" / "legacy" / "file.txt").exists() diff --git a/tests/test_tracer_run_id_resolve.py b/tests/test_tracer_run_id_resolve.py new file mode 100644 index 00000000..df6226f2 --- /dev/null +++ b/tests/test_tracer_run_id_resolve.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from fetchgraph.tracer import cli + + +def _write_jsonl(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload) + "\n", encoding="utf-8") + + +def _write_history_entry(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(payload) + "\n") + + +def _make_run_case(data_dir: Path, run_dir_name: str, case_id: str) -> tuple[Path, Path]: + run_dir = data_dir / ".runs" / "runs" / run_dir_name + case_dir = run_dir / "cases" / f"{case_id}_x" + case_dir.mkdir(parents=True, exist_ok=True) + events_path = case_dir / "events.jsonl" + _write_jsonl( + events_path, + { + "type": "replay_case", + "id": "replay_1", + "v": 2, + "input": {}, + "observed": {}, + }, + ) + return run_dir, case_dir + + +def test_export_case_bundle_resolves_run_id_from_history(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + out_dir = tmp_path / "out" + run_dir, _ = _make_run_case(data_dir, "run_folder", "agg_003") + history_path = data_dir / ".runs" / "history.jsonl" + _write_history_entry(history_path, {"run_id": "abc123", "run_dir": str(run_dir)}) + + exit_code = cli.main( + [ + "export-case-bundle", + "--id", + "replay_1", + "--out", + str(out_dir), + "--case", + "agg_003", + "--data", + str(data_dir), + "--run-id", + "abc123", + ] + ) + + assert exit_code == 0 + assert list(out_dir.glob("*.case.json")) + + +def test_export_case_bundle_resolves_run_id_from_run_meta(tmp_path: Path) -> None: + data_dir = tmp_path / "data" + out_dir = tmp_path / "out" + run_dir, _ = _make_run_case(data_dir, "run_folder_meta", "agg_003") + (run_dir / "run_meta.json").write_text(json.dumps({"run_id": "meta123"}), encoding="utf-8") + + exit_code = cli.main( + [ + "export-case-bundle", + "--id", + "replay_1", + "--out", + str(out_dir), + "--case", + "agg_003", + "--data", + str(data_dir), + "--run-id", + "meta123", + ] + ) + + assert exit_code == 0 + assert list(out_dir.glob("*.case.json"))