Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/320.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
History entries now use the manifest ``name:`` field as the match key for sparkline trends and incremental filtering, instead of the manifest filename. Projects that set ``name: my-project`` in their manifest YAML get a stable, rename-proof identity across runs. Projects without a ``name:`` field continue to use the filename (backward-compatible).
16 changes: 12 additions & 4 deletions src/raki/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,13 @@ def run(
out.print(f"[red]Error loading manifest: {redact_sensitive(str(exc))}[/red]")
raise SystemExit(2) from exc

# Compute the effective manifest identity for history filtering.
# Prefer the logical ``name:`` field from the manifest YAML (stable across
# file renames) and fall back to the filename when name is empty.
from raki.report.history import effective_manifest_name

effective_manifest_id: str | None = effective_manifest_name(manifest.name, manifest_file)

# Validate manifest judge.provider against allowed providers early,
# before it reaches MetricConfig where it would raise a raw Pydantic error.
if manifest.judge is not None and manifest.judge.provider is not None:
Expand Down Expand Up @@ -427,7 +434,7 @@ def run(
try:
from raki.report.history import load_seen_session_ids

seen_ids = load_seen_session_ids(_resolved_history_path, manifest=manifest_file.name)
seen_ids = load_seen_session_ids(_resolved_history_path, manifest=effective_manifest_id)
if not incremental and not rerun_all and not quiet and seen_ids:
out.print(
f"[yellow]Warning: {len(seen_ids)} session(s) were already evaluated in a "
Expand Down Expand Up @@ -547,14 +554,14 @@ def run(
existing_entries = load_history(_resolved_history_path)
prior = find_last_matching_entry(
existing_entries,
manifest_file.name,
effective_manifest_id,
len(dataset.samples),
)
if prior is not None:
time_ago = format_time_ago(prior.timestamp)
out.print(
f"[yellow]Warning: This exact evaluation was already run {time_ago} "
f"(manifest={manifest_file.name}, sessions={len(dataset.samples)}). "
f"(manifest={effective_manifest_id}, sessions={len(dataset.samples)}). "
f"Use --force to run again.[/yellow]"
)
return
Expand Down Expand Up @@ -629,7 +636,7 @@ def run(
}
run_sparklines = build_sparkline_data(
history_entries,
manifest=manifest_file.name,
manifest=effective_manifest_id,
metric_polarity=_polarity,
)
except Exception:
Expand Down Expand Up @@ -704,6 +711,7 @@ def run(
history_path,
sessions_count=len(dataset.samples),
manifest_file=manifest_file,
manifest_name=manifest.name,
)
except Exception as exc:
out.print(f"[yellow]Warning: Failed to write history log: {exc}[/yellow]")
Expand Down
47 changes: 43 additions & 4 deletions src/raki/report/history.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,35 @@
from raki.model.report import EvalReport


def effective_manifest_name(
manifest_name: str | None,
manifest_file: Path | None,
) -> str | None:
"""Return the logical manifest identifier to store in a history entry.

Priority:
1. *manifest_name* — the ``name:`` field from the manifest YAML — when it
is a non-empty string. This gives projects a stable, rename-proof
identity that does not change if the manifest file is renamed.
2. *manifest_file*.name (the filename) when *manifest_name* is absent or
empty — preserves backward-compatibility for projects that have not yet
set ``name:`` in their manifest.
3. ``None`` when both arguments are absent/empty.

Args:
manifest_name: The ``EvalManifest.name`` value, or ``None``.
manifest_file: The ``Path`` of the manifest file, or ``None``.

Returns:
A non-empty string, or ``None`` when no identifier is available.
"""
if manifest_name:
return manifest_name
if manifest_file is not None:
return manifest_file.name
return None


def _git_sha() -> str | None:
"""Return the short git SHA of HEAD, or None if not in a git repo."""
try:
Expand Down Expand Up @@ -62,6 +91,7 @@ def append_history_entry(
*,
sessions_count: int,
manifest_file: Path | None = None,
manifest_name: str | None = None,
) -> None:
"""Append a single ``HistoryEntry`` line to the JSONL history file.

Expand All @@ -71,7 +101,12 @@ def append_history_entry(
directories) on first call. Subsequent calls append without
overwriting existing entries.
sessions_count: Number of sessions that were evaluated in this run.
manifest_file: Path to the manifest file used for this run (basename stored).
manifest_file: Path to the manifest file used for this run.
manifest_name: Logical project name from ``EvalManifest.name``. When
non-empty, this value is stored in the ``manifest`` field instead
of the filename, giving the project a stable, rename-proof
identity across runs. Falls back to ``manifest_file.name`` when
absent or empty.

Raises:
ValueError: If ``history_path`` is a symlink (security guard).
Expand All @@ -95,7 +130,7 @@ def append_history_entry(
timestamp=report.timestamp,
sessions_count=sessions_count,
metrics=metrics_dict,
manifest=manifest_file.name if manifest_file is not None else None,
manifest=effective_manifest_name(manifest_name, manifest_file),
config_hash=_config_hash(report.config),
git_sha=_git_sha(),
warning_count=len(report.warnings),
Expand Down Expand Up @@ -170,7 +205,7 @@ def import_history_entry(

def find_last_matching_entry(
entries: list[HistoryEntry],
manifest: str,
manifest: str | None,
sessions_count: int,
) -> HistoryEntry | None:
"""Return the most recent history entry matching both *manifest* and *sessions_count*.
Expand All @@ -180,13 +215,17 @@ def find_last_matching_entry(

Args:
entries: Existing history entries to search (any order).
manifest: Manifest basename to match against ``entry.manifest``.
manifest: Manifest identifier (logical project name or filename) to match
against ``entry.manifest``. When ``None``, no entries can match and
``None`` is returned immediately.
sessions_count: Session count to match against ``entry.sessions_count``.

Returns:
The entry with the highest ``timestamp`` that matches both criteria,
or ``None`` when no entry matches.
"""
if manifest is None:
return None
matching = [
entry
for entry in entries
Expand Down
Loading
Loading