diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a80974d..78dcb72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -325,6 +325,26 @@ jobs: || { echo "FAIL: streaming demo did not pass its assertions"; exit 1; } echo "v0.2 Streaming exit criterion holds: $CHUNKS chunks live, one recorded completion, replay with zero model calls." + # The v0.2 timeline exit criterion (spec §12): an execution's folded state renders to a read-only + # HTML report. "self-contained: true" is the load-bearing assertion — a report with no scripts and + # no external references is one you can attach to a ticket or open from file:// years later. + # "deterministic: true" pins that it stays a pure fold of the log, not a live view. + - name: v0.2 Timeline exit demo (folded execution renders to a self-contained HTML report) + run: | + set -uo pipefail + CP="catalyst-api/target/classes:$(cat catalyst-api/target/cp.txt)" + + echo "--- render an execution's timeline ---" + OUT="$(java -cp "$CP" com.cajunsystems.catalyst.api.Demo timeline)" + echo "$OUT" + echo "$OUT" | grep -q "self-contained (no scripts, no external refs): true" \ + || { echo "FAIL: the report referenced something external"; exit 1; } + echo "$OUT" | grep -q "deterministic for a given log: true" \ + || { echo "FAIL: the report is not a pure fold of the log"; exit 1; } + echo "$OUT" | grep -q "\[timeline\] timeline criterion holds" \ + || { echo "FAIL: timeline demo did not pass its assertions"; exit 1; } + echo "v0.2 Timeline exit criterion holds: folded execution rendered to a portable HTML report." + # The -javaagent launch the README documents, exercised the way a user would: the agent jar alone, # with NO Byte Buddy on the application classpath (a -javaagent jar does not get its Maven # dependencies). This gates two failure modes that are invisible to the in-process tests — an diff --git a/CLAUDE.md b/CLAUDE.md index 730fa79..f129e14 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -40,6 +40,9 @@ intended source, but if `jitpack.io` is blocked, install Gumbo locally first: task packages into calls on `catalyst-core`'s `AutoCapture` bridge. The bridge lives in **core**, not here, so instrumented classes link without the agent. Depends on Byte Buddy; tests fork a JVM per class (agent installs are JVM-wide). +- `catalyst-timeline` — `TimelineReport`: renders a folded `ExecutionState` as a self-contained HTML + page (header + roll-ups + step table). Read-only, post-hoc, pure function of the log; depends on + `catalyst-core` only — no templating engine, no web server. All interpolated log content is escaped. - `catalyst-otel` — `CatalystTracer`: folds an execution's event log into an OpenTelemetry trace (root span + per-boundary child spans + lifecycle annotations). Read-only, post-hoc, no runtime hook. Depends on the OpenTelemetry **API** only; the app supplies the SDK + exporter. Tested offline with @@ -134,6 +137,13 @@ intended source, but if `jitpack.io` is blocked, install Gumbo locally first: executes between `CompletionRequested` and `CompletionReceived`, and a capture there would append into that gap and break replay. `LangChain4jModel.streaming(...)` bridges LangChain4j's async callbacks over a `BlockingQueue` so the sink runs on the **task's** thread, not the provider's. +- **v0.2 Timeline UI** — `TimelineAcceptanceTest` + `Demo timeline`: a folded execution renders to a + self-contained HTML report via `TimelineReport.html(state)` / `writeTo(state, path)`. Same shape as + the OTel exporter: consumes `inspect(id)` (a fold), no runtime hook, so the log stays the only source + of truth. Two properties are gated because they are what make a report useful: **self-contained** + (no scripts/external refs — portable, openable from `file://`) and **deterministic** (a pure fold, so + reports diff cleanly). Everything interpolated is HTML-escaped — tool names, effect labels and + payloads are log content. Oversized payloads are elided at 2000 chars rather than inlined. - **v0.2 Observability / OTel exporter** — `OtelAcceptanceTest` + `Demo otel`: an execution's event log folds into one OpenTelemetry trace via `catalyst-otel`'s `CatalystTracer.export(id, events)` — a root span for the run, a child span per boundary (model/tool/effect/memory; model/tool carry real latency, diff --git a/README.md b/README.md index 80c7704..36f4d39 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,9 @@ log) provides durability. incrementally while recording the assembled result, so a streamed execution writes the same log a non-streaming one does and replays with zero provider calls. See [Streaming](#streaming-completions). +- **Timeline report (v0.2)** — `TimelineReport.html(runtime.inspect(id))` renders an execution as a + self-contained HTML page: status, token/cost roll-ups, and the step-by-step trajectory. Read-only and + post-hoc, so any recorded execution renders. See [Timeline reports](#timeline-reports). - **Auto-capture (v0.2)** — attach `catalyst-agent` and task code no longer has to wrap its nondeterminism: `Instant.now()`, `UUID.randomUUID()`, `Math.random()` and `Random` draws are rewritten at their call sites into recorded boundaries, so a task written with plain JDK calls @@ -79,6 +82,7 @@ Deferred to later milestones (schema slots already reserved so no breaking chang | `catalyst-tools` | `ClockTool`, `CalculatorTool` | | `catalyst-langchain4j` | `LangChain4jModel`: adapts any LangChain4j `ChatModel` to Catalyst's `Model` | | `catalyst-otel` | `CatalystTracer`: folds an execution's log into an OpenTelemetry trace (API-only; app supplies the SDK) | +| `catalyst-timeline` | `TimelineReport`: renders a folded execution as a self-contained HTML timeline (no deps beyond core) | | `catalyst-agent` | `AutoCaptureAgent`: rewrites the JDK's nondeterministic call sites in task code into recorded boundaries | | `catalyst-api` | Thin facade: `Catalyst.embedded(path)`, builders, `Serializers` | @@ -236,6 +240,34 @@ Two things to know: request and its result, which no replay can match. Asserted automatically in `StreamingAcceptanceTest` and gated in CI by `Demo streaming`. +## Timeline reports + +The log already contains everything an execution did, so a timeline is a fold, not an instrumentation +layer. `catalyst-timeline` renders that fold as a page: + +```java +ExecutionState state = runtime.inspect(id); +TimelineReport.writeTo(state, Path.of("build/reports/" + id.value() + ".html")); +``` + +You get a status header, the roll-ups from `timelineView()` (model/tool calls, prompt and completion +tokens, cost, boundary latency, wall clock), and the step-by-step trajectory — each boundary with its +kind, label, offset from start, latency, and recorded payload in a collapsed block. + +The design constraints are the interesting part: + +- **Read-only and post-hoc**, exactly like the OTel exporter. It consumes an `ExecutionState` and + installs no runtime hook, so an execution recorded months ago renders just as well as a fresh one. +- **A pure function of the log.** Two renders of the same execution are byte-identical, which is what + makes a report safe to diff or commit as a build artifact. +- **Self-contained.** Inline CSS, no scripts, no fonts, no images — it opens from a `file://` URL with + nothing else present, so you can attach one to a ticket. +- **Everything is escaped.** Tool names, effect labels and recorded payloads are log content, and a + report is likely to be opened by someone other than whoever produced the execution. + +The module depends on `catalyst-core` and nothing else: no templating engine, no web server. + +Asserted automatically in `TimelineAcceptanceTest` and gated in CI by `Demo timeline`. ## Auto-capture: nondeterminism without the ceremony @@ -302,9 +334,9 @@ and **M2** (branch + diff). ## Roadmap -v0.1 is complete. See [ROADMAP.md](ROADMAP.md) for what's next — v0.2 (largely shipped: snapshots, -blob store, retry semantics, the OTel exporter, built-in tools, the auto-capture agent, streaming -completions; remaining: the timeline UI) and v1 (agents, the +v0.1 is complete, and v0.2 with it. See [ROADMAP.md](ROADMAP.md) for what's next — v0.2 (shipped: +snapshots, blob store, retry semantics, the OTel exporter, built-in tools, the auto-capture agent, +streaming completions, the timeline report) and v1 (agents, the `WAITING`/signal APIs and human-in-the-loop via Boudin, distributed execution over a Gumbo cluster, and the replay-driven eval harness). diff --git a/ROADMAP.md b/ROADMAP.md index 6f125cc..061f5f3 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -37,8 +37,8 @@ auto-capture agent, per-execution locking, streaming, and observability sequence guide, not a contract — it flexes as we learn. Snapshots, the cancellation event (①), the task registry / standalone `resume(id)` (②), the built-in HTTP + Filesystem tools (③), generic-collection payloads (④), the blob store (⑤), schema evolution, per-execution locking, retry semantics, the OTel -exporter, the auto-capture agent, and streaming completions have shipped; the timeline UI is the last -remaining v0.2 item. +exporter, the auto-capture agent, streaming completions, and the timeline UI have all shipped — +**v0.2 is complete**. ### Durability & storage (spec §8) - ✅ **Snapshots** — periodic fold checkpoints so long executions don't re-fold the whole log on @@ -186,7 +186,21 @@ remaining v0.2 item. `runtime.log().read(id)`, needing no runtime hook — so the log genuinely *is* the trace. The module depends on the OpenTelemetry **API** only; the app supplies the SDK + a real OTLP exporter (the same shape as the LangChain4j adapter). Gated by the v0.2 OTel exit demo in CI. -- **Timeline UI** — a read-only view over `inspect(id).timelineView()` / `Trajectory`. +- ✅ **Timeline UI** — `catalyst-timeline`'s `TimelineReport.html(state)` renders a folded + `ExecutionState` as a **self-contained HTML page**: a status header, the roll-ups + (`timelineView()` — model/tool counts, tokens, cost, latency, wall clock) and the step-by-step + trajectory with each boundary's label, offset, latency and recorded payload. Read-only and post-hoc, + the same shape as the OTel exporter — it consumes a fold of the log and installs no runtime hook, so + any execution renders without having cooperated in advance, and the log stays the single source of + truth. Because the input is a fold, the output is a **pure function** of the log: two renders of one + execution are byte-identical, which is what makes a report safe to diff or commit as a build + artifact. The page references nothing external (inline CSS, no scripts/fonts/images) so it opens from + `file://` years later; everything interpolated is HTML-escaped, since tool names, effect labels and + recorded payloads are log content and a report gets opened in a browser by someone who did not + produce the execution. The module depends on `catalyst-core` alone — no templating engine, no web + server. Gated by the v0.2 Timeline exit demo in CI. *Deferred (noted): latency bars and a + `TrajectoryDiff` view (the M2 branch comparison) — the table is the read-only view the roadmap asked + for; visual timing and diff rendering are polish on top.* --- diff --git a/catalyst-api/pom.xml b/catalyst-api/pom.xml index 3da2f7d..62a8074 100644 --- a/catalyst-api/pom.xml +++ b/catalyst-api/pom.xml @@ -35,6 +35,10 @@ com.cajunsystems catalyst-otel + + com.cajunsystems + catalyst-timeline + + + com.cajunsystems + catalyst-core + + + + com.cajunsystems + catalyst-runtime + test + + + diff --git a/catalyst-timeline/src/main/java/com/cajunsystems/catalyst/timeline/TimelineReport.java b/catalyst-timeline/src/main/java/com/cajunsystems/catalyst/timeline/TimelineReport.java new file mode 100644 index 0000000..4d28ea0 --- /dev/null +++ b/catalyst-timeline/src/main/java/com/cajunsystems/catalyst/timeline/TimelineReport.java @@ -0,0 +1,313 @@ +package com.cajunsystems.catalyst.timeline; + +import com.cajunsystems.catalyst.Status; +import com.cajunsystems.catalyst.engine.ExecutionState; +import com.cajunsystems.catalyst.engine.Timeline; +import com.cajunsystems.catalyst.engine.TimelineStep; +import com.fasterxml.jackson.databind.JsonNode; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.Locale; + +/** + * Renders an execution's folded timeline as a self-contained HTML page (spec §12) — the read-only view + * over {@code runtime.inspect(id)}. + * + *

Read-only and post-hoc, the same shape as {@code catalyst-otel}: it consumes an + * {@link ExecutionState}, which is itself nothing but a fold of the event log, and installs no runtime + * hook. The log remains the only source of truth; this renders it. Because the input is a fold, the + * report for a given log is deterministic — the same execution always produces the same page. + * + *

The output has no external references at all: styles are inline, there are no scripts, images or + * fonts to fetch. A report can be attached to a ticket, committed as a build artifact, or opened from + * a file:// URL with nothing else present. + * + *

Escaping

+ * Everything interpolated comes from the log — task types, tool names, effect labels, recorded + * payloads, error strings — so all of it is HTML-escaped. Log content is data, and a report is likely + * to be opened in a browser by someone other than whoever produced the execution. + */ +public final class TimelineReport { + + /** Recorded payloads can be large; past this many characters a detail is elided in the report. */ + private static final int MAX_DETAIL_CHARS = 2_000; + + private TimelineReport() {} + + /** Renders {@code state} as a complete HTML document. */ + public static String html(ExecutionState state) { + if (state == null) throw new IllegalArgumentException("state"); + Timeline timeline = state.timelineView(); + + StringBuilder out = new StringBuilder(8_192); + out.append("\n\n\n"); + out.append("\n"); + out.append("\n"); + out.append("Catalyst execution ").append(escape(idOf(state))).append("\n"); + out.append("\n"); + out.append("\n\n"); + + appendHeader(out, state); + appendSummary(out, state, timeline); + appendSteps(out, state); + appendFooter(out); + + out.append("\n\n"); + return out.toString(); + } + + /** Renders {@code state} and writes it to {@code file}, creating parent directories as needed. */ + public static Path writeTo(ExecutionState state, Path file) { + try { + Path parent = file.toAbsolutePath().getParent(); + if (parent != null) Files.createDirectories(parent); + Files.writeString(file, html(state), StandardCharsets.UTF_8); + return file; + } catch (IOException e) { + throw new UncheckedIOException("Failed to write timeline report to " + file, e); + } + } + + // ── Sections ─────────────────────────────────────────────────────────────── + + private static void appendHeader(StringBuilder out, ExecutionState state) { + String status = state.status() == null ? "UNKNOWN" : state.status().name(); + out.append("
\n"); + out.append("

").append(escape(taskTypeOf(state))).append("

\n"); + out.append("

").append(escape(idOf(state))).append("

\n"); + out.append("

") + .append(escape(status)).append(""); + out.append(" ") + .append("attempt ").append(state.attempt()) + .append(" · ").append(state.retries()).append(" retr") + .append(state.retries() == 1 ? "y" : "ies") + .append("

\n"); + if (state.error() != null && !state.error().isBlank()) { + out.append("

").append(escape(state.error())).append("

\n"); + } + out.append("
\n"); + } + + private static void appendSummary(StringBuilder out, ExecutionState state, Timeline timeline) { + out.append("
\n
\n"); + tile(out, "Model calls", String.valueOf(timeline.modelCalls())); + tile(out, "Tool calls", String.valueOf(timeline.toolCalls())); + tile(out, "Prompt tokens", String.valueOf(timeline.promptTokens())); + tile(out, "Completion tokens", String.valueOf(timeline.completionTokens())); + tile(out, "Cost", String.format(Locale.ROOT, "$%.6f", timeline.totalCostUsd())); + tile(out, "Boundary latency", millis(timeline.totalLatencyMillis())); + tile(out, "Wall clock", wallClock(state)); + tile(out, "Steps", String.valueOf(state.timeline().size())); + out.append("
\n
\n"); + } + + private static void tile(StringBuilder out, String label, String value) { + out.append("
").append(escape(label)).append("
") + .append(escape(value)).append("
\n"); + } + + private static void appendSteps(StringBuilder out, ExecutionState state) { + out.append("
\n

Timeline

\n"); + if (state.timeline().isEmpty()) { + out.append("

No steps recorded.

\n
\n"); + return; + } + // The table scrolls inside its own container so a long label or a wide payload never forces + // the page itself to scroll sideways. + out.append("
\n\n"); + out.append("") + .append(""); + out.append("\n\n"); + + Instant start = state.startedAt(); + for (TimelineStep step : state.timeline()) { + out.append(""); + out.append(""); + out.append(""); + out.append(""); + out.append(""); + out.append(""); + out.append(""); + out.append("\n"); + } + out.append("\n
seqkindlabelt+latencydetail
").append(step.seq()).append("") + .append(escape(step.kind() == null ? "?" : step.kind().name())).append("").append(escape(step.label() == null ? "" : step.label())).append("").append(escape(offset(start, step.at()))).append("") + .append(step.latencyMillis() > 0 ? escape(millis(step.latencyMillis())) : "") + .append("").append(detail(step.detail())).append("
\n
\n\n"); + } + + private static void appendFooter(StringBuilder out) { + out.append("\n"); + } + + // ── Values ───────────────────────────────────────────────────────────────── + + private static String idOf(ExecutionState state) { + return state.id() == null ? "(unknown id)" : state.id().value(); + } + + private static String taskTypeOf(ExecutionState state) { + return state.taskType() == null || state.taskType().isBlank() ? "(unknown task)" : state.taskType(); + } + + /** A recorded payload, rendered in a collapsed block so a big completion cannot swamp the page. */ + private static String detail(JsonNode node) { + if (node == null || node.isNull()) return ""; + String text = node.toString(); + boolean elided = text.length() > MAX_DETAIL_CHARS; + int cut = elided ? truncationPoint(text) : text.length(); + StringBuilder sb = new StringBuilder(); + sb.append("
").append(text.length()).append(" chars
")
+                .append(escape(text.substring(0, cut)));
+        if (elided) sb.append("\n… elided (").append(text.length() - cut).append(" more)");
+        sb.append("
"); + return sb.toString(); + } + + /** + * Where to cut an oversized payload, never between the halves of a surrogate pair. A cut landing + * inside one leaves a lone surrogate, which is unmappable in UTF-8 and reaches the page as a + * replacement character — a visibly corrupted last character in what is supposed to be a faithful + * record of a payload. Emoji and other supplementary characters turn up in completions routinely, + * so the boundary is worth respecting. + */ + private static int truncationPoint(String text) { + int cut = Math.min(MAX_DETAIL_CHARS, text.length()); + if (cut > 0 && cut < text.length() && Character.isHighSurrogate(text.charAt(cut - 1))) { + cut--; // the pair straddles the cut: drop its leading half too + } + return cut; + } + + private static String offset(Instant start, Instant at) { + if (start == null || at == null) return ""; + long ms = Duration.between(start, at).toMillis(); + return ms < 0 ? "" : "+" + millis(ms); + } + + private static String millis(long ms) { + if (ms < 1_000) return ms + " ms"; + return String.format(Locale.ROOT, "%.2f s", ms / 1_000.0); + } + + private static String wallClock(ExecutionState state) { + if (state.startedAt() == null) return "—"; + Instant end = state.endedAt(); + if (end == null) return "in flight"; + return millis(Duration.between(state.startedAt(), end).toMillis()); + } + + private static String statusClass(Status status) { + if (status == null) return "unknown"; + return switch (status) { + case COMPLETED -> "ok"; + case FAILED -> "bad"; + case CANCELLED -> "warn"; + default -> "running"; + }; + } + + private static String kindClass(TimelineStep.Kind kind) { + if (kind == null) return "other"; + return switch (kind) { + case MODEL -> "model"; + case TOOL -> "tool"; + case EFFECT -> "effect"; + case MEMORY_READ, MEMORY_WRITE -> "memory"; + case FAILED, CANCELLED -> "bad"; + case COMPLETED -> "ok"; + case RETRY, PAUSED, BRANCHED -> "warn"; + default -> "other"; + }; + } + + /** + * Escapes the five characters that can break out of HTML text or an attribute. Applied to every + * interpolated value without exception — log content is data, never markup. + */ + static String escape(String raw) { + if (raw == null) return ""; + StringBuilder sb = new StringBuilder(raw.length() + 16); + for (int i = 0; i < raw.length(); i++) { + char c = raw.charAt(i); + switch (c) { + case '&' -> sb.append("&"); + case '<' -> sb.append("<"); + case '>' -> sb.append(">"); + case '"' -> sb.append("""); + case '\'' -> sb.append("'"); + default -> sb.append(c); + } + } + return sb.toString(); + } + + private static String css() { + return """ + :root { color-scheme: light dark; + --bg:#fff; --fg:#1a1a1a; --muted:#6b7280; --line:#e5e7eb; --panel:#f9fafb; } + @media (prefers-color-scheme: dark) { + :root { --bg:#0f1115; --fg:#e6e6e6; --muted:#9aa1ab; --line:#252a33; --panel:#161a21; } + } + * { box-sizing: border-box; } + body { margin:0; padding:2rem 1.25rem; background:var(--bg); color:var(--fg); + font:15px/1.55 ui-sans-serif,system-ui,-apple-system,"Segoe UI",Roboto,sans-serif; + max-width:70rem; margin-inline:auto; } + h1 { font-size:1.4rem; margin:0 0 .2rem; } + h2 { font-size:1rem; text-transform:uppercase; letter-spacing:.06em; + color:var(--muted); margin:2rem 0 .6rem; } + .id { font-family:ui-monospace,SFMono-Regular,Menlo,monospace; color:var(--muted); + margin:0 0 .6rem; font-size:.85rem; } + .muted { color:var(--muted); } + .error { background:var(--panel); border-left:3px solid #dc2626; padding:.6rem .8rem; + font-family:ui-monospace,SFMono-Regular,Menlo,monospace; font-size:.85rem; + white-space:pre-wrap; overflow-wrap:anywhere; } + .badge { display:inline-block; padding:.15rem .55rem; border-radius:999px; + font-size:.75rem; font-weight:600; letter-spacing:.04em; } + .badge.ok { background:#dcfce7; color:#166534; } + .badge.bad { background:#fee2e2; color:#991b1b; } + .badge.warn { background:#fef3c7; color:#92400e; } + .badge.running,.badge.unknown { background:#e0e7ff; color:#3730a3; } + .summary dl { display:grid; gap:.75rem; margin:1.5rem 0 0; + grid-template-columns:repeat(auto-fit,minmax(9rem,1fr)); } + .summary div { background:var(--panel); border:1px solid var(--line); + border-radius:.5rem; padding:.6rem .75rem; } + .summary dt { font-size:.72rem; text-transform:uppercase; letter-spacing:.05em; + color:var(--muted); } + .summary dd { margin:.15rem 0 0; font-size:1.15rem; + font-variant-numeric:tabular-nums; } + .scroll { overflow-x:auto; } + table { border-collapse:collapse; width:100%; font-size:.88rem; } + th,td { text-align:left; padding:.45rem .6rem; border-bottom:1px solid var(--line); + vertical-align:top; } + th { font-size:.72rem; text-transform:uppercase; letter-spacing:.05em; + color:var(--muted); font-weight:600; } + td.num,th.num { text-align:right; font-variant-numeric:tabular-nums; + white-space:nowrap; } + .kind { font-size:.7rem; font-weight:600; letter-spacing:.04em; + padding:.1rem .4rem; border-radius:.25rem; white-space:nowrap; } + .kind.model { background:#e0e7ff; color:#3730a3; } + .kind.tool { background:#cffafe; color:#155e75; } + .kind.effect { background:#f3e8ff; color:#6b21a8; } + .kind.memory { background:#ecfccb; color:#3f6212; } + .kind.ok { background:#dcfce7; color:#166534; } + .kind.bad { background:#fee2e2; color:#991b1b; } + .kind.warn { background:#fef3c7; color:#92400e; } + .kind.other { background:var(--panel); color:var(--muted); } + details pre { margin:.4rem 0 0; padding:.5rem; background:var(--panel); + border-radius:.35rem; font-size:.78rem; white-space:pre-wrap; + overflow-wrap:anywhere; max-height:22rem; overflow:auto; } + summary { cursor:pointer; color:var(--muted); font-size:.78rem; } + footer { margin-top:2.5rem; padding-top:1rem; border-top:1px solid var(--line); + font-size:.8rem; } + """; + } +} diff --git a/catalyst-timeline/src/test/java/com/cajunsystems/catalyst/timeline/TimelineReportTest.java b/catalyst-timeline/src/test/java/com/cajunsystems/catalyst/timeline/TimelineReportTest.java new file mode 100644 index 0000000..2e185c1 --- /dev/null +++ b/catalyst-timeline/src/test/java/com/cajunsystems/catalyst/timeline/TimelineReportTest.java @@ -0,0 +1,205 @@ +package com.cajunsystems.catalyst.timeline; + +import com.cajunsystems.catalyst.ExecutionOptions; +import com.cajunsystems.catalyst.Task; +import com.cajunsystems.catalyst.Tool; +import com.cajunsystems.catalyst.engine.ExecutionState; +import com.cajunsystems.catalyst.mock.MockModel; +import com.cajunsystems.catalyst.model.CompletionRequest; +import com.cajunsystems.catalyst.model.Prompt; +import com.cajunsystems.catalyst.runtime.CatalystRuntime; +import com.cajunsystems.catalyst.runtime.EventLogs; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class TimelineReportTest { + + record In(String v) {} + + /** A tool whose name and output are attacker-controlled as far as the report is concerned. */ + record EchoTool(String name) implements Tool { + @Override public Class inputType() { return In.class; } + @Override public String apply(In in) { return "echo:" + in.v(); } + } + + private static ExecutionState runAndInspect(CatalystRuntime runtime, Task task) { + var handle = runtime.execute(task, ExecutionOptions.withKey("k")); + handle.result(); + return runtime.inspect(handle.id()); + } + + @Test + void rendersASelfContainedReportOfTheFoldedTimeline() { + try (CatalystRuntime runtime = CatalystRuntime.builder() + .log(EventLogs.inMemory()).model(MockModel.alwaysReturn("hello")).build()) { + + Tool echo = new EchoTool("echo"); + ExecutionState state = runAndInspect(runtime, ctx -> { + String m = ctx.model().complete( + CompletionRequest.of(Prompt.builder().user("hi").build())).message(); + String t = ctx.call(echo, new In("x")); + ctx.memory().put("note", "remembered"); + return m + "|" + t; + }); + + String html = TimelineReport.html(state); + + assertThat(html).startsWith("").endsWith("\n"); + assertThat(html).contains(state.id().value()); + assertThat(html).contains("COMPLETED"); + // Every boundary kind the execution produced shows up as a step. + assertThat(html).contains(">MODEL<").contains(">TOOL<").contains(">MEMORY_WRITE<"); + assertThat(html).contains("echo"); + + // Self-contained: no network references of any kind, and no scripts. + assertThat(html).doesNotContain("http://").doesNotContain("https://") + .doesNotContain(" nasty = new EchoTool(""); + ExecutionState state = runAndInspect(runtime, ctx -> { + ctx.effect("", () -> "value"); + return ctx.call(nasty, new In("")); + }); + + String html = TimelineReport.html(state); + + assertThat(html).doesNotContain("&'")) + .isEqualTo("<a href="x">&'</a>"); + assertThat(TimelineReport.escape(null)).isEmpty(); + assertThat(TimelineReport.escape("plain")).isEqualTo("plain"); + } + + @Test + void reportsAFailedExecutionWithItsError() { + try (CatalystRuntime runtime = CatalystRuntime.builder() + .log(EventLogs.inMemory()).build()) { + + var handle = runtime.execute((Task) ctx -> { + throw new IllegalStateException("boom <&>"); + }, ExecutionOptions.withKey("k")); + try { + handle.result(); + } catch (Throwable expected) { + // terminal failure is the point + } + + String html = TimelineReport.html(runtime.inspect(handle.id())); + assertThat(html).contains("FAILED").contains("boom <&>"); + assertThat(html).doesNotContain("boom <&>"); + } + } + + @Test + void isDeterministicForAGivenLog() { + // The report is a pure function of a fold, so the same execution renders identically. That is + // what makes it safe to diff two reports or commit one as a build artifact. + try (CatalystRuntime runtime = CatalystRuntime.builder() + .log(EventLogs.inMemory()).model(MockModel.alwaysReturn("hello")).build()) { + + ExecutionState state = runAndInspect(runtime, ctx -> ctx.model().complete( + CompletionRequest.of(Prompt.builder().user("hi").build())).message()); + + assertThat(TimelineReport.html(state)).isEqualTo(TimelineReport.html(state)); + } + } + + @Test + void elidesAnOversizedPayloadInsteadOfInliningIt() { + String big = "x".repeat(50_000); + try (CatalystRuntime runtime = CatalystRuntime.builder() + .log(EventLogs.inMemory()).build()) { + + ExecutionState state = runAndInspect(runtime, ctx -> ctx.effect("big", () -> big)); + + String html = TimelineReport.html(state); + assertThat(html).contains("elided"); + assertThat(html.length()).isLessThan(big.length()); // the page did not swallow the payload + } + } + + @Test + void writesTheReportToAFileCreatingParentDirectories(@TempDir Path dir) throws Exception { + try (CatalystRuntime runtime = CatalystRuntime.builder() + .log(EventLogs.inMemory()).model(MockModel.alwaysReturn("hello")).build()) { + + ExecutionState state = runAndInspect(runtime, ctx -> ctx.model().complete( + CompletionRequest.of(Prompt.builder().user("hi").build())).message()); + + Path target = dir.resolve("nested/report.html"); + Path written = TimelineReport.writeTo(state, target); + + assertThat(written).isEqualTo(target); + assertThat(Files.readString(target)).isEqualTo(TimelineReport.html(state)); + } + } + + @Test + void rejectsANullState() { + assertThatThrownBy(() -> TimelineReport.html(null)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void elidesOnACodePointBoundarySoNoCharacterIsCorrupted() { + // The payload is serialized into a JSON envelope before truncation, so the exact padding that + // puts a surrogate pair across the 2000-char cut is not something to guess at. Sweep a range + // of lengths around the boundary: at least one lands mid-pair, and none may corrupt. + // "\uD83D\uDE80" is a rocket — two UTF-16 chars, one code point. + String rocket = "\uD83D\uDE80"; + + for (int pad = 1_950; pad <= 2_010; pad++) { + String payload = "a".repeat(pad) + rocket.repeat(100); + try (CatalystRuntime runtime = CatalystRuntime.builder() + .log(EventLogs.inMemory()).build()) { + + ExecutionState state = runAndInspect(runtime, ctx -> ctx.effect("emoji", () -> payload)); + String html = TimelineReport.html(state); + + assertThat(html).as("pad=%d should elide", pad).contains("elided"); + assertNoLoneSurrogates(html, pad); + // A lone surrogate is unmappable in UTF-8, so it would not survive this round-trip. + byte[] utf8 = html.getBytes(java.nio.charset.StandardCharsets.UTF_8); + assertThat(new String(utf8, java.nio.charset.StandardCharsets.UTF_8)) + .as("pad=%d must survive a UTF-8 round-trip", pad).isEqualTo(html); + } + } + } + + private static void assertNoLoneSurrogates(String html, int pad) { + for (int i = 0; i < html.length(); i++) { + char c = html.charAt(i); + if (Character.isHighSurrogate(c)) { + assertThat(i + 1 < html.length() && Character.isLowSurrogate(html.charAt(i + 1))) + .as("pad=%d left a lone high surrogate at %d", pad, i).isTrue(); + } else if (Character.isLowSurrogate(c)) { + assertThat(i > 0 && Character.isHighSurrogate(html.charAt(i - 1))) + .as("pad=%d left a lone low surrogate at %d", pad, i).isTrue(); + } + } + } +} diff --git a/pom.xml b/pom.xml index 498720c..af6f1ab 100644 --- a/pom.xml +++ b/pom.xml @@ -21,6 +21,7 @@ catalyst-tools catalyst-langchain4j catalyst-otel + catalyst-timeline catalyst-api @@ -87,6 +88,11 @@ catalyst-agent ${catalyst.version} + + com.cajunsystems + catalyst-timeline + ${catalyst.version} +