From b8e756993211deff48899f4d5cf8ccf1ef3ff50e Mon Sep 17 00:00:00 2001 From: fullsend-code Date: Fri, 8 May 2026 14:27:54 +0000 Subject: [PATCH 1/2] feat(#704): surface agent errors in workflow logs When an agent run fails, the root cause error is buried in the transcript JSONL artifact. Operators must download and manually search the artifact to diagnose failures. This adds two improvements: 1. In-process error surfacing (run.go): After transcript extraction, parse the JSONL for result events with is_error=true and emit ::error:: annotations to stderr. These appear in the GitHub Actions job summary, making failures diagnosable without downloading artifacts. 2. Standalone extraction script (extract-transcript-error.sh): A shell script that extracts errors from transcript files or directories. Usable by post-scripts, the triage agent (after downloading artifacts via gh run download), and operators debugging failed runs. The transcript parser is intentionally lightweight: it scans only for result-type lines (fast string contains check) and parses the last one. Error messages are truncated to 2000 chars and sanitized to prevent GHA command injection. New files: - internal/cli/transcript.go: JSONL parsing and error emission - internal/cli/transcript_test.go: 15 unit tests - internal/scaffold/.../extract-transcript-error.sh: standalone tool Modified: - internal/cli/run.go: call extractTranscriptErrors after iteration loop when agent exits non-zero Closes #704 Signed-off-by: fullsend-code --- internal/cli/run.go | 15 +- internal/cli/transcript.go | 146 ++++++++++ internal/cli/transcript_test.go | 268 ++++++++++++++++++ .../scripts/extract-transcript-error.sh | 105 +++++++ 4 files changed, 533 insertions(+), 1 deletion(-) create mode 100644 internal/cli/transcript.go create mode 100644 internal/cli/transcript_test.go create mode 100755 internal/scaffold/fullsend-repo/scripts/extract-transcript-error.sh diff --git a/internal/cli/run.go b/internal/cli/run.go index 74424877..b47f9aa0 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -589,7 +589,20 @@ func runAgent(agentName, fullsendDir, outputBase, targetRepo, fullsendBinary str } } - // 9e. Post-agent output scan — redact secrets from extracted output. + // 9e-bis. Surface transcript errors in workflow logs (GitHub Actions). + // When the agent exits non-zero, parse transcript JSONL files and emit + // ::error:: annotations so operators can diagnose failures without + // downloading artifacts. See #704. + if lastExitCode != 0 { + lastIterDir := filepath.Join(runDir, fmt.Sprintf("iteration-%d", runCount)) + lastTranscriptDir := filepath.Join(lastIterDir, "transcripts") + if errorSummaries := extractTranscriptErrors(lastTranscriptDir); len(errorSummaries) > 0 { + printer.StepWarn(fmt.Sprintf("Found %d transcript error(s) — emitting to workflow log", len(errorSummaries))) + emitTranscriptErrors(os.Stderr, errorSummaries) + } + } + + // 9f. Post-agent output scan — redact secrets from extracted output. if h.SecurityEnabled() { printer.StepStart("Running post-agent output scan") if err := scanOutputFiles(runDir, traceID, printer); err != nil { diff --git a/internal/cli/transcript.go b/internal/cli/transcript.go new file mode 100644 index 00000000..64634420 --- /dev/null +++ b/internal/cli/transcript.go @@ -0,0 +1,146 @@ +package cli + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +const ( + // maxTranscriptErrorLength is the maximum length of an error message + // emitted via ::error:: to avoid overwhelming workflow logs. + maxTranscriptErrorLength = 2000 + + // maxTranscriptLineSize is the maximum size of a single JSONL line + // we will attempt to parse. Lines larger than this are skipped to + // avoid excessive memory use on very large tool outputs. + maxTranscriptLineSize = 1024 * 1024 // 1 MB +) + +// transcriptResult represents the final result event in a Claude Code +// stream-json transcript. This is the last event emitted and indicates +// whether the session ended in error. +type transcriptResult struct { + Type string `json:"type"` + Subtype string `json:"subtype,omitempty"` + IsError bool `json:"is_error"` + Result string `json:"result,omitempty"` +} + +// transcriptErrorSummary holds extracted error information from a transcript. +type transcriptErrorSummary struct { + // Source is the transcript filename the error was found in. + Source string + // IsError is true when the result event has is_error set. + IsError bool + // ErrorMessage is the error text from the result event. + ErrorMessage string + // Subtype is the result subtype (e.g. "error_max_turns"). + Subtype string +} + +// extractTranscriptErrors scans all JSONL files in transcriptDir for +// result events with errors. Returns a summary for each transcript that +// contains an error result. Files that cannot be read or parsed are +// silently skipped — transcript extraction is best-effort. +func extractTranscriptErrors(transcriptDir string) []transcriptErrorSummary { + entries, err := os.ReadDir(transcriptDir) + if err != nil { + return nil + } + + var summaries []transcriptErrorSummary + + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".jsonl") { + continue + } + path := filepath.Join(transcriptDir, entry.Name()) + if summary, ok := parseTranscriptFile(path); ok && summary.IsError { + summaries = append(summaries, summary) + } + } + + return summaries +} + +// parseTranscriptFile reads a JSONL transcript and returns the last result +// event, if any. The second return value is false if no result event was found. +func parseTranscriptFile(path string) (transcriptErrorSummary, bool) { + f, err := os.Open(path) + if err != nil { + return transcriptErrorSummary{}, false + } + defer f.Close() + + var lastResult *transcriptResult + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 0, 64*1024), maxTranscriptLineSize) + + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + + // Quick check: only parse lines that look like result events. + // This avoids unmarshalling every line in potentially large transcripts. + if !isResultLine(line) { + continue + } + + var result transcriptResult + if err := json.Unmarshal(line, &result); err != nil { + continue + } + if result.Type == "result" { + lastResult = &result + } + } + + if lastResult == nil { + return transcriptErrorSummary{}, false + } + + return transcriptErrorSummary{ + Source: filepath.Base(path), + IsError: lastResult.IsError, + ErrorMessage: truncateError(lastResult.Result), + Subtype: lastResult.Subtype, + }, true +} + +// isResultLine does a fast prefix/contains check to avoid parsing every +// JSONL line. Claude Code transcripts can be very large. +func isResultLine(line []byte) bool { + // Result events contain "type":"result" or "type": "result". + return strings.Contains(string(line), `"type":"result"`) || + strings.Contains(string(line), `"type": "result"`) +} + +// truncateError trims an error message to maxTranscriptErrorLength. +// If truncated, appends an ellipsis indicator. +func truncateError(msg string) string { + if len(msg) <= maxTranscriptErrorLength { + return msg + } + return msg[:maxTranscriptErrorLength] + "… (truncated)" +} + +// emitTranscriptErrors writes ::error:: annotations for each transcript +// error summary. These appear in the GitHub Actions job summary, making +// agent failures diagnosable without downloading artifacts. +func emitTranscriptErrors(w io.Writer, summaries []transcriptErrorSummary) { + for _, s := range summaries { + // Sanitize the error message to prevent GHA command injection. + msg := sanitizeOutput(s.ErrorMessage) + if msg == "" { + msg = fmt.Sprintf("agent terminated with error (subtype: %s)", s.Subtype) + } + fmt.Fprintf(w, "::error title=Agent Error (%s)::%s\n", s.Source, msg) + } +} diff --git a/internal/cli/transcript_test.go b/internal/cli/transcript_test.go new file mode 100644 index 00000000..385dd54f --- /dev/null +++ b/internal/cli/transcript_test.go @@ -0,0 +1,268 @@ +package cli + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestParseTranscriptFile_ErrorResult(t *testing.T) { + dir := t.TempDir() + content := `{"type":"system","subtype":"init","session_id":"abc123"} +{"type":"assistant","content":[{"type":"text","text":"Working on it..."}]} +{"type":"result","subtype":"error_max_turns","is_error":true,"result":"Agent reached maximum number of turns","session_id":"abc123"} +` + path := filepath.Join(dir, "test-transcript.jsonl") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + summary, ok := parseTranscriptFile(path) + if !ok { + t.Fatal("expected result event to be found") + } + if !summary.IsError { + t.Error("expected IsError to be true") + } + if summary.Subtype != "error_max_turns" { + t.Errorf("expected subtype 'error_max_turns', got %q", summary.Subtype) + } + if summary.ErrorMessage != "Agent reached maximum number of turns" { + t.Errorf("unexpected error message: %q", summary.ErrorMessage) + } +} + +func TestParseTranscriptFile_SuccessResult(t *testing.T) { + dir := t.TempDir() + content := `{"type":"system","subtype":"init","session_id":"abc123"} +{"type":"result","subtype":"success","is_error":false,"result":"Task completed","session_id":"abc123"} +` + path := filepath.Join(dir, "test-transcript.jsonl") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + summary, ok := parseTranscriptFile(path) + if !ok { + t.Fatal("expected result event to be found") + } + if summary.IsError { + t.Error("expected IsError to be false for success result") + } +} + +func TestParseTranscriptFile_NoResult(t *testing.T) { + dir := t.TempDir() + content := `{"type":"system","subtype":"init","session_id":"abc123"} +{"type":"assistant","content":[{"type":"text","text":"Working..."}]} +` + path := filepath.Join(dir, "test-transcript.jsonl") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + _, ok := parseTranscriptFile(path) + if ok { + t.Error("expected no result event") + } +} + +func TestParseTranscriptFile_EmptyFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "empty.jsonl") + if err := os.WriteFile(path, []byte(""), 0o644); err != nil { + t.Fatal(err) + } + + _, ok := parseTranscriptFile(path) + if ok { + t.Error("expected no result from empty file") + } +} + +func TestParseTranscriptFile_MissingFile(t *testing.T) { + _, ok := parseTranscriptFile("/nonexistent/path.jsonl") + if ok { + t.Error("expected failure for missing file") + } +} + +func TestParseTranscriptFile_LastResultWins(t *testing.T) { + dir := t.TempDir() + // Two result events — the last one should win. + content := `{"type":"result","subtype":"success","is_error":false,"result":"first attempt ok"} +{"type":"result","subtype":"error_max_turns","is_error":true,"result":"second attempt failed"} +` + path := filepath.Join(dir, "multi-result.jsonl") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + summary, ok := parseTranscriptFile(path) + if !ok { + t.Fatal("expected result event to be found") + } + if !summary.IsError { + t.Error("expected last result (error) to win") + } + if summary.ErrorMessage != "second attempt failed" { + t.Errorf("unexpected error message: %q", summary.ErrorMessage) + } +} + +func TestParseTranscriptFile_TypeWithSpace(t *testing.T) { + dir := t.TempDir() + // Some JSON encoders add a space after the colon. + content := `{"type": "result", "subtype": "error_max_turns", "is_error": true, "result": "failed with space"} +` + path := filepath.Join(dir, "spaced.jsonl") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + summary, ok := parseTranscriptFile(path) + if !ok { + t.Fatal("expected result event to be found") + } + if !summary.IsError { + t.Error("expected IsError to be true") + } + if summary.ErrorMessage != "failed with space" { + t.Errorf("unexpected error message: %q", summary.ErrorMessage) + } +} + +func TestExtractTranscriptErrors_MultiplFiles(t *testing.T) { + dir := t.TempDir() + + // File 1: error result. + err1 := `{"type":"result","subtype":"error_max_turns","is_error":true,"result":"agent timed out"}` + if err := os.WriteFile(filepath.Join(dir, "agent1.jsonl"), []byte(err1), 0o644); err != nil { + t.Fatal(err) + } + + // File 2: success result (should not appear in summaries). + ok2 := `{"type":"result","subtype":"success","is_error":false,"result":"all good"}` + if err := os.WriteFile(filepath.Join(dir, "agent2.jsonl"), []byte(ok2), 0o644); err != nil { + t.Fatal(err) + } + + // File 3: not a JSONL file (should be skipped). + if err := os.WriteFile(filepath.Join(dir, "readme.txt"), []byte("not jsonl"), 0o644); err != nil { + t.Fatal(err) + } + + summaries := extractTranscriptErrors(dir) + if len(summaries) != 1 { + t.Fatalf("expected 1 error summary, got %d", len(summaries)) + } + if summaries[0].Source != "agent1.jsonl" { + t.Errorf("unexpected source: %q", summaries[0].Source) + } + if summaries[0].ErrorMessage != "agent timed out" { + t.Errorf("unexpected error message: %q", summaries[0].ErrorMessage) + } +} + +func TestExtractTranscriptErrors_EmptyDir(t *testing.T) { + dir := t.TempDir() + summaries := extractTranscriptErrors(dir) + if len(summaries) != 0 { + t.Errorf("expected no summaries for empty dir, got %d", len(summaries)) + } +} + +func TestExtractTranscriptErrors_MissingDir(t *testing.T) { + summaries := extractTranscriptErrors("/nonexistent/dir") + if summaries != nil { + t.Errorf("expected nil for missing dir, got %v", summaries) + } +} + +func TestTruncateError(t *testing.T) { + short := "short error" + if got := truncateError(short); got != short { + t.Errorf("short message should not be truncated: %q", got) + } + + long := strings.Repeat("x", maxTranscriptErrorLength+100) + got := truncateError(long) + if len(got) > maxTranscriptErrorLength+20 { + t.Errorf("truncated message too long: %d", len(got)) + } + if !strings.HasSuffix(got, "… (truncated)") { + t.Errorf("truncated message should end with ellipsis indicator: %q", got) + } +} + +func TestEmitTranscriptErrors(t *testing.T) { + summaries := []transcriptErrorSummary{ + { + Source: "code-transcript.jsonl", + IsError: true, + ErrorMessage: "Agent reached maximum turns", + Subtype: "error_max_turns", + }, + } + + var buf bytes.Buffer + emitTranscriptErrors(&buf, summaries) + + output := buf.String() + if !strings.Contains(output, "::error title=Agent Error (code-transcript.jsonl)::") { + t.Errorf("expected ::error:: annotation, got: %q", output) + } + if !strings.Contains(output, "Agent reached maximum turns") { + t.Errorf("expected error message in output, got: %q", output) + } +} + +func TestEmitTranscriptErrors_EmptyMessage(t *testing.T) { + summaries := []transcriptErrorSummary{ + { + Source: "test.jsonl", + IsError: true, + Subtype: "error_unknown", + }, + } + + var buf bytes.Buffer + emitTranscriptErrors(&buf, summaries) + + output := buf.String() + if !strings.Contains(output, "agent terminated with error (subtype: error_unknown)") { + t.Errorf("expected fallback message, got: %q", output) + } +} + +func TestEmitTranscriptErrors_NoSummaries(t *testing.T) { + var buf bytes.Buffer + emitTranscriptErrors(&buf, nil) + + if buf.Len() != 0 { + t.Errorf("expected no output for nil summaries, got: %q", buf.String()) + } +} + +func TestIsResultLine(t *testing.T) { + tests := []struct { + line string + want bool + }{ + {`{"type":"result","is_error":true}`, true}, + {`{"type": "result", "is_error": true}`, true}, + {`{"type":"assistant","content":[]}`, false}, + {`{"type":"system","subtype":"init"}`, false}, + {`not json at all`, false}, + {``, false}, + } + + for _, tt := range tests { + got := isResultLine([]byte(tt.line)) + if got != tt.want { + t.Errorf("isResultLine(%q) = %v, want %v", tt.line, got, tt.want) + } + } +} diff --git a/internal/scaffold/fullsend-repo/scripts/extract-transcript-error.sh b/internal/scaffold/fullsend-repo/scripts/extract-transcript-error.sh new file mode 100755 index 00000000..e47e3c97 --- /dev/null +++ b/internal/scaffold/fullsend-repo/scripts/extract-transcript-error.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# extract-transcript-error.sh — Extract errors from agent transcript JSONL files. +# +# Reads transcript JSONL files (Claude Code stream-json format) and extracts +# the final result event. If the result indicates an error, prints a summary +# suitable for GitHub Actions annotations or human consumption. +# +# Usage: +# extract-transcript-error.sh +# +# When given a directory, processes all .jsonl files in it. +# When given a file, processes just that file. +# +# Exit codes: +# 0 — no errors found (or no transcript files) +# 1 — at least one transcript contains an error result +# 2 — usage error (bad arguments) +# +# This script can be used by: +# - Post-scripts to surface errors in workflow logs +# - The triage agent to extract errors from downloaded artifacts +# - Operators debugging failed agent runs +# +# Example with artifact download: +# gh run download -n agent-transcripts -D /tmp/transcripts +# extract-transcript-error.sh /tmp/transcripts/ + +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +TARGET="$1" +FOUND_ERROR=0 +MAX_ERROR_LENGTH=2000 + +# extract_error processes a single JSONL file and prints any error found. +extract_error() { + local file="$1" + local basename + basename="$(basename "$file")" + + # Find the last result line in the file. + # Claude Code transcripts end with a result event. + local last_result + last_result="$(grep -E '"type"\s*:\s*"result"' "$file" | tail -1)" || true + + if [[ -z "$last_result" ]]; then + return + fi + + # Check if the result indicates an error. + local is_error + is_error="$(echo "$last_result" | jq -r '.is_error // false' 2>/dev/null)" || true + + if [[ "$is_error" != "true" ]]; then + return + fi + + FOUND_ERROR=1 + + local error_msg + error_msg="$(echo "$last_result" | jq -r '.result // "unknown error"' 2>/dev/null)" || error_msg="unknown error" + + local subtype + subtype="$(echo "$last_result" | jq -r '.subtype // "unknown"' 2>/dev/null)" || subtype="unknown" + + # Truncate long error messages. + if [[ ${#error_msg} -gt $MAX_ERROR_LENGTH ]]; then + error_msg="${error_msg:0:$MAX_ERROR_LENGTH}... (truncated)" + fi + + echo "--- Error in ${basename} ---" + echo "Subtype: ${subtype}" + echo "Message: ${error_msg}" + echo "" + + # Emit GHA annotation if running in GitHub Actions. + if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then + # Sanitize for GHA: replace :: with : : to prevent command injection. + local safe_msg="${error_msg//::/ :}" + echo "::error title=Agent Error (${basename})::${safe_msg}" + fi +} + +if [[ -d "$TARGET" ]]; then + # Process all JSONL files in the directory. + for f in "$TARGET"/*.jsonl; do + [[ -f "$f" ]] || continue + extract_error "$f" + done +elif [[ -f "$TARGET" ]]; then + extract_error "$TARGET" +else + echo "Error: $TARGET is not a file or directory" >&2 + exit 2 +fi + +if [[ $FOUND_ERROR -eq 1 ]]; then + exit 1 +fi + +exit 0 From 1a4e16688b0bc62a3b42978eae2e5ba415645351 Mon Sep 17 00:00:00 2001 From: fullsend-fix Date: Fri, 8 May 2026 19:28:17 +0000 Subject: [PATCH 2/2] fix: address review feedback on PR #764 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Sanitize s.Source and s.Subtype in emitTranscriptErrors to prevent GHA command injection via crafted filenames or subtypes [critical] - Add %0A/%0D sanitization to shell script's GHA annotation output to match the Go sanitizeOutput behavior [important] - Fix truncateError to walk back to valid UTF-8 rune boundary before truncating, avoiding incomplete multi-byte characters [moderate] - Switch isResultLine from strings.Contains(string(line)) to bytes.Contains to avoid unnecessary allocations [minor] - Fix typo: MultiplFiles → MultipleFiles in test name [nit] Addresses review feedback on #764 Signed-off-by: fullsend-fix --- internal/cli/transcript.go | 19 +++++++++++++------ internal/cli/transcript_test.go | 2 +- .../scripts/extract-transcript-error.sh | 6 +++++- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/internal/cli/transcript.go b/internal/cli/transcript.go index 64634420..d5cd0107 100644 --- a/internal/cli/transcript.go +++ b/internal/cli/transcript.go @@ -2,12 +2,14 @@ package cli import ( "bufio" + "bytes" "encoding/json" "fmt" "io" "os" "path/filepath" "strings" + "unicode/utf8" ) const ( @@ -118,17 +120,22 @@ func parseTranscriptFile(path string) (transcriptErrorSummary, bool) { // JSONL line. Claude Code transcripts can be very large. func isResultLine(line []byte) bool { // Result events contain "type":"result" or "type": "result". - return strings.Contains(string(line), `"type":"result"`) || - strings.Contains(string(line), `"type": "result"`) + return bytes.Contains(line, []byte(`"type":"result"`)) || + bytes.Contains(line, []byte(`"type": "result"`)) } // truncateError trims an error message to maxTranscriptErrorLength. -// If truncated, appends an ellipsis indicator. +// If truncated, walks back to a valid UTF-8 rune boundary before +// appending an ellipsis indicator. func truncateError(msg string) string { if len(msg) <= maxTranscriptErrorLength { return msg } - return msg[:maxTranscriptErrorLength] + "… (truncated)" + truncated := msg[:maxTranscriptErrorLength] + for len(truncated) > 0 && !utf8.Valid([]byte(truncated)) { + truncated = truncated[:len(truncated)-1] + } + return truncated + "… (truncated)" } // emitTranscriptErrors writes ::error:: annotations for each transcript @@ -139,8 +146,8 @@ func emitTranscriptErrors(w io.Writer, summaries []transcriptErrorSummary) { // Sanitize the error message to prevent GHA command injection. msg := sanitizeOutput(s.ErrorMessage) if msg == "" { - msg = fmt.Sprintf("agent terminated with error (subtype: %s)", s.Subtype) + msg = fmt.Sprintf("agent terminated with error (subtype: %s)", sanitizeOutput(s.Subtype)) } - fmt.Fprintf(w, "::error title=Agent Error (%s)::%s\n", s.Source, msg) + fmt.Fprintf(w, "::error title=Agent Error (%s)::%s\n", sanitizeOutput(s.Source), msg) } } diff --git a/internal/cli/transcript_test.go b/internal/cli/transcript_test.go index 385dd54f..232ba9d8 100644 --- a/internal/cli/transcript_test.go +++ b/internal/cli/transcript_test.go @@ -134,7 +134,7 @@ func TestParseTranscriptFile_TypeWithSpace(t *testing.T) { } } -func TestExtractTranscriptErrors_MultiplFiles(t *testing.T) { +func TestExtractTranscriptErrors_MultipleFiles(t *testing.T) { dir := t.TempDir() // File 1: error result. diff --git a/internal/scaffold/fullsend-repo/scripts/extract-transcript-error.sh b/internal/scaffold/fullsend-repo/scripts/extract-transcript-error.sh index e47e3c97..84a263e0 100755 --- a/internal/scaffold/fullsend-repo/scripts/extract-transcript-error.sh +++ b/internal/scaffold/fullsend-repo/scripts/extract-transcript-error.sh @@ -79,8 +79,12 @@ extract_error() { # Emit GHA annotation if running in GitHub Actions. if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then - # Sanitize for GHA: replace :: with : : to prevent command injection. + # Sanitize for GHA: replace :: and URL-encoded newlines to prevent command injection. local safe_msg="${error_msg//::/ :}" + safe_msg="${safe_msg//%0A/ }" + safe_msg="${safe_msg//%0a/ }" + safe_msg="${safe_msg//%0D/ }" + safe_msg="${safe_msg//%0d/ }" echo "::error title=Agent Error (${basename})::${safe_msg}" fi }