diff --git a/internal/cli/run.go b/internal/cli/run.go index 744248774..b47f9aa0a 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -589,7 +589,20 @@ func runAgent(agentName, fullsendDir, outputBase, targetRepo, fullsendBinary str } } - // 9e. Post-agent output scan — redact secrets from extracted output. + // 9e-bis. Surface transcript errors in workflow logs (GitHub Actions). + // When the agent exits non-zero, parse transcript JSONL files and emit + // ::error:: annotations so operators can diagnose failures without + // downloading artifacts. See #704. + if lastExitCode != 0 { + lastIterDir := filepath.Join(runDir, fmt.Sprintf("iteration-%d", runCount)) + lastTranscriptDir := filepath.Join(lastIterDir, "transcripts") + if errorSummaries := extractTranscriptErrors(lastTranscriptDir); len(errorSummaries) > 0 { + printer.StepWarn(fmt.Sprintf("Found %d transcript error(s) — emitting to workflow log", len(errorSummaries))) + emitTranscriptErrors(os.Stderr, errorSummaries) + } + } + + // 9f. Post-agent output scan — redact secrets from extracted output. if h.SecurityEnabled() { printer.StepStart("Running post-agent output scan") if err := scanOutputFiles(runDir, traceID, printer); err != nil { diff --git a/internal/cli/transcript.go b/internal/cli/transcript.go new file mode 100644 index 000000000..d5cd01079 --- /dev/null +++ b/internal/cli/transcript.go @@ -0,0 +1,153 @@ +package cli + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "unicode/utf8" +) + +const ( + // maxTranscriptErrorLength is the maximum length of an error message + // emitted via ::error:: to avoid overwhelming workflow logs. + maxTranscriptErrorLength = 2000 + + // maxTranscriptLineSize is the maximum size of a single JSONL line + // we will attempt to parse. Lines larger than this are skipped to + // avoid excessive memory use on very large tool outputs. + maxTranscriptLineSize = 1024 * 1024 // 1 MB +) + +// transcriptResult represents the final result event in a Claude Code +// stream-json transcript. This is the last event emitted and indicates +// whether the session ended in error. +type transcriptResult struct { + Type string `json:"type"` + Subtype string `json:"subtype,omitempty"` + IsError bool `json:"is_error"` + Result string `json:"result,omitempty"` +} + +// transcriptErrorSummary holds extracted error information from a transcript. +type transcriptErrorSummary struct { + // Source is the transcript filename the error was found in. + Source string + // IsError is true when the result event has is_error set. + IsError bool + // ErrorMessage is the error text from the result event. + ErrorMessage string + // Subtype is the result subtype (e.g. "error_max_turns"). + Subtype string +} + +// extractTranscriptErrors scans all JSONL files in transcriptDir for +// result events with errors. Returns a summary for each transcript that +// contains an error result. Files that cannot be read or parsed are +// silently skipped — transcript extraction is best-effort. +func extractTranscriptErrors(transcriptDir string) []transcriptErrorSummary { + entries, err := os.ReadDir(transcriptDir) + if err != nil { + return nil + } + + var summaries []transcriptErrorSummary + + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".jsonl") { + continue + } + path := filepath.Join(transcriptDir, entry.Name()) + if summary, ok := parseTranscriptFile(path); ok && summary.IsError { + summaries = append(summaries, summary) + } + } + + return summaries +} + +// parseTranscriptFile reads a JSONL transcript and returns the last result +// event, if any. The second return value is false if no result event was found. +func parseTranscriptFile(path string) (transcriptErrorSummary, bool) { + f, err := os.Open(path) + if err != nil { + return transcriptErrorSummary{}, false + } + defer f.Close() + + var lastResult *transcriptResult + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 0, 64*1024), maxTranscriptLineSize) + + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + + // Quick check: only parse lines that look like result events. + // This avoids unmarshalling every line in potentially large transcripts. + if !isResultLine(line) { + continue + } + + var result transcriptResult + if err := json.Unmarshal(line, &result); err != nil { + continue + } + if result.Type == "result" { + lastResult = &result + } + } + + if lastResult == nil { + return transcriptErrorSummary{}, false + } + + return transcriptErrorSummary{ + Source: filepath.Base(path), + IsError: lastResult.IsError, + ErrorMessage: truncateError(lastResult.Result), + Subtype: lastResult.Subtype, + }, true +} + +// isResultLine does a fast prefix/contains check to avoid parsing every +// JSONL line. Claude Code transcripts can be very large. +func isResultLine(line []byte) bool { + // Result events contain "type":"result" or "type": "result". + return bytes.Contains(line, []byte(`"type":"result"`)) || + bytes.Contains(line, []byte(`"type": "result"`)) +} + +// truncateError trims an error message to maxTranscriptErrorLength. +// If truncated, walks back to a valid UTF-8 rune boundary before +// appending an ellipsis indicator. +func truncateError(msg string) string { + if len(msg) <= maxTranscriptErrorLength { + return msg + } + truncated := msg[:maxTranscriptErrorLength] + for len(truncated) > 0 && !utf8.Valid([]byte(truncated)) { + truncated = truncated[:len(truncated)-1] + } + return truncated + "… (truncated)" +} + +// emitTranscriptErrors writes ::error:: annotations for each transcript +// error summary. These appear in the GitHub Actions job summary, making +// agent failures diagnosable without downloading artifacts. +func emitTranscriptErrors(w io.Writer, summaries []transcriptErrorSummary) { + for _, s := range summaries { + // Sanitize the error message to prevent GHA command injection. + msg := sanitizeOutput(s.ErrorMessage) + if msg == "" { + msg = fmt.Sprintf("agent terminated with error (subtype: %s)", sanitizeOutput(s.Subtype)) + } + fmt.Fprintf(w, "::error title=Agent Error (%s)::%s\n", sanitizeOutput(s.Source), msg) + } +} diff --git a/internal/cli/transcript_test.go b/internal/cli/transcript_test.go new file mode 100644 index 000000000..232ba9d85 --- /dev/null +++ b/internal/cli/transcript_test.go @@ -0,0 +1,268 @@ +package cli + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestParseTranscriptFile_ErrorResult(t *testing.T) { + dir := t.TempDir() + content := `{"type":"system","subtype":"init","session_id":"abc123"} +{"type":"assistant","content":[{"type":"text","text":"Working on it..."}]} +{"type":"result","subtype":"error_max_turns","is_error":true,"result":"Agent reached maximum number of turns","session_id":"abc123"} +` + path := filepath.Join(dir, "test-transcript.jsonl") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + summary, ok := parseTranscriptFile(path) + if !ok { + t.Fatal("expected result event to be found") + } + if !summary.IsError { + t.Error("expected IsError to be true") + } + if summary.Subtype != "error_max_turns" { + t.Errorf("expected subtype 'error_max_turns', got %q", summary.Subtype) + } + if summary.ErrorMessage != "Agent reached maximum number of turns" { + t.Errorf("unexpected error message: %q", summary.ErrorMessage) + } +} + +func TestParseTranscriptFile_SuccessResult(t *testing.T) { + dir := t.TempDir() + content := `{"type":"system","subtype":"init","session_id":"abc123"} +{"type":"result","subtype":"success","is_error":false,"result":"Task completed","session_id":"abc123"} +` + path := filepath.Join(dir, "test-transcript.jsonl") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + summary, ok := parseTranscriptFile(path) + if !ok { + t.Fatal("expected result event to be found") + } + if summary.IsError { + t.Error("expected IsError to be false for success result") + } +} + +func TestParseTranscriptFile_NoResult(t *testing.T) { + dir := t.TempDir() + content := `{"type":"system","subtype":"init","session_id":"abc123"} +{"type":"assistant","content":[{"type":"text","text":"Working..."}]} +` + path := filepath.Join(dir, "test-transcript.jsonl") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + _, ok := parseTranscriptFile(path) + if ok { + t.Error("expected no result event") + } +} + +func TestParseTranscriptFile_EmptyFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "empty.jsonl") + if err := os.WriteFile(path, []byte(""), 0o644); err != nil { + t.Fatal(err) + } + + _, ok := parseTranscriptFile(path) + if ok { + t.Error("expected no result from empty file") + } +} + +func TestParseTranscriptFile_MissingFile(t *testing.T) { + _, ok := parseTranscriptFile("/nonexistent/path.jsonl") + if ok { + t.Error("expected failure for missing file") + } +} + +func TestParseTranscriptFile_LastResultWins(t *testing.T) { + dir := t.TempDir() + // Two result events — the last one should win. + content := `{"type":"result","subtype":"success","is_error":false,"result":"first attempt ok"} +{"type":"result","subtype":"error_max_turns","is_error":true,"result":"second attempt failed"} +` + path := filepath.Join(dir, "multi-result.jsonl") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + summary, ok := parseTranscriptFile(path) + if !ok { + t.Fatal("expected result event to be found") + } + if !summary.IsError { + t.Error("expected last result (error) to win") + } + if summary.ErrorMessage != "second attempt failed" { + t.Errorf("unexpected error message: %q", summary.ErrorMessage) + } +} + +func TestParseTranscriptFile_TypeWithSpace(t *testing.T) { + dir := t.TempDir() + // Some JSON encoders add a space after the colon. + content := `{"type": "result", "subtype": "error_max_turns", "is_error": true, "result": "failed with space"} +` + path := filepath.Join(dir, "spaced.jsonl") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + summary, ok := parseTranscriptFile(path) + if !ok { + t.Fatal("expected result event to be found") + } + if !summary.IsError { + t.Error("expected IsError to be true") + } + if summary.ErrorMessage != "failed with space" { + t.Errorf("unexpected error message: %q", summary.ErrorMessage) + } +} + +func TestExtractTranscriptErrors_MultipleFiles(t *testing.T) { + dir := t.TempDir() + + // File 1: error result. + err1 := `{"type":"result","subtype":"error_max_turns","is_error":true,"result":"agent timed out"}` + if err := os.WriteFile(filepath.Join(dir, "agent1.jsonl"), []byte(err1), 0o644); err != nil { + t.Fatal(err) + } + + // File 2: success result (should not appear in summaries). + ok2 := `{"type":"result","subtype":"success","is_error":false,"result":"all good"}` + if err := os.WriteFile(filepath.Join(dir, "agent2.jsonl"), []byte(ok2), 0o644); err != nil { + t.Fatal(err) + } + + // File 3: not a JSONL file (should be skipped). + if err := os.WriteFile(filepath.Join(dir, "readme.txt"), []byte("not jsonl"), 0o644); err != nil { + t.Fatal(err) + } + + summaries := extractTranscriptErrors(dir) + if len(summaries) != 1 { + t.Fatalf("expected 1 error summary, got %d", len(summaries)) + } + if summaries[0].Source != "agent1.jsonl" { + t.Errorf("unexpected source: %q", summaries[0].Source) + } + if summaries[0].ErrorMessage != "agent timed out" { + t.Errorf("unexpected error message: %q", summaries[0].ErrorMessage) + } +} + +func TestExtractTranscriptErrors_EmptyDir(t *testing.T) { + dir := t.TempDir() + summaries := extractTranscriptErrors(dir) + if len(summaries) != 0 { + t.Errorf("expected no summaries for empty dir, got %d", len(summaries)) + } +} + +func TestExtractTranscriptErrors_MissingDir(t *testing.T) { + summaries := extractTranscriptErrors("/nonexistent/dir") + if summaries != nil { + t.Errorf("expected nil for missing dir, got %v", summaries) + } +} + +func TestTruncateError(t *testing.T) { + short := "short error" + if got := truncateError(short); got != short { + t.Errorf("short message should not be truncated: %q", got) + } + + long := strings.Repeat("x", maxTranscriptErrorLength+100) + got := truncateError(long) + if len(got) > maxTranscriptErrorLength+20 { + t.Errorf("truncated message too long: %d", len(got)) + } + if !strings.HasSuffix(got, "… (truncated)") { + t.Errorf("truncated message should end with ellipsis indicator: %q", got) + } +} + +func TestEmitTranscriptErrors(t *testing.T) { + summaries := []transcriptErrorSummary{ + { + Source: "code-transcript.jsonl", + IsError: true, + ErrorMessage: "Agent reached maximum turns", + Subtype: "error_max_turns", + }, + } + + var buf bytes.Buffer + emitTranscriptErrors(&buf, summaries) + + output := buf.String() + if !strings.Contains(output, "::error title=Agent Error (code-transcript.jsonl)::") { + t.Errorf("expected ::error:: annotation, got: %q", output) + } + if !strings.Contains(output, "Agent reached maximum turns") { + t.Errorf("expected error message in output, got: %q", output) + } +} + +func TestEmitTranscriptErrors_EmptyMessage(t *testing.T) { + summaries := []transcriptErrorSummary{ + { + Source: "test.jsonl", + IsError: true, + Subtype: "error_unknown", + }, + } + + var buf bytes.Buffer + emitTranscriptErrors(&buf, summaries) + + output := buf.String() + if !strings.Contains(output, "agent terminated with error (subtype: error_unknown)") { + t.Errorf("expected fallback message, got: %q", output) + } +} + +func TestEmitTranscriptErrors_NoSummaries(t *testing.T) { + var buf bytes.Buffer + emitTranscriptErrors(&buf, nil) + + if buf.Len() != 0 { + t.Errorf("expected no output for nil summaries, got: %q", buf.String()) + } +} + +func TestIsResultLine(t *testing.T) { + tests := []struct { + line string + want bool + }{ + {`{"type":"result","is_error":true}`, true}, + {`{"type": "result", "is_error": true}`, true}, + {`{"type":"assistant","content":[]}`, false}, + {`{"type":"system","subtype":"init"}`, false}, + {`not json at all`, false}, + {``, false}, + } + + for _, tt := range tests { + got := isResultLine([]byte(tt.line)) + if got != tt.want { + t.Errorf("isResultLine(%q) = %v, want %v", tt.line, got, tt.want) + } + } +} diff --git a/internal/scaffold/fullsend-repo/scripts/extract-transcript-error.sh b/internal/scaffold/fullsend-repo/scripts/extract-transcript-error.sh new file mode 100755 index 000000000..84a263e0d --- /dev/null +++ b/internal/scaffold/fullsend-repo/scripts/extract-transcript-error.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# extract-transcript-error.sh — Extract errors from agent transcript JSONL files. +# +# Reads transcript JSONL files (Claude Code stream-json format) and extracts +# the final result event. If the result indicates an error, prints a summary +# suitable for GitHub Actions annotations or human consumption. +# +# Usage: +# extract-transcript-error.sh +# +# When given a directory, processes all .jsonl files in it. +# When given a file, processes just that file. +# +# Exit codes: +# 0 — no errors found (or no transcript files) +# 1 — at least one transcript contains an error result +# 2 — usage error (bad arguments) +# +# This script can be used by: +# - Post-scripts to surface errors in workflow logs +# - The triage agent to extract errors from downloaded artifacts +# - Operators debugging failed agent runs +# +# Example with artifact download: +# gh run download -n agent-transcripts -D /tmp/transcripts +# extract-transcript-error.sh /tmp/transcripts/ + +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +TARGET="$1" +FOUND_ERROR=0 +MAX_ERROR_LENGTH=2000 + +# extract_error processes a single JSONL file and prints any error found. +extract_error() { + local file="$1" + local basename + basename="$(basename "$file")" + + # Find the last result line in the file. + # Claude Code transcripts end with a result event. + local last_result + last_result="$(grep -E '"type"\s*:\s*"result"' "$file" | tail -1)" || true + + if [[ -z "$last_result" ]]; then + return + fi + + # Check if the result indicates an error. + local is_error + is_error="$(echo "$last_result" | jq -r '.is_error // false' 2>/dev/null)" || true + + if [[ "$is_error" != "true" ]]; then + return + fi + + FOUND_ERROR=1 + + local error_msg + error_msg="$(echo "$last_result" | jq -r '.result // "unknown error"' 2>/dev/null)" || error_msg="unknown error" + + local subtype + subtype="$(echo "$last_result" | jq -r '.subtype // "unknown"' 2>/dev/null)" || subtype="unknown" + + # Truncate long error messages. + if [[ ${#error_msg} -gt $MAX_ERROR_LENGTH ]]; then + error_msg="${error_msg:0:$MAX_ERROR_LENGTH}... (truncated)" + fi + + echo "--- Error in ${basename} ---" + echo "Subtype: ${subtype}" + echo "Message: ${error_msg}" + echo "" + + # Emit GHA annotation if running in GitHub Actions. + if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then + # Sanitize for GHA: replace :: and URL-encoded newlines to prevent command injection. + local safe_msg="${error_msg//::/ :}" + safe_msg="${safe_msg//%0A/ }" + safe_msg="${safe_msg//%0a/ }" + safe_msg="${safe_msg//%0D/ }" + safe_msg="${safe_msg//%0d/ }" + echo "::error title=Agent Error (${basename})::${safe_msg}" + fi +} + +if [[ -d "$TARGET" ]]; then + # Process all JSONL files in the directory. + for f in "$TARGET"/*.jsonl; do + [[ -f "$f" ]] || continue + extract_error "$f" + done +elif [[ -f "$TARGET" ]]; then + extract_error "$TARGET" +else + echo "Error: $TARGET is not a file or directory" >&2 + exit 2 +fi + +if [[ $FOUND_ERROR -eq 1 ]]; then + exit 1 +fi + +exit 0