trustworthy content-stream tokenization and --ops output

cmd/cli/cmd_stream.go

@@ -357,6 +357,14 @@ func emitOps(inspector *pdfcore.Inspector, result *pdfcore.ContentStreamData, pa
 	// exactly one JSON object per line (NDJSON). --pretty does not apply.
 	enc := json.NewEncoder(os.Stdout)
 	for _, fl := range result.Formatted {
+		// --ops is operator-centric NDJSON: one object per operator. Comment
+		// lines and trailing dangling-operand runs carry Operator == "" and are
+		// not operators; skipping them upholds the one-object-per-operator
+		// contract (a phantom {"op":""} record would breach it). --json is
+		// unaffected -- it serializes the full Formatted slice.
+		if fl.Operator == "" {
+			continue
+		}
 		line := opsLine{
 			Op:           fl.Operator,
 			Params:       operandValues(fl),

internal/pdfcore/objectsource.go

@@ -300,6 +300,10 @@ func writeDict(w *sourceWriter, d pdfcpu_types.Dict, depth int) {
 // element per line with hanging indentation, matching the AC1 example.
 func writeArray(w *sourceWriter, arr pdfcpu_types.Array, depth int) {
 	if isShortSimpleArray(arr) {
+		// F4 (known, near-unreachable gap): unlike the long-array path, the short
+		// form drops cap-truncated elements WITHOUT a "... [truncated]" marker.
+		// Only reachable with <= threshold simple scalars exceeding the 256KB cap
+		// (e.g. a single ~250k-digit number), which is not a PDF found in nature.
 		// WriteAlways for the brackets; pair with the closing `]` so the
 		// short form is well-formed even when cap-truncated mid-element.
 		w.WriteAlways("[")

internal/pdfcore/stream.go

@@ -348,11 +348,14 @@ func tokenizeContentStream(raw string) []Token {
 			continue
 		}
 
-		// Number: optional '-', then digits/dot, or '.' then digits.
-		// Also handle negative numbers: '-' followed by digit or '.'.
+		// Number: optional sign ('+' or '-'), then digits/dot, or '.' then digits.
+		// ISO 32000-1 7.3.3 permits a leading '+' or '-' on integers and reals.
 		if isNumberStart(raw, i, n) {
 			j := i
-			if raw[j] == '-' {
+			// Consume the leading sign so it is part of the number token, not a
+			// stray first char of the digit scan. Both '+' and '-' must be
+			// handled here or isNumberStart accepting '+' yields a malformed token.
+			if raw[j] == '-' || raw[j] == '+' {
 				j++
 			}
 			hasDot := false
@@ -417,11 +420,32 @@ func tokenizeContentStream(raw string) []Token {
 						// Check following byte is whitespace or EOF
 						afterEI := k + 2
 						if afterEI >= n || isWSByte(raw[afterEI]) {
-							// Emit data token (without the preceding whitespace)
+							// Emit data token (without the preceding whitespace).
+							// endIdx = k-1 already drops the single delimiter byte
+							// adjacent to 'E' (guaranteed whitespace by the check
+							// above).
 							endIdx := k - 1
 							if endIdx < dataStart {
 								endIdx = dataStart
 							}
+							// F3: if that delimiter is the LF of a "\r\nEI" CRLF pair,
+							// also drop the CR so the opaque data token carries no
+							// stray trailing '\r'. Bounded to a single EOL delimiter
+							// (one CRLF unit) -- deliberately NOT an unbounded
+							// whitespace run, which would strip whitespace-valued
+							// bytes (e.g. a 0x20 sample) that are legitimately part of
+							// the opaque inline-image payload.
+							// Irreducible caveat: with no inline-image length model
+							// (EI is found heuristically), a payload legitimately
+							// ending in a real 0x0D byte before a bare-LF delimiter
+							// ("...<0x0D>\nEI") is indistinguishable from a CRLF and
+							// that byte is dropped. Accepted tradeoff: bounding an
+							// opaque display token to a length model is out of scope,
+							// and not stripping the CR would reintroduce the stray
+							// '\r' this fix removes.
+							if endIdx > dataStart && raw[endIdx] == '\n' && raw[endIdx-1] == '\r' {
+								endIdx--
+							}
 							data := raw[dataStart:endIdx]
 							if len(data) > 0 {
 								tokens = append(tokens, Token{Type: "string", Value: data, Line: dataLine, Col: dataCol})
@@ -479,7 +503,9 @@ func isNumberStart(raw string, i, n int) bool {
 	if ch == '.' {
 		return i+1 < n && raw[i+1] >= '0' && raw[i+1] <= '9'
 	}
-	if ch == '-' {
+	// A leading '+' or '-' is a number start only when followed by a digit, or
+	// by '.' + digit (ISO 32000-1 7.3.3). A bare sign stays a word/operator.
+	if ch == '-' || ch == '+' {
 		if i+1 < n {
 			next := raw[i+1]
 			if next >= '0' && next <= '9' {

internal/pdfcore/stream_test.go

@@ -887,6 +887,139 @@ func TestTokenizeInlineImagePayloadOpaque(t *testing.T) {
 	}
 }
 
+// ---------------------------------------------------------------------------
+// 14.1-UNIT-002 [P2] (risk R-14-13): inline-image data token has no trailing
+// whitespace (F3). A "\r\nEI" sequence must drop the CR as well as the LF
+// delimiter (one CRLF unit), not leave the stray '\r' the single-byte strip
+// left behind, so the opaque data token carries no trailing EOL fragment.
+// ---------------------------------------------------------------------------
+
+func TestTokenizeInlineImageTrailingWhitespaceStripped(t *testing.T) {
+	input := "q\r\nBI /W 1 /H 1 ID\x00\x01\x02\r\nEI\nQ"
+	toks := tokenizeContentStream(input)
+
+	idIdx, eiIdx := -1, -1
+	for i, tk := range toks {
+		if tk.Type == "operator" && tk.Value == "ID" && idIdx == -1 {
+			idIdx = i
+		}
+		if tk.Type == "operator" && tk.Value == "EI" && eiIdx == -1 && idIdx != -1 && i > idIdx {
+			eiIdx = i
+		}
+	}
+	if idIdx == -1 || eiIdx == -1 {
+		t.Fatalf("[14.1-UNIT-002] expected ID and EI operators; got tokens=%+v", toks)
+	}
+	if eiIdx-idIdx != 2 {
+		t.Fatalf("[14.1-UNIT-002] expected exactly one payload token between ID and EI; got %d, sequence=%+v",
+			eiIdx-idIdx-1, toks[idIdx:eiIdx+1])
+	}
+	payload := toks[idIdx+1]
+	if payload.Value != "\x00\x01\x02" {
+		t.Errorf("[14.1-UNIT-002] payload Value = %q, want %q (\"\\r\\n\" CRLF delimiter before EI must be stripped, CR included)", payload.Value, "\x00\x01\x02")
+	}
+	if strings.HasSuffix(payload.Value, "\r") || strings.HasSuffix(payload.Value, "\n") {
+		t.Errorf("[14.1-UNIT-002] payload Value = %q retains trailing whitespace", payload.Value)
+	}
+}
+
+// 14.1-UNIT-002b: a whitespace-valued byte that is genuinely part of the opaque
+// inline-image payload (here a 0x20 sample immediately before the single-LF
+// delimiter) must be preserved. The EOL strip is bounded to one delimiter
+// (CRLF unit), not an unbounded whitespace run that would eat real data bytes.
+func TestTokenizeInlineImagePayloadWhitespaceValuedByteKept(t *testing.T) {
+	input := "q\nBI /W 1 /H 1 ID\x00\x20\nEI\nQ"
+	toks := tokenizeContentStream(input)
+
+	idIdx, eiIdx := -1, -1
+	for i, tk := range toks {
+		if tk.Type == "operator" && tk.Value == "ID" && idIdx == -1 {
+			idIdx = i
+		}
+		if tk.Type == "operator" && tk.Value == "EI" && eiIdx == -1 && idIdx != -1 && i > idIdx {
+			eiIdx = i
+		}
+	}
+	if idIdx == -1 || eiIdx == -1 {
+		t.Fatalf("[14.1-UNIT-002b] expected ID and EI operators; got tokens=%+v", toks)
+	}
+	if eiIdx-idIdx != 2 {
+		t.Fatalf("[14.1-UNIT-002b] expected exactly one payload token between ID and EI; got %d, sequence=%+v",
+			eiIdx-idIdx-1, toks[idIdx:eiIdx+1])
+	}
+	payload := toks[idIdx+1]
+	if payload.Value != "\x00\x20" {
+		t.Errorf("[14.1-UNIT-002b] payload Value = %q, want %q (whitespace-valued payload byte before the single-LF delimiter must be preserved, not stripped)", payload.Value, "\x00\x20")
+	}
+}
+
+// ---------------------------------------------------------------------------
+// 14.1-UNIT-001 [P1] (risk R-14-02): leading-sign number tokenization (F1).
+//
+// RED PHASE (Story 14-1): ISO 32000-1 7.3.3 permits a leading '+' (as well as
+// '-') on integers and reals. The current tokenizer accepts a leading '-' but
+// NOT a leading '+', so a spec-valid operand like "+5" falls through to the
+// word/operator branch and is mis-emitted as an OPERATOR token. This
+// table-driven test feeds bare number byte-slices and asserts each signed
+// value classifies as a SINGLE "number" token.
+//
+// Expected failures against baseline code: the "+5", "+.5", and "+5.0" cases
+// (mislabeled operator, and the "+5.0" case additionally split by the word
+// scan). The "-3" / "-.5" cases already pass. The bare "+"/"-" cases (a sign
+// not followed by a digit or ".digit") must REMAIN word/operator tokens.
+// ---------------------------------------------------------------------------
+
+func TestTokenizeLeadingSignNumbers(t *testing.T) {
+	numberCases := []struct {
+		name  string
+		input string
+		want  string
+	}{
+		{"leading plus integer", "+5", "+5"},
+		{"leading plus leading-dot real", "+.5", "+.5"},
+		{"leading minus integer", "-3", "-3"},
+		{"leading minus leading-dot real", "-.5", "-.5"},
+		{"leading plus real", "+5.0", "+5.0"},
+	}
+	for _, tc := range numberCases {
+		t.Run(tc.name, func(t *testing.T) {
+			tokens := tokenizeContentStream(tc.input)
+			if len(tokens) != 1 {
+				t.Fatalf("[14.1-UNIT-001] %q: got %d tokens, want exactly 1 (single number token); tokens=%+v",
+					tc.input, len(tokens), tokens)
+			}
+			if tokens[0].Type != "number" {
+				t.Errorf("[14.1-UNIT-001] %q: token Type = %q, want \"number\"", tc.input, tokens[0].Type)
+			}
+			if tokens[0].Value != tc.want {
+				t.Errorf("[14.1-UNIT-001] %q: token Value = %q, want %q", tc.input, tokens[0].Value, tc.want)
+			}
+		})
+	}
+
+	// A bare sign not followed by a digit or ".digit" stays a word/operator
+	// token (unchanged behavior). Guards against an over-broad fix.
+	bareCases := []struct {
+		name  string
+		input string
+		want  string
+	}{
+		{"bare plus stays operator", "+ 10 Td", "+"},
+		{"bare minus stays operator", "- 10 Td", "-"},
+	}
+	for _, tc := range bareCases {
+		t.Run(tc.name, func(t *testing.T) {
+			tokens := tokenizeContentStream(tc.input)
+			if len(tokens) == 0 {
+				t.Fatalf("[14.1-UNIT-001] %q: got 0 tokens, want at least 1", tc.input)
+			}
+			if tokens[0].Type != "operator" || tokens[0].Value != tc.want {
+				t.Errorf("[14.1-UNIT-001] %q: token[0] = %+v, want operator(%s)", tc.input, tokens[0], tc.want)
+			}
+		})
+	}
+}
+
 // ---------------------------------------------------------------------------
 // 3.3-BENCH-001 [P2]: Benchmark tokenizer on a ~100KB content stream.
 // Verifies O(n) performance and detects regressions.

testdata/correctness/README.md

@@ -80,6 +80,45 @@ Pre-fix `parseDifferences` appended `/a` at code -1 and `/b` at code 999.
 Post-fix both entries are skipped silently; only the `/space` entry at code
 32 (and the implicit code 33 if a name follows) populates the encoding table.
 
+## leading-plus.pdf
+
+Single-page PDF whose content stream carries leading-`+` signed operands
+(Story 14-1, F1):
+
+```
+obj 4: << /Length 16 >> stream
++5 0 0 +5 0 0 cm
+endstream
+```
+
+ISO 32000-1 7.3.3 permits a leading `+` on integers and reals. Pre-fix
+`isNumberStart` (and the number-consumption loop) accept a leading `-` but not
+a leading `+`, so each `+5` falls through to the word/operator branch and is
+mis-emitted as an OPERATOR token; `dump stream --json` / `--ops` report a bogus
+operator `+5` and `Format()` mis-groups the transform. Post-fix each `+5` is a
+`number` token and `cm` is the sole operator for the line. 4 objects total.
+
+## comment-and-dangling.pdf
+
+Single-page PDF whose content stream contains a `%` comment line and a trailing
+dangling operand run with no terminating operator (Story 14-1, F2):
+
+```
+obj 4: << /Length 40 >> stream
+% leading comment
+1 0 0 1 10 20 cm
+30 40
+endstream
+```
+
+`Format()` emits the comment line and the trailing `30 40` run as
+`FormattedLine`s with `Operator == ""`. Pre-fix `emitOps` iterates `Formatted`
+with no guard, so `dump stream --ops` emits two phantom `{"op":"","params":...}`
+NDJSON records, breaching the documented one-object-per-operator contract.
+Post-fix `--ops` skips empty-operator lines and emits only the single `cm`
+record. `--json` is unaffected (it serializes the full formatted slice,
+comments included). 4 objects total.
+
 ## cjk-cmap-carry.pdf
 
 Type0 composite font referencing a ToUnicode CMap with a bfrange whose

tests/14-1-trustworthy-stream-op-output/go.mod

@@ -0,0 +1,3 @@
+module trustworthy-stream-op-output-test
+
+go 1.25

tests/14-1-trustworthy-stream-op-output/helpers_test.go

@@ -0,0 +1,109 @@
+// Story 14-1 RED-PHASE acceptance harness for the machine-contract surface of
+// `dump stream --json` / `--ops`.
+//
+// Black-box: build the pdfdebug CLI binary and run it as a subprocess against
+// committed fixtures in testdata/correctness/. These tests assert the EXPECTED
+// post-implementation contract behavior and MUST FAIL against the current
+// binary (leading-sign operands mislabeled as operators; --ops emitting
+// empty-operator records) until Story 14-1 is implemented. They fail at
+// RUNTIME (wrong token type / phantom NDJSON record), not at compile time, so
+// the main module keeps building green. This module has its own go.mod and is
+// not part of the main build (mirrors tests/13-6-structural-diff).
+//
+// Naming: 14.1-INTG-NNN [Px] per the story Testing Requirements (AC6).
+//
+// Run: cd tests/14-1-trustworthy-stream-op-output && go test -v -count=1 ./...
+package trustworthy_stream_test
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+)
+
+// projectRoot walks up from the test directory to find the main module's go.mod.
+func projectRoot(t *testing.T) string {
+	t.Helper()
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("failed to get working directory: %v", err)
+	}
+	for {
+		goModPath := filepath.Join(dir, "go.mod")
+		if content, err := os.ReadFile(goModPath); err == nil {
+			if strings.Contains(string(content), "module unidoc-pdf-debugger") {
+				return dir
+			}
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			t.Fatalf("could not find project root (no go.mod with module unidoc-pdf-debugger found)")
+		}
+		dir = parent
+	}
+}
+
+// fixturePath returns the absolute path to a committed correctness-corpus fixture.
+func fixturePath(t *testing.T, name string) string {
+	t.Helper()
+	return filepath.Join(projectRoot(t), "testdata", "correctness", name)
+}
+
+var (
+	cliBuildOnce sync.Once
+	cliBinPath   string
+	cliBuildErr  string
+)
+
+// buildCLI compiles the CLI binary once per test package and returns its path.
+// Cached via sync.Once: the binary is identical for every test in the module.
+func buildCLI(t *testing.T) string {
+	t.Helper()
+	cliBuildOnce.Do(func() {
+		root := projectRoot(t)
+		binName := "pdfdebug"
+		if runtime.GOOS == "windows" {
+			binName += ".exe"
+		}
+		tmpDir, err := os.MkdirTemp("", "pdfdebug-cli-")
+		if err != nil {
+			cliBuildErr = "failed to create temp dir: " + err.Error()
+			return
+		}
+		binPath := filepath.Join(tmpDir, binName)
+		cmd := exec.Command("go", "build", "-o", binPath, "./cmd/cli/")
+		cmd.Dir = root
+		if output, err := cmd.CombinedOutput(); err != nil {
+			cliBuildErr = "failed to build CLI binary: " + err.Error() + "\n" + string(output)
+			return
+		}
+		cliBinPath = binPath
+	})
+	if cliBuildErr != "" {
+		t.Fatalf("%s", cliBuildErr)
+	}
+	return cliBinPath
+}
+
+// runCLI executes the CLI binary with args and returns stdout, stderr, exit code.
+func runCLI(t *testing.T, binPath string, args ...string) (stdout, stderr string, exitCode int) {
+	t.Helper()
+	cmd := exec.Command(binPath, args...)
+	var outBuf, errBuf strings.Builder
+	cmd.Stdout = &outBuf
+	cmd.Stderr = &errBuf
+	err := cmd.Run()
+	exitCode = 0
+	if err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			exitCode = exitErr.ExitCode()
+		} else {
+			t.Fatalf("failed to run CLI: %v", err)
+		}
+	}
+	return outBuf.String(), errBuf.String(), exitCode
+}