From 427cd946d8968e93baf5c549d1b599c4aa97f83b Mon Sep 17 00:00:00 2001 From: Ade Anom A Date: Wed, 15 Jul 2026 17:19:36 +0700 Subject: [PATCH] fix(14-1): trustworthy content-stream tokenization and --ops output - F1: tokenize leading-sign numeric operands (+5, +.5) as numbers not operators (isNumberStart + number-consumption loop), fixing --json/--ops mislabeling of ISO 32000-1 7.3.3 signed operands. - F2: emitOps skips empty-operator FormattedLines so --ops NDJSON no longer emits phantom {"op":""} records for comment/dangling lines. - F3: strip the CRLF EOL delimiter (bounded to one unit) before inline-image EI, removing the stray \r without eating whitespace-valued payload bytes. - F4: document the short-array cap-truncation marker gap (comment only). - Tests: 14.1-UNIT-001/-002/-002b + tests/14-1 acceptance module (INTG-000/-001/-002/-003) with two hand-authored fixtures. --- cmd/cli/cmd_stream.go | 8 + internal/pdfcore/objectsource.go | 4 + internal/pdfcore/stream.go | 36 ++- internal/pdfcore/stream_test.go | 133 +++++++++++ testdata/correctness/README.md | 39 ++++ testdata/correctness/comment-and-dangling.pdf | Bin 0 -> 455 bytes testdata/correctness/leading-plus.pdf | Bin 0 -> 431 bytes .../14-1-trustworthy-stream-op-output/go.mod | 3 + .../helpers_test.go | 109 +++++++++ .../stream_contract_test.go | 214 ++++++++++++++++++ 10 files changed, 541 insertions(+), 5 deletions(-) create mode 100644 testdata/correctness/comment-and-dangling.pdf create mode 100644 testdata/correctness/leading-plus.pdf create mode 100644 tests/14-1-trustworthy-stream-op-output/go.mod create mode 100644 tests/14-1-trustworthy-stream-op-output/helpers_test.go create mode 100644 tests/14-1-trustworthy-stream-op-output/stream_contract_test.go diff --git a/cmd/cli/cmd_stream.go b/cmd/cli/cmd_stream.go index eab09ec..115143b 100644 --- a/cmd/cli/cmd_stream.go +++ b/cmd/cli/cmd_stream.go @@ -357,6 +357,14 @@ func emitOps(inspector *pdfcore.Inspector, result *pdfcore.ContentStreamData, pa // exactly one JSON object per line (NDJSON). --pretty does not apply. enc := json.NewEncoder(os.Stdout) for _, fl := range result.Formatted { + // --ops is operator-centric NDJSON: one object per operator. Comment + // lines and trailing dangling-operand runs carry Operator == "" and are + // not operators; skipping them upholds the one-object-per-operator + // contract (a phantom {"op":""} record would breach it). --json is + // unaffected -- it serializes the full Formatted slice. + if fl.Operator == "" { + continue + } line := opsLine{ Op: fl.Operator, Params: operandValues(fl), diff --git a/internal/pdfcore/objectsource.go b/internal/pdfcore/objectsource.go index 755282b..86a2d67 100644 --- a/internal/pdfcore/objectsource.go +++ b/internal/pdfcore/objectsource.go @@ -300,6 +300,10 @@ func writeDict(w *sourceWriter, d pdfcpu_types.Dict, depth int) { // element per line with hanging indentation, matching the AC1 example. func writeArray(w *sourceWriter, arr pdfcpu_types.Array, depth int) { if isShortSimpleArray(arr) { + // F4 (known, near-unreachable gap): unlike the long-array path, the short + // form drops cap-truncated elements WITHOUT a "... [truncated]" marker. + // Only reachable with <= threshold simple scalars exceeding the 256KB cap + // (e.g. a single ~250k-digit number), which is not a PDF found in nature. // WriteAlways for the brackets; pair with the closing `]` so the // short form is well-formed even when cap-truncated mid-element. w.WriteAlways("[") diff --git a/internal/pdfcore/stream.go b/internal/pdfcore/stream.go index a47a4c9..04af35a 100644 --- a/internal/pdfcore/stream.go +++ b/internal/pdfcore/stream.go @@ -348,11 +348,14 @@ func tokenizeContentStream(raw string) []Token { continue } - // Number: optional '-', then digits/dot, or '.' then digits. - // Also handle negative numbers: '-' followed by digit or '.'. + // Number: optional sign ('+' or '-'), then digits/dot, or '.' then digits. + // ISO 32000-1 7.3.3 permits a leading '+' or '-' on integers and reals. if isNumberStart(raw, i, n) { j := i - if raw[j] == '-' { + // Consume the leading sign so it is part of the number token, not a + // stray first char of the digit scan. Both '+' and '-' must be + // handled here or isNumberStart accepting '+' yields a malformed token. + if raw[j] == '-' || raw[j] == '+' { j++ } hasDot := false @@ -417,11 +420,32 @@ func tokenizeContentStream(raw string) []Token { // Check following byte is whitespace or EOF afterEI := k + 2 if afterEI >= n || isWSByte(raw[afterEI]) { - // Emit data token (without the preceding whitespace) + // Emit data token (without the preceding whitespace). + // endIdx = k-1 already drops the single delimiter byte + // adjacent to 'E' (guaranteed whitespace by the check + // above). endIdx := k - 1 if endIdx < dataStart { endIdx = dataStart } + // F3: if that delimiter is the LF of a "\r\nEI" CRLF pair, + // also drop the CR so the opaque data token carries no + // stray trailing '\r'. Bounded to a single EOL delimiter + // (one CRLF unit) -- deliberately NOT an unbounded + // whitespace run, which would strip whitespace-valued + // bytes (e.g. a 0x20 sample) that are legitimately part of + // the opaque inline-image payload. + // Irreducible caveat: with no inline-image length model + // (EI is found heuristically), a payload legitimately + // ending in a real 0x0D byte before a bare-LF delimiter + // ("...<0x0D>\nEI") is indistinguishable from a CRLF and + // that byte is dropped. Accepted tradeoff: bounding an + // opaque display token to a length model is out of scope, + // and not stripping the CR would reintroduce the stray + // '\r' this fix removes. + if endIdx > dataStart && raw[endIdx] == '\n' && raw[endIdx-1] == '\r' { + endIdx-- + } data := raw[dataStart:endIdx] if len(data) > 0 { tokens = append(tokens, Token{Type: "string", Value: data, Line: dataLine, Col: dataCol}) @@ -479,7 +503,9 @@ func isNumberStart(raw string, i, n int) bool { if ch == '.' { return i+1 < n && raw[i+1] >= '0' && raw[i+1] <= '9' } - if ch == '-' { + // A leading '+' or '-' is a number start only when followed by a digit, or + // by '.' + digit (ISO 32000-1 7.3.3). A bare sign stays a word/operator. + if ch == '-' || ch == '+' { if i+1 < n { next := raw[i+1] if next >= '0' && next <= '9' { diff --git a/internal/pdfcore/stream_test.go b/internal/pdfcore/stream_test.go index 7fb1ec0..5de6d53 100644 --- a/internal/pdfcore/stream_test.go +++ b/internal/pdfcore/stream_test.go @@ -887,6 +887,139 @@ func TestTokenizeInlineImagePayloadOpaque(t *testing.T) { } } +// --------------------------------------------------------------------------- +// 14.1-UNIT-002 [P2] (risk R-14-13): inline-image data token has no trailing +// whitespace (F3). A "\r\nEI" sequence must drop the CR as well as the LF +// delimiter (one CRLF unit), not leave the stray '\r' the single-byte strip +// left behind, so the opaque data token carries no trailing EOL fragment. +// --------------------------------------------------------------------------- + +func TestTokenizeInlineImageTrailingWhitespaceStripped(t *testing.T) { + input := "q\r\nBI /W 1 /H 1 ID\x00\x01\x02\r\nEI\nQ" + toks := tokenizeContentStream(input) + + idIdx, eiIdx := -1, -1 + for i, tk := range toks { + if tk.Type == "operator" && tk.Value == "ID" && idIdx == -1 { + idIdx = i + } + if tk.Type == "operator" && tk.Value == "EI" && eiIdx == -1 && idIdx != -1 && i > idIdx { + eiIdx = i + } + } + if idIdx == -1 || eiIdx == -1 { + t.Fatalf("[14.1-UNIT-002] expected ID and EI operators; got tokens=%+v", toks) + } + if eiIdx-idIdx != 2 { + t.Fatalf("[14.1-UNIT-002] expected exactly one payload token between ID and EI; got %d, sequence=%+v", + eiIdx-idIdx-1, toks[idIdx:eiIdx+1]) + } + payload := toks[idIdx+1] + if payload.Value != "\x00\x01\x02" { + t.Errorf("[14.1-UNIT-002] payload Value = %q, want %q (\"\\r\\n\" CRLF delimiter before EI must be stripped, CR included)", payload.Value, "\x00\x01\x02") + } + if strings.HasSuffix(payload.Value, "\r") || strings.HasSuffix(payload.Value, "\n") { + t.Errorf("[14.1-UNIT-002] payload Value = %q retains trailing whitespace", payload.Value) + } +} + +// 14.1-UNIT-002b: a whitespace-valued byte that is genuinely part of the opaque +// inline-image payload (here a 0x20 sample immediately before the single-LF +// delimiter) must be preserved. The EOL strip is bounded to one delimiter +// (CRLF unit), not an unbounded whitespace run that would eat real data bytes. +func TestTokenizeInlineImagePayloadWhitespaceValuedByteKept(t *testing.T) { + input := "q\nBI /W 1 /H 1 ID\x00\x20\nEI\nQ" + toks := tokenizeContentStream(input) + + idIdx, eiIdx := -1, -1 + for i, tk := range toks { + if tk.Type == "operator" && tk.Value == "ID" && idIdx == -1 { + idIdx = i + } + if tk.Type == "operator" && tk.Value == "EI" && eiIdx == -1 && idIdx != -1 && i > idIdx { + eiIdx = i + } + } + if idIdx == -1 || eiIdx == -1 { + t.Fatalf("[14.1-UNIT-002b] expected ID and EI operators; got tokens=%+v", toks) + } + if eiIdx-idIdx != 2 { + t.Fatalf("[14.1-UNIT-002b] expected exactly one payload token between ID and EI; got %d, sequence=%+v", + eiIdx-idIdx-1, toks[idIdx:eiIdx+1]) + } + payload := toks[idIdx+1] + if payload.Value != "\x00\x20" { + t.Errorf("[14.1-UNIT-002b] payload Value = %q, want %q (whitespace-valued payload byte before the single-LF delimiter must be preserved, not stripped)", payload.Value, "\x00\x20") + } +} + +// --------------------------------------------------------------------------- +// 14.1-UNIT-001 [P1] (risk R-14-02): leading-sign number tokenization (F1). +// +// RED PHASE (Story 14-1): ISO 32000-1 7.3.3 permits a leading '+' (as well as +// '-') on integers and reals. The current tokenizer accepts a leading '-' but +// NOT a leading '+', so a spec-valid operand like "+5" falls through to the +// word/operator branch and is mis-emitted as an OPERATOR token. This +// table-driven test feeds bare number byte-slices and asserts each signed +// value classifies as a SINGLE "number" token. +// +// Expected failures against baseline code: the "+5", "+.5", and "+5.0" cases +// (mislabeled operator, and the "+5.0" case additionally split by the word +// scan). The "-3" / "-.5" cases already pass. The bare "+"/"-" cases (a sign +// not followed by a digit or ".digit") must REMAIN word/operator tokens. +// --------------------------------------------------------------------------- + +func TestTokenizeLeadingSignNumbers(t *testing.T) { + numberCases := []struct { + name string + input string + want string + }{ + {"leading plus integer", "+5", "+5"}, + {"leading plus leading-dot real", "+.5", "+.5"}, + {"leading minus integer", "-3", "-3"}, + {"leading minus leading-dot real", "-.5", "-.5"}, + {"leading plus real", "+5.0", "+5.0"}, + } + for _, tc := range numberCases { + t.Run(tc.name, func(t *testing.T) { + tokens := tokenizeContentStream(tc.input) + if len(tokens) != 1 { + t.Fatalf("[14.1-UNIT-001] %q: got %d tokens, want exactly 1 (single number token); tokens=%+v", + tc.input, len(tokens), tokens) + } + if tokens[0].Type != "number" { + t.Errorf("[14.1-UNIT-001] %q: token Type = %q, want \"number\"", tc.input, tokens[0].Type) + } + if tokens[0].Value != tc.want { + t.Errorf("[14.1-UNIT-001] %q: token Value = %q, want %q", tc.input, tokens[0].Value, tc.want) + } + }) + } + + // A bare sign not followed by a digit or ".digit" stays a word/operator + // token (unchanged behavior). Guards against an over-broad fix. + bareCases := []struct { + name string + input string + want string + }{ + {"bare plus stays operator", "+ 10 Td", "+"}, + {"bare minus stays operator", "- 10 Td", "-"}, + } + for _, tc := range bareCases { + t.Run(tc.name, func(t *testing.T) { + tokens := tokenizeContentStream(tc.input) + if len(tokens) == 0 { + t.Fatalf("[14.1-UNIT-001] %q: got 0 tokens, want at least 1", tc.input) + } + if tokens[0].Type != "operator" || tokens[0].Value != tc.want { + t.Errorf("[14.1-UNIT-001] %q: token[0] = %+v, want operator(%s)", tc.input, tokens[0], tc.want) + } + }) + } +} + // --------------------------------------------------------------------------- // 3.3-BENCH-001 [P2]: Benchmark tokenizer on a ~100KB content stream. // Verifies O(n) performance and detects regressions. diff --git a/testdata/correctness/README.md b/testdata/correctness/README.md index e048f94..3068cbf 100644 --- a/testdata/correctness/README.md +++ b/testdata/correctness/README.md @@ -80,6 +80,45 @@ Pre-fix `parseDifferences` appended `/a` at code -1 and `/b` at code 999. Post-fix both entries are skipped silently; only the `/space` entry at code 32 (and the implicit code 33 if a name follows) populates the encoding table. +## leading-plus.pdf + +Single-page PDF whose content stream carries leading-`+` signed operands +(Story 14-1, F1): + +``` +obj 4: << /Length 16 >> stream ++5 0 0 +5 0 0 cm +endstream +``` + +ISO 32000-1 7.3.3 permits a leading `+` on integers and reals. Pre-fix +`isNumberStart` (and the number-consumption loop) accept a leading `-` but not +a leading `+`, so each `+5` falls through to the word/operator branch and is +mis-emitted as an OPERATOR token; `dump stream --json` / `--ops` report a bogus +operator `+5` and `Format()` mis-groups the transform. Post-fix each `+5` is a +`number` token and `cm` is the sole operator for the line. 4 objects total. + +## comment-and-dangling.pdf + +Single-page PDF whose content stream contains a `%` comment line and a trailing +dangling operand run with no terminating operator (Story 14-1, F2): + +``` +obj 4: << /Length 40 >> stream +% leading comment +1 0 0 1 10 20 cm +30 40 +endstream +``` + +`Format()` emits the comment line and the trailing `30 40` run as +`FormattedLine`s with `Operator == ""`. Pre-fix `emitOps` iterates `Formatted` +with no guard, so `dump stream --ops` emits two phantom `{"op":"","params":...}` +NDJSON records, breaching the documented one-object-per-operator contract. +Post-fix `--ops` skips empty-operator lines and emits only the single `cm` +record. `--json` is unaffected (it serializes the full formatted slice, +comments included). 4 objects total. + ## cjk-cmap-carry.pdf Type0 composite font referencing a ToUnicode CMap with a bfrange whose diff --git a/testdata/correctness/comment-and-dangling.pdf b/testdata/correctness/comment-and-dangling.pdf new file mode 100644 index 0000000000000000000000000000000000000000..37040993b0db0d59149d38c2407cf1e50e087841 GIT binary patch literal 455 zcmZWm%TB{E5WM><=8_}$k+=}4>VZcufT#+$ii2s~5R}-+F;d~{@uqGrMDF ztDEJSJTI7pfPZ~w^EvqQ*M|kaHqo@c249)l_E3qi#$v&&tB6Sbf5Cyr@0zN|CMWt! za`B%oLY@d|G8NLAU=#JQhdwniNn1+m7WTNhsea8I$gV-7x{p#9v{ib*{0)5i2oF y;f|QToM2Mw38qBKOhwulOw-yh{KHf8ZK1)}-qRC~dHZNW9BxvjVp(?kxMV;2Uv$j? literal 0 HcmV?d00001 diff --git a/testdata/correctness/leading-plus.pdf b/testdata/correctness/leading-plus.pdf new file mode 100644 index 0000000000000000000000000000000000000000..277a3c21a18aad00a42ccc3be7650beac7e723ad GIT binary patch literal 431 zcmZXQO-}+b5Qgvl74KzFMEe0+LLytYI$A%slf> z-zgTi^9y-7ViE%W?SoCH;4j~w7W~E}v-LIj!qhfGWnhWfj9FI^NcsPP1IO>1Dq=Mx z{57liFP9*DgVZ@6q&2~2^st9N+Nv?vevg&Nd?^*KCaTqQDPs{MTF{fF{3!ciU7g+` zOX|lItl6=F&VWGotas$2YjuwH!ETskC(BMn&=~SJsm`q6RpHw+!MRE5uXTJ*#ZKnVL literal 0 HcmV?d00001 diff --git a/tests/14-1-trustworthy-stream-op-output/go.mod b/tests/14-1-trustworthy-stream-op-output/go.mod new file mode 100644 index 0000000..57f00e0 --- /dev/null +++ b/tests/14-1-trustworthy-stream-op-output/go.mod @@ -0,0 +1,3 @@ +module trustworthy-stream-op-output-test + +go 1.25 diff --git a/tests/14-1-trustworthy-stream-op-output/helpers_test.go b/tests/14-1-trustworthy-stream-op-output/helpers_test.go new file mode 100644 index 0000000..2503261 --- /dev/null +++ b/tests/14-1-trustworthy-stream-op-output/helpers_test.go @@ -0,0 +1,109 @@ +// Story 14-1 RED-PHASE acceptance harness for the machine-contract surface of +// `dump stream --json` / `--ops`. +// +// Black-box: build the pdfdebug CLI binary and run it as a subprocess against +// committed fixtures in testdata/correctness/. These tests assert the EXPECTED +// post-implementation contract behavior and MUST FAIL against the current +// binary (leading-sign operands mislabeled as operators; --ops emitting +// empty-operator records) until Story 14-1 is implemented. They fail at +// RUNTIME (wrong token type / phantom NDJSON record), not at compile time, so +// the main module keeps building green. This module has its own go.mod and is +// not part of the main build (mirrors tests/13-6-structural-diff). +// +// Naming: 14.1-INTG-NNN [Px] per the story Testing Requirements (AC6). +// +// Run: cd tests/14-1-trustworthy-stream-op-output && go test -v -count=1 ./... +package trustworthy_stream_test + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" +) + +// projectRoot walks up from the test directory to find the main module's go.mod. +func projectRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatalf("failed to get working directory: %v", err) + } + for { + goModPath := filepath.Join(dir, "go.mod") + if content, err := os.ReadFile(goModPath); err == nil { + if strings.Contains(string(content), "module unidoc-pdf-debugger") { + return dir + } + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatalf("could not find project root (no go.mod with module unidoc-pdf-debugger found)") + } + dir = parent + } +} + +// fixturePath returns the absolute path to a committed correctness-corpus fixture. +func fixturePath(t *testing.T, name string) string { + t.Helper() + return filepath.Join(projectRoot(t), "testdata", "correctness", name) +} + +var ( + cliBuildOnce sync.Once + cliBinPath string + cliBuildErr string +) + +// buildCLI compiles the CLI binary once per test package and returns its path. +// Cached via sync.Once: the binary is identical for every test in the module. +func buildCLI(t *testing.T) string { + t.Helper() + cliBuildOnce.Do(func() { + root := projectRoot(t) + binName := "pdfdebug" + if runtime.GOOS == "windows" { + binName += ".exe" + } + tmpDir, err := os.MkdirTemp("", "pdfdebug-cli-") + if err != nil { + cliBuildErr = "failed to create temp dir: " + err.Error() + return + } + binPath := filepath.Join(tmpDir, binName) + cmd := exec.Command("go", "build", "-o", binPath, "./cmd/cli/") + cmd.Dir = root + if output, err := cmd.CombinedOutput(); err != nil { + cliBuildErr = "failed to build CLI binary: " + err.Error() + "\n" + string(output) + return + } + cliBinPath = binPath + }) + if cliBuildErr != "" { + t.Fatalf("%s", cliBuildErr) + } + return cliBinPath +} + +// runCLI executes the CLI binary with args and returns stdout, stderr, exit code. +func runCLI(t *testing.T, binPath string, args ...string) (stdout, stderr string, exitCode int) { + t.Helper() + cmd := exec.Command(binPath, args...) + var outBuf, errBuf strings.Builder + cmd.Stdout = &outBuf + cmd.Stderr = &errBuf + err := cmd.Run() + exitCode = 0 + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + exitCode = exitErr.ExitCode() + } else { + t.Fatalf("failed to run CLI: %v", err) + } + } + return outBuf.String(), errBuf.String(), exitCode +} diff --git a/tests/14-1-trustworthy-stream-op-output/stream_contract_test.go b/tests/14-1-trustworthy-stream-op-output/stream_contract_test.go new file mode 100644 index 0000000..1476f48 --- /dev/null +++ b/tests/14-1-trustworthy-stream-op-output/stream_contract_test.go @@ -0,0 +1,214 @@ +package trustworthy_stream_test + +// Story 14.1 -- machine-contract correctness of `dump stream --json` / `--ops`. +// RED PHASE: the tokenizer mislabels leading-'+' operands and --ops emits +// empty-operator records; the two contract tests below fail at runtime against +// the current binary. The fixture-sanity test passes today and guards the +// suite against an eternally-red (unparseable) fixture. + +import ( + "encoding/json" + "strings" + "testing" +) + +// token mirrors the pdfcore.Token JSON wire shape emitted by `dump stream --json`. +type token struct { + Type string `json:"type"` + Value string `json:"value"` +} + +// formattedLine mirrors the pdfcore.FormattedLine JSON wire shape. +type formattedLine struct { + Tokens []token `json:"tokens"` + Operator string `json:"operator"` +} + +// streamJSON is the `dump stream --json` top-level object (subset). +type streamJSON struct { + Tokenized []token `json:"tokenized"` + Formatted []formattedLine `json:"formatted"` +} + +// opsRecord is one `dump stream --ops` NDJSON record (subset). +type opsRecord struct { + Op string `json:"op"` + Params []string `json:"params"` +} + +// --------------------------------------------------------------------------- +// 14.1-INTG-000 [P0] fixture sanity: the hand-authored corpus fixtures must +// parse through the EXISTING open path (dump objects, exit 0). Passes TODAY; +// guards the suite against an eternally-red broken fixture (13-4/13-5/13-6 +// precedent). +// --------------------------------------------------------------------------- + +func TestStream_FixturesParseThroughOpenPath(t *testing.T) { + bin := buildCLI(t) + for _, name := range []string{"leading-plus.pdf", "comment-and-dangling.pdf"} { + _, stderr, ec := runCLI(t, bin, "dump", "objects", fixturePath(t, name)) + if ec != 0 { + t.Fatalf("[P0] 14.1-INTG-000: fixture %q rejected by the existing open path (exit %d): %s", name, ec, stderr) + } + } +} + +// --------------------------------------------------------------------------- +// 14.1-INTG-001 [P1] (F1, risk R-14-02): `dump stream --json` on +// leading-plus.pdf (content stream "+5 0 0 +5 0 0 cm"). +// +// RED today: the tokenizer emits "+5" as an OPERATOR token, so Format groups +// it as a standalone operation and --json reports a bogus operator "+5". +// Post-fix: each "+5" is a "number" token, "cm" is the sole operator, and no +// token/operator is named "+5" as an operator. +// --------------------------------------------------------------------------- + +func TestStream_LeadingPlusJSON(t *testing.T) { + bin := buildCLI(t) + stdout, stderr, ec := runCLI(t, bin, "dump", "stream", "--json", "--page", "1", fixturePath(t, "leading-plus.pdf")) + if ec != 0 { + t.Fatalf("[P1] 14.1-INTG-001: --json must exit 0, got %d\nstdout: %s\nstderr: %s", ec, stdout, stderr) + } + + var res streamJSON + if err := json.Unmarshal([]byte(stdout), &res); err != nil { + t.Fatalf("[P1] 14.1-INTG-001: failed to parse --json output: %v\nraw: %s", err, stdout) + } + + // Every token whose value is "+5" must be classified as a number, never an + // operator. This is the headline red assertion. + plusCount := 0 + for _, tk := range res.Tokenized { + if tk.Value == "+5" { + plusCount++ + if tk.Type != "number" { + t.Errorf("[P1] 14.1-INTG-001: token %q has Type %q, want \"number\" (leading '+' mislabeled)", tk.Value, tk.Type) + } + } + } + if plusCount != 2 { + t.Errorf("[P1] 14.1-INTG-001: found %d \"+5\" tokens, want 2 (content stream is \"+5 0 0 +5 0 0 cm\")", plusCount) + } + + // No formatted line may be named "+5"; exactly the "cm" operator terminates + // the transform. + sawCM := false + for _, fl := range res.Formatted { + if fl.Operator == "+5" { + t.Errorf("[P1] 14.1-INTG-001: a formatted operation is named operator %q; signed operands must not flush as their own operation", fl.Operator) + } + if fl.Operator == "cm" { + sawCM = true + } + } + if !sawCM { + t.Errorf("[P1] 14.1-INTG-001: no formatted operation with operator \"cm\"; the six signed operands should group under the single cm operator") + } +} + +// --------------------------------------------------------------------------- +// 14.1-INTG-002 [P1] (F2, risk R-14-03): `dump stream --ops` on +// comment-and-dangling.pdf (a "% comment" line plus a trailing dangling +// operand run with no operator). +// +// RED today: emitOps iterates Formatted with no guard on Operator, so the +// comment line and the dangling operands each yield a phantom {"op":""} NDJSON +// record. Post-fix: zero records with op == "" -- every emitted record has a +// non-empty op -- and the real "cm" operator still emits exactly one record. +// --------------------------------------------------------------------------- + +func TestStream_CommentAndDanglingOps(t *testing.T) { + bin := buildCLI(t) + stdout, stderr, ec := runCLI(t, bin, "dump", "stream", "--ops", "--page", "1", fixturePath(t, "comment-and-dangling.pdf")) + if ec != 0 { + t.Fatalf("[P1] 14.1-INTG-002: --ops must exit 0, got %d\nstdout: %s\nstderr: %s", ec, stdout, stderr) + } + + emptyOps := 0 + sawCM := false + records := 0 + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + records++ + var rec opsRecord + if err := json.Unmarshal([]byte(line), &rec); err != nil { + t.Fatalf("[P1] 14.1-INTG-002: NDJSON line is not valid JSON: %v\nline: %s", err, line) + } + if rec.Op == "" { + emptyOps++ + t.Errorf("[P1] 14.1-INTG-002: NDJSON record with empty op (phantom no-op record breaches the one-object-per-operator contract): %s", line) + } + if rec.Op == "cm" { + sawCM = true + } + } + if emptyOps != 0 { + t.Errorf("[P1] 14.1-INTG-002: %d record(s) with op == \"\"; want 0 (comment and dangling-operand lines must not emit)", emptyOps) + } + if !sawCM { + t.Errorf("[P1] 14.1-INTG-002: no record with op == \"cm\"; the real operator must still emit (got %d records)", records) + } +} + +// --------------------------------------------------------------------------- +// 14.1-INTG-003 [P2] (F1 cascade on the --ops surface, risk R-14-02): closes +// traceability gap G1. AC2 names an --ops-observable signal for leading-plus.pdf +// ("a single cm record whose params are the signed numbers, and NO +5 record"), +// but that signal was only covered transitively. This drives +// `dump stream --ops --page 1 leading-plus.pdf` directly (content stream +// "+5 0 0 +5 0 0 cm") and asserts the operator-centric NDJSON contract: exactly +// one record, op == "cm", its params are the six signed operands (two "+5"s as +// bare strings), and no record is named op == "+5". +// --------------------------------------------------------------------------- + +func TestStream_LeadingPlusOps(t *testing.T) { + bin := buildCLI(t) + stdout, stderr, ec := runCLI(t, bin, "dump", "stream", "--ops", "--page", "1", fixturePath(t, "leading-plus.pdf")) + if ec != 0 { + t.Fatalf("[P2] 14.1-INTG-003: --ops must exit 0, got %d\nstdout: %s\nstderr: %s", ec, stdout, stderr) + } + + var cmRecords, plusRecords int + var cmParams []string + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + var rec opsRecord + if err := json.Unmarshal([]byte(line), &rec); err != nil { + t.Fatalf("[P2] 14.1-INTG-003: NDJSON line is not valid JSON: %v\nline: %s", err, line) + } + // A "+5" must never surface as an operator record: it is a numeric operand. + if rec.Op == "+5" { + plusRecords++ + t.Errorf("[P2] 14.1-INTG-003: NDJSON record named op == %q; a signed operand must not flush as its own operation: %s", rec.Op, line) + } + if rec.Op == "cm" { + cmRecords++ + cmParams = rec.Params + } + } + + if plusRecords != 0 { + t.Errorf("[P2] 14.1-INTG-003: %d record(s) named op == \"+5\"; want 0", plusRecords) + } + if cmRecords != 1 { + t.Fatalf("[P2] 14.1-INTG-003: found %d \"cm\" records, want exactly 1 (the six signed operands group under a single cm)", cmRecords) + } + + // The cm record's params are the six operands verbatim, with the leading '+' + // preserved as bare strings (--ops params are raw token values, not typed). + want := []string{"+5", "0", "0", "+5", "0", "0"} + if len(cmParams) != len(want) { + t.Fatalf("[P2] 14.1-INTG-003: cm record has %d params %v, want %d %v", len(cmParams), cmParams, len(want), want) + } + for i := range want { + if cmParams[i] != want[i] { + t.Errorf("[P2] 14.1-INTG-003: cm param[%d] = %q, want %q (full: %v)", i, cmParams[i], want[i], cmParams) + } + } +}