Skip to content

parse-reader: splice multi-line def := continuations (eigentrust pitfall #15) #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hierophantos merged 1 commit into from
Apr 30, 2026
Merged

parse-reader: splice multi-line def := continuations (eigentrust pitfall #15) #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 81 additions & 10 deletions racket/prologos/parse-reader.rkt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1973,18 +1973,22 @@
;; to correctly handle both indent grouping and cross-line brackets.
;; Indent grouping is recovered by treating child-node boundaries as
;; implicit wrapping points when not inside an open bracket.
;;
;; Some forms need their indent-grouped continuation lines to be SPLICED
;; into the parent token stream rather than wrapped as sub-lists, so that
;; binding/arrow markers placed on continuation lines remain at the top
;; level where the elaborator and preparser can find them. See:
;; - `flatten-with-boundaries/spec`: splices type-signature continuations
;; so `->` on its own line is visible to `decompose-spec-type`.
;; - `flatten-with-boundaries/def`: splices ONLY continuations whose first
;; token is `:=`, so `def name : T\n := body` is read identically to
;; `def name : T := body`.
(define (tree-node->stx-elements node source source-str)
;; Collect tokens with indent-boundary markers. For a `spec` form, the
;; type-signature continuation lines are inlined directly (rather than
;; wrapped as sub-lists) so that arrows like `-> Result` placed on their
;; own continuation line remain at the top level of the spec body — where
;; `decompose-spec-type` looks for them. Metadata-style continuation lines
;; (whose first token is a keyword-like symbol such as `:doc`) are still
;; wrapped so the `process-spec` metadata loop recognizes them.
(define items
(if (spec-form-node? node)
(flatten-with-boundaries/spec node)
(flatten-with-boundaries node)))
(cond
[(spec-form-node? node) (flatten-with-boundaries/spec node)]
[(def-form-node? node) (flatten-with-boundaries/def node)]
[else (flatten-with-boundaries node)]))
(define vec (list->vector items))
(define-values (elems _end)
(group-items vec 0 (vector-length vec) #f source source-str))
Expand Down Expand Up @@ -2021,6 +2025,22 @@
(or (equal? lex "spec")
(equal? lex "spec-")))))))))

;; Recognize a parse-tree-node whose first token is `def` or `def-`.
;; Used by `tree-node->stx-elements` to apply the def-specific splice
;; rule. Mirrors `spec-form-node?`: the node's tag may not yet be
;; refined to `'def` (the private `def-` variant is registered as a
;; preparser keyword but has no tag rule), so we look at the first
;; child token directly.
(define (def-form-node? node)
(and (parse-tree-node? node)
(let ([children (parse-tree-node-children node)])
(and (> (rrb-size children) 0)
(let ([first (rrb-get children 0)])
(and (token-entry? first)
(let ([lex (token-entry-lexeme first)])
(or (equal? lex "def")
(equal? lex "def-")))))))))

;; Flatten variant for `spec` forms: top-level child lines are SPLICED into
;; the parent token stream (no `indent-open`/`indent-close` wrapping), so a
;; multi-line type signature like
Expand Down Expand Up @@ -2049,6 +2069,45 @@
inner)]
[else '()]))))

;; Flatten variant for `def`/`def-` forms: top-level child lines are
;; SPLICED into the parent token stream ONLY when the line begins with
;; the binding marker `:=`. This makes
;; def name : T
;; := body
;; read identically to
;; def name : T := body
;; so `expand-def-assign` (which only scans the top level for `:=`)
;; picks up the assignment.
;;
;; All other continuation lines (a bare body expression, a continued
;; bracketed application, etc.) remain wrapped as sub-lists, preserving
;; the existing semantics where
;; def name
;; + 1 2
;; reads as `(def name (+ 1 2))` with the body grouped as one
;; application. This is intentionally narrower than the spec splice
;; rule: spec needs to expose `->` from anywhere in a multi-line type
;; signature, while def only needs to expose `:=` (which is always the
;; head of its continuation when line-broken in the natural way).
(define (flatten-with-boundaries/def node)
(define children (parse-tree-node-children node))
(define n (rrb-size children))
(apply append
(for/list ([i (in-range n)])
(define child (rrb-get children i))
(cond
[(token-entry? child) (list child)]
[(parse-tree-node? child)
(define inner (flatten-with-boundaries child))
(if (continuation-starts-with-assign? inner)
;; `:= body` continuation — splice tokens directly so `:=`
;; appears at the top level of the def form.
inner
;; Other continuation (bare body, application chain,
;; etc.): keep wrapped as a sub-list.
(append (list 'indent-open) inner (list 'indent-close)))]
[else '()]))))

;; Does this flattened token list start with a keyword-like token (lexeme
;; beginning with `:`)? Used to distinguish metadata continuations from
;; type-signature continuations in spec forms.
Expand All @@ -2064,6 +2123,18 @@
(char=? (string-ref lex 0) #\:)))]
[else #f])))

;; Does this flattened token list start with the assignment token `:=`?
;; Used by `flatten-with-boundaries/def` to identify continuations that
;; should be spliced into the def's top-level token stream.
(define (continuation-starts-with-assign? items)
(let loop ([items items])
(cond
[(null? items) #f]
[(eq? (car items) 'indent-open) (loop (cdr items))]
[(token-entry? (car items))
(equal? (token-entry-lexeme (car items)) ":=")]
[else #f])))

;; Lookahead: check if there's a matching rangle before the current scope closes.
;; Scans forward tracking nesting depth for <> pairs.
(define (has-matching-rangle? vec start end close-type)
Expand Down
222 changes: 222 additions & 0 deletions racket/prologos/tests/test-def-multiline-ws.rkt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,222 @@
#lang racket/base

;;;
;;; Regression tests for multi-line `def NAME : TYPE := BODY` forms in WS mode.
;;;
;;; Background — the eigentrust pitfalls doc (2026-04-23) #15 reported that a
;;; `def` with a type annotation split across two lines like
;;;
;;; def c-asym-3 : [List [List Rat]]
;;; := '['[rz 1/2 1/2] '[rz rz ro] '[ro rz rz]]
;;;
;;; would silently misparse: the indented continuation `:= BODY` was wrapped
;;; by the WS reader as a sub-list `(:= BODY)`, leaving the def datum as
;;;
;;; (def c : (List ...) (:= ($list-literal ...)))
;;;
;;; instead of the one-line equivalent
;;;
;;; (def c : (List ...) := ($list-literal ...))
;;;
;;; `expand-def-assign` only scans the TOP LEVEL of the def body for `:=`
;;; (via `memq`), so the buried `:=` was invisible. The 4-element fallback
;;; in `parse-def` then treated `(:= ($list-literal ...))` as the BODY of
;;; `def`, which on elaboration tried to apply `:=` to `($list-literal ...)`
;;; and reported `Unbound variable: $list-literal`. Worse, downstream
;;; references to the (never-bound) `c` produced spurious unbound-variable
;;; errors and benchmarks measuring `reduce_ms` reported 0 because the
;;; intended computation never ran.
;;;
;;; Fix shape (a in the eigentrust pitfalls doc): `parse-reader.rkt` recognises
;;; `def`/`def-` form nodes in `tree-node->stx-elements` and SPLICES any
;;; indent-grouped continuation line whose first token is `:=` directly
;;; into the parent token stream. All other continuations (bare bodies,
;;; bracketed application chains) remain wrapped, preserving today's
;;; semantics for forms like `(def c\n + 1 2)` → `(def c (+ 1 2))`.
;;;

(require rackunit
racket/list
"../parse-reader.rkt"
"test-support.rkt")

;; ========================================
;; Helpers
;; ========================================

;; Read a single WS datum.
(define (ws-read s)
(define in (open-input-string s))
(prologos-read in))

;; Read all WS datums until eof.
(define (ws-read-all s)
(define in (open-input-string s))
(let loop ([acc '()])
(define d (prologos-read in))
(if (eof-object? d) (reverse acc) (loop (cons d acc)))))

;; ========================================
;; Datum-level checks: one-line and two-line forms produce identical surf
;; ========================================

(test-case "two-line def with `:= BODY` on continuation matches one-line"
(define one-line (ws-read "def c : T := 42"))
(define two-line (ws-read "def c : T\n := 42"))
(check-equal? two-line one-line)
(check-equal? two-line '(def c : T := 42))
;; The `:=` MUST appear at the top level so `expand-def-assign`'s
;; `(memq ':= datum)` test fires. memq returns a non-#f tail (truthy)
;; when `:=` is a top-level element of the form.
(check-true (and (memq ':= two-line) #t)
"`:=` must be at the top level, not buried in a sub-list"))

(test-case "two-line def with `:=` on its own line (body on third line)"
;; Rare but valid: split the assignment marker AND the body.
(define d (ws-read "def c : T\n :=\n 42"))
(check-equal? d '(def c : T := 42))
(check-true (and (memq ':= d) #t)))

(test-case "two-line def with multi-token list-literal body"
;; Reproducer matching the eigentrust pitfalls doc's primary example.
;; Slash-containing rationals carry the `$rat-literal` sentinel from
;; the WS reader's number tokenizer (see eigentrust pitfalls doc #3).
(define d (ws-read "def c : [List Rat]\n := '[1/2 1/2 1/2]"))
(check-equal?
d
'(def c : (List Rat) := ($list-literal ($rat-literal 1/2)
($rat-literal 1/2)
($rat-literal 1/2))))
(check-true (and (memq ':= d) #t)))

(test-case "two-line def with bracketed multi-line body after `:=`"
;; The body itself spans lines as a bracketed group — must remain a
;; single sub-list and the `:=` must remain spliced.
(define d (ws-read "def c : T\n := [+\n 1\n 2]"))
(check-equal? d '(def c : T := (+ 1 2))))

(test-case "one-line def is unchanged by the splice rule"
(check-equal? (ws-read "def c : T := 42") '(def c : T := 42))
(check-equal? (ws-read "def c := 42") '(def c := 42))
(check-equal? (ws-read "def c 42") '(def c 42)))

(test-case "def with bare body on continuation (no `:=`) is still wrapped"
;; Without `:=`, the continuation line is a body expression. It must
;; remain grouped (otherwise `(def c\n + 1 2)` would become
;; `(def c + 1 2)` — a 4-arg def parse error rather than `(def c (+ 1 2))`).
(check-equal? (ws-read "def c\n 42") '(def c 42))
(check-equal? (ws-read "def c\n + 1 2") '(def c (+ 1 2)))
(check-equal? (ws-read "def c\n [+ 1 2]") '(def c (+ 1 2)))
;; Multi-line bracketed body — also stays grouped.
(check-equal? (ws-read "def c\n [+\n 1\n 2]") '(def c (+ 1 2))))

(test-case "private def- gets the same splice treatment"
(define one-line (ws-read "def- c : T := 42"))
(define two-line (ws-read "def- c : T\n := 42"))
(check-equal? two-line one-line)
(check-equal? two-line '(def- c : T := 42)))

;; Non-regression: confirm the splice rule does NOT touch other forms.
(test-case "non-def forms are unaffected by the def splice rule"
;; `defn` body: continuation MUST stay wrapped — defn relies on the
;; sub-list grouping to recover its body shape.
(define defn-d (ws-read "defn foo [x]\n + x 1"))
(check-equal? defn-d '(defn foo (x) (+ x 1)))
;; A plain top-level expression with a continuation also stays wrapped.
(define expr-d (ws-read "let x := 5\n + x 1"))
;; (Just check the form parses without error and `:=` is at top level
;; exactly where the user wrote it — the splice rule is def-only.)
(check-equal? (car expr-d) 'let))

;; ========================================
;; End-to-end: two-line def actually evaluates downstream expressions
;; ========================================

;; The eigentrust pitfalls doc's headline failure is silent suppression: with the bug,
;; the def's body never runs, and any expression referencing the bound name
;; reports "Unbound variable". With the fix, the bound name evaluates as
;; expected.

;; Note: every E2E program needs `ns NAME` so the prelude (List, Rat,
;; numeric traits, ...) loads — `run-ns-ws-last` starts with an empty
;; prelude env per the standard test fixture pattern.

(define two-line-program
(string-append
"ns test-def-multiline-two\n"
"def d : [List Rat]\n"
" := '[1/2 1/2 1/2]\n"
"d\n"))

(define one-line-program
(string-append
"ns test-def-multiline-one\n"
"def d : [List Rat] := '[1/2 1/2 1/2]\n"
"d\n"))

(test-case "two-line def: bound name evaluates (not 'Unbound variable d')"
(define result (run-ns-ws-last two-line-program))
;; result should be a string describing the value+type, NOT a prologos-error
(check-true (string? result)
(format "expected evaluation result string, got: ~v" result))
;; The value should print as the list literal we bound.
(check-true (regexp-match? #rx"1/2 1/2 1/2" result)
(format "expected bound value to print, got: ~v" result)))

(test-case "two-line def: result matches one-line def"
(define r-two (run-ns-ws-last two-line-program))
(define r-one (run-ns-ws-last one-line-program))
(check-equal? r-two r-one
"two-line and one-line def must produce identical results"))

(test-case "two-line def: mid-program references resolve"
;; Variant exercising several defs with a downstream expression that
;; depends on every one of them.
(define program
(string-append
"ns test-def-multiline-mid\n"
"def x : Int\n"
" := 10\n"
"def y : Int\n"
" := 20\n"
"+ x y\n"))
(define result (run-ns-ws-last program))
(check-true (string? result)
(format "expected `+ x y` to evaluate, got: ~v" result))
(check-true (regexp-match? #rx"30" result)
(format "expected `30` in result, got: ~v" result)))

;; ========================================
;; Regression: the silently-suppressed evaluation case
;; ========================================
;;
;; Pre-fix behaviour (verified manually before applying the fix):
;; Results count: 4
;; [0] => "c : [List Rat] defined."
;; [1] => (unbound-variable-error ... "Unbound variable" '$list-literal)
;; [2] => "'[1/2 1/2 1/2] : [List Rat]" ; c (one-line) ok
;; [3] => (unbound-variable-error ... "Unbound variable" 'd)
;;
;; This test pins the post-fix behaviour: ALL four results are non-error,
;; and the two def forms produce equivalent value strings.

(test-case "downstream expressions after two-line def all evaluate cleanly"
(define program
(string-append
"ns test-def-multiline-mixed\n"
"def c : [List Rat] := '[1/2 1/2 1/2]\n" ;; one-line baseline
"def d : [List Rat]\n"
" := '[1/2 1/2 1/2]\n" ;; two-line
"c\n"
"d\n"))
(define results (run-ns-ws-all program))
(check-equal? (length results) 4
(format "expected 4 results, got: ~a (~v)" (length results) results))
(for ([r (in-list results)] [i (in-naturals)])
(check-true (string? r)
(format "result [~a] expected string, got: ~v" i r)))
;; The two def status strings should match (modulo whitespace/name).
(define c-result (list-ref results 2))
(define d-result (list-ref results 3))
(check-equal? c-result d-result
"c (one-line) and d (two-line) must evaluate to the same value"))
Loading