diff --git a/.github/workflows/Downstream.yml b/.github/workflows/Downstream.yml index ca58342..8d4bee3 100644 --- a/.github/workflows/Downstream.yml +++ b/.github/workflows/Downstream.yml @@ -12,14 +12,8 @@ jobs: strategy: fail-fast: false matrix: - include: - # XLSX's XML-v0.4 adaptation (JuliaData/XLSX.jl#415). The *registered* XLSX - # pins XML 0.3, so testing it against this branch fails by design; test the - # adaptation PR instead. Once #415 merges and releases, point this back at - # the registered package (PackageSpec(name=...) as before). - - package: XLSX - repository: JuliaData/XLSX.jl - ref: refs/pull/415/head + package: + - "XLSX" steps: - uses: actions/checkout@v7 - uses: julia-actions/setup-julia@v3 @@ -28,21 +22,17 @@ jobs: arch: x64 show-versioninfo: true - uses: julia-actions/julia-buildpkg@latest - - uses: actions/checkout@v7 - with: - repository: ${{ matrix.repository }} - ref: ${{ matrix.ref }} - path: downstream - name: Load this and run the downstream tests shell: julia --color=yes {0} run: | using Pkg Pkg.Registry.update() Pkg.activate(;temp=true) - # this branch's XML + the downstream package's adaptation branch + # force it to use this PR's version of the package + ENV["JULIA_PKG_DEVDIR"]= mktempdir() Pkg.develop([ PackageSpec(path="."), - PackageSpec(path="downstream"), + PackageSpec(name="${{ matrix.package }}"), ]) Pkg.update() Pkg.test("${{ matrix.package }}") diff --git a/CHANGELOG.md b/CHANGELOG.md index cd57c99..58d5b4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **`FlatNode` — a fourth reader: read-only columnar full-DOM** (`parse(xml, FlatNode)`, + `read(file, FlatNode)`). The whole document is materialized once into a contiguous store of + isbits records (zero-copy byte ranges into the retained source; text/attribute values + entity-decoded on access), so building is fast, random access is O(1), and the GC sees a + handful of arrays instead of one object per node — *`Node`'s read half at `Cursor`'s GC + cost*. Extras over `Node`: O(1) `parent`, O(depth) 1-arg `depth`. By design: read-only, + whole-store retention, 2 GiB/`typemax(Int32)` limits (use `Node` beyond). `==`/`hash` on + `FlatNode` are positional identity (same store, same node). `Node(flatnode)` materializes + a handle as a mutable `Node`; `XML.write` accepts `FlatNode` directly. Same + well-formedness levels and error messages as the `Node` parser (#82). + ### Internal - Source layout: the `src/XML.jl` monolith (1409 lines) is split into dedicated files — diff --git a/README.md b/README.md index 2a65002..020fbd8 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,25 @@ doc[end][2] # Second child of root
+# Choosing a reader + +XML.jl ships four readers behind one set of accessors (`nodetype`, `tag`, `attributes`, +`value`, `children`, `eachelement`, …). Pick by what you do with the document: + +| Reader | DOM materialized? | Revisit a node | Mutable | GC cost | +|---|---|---|---|---| +| `Cursor` | no (nothing) | impossible (forward-only) | — | ~0 | +| `LazyNode` | virtual | re-decode per visit (pay-per-traversal) | no | ~0 | +| `FlatNode` | yes, compact (columnar) | O(1), paid once | no | ~0 | +| `Node` | yes, objects | O(1), paid once | **yes** | high | + +Rules of thumb: one forward pass → `Cursor` · extract a little, memory-tight → `LazyNode` · +read-heavy full document, repeated traversals → `FlatNode` (*`Node`'s read half at `Cursor`'s +GC cost*) · build or edit documents → `Node`. `FlatNode` and `LazyNode` retain the source +string as long as any handle lives. + +
+ # `Node` Interface Every node in the XML DOM is represented by `Node`, a single type parametrized on its string storage. diff --git a/benchmarks/flatnode_bench.jl b/benchmarks/flatnode_bench.jl new file mode 100644 index 0000000..d94af12 --- /dev/null +++ b/benchmarks/flatnode_bench.jl @@ -0,0 +1,72 @@ +# FlatNode exhaustive in-package benchmark — by regime, min of N runs, vs Node/LazyNode/EzXML. +# julia benchmarks/flatnode_bench.jl (self-contained temp env: dev XML from this checkout + EzXML) +using Pkg +Pkg.activate(; temp = true, io = devnull) +Pkg.develop(path = joinpath(@__DIR__, ".."), io = devnull) +Pkg.add("EzXML", io = devnull) +using XML, EzXML + +const xmark = joinpath(@__DIR__, "data", "xmark.xml") +const xml = read(xmark, String) +println("corpus: ", basename(xmark), " (", round(ncodeunits(xml) / 2^20, digits = 1), " MiB)") + +minN(f, n = 5) = minimum((GC.gc(); @elapsed f()) for _ in 1:n) +allocs(f) = (GC.gc(); Base.gc_num().allocd; a0 = Base.gc_bytes(); f(); Base.gc_bytes() - a0) + +# ── build ── +fbuild() = parse(xml, FlatNode; wellformed = :lenient) +nbuild() = parse(xml, Node; wellformed = :lenient) +ezbuild() = EzXML.parsexml(xml) +for (nm, f) in ["FlatNode" => fbuild, "Node" => nbuild, "EzXML" => ezbuild] + println("build ", rpad(nm, 9), lpad(round(minN(f) * 1000, digits = 1), 8), " ms") +end + +# ── traverse (count nodes via each reader's handles) ── +const F = fbuild(); const N = nbuild(); const EZ = ezbuild() +function fwalk(n, c = Ref(0)) + c[] += 1 + for ch in XML.eachchildnode(n); fwalk(ch, c); end + c[] +end +function nwalk(n, c = Ref(0)) + c[] += 1 + for ch in children(n); nwalk(ch, c); end + c[] +end +function curwalk() + c = Cursor(xml); n = 0 + while next!(c) !== nothing; n += 1; end + n +end +println("nodes: flat=", fwalk(FlatNode(F.store, Int32(1))), " node=", nwalk(N)) +for (nm, f) in ["FlatNode" => () -> fwalk(FlatNode(F.store, Int32(1))), "Node" => () -> nwalk(N), "Cursor" => curwalk] + println("walk ", rpad(nm, 9), lpad(round(minN(f) * 1000, digits = 2), 8), " ms") +end + +# ── extract (tag/value byte sums through the public accessors) ── +function fextract(n, acc = Ref(0)) + t = tag(n); v = value(n) + t === nothing || (acc[] += ncodeunits(t)); v === nothing || (acc[] += ncodeunits(v)) + for ch in XML.eachchildnode(n); fextract(ch, acc); end + acc[] +end +function nextract(n, acc = Ref(0)) + t = tag(n); v = value(n) + t === nothing || (acc[] += ncodeunits(t)); v === nothing || (acc[] += ncodeunits(v)) + for ch in children(n); nextract(ch, acc); end + acc[] +end +println("extract sums: flat=", fextract(FlatNode(F.store, Int32(1))), " node=", nextract(N)) +for (nm, f) in ["FlatNode" => () -> fextract(FlatNode(F.store, Int32(1))), "Node" => () -> nextract(N)] + println("extract ", rpad(nm, 9), lpad(round(minN(f) * 1000, digits = 2), 8), " ms") +end + +# ── retained + build allocations ── +println("retained FlatNode ", lpad(round(Base.summarysize(F.store) / 2^20, digits = 1), 7), " MiB ", + "Node ", lpad(round(Base.summarysize(N) / 2^20, digits = 1), 7), " MiB") +println("buildalloc FlatNode ", lpad(round(allocs(fbuild) / 2^20, digits = 1), 6), " MiB ", + "Node ", lpad(round(allocs(nbuild) / 2^20, digits = 1), 6), " MiB") + +# ── GC pressure: full collection time with the tree live ── +gcms(x) = (GC.gc(); t = @elapsed GC.gc(true); t * 1000) +println("GC full with FlatNode live ", round(gcms(F), digits = 2), " ms with Node live ", round(gcms(N), digits = 2), " ms") diff --git a/src/XML.jl b/src/XML.jl index 389b974..952af78 100644 --- a/src/XML.jl +++ b/src/XML.jl @@ -1,7 +1,7 @@ module XML export - Node, LazyNode, NodeType, Attributes, + Node, LazyNode, FlatNode, NodeType, Attributes, CData, Comment, Declaration, Document, DTD, Element, ProcessingInstruction, Text, nodetype, tag, attributes, value, children, children!, eachchildnode, eachattribute, eachelement, elements, @@ -23,7 +23,7 @@ include("node.jl") # NodeType, Attributes, Node + accessors/navigation/equ include("xpath.jl") # xpath over Node trees (needs Node + accessors) include("lazynode.jl") # LazyNode reader (needs NodeType/Attributes; extends the generic accessors) include("cursor.jl") # Cursor pull reader (needs NodeType/Attributes) -# (flatnode.jl will slot here — read-only columnar reader, planned for v0.5) +include("flatnode.jl") # FlatNode read-only columnar reader (needs node.jl types; checks live in parse.jl) include("write.jl") # XML writer: _write_xml/_write_escaped + XML.write entry points include("parse.jl") # BOM normalization, Base.read entry points, the VPA parser include("dtd.jl") # DTD/DOCTYPE parsing (independent: uses only the tokenizer) diff --git a/src/flatnode.jl b/src/flatnode.jl new file mode 100644 index 0000000..241b7aa --- /dev/null +++ b/src/flatnode.jl @@ -0,0 +1,461 @@ +#-----------------------------------------------------------------------------# FlatNode — read-only columnar full-DOM reader +# The whole document is materialized ONCE into a FlatStore: one contiguous Vector of isbits +# records with integer index links (parent / first_child / next_sibling) plus a side Vector +# for attributes — instead of Node's per-node heap objects. Zero-copy: records hold byte +# ranges into the retained source; text/attribute values are entity-decoded at access. +# A FlatNode is a lightweight handle (store, index); the Document node is index 1. + +struct _FlatAttr # isbits, 16 B + name_offset::Int32; name_len::Int32 + value_offset::Int32; value_len::Int32 # value_len < 0 ⇒ value carries entities (decode at access) +end + +struct _FlatRec # isbits, no pointers + kind::NodeType + parent::Int32 + first_child::Int32 + next_sibling::Int32 + name_offset::Int32; name_len::Int32 # Element tag / PI target + value_offset::Int32; value_len::Int32 # content; offset == -1 ⇒ NO value (vs empty ""); len < 0 ⇒ entities + attr_first::Int32; attr_count::Int32 +end + +struct FlatStore + source::String + recs::Vector{_FlatRec} # recs[1] = the Document node + attrs::Vector{_FlatAttr} + spans::Vector{NTuple{2,Int32}} # per-record source span (start, end) — 0-based, half-open +end + +""" + FlatNode + +Read-only handle into a [`FlatStore`](@ref) — XML.jl's fourth reader, alongside `Node` +(mutable DOM), `LazyNode` (pay-per-traversal) and `Cursor` (pull streaming): *`Node`'s read +half at `Cursor`'s GC cost*. + + doc = parse(xml, FlatNode) # or read(filename, FlatNode) + root = only(eachelement(doc)) + for el in eachelement(root) + tag(el), attributes(el), value(el) + end + +The whole document is parsed once into a contiguous columnar store (a `Vector` of isbits +records indexing into the retained source string), so building is fast, random access is +O(1), repeated traversals never re-decode structure, and the garbage collector sees a +handful of arrays instead of one object per node. Text and attribute values are zero-copy +`SubString`s, entity-decoded on access. + +Compared with `Node`: +- **read-only** — no `push!`/`setindex!`; build documents with `Node` / [`h`](@ref). +- `parent(node)` and `depth(node)` work directly (O(1) / O(depth)) — the store keeps + parent links, which `Node` does not. +- `==` and `hash` are **positional identity**: two `FlatNode`s are equal iff they point at + the same node of the same store (compare content by converting: `Node(a) == Node(b)`). +- retention is all-or-nothing: any live handle keeps the whole store (and source) alive. +- documents are limited to 2 GiB / `typemax(Int32)` nodes; parse with `Node` beyond that. + +`Node(flatnode)` materializes a handle (and its subtree) as an ordinary mutable `Node`; +`XML.write` accepts a `FlatNode` directly. +""" +struct FlatNode + store::FlatStore + i::Int32 +end + +Base.:(==)(a::FlatNode, b::FlatNode) = a.store === b.store && a.i == b.i +Base.hash(a::FlatNode, h::UInt) = hash(a.i, hash(objectid(a.store), h)) + +@inline _rec(n::FlatNode) = @inbounds n.store.recs[n.i] +@inline _fsub(store::FlatStore, off::Int32, len::Int32) = + @inbounds SubString(store.source, off + 1, prevind(store.source, off + len + 1)) + +# byte range of a SubString into its (root) parent string +@inline _frng(s::SubString{String}) = (Int32(s.offset), Int32(s.ncodeunits)) + +# record-update helpers (immutable records; rebuild in place) +@inline _fset_first_child(n::_FlatRec, i::Int32) = + _FlatRec(n.kind, n.parent, i, n.next_sibling, n.name_offset, n.name_len, n.value_offset, n.value_len, n.attr_first, n.attr_count) +@inline _fset_next_sibling(n::_FlatRec, i::Int32) = + _FlatRec(n.kind, n.parent, n.first_child, i, n.name_offset, n.name_len, n.value_offset, n.value_len, n.attr_first, n.attr_count) +@inline _fset_value(n::_FlatRec, off::Int32, len::Int32) = + _FlatRec(n.kind, n.parent, n.first_child, n.next_sibling, n.name_offset, n.name_len, off, len, n.attr_first, n.attr_count) +@inline _fadd_attr(n::_FlatRec, first::Int32) = + _FlatRec(n.kind, n.parent, n.first_child, n.next_sibling, n.name_offset, n.name_len, n.value_offset, n.value_len, + n.attr_first == 0 ? first : n.attr_first, n.attr_count + Int32(1)) + +# wire record `idx` as the next child of the current open parent; returns the parent index +@inline function _fattach!(recs::Vector{_FlatRec}, pstack::Vector{Int32}, lastchild::Vector{Int32}, idx::Int32) + p = @inbounds pstack[end] + lc = @inbounds lastchild[end] + if lc == 0 + @inbounds recs[p] = _fset_first_child(recs[p], idx) + else + @inbounds recs[lc] = _fset_next_sibling(recs[lc], idx) + end + @inbounds lastchild[end] = idx + p +end + +# Token-stream → FlatStore builder: the same single visibly-pushdown pass as `_parse` +# (see parse.jl), with the same well-formedness checks at the same token points — gated by +# `Val{W}` so :lenient pays nothing — but appending isbits records instead of heap nodes. +function _flat_parse(xml::String, ::Val{W}) where {W} + ncodeunits(xml) <= typemax(Int32) || + error("FlatNode stores byte offsets as Int32: source is larger than 2 GiB. Use `parse(xml, Node)`.") + recs = _FlatRec[] + attrs = _FlatAttr[] + spans = NTuple{2,Int32}[] + sizehint!(recs, ncodeunits(xml) >> 4) + sizehint!(spans, ncodeunits(xml) >> 4) + push!(recs, _FlatRec(Document, Int32(0), Int32(0), Int32(0), Int32(0), Int32(0), Int32(-1), Int32(0), Int32(0), Int32(0))) + push!(spans, (Int32(0), Int32(ncodeunits(xml)))) + pstack = Int32[1] + lastchild = Int32[0] + K = TokenKinds + + cur = Int32(0) # record receiving ATTR_* (open element or XML declaration) + pend = Int32(0) # record awaiting its content / span end — patched in place + pan_off = Int32(0); pan_len = Int32(0) # pending attribute name + open_off = Int32(0) # offset of the last COMMENT/CDATA/DOCTYPE opening marker + tok_end(t) = Int32(t.offset + t.ncodeunits) + + for token in tokenize(xml) + k = token.kind + if k === K.TEXT + rawtext = raw(token, xml) + W === :strict && _check_chars_strict(rawtext) + W === :strict && token.has_entities && _check_charrefs_strict(rawtext) + off, len = _frng(rawtext) + token.has_entities && (len = -len) + idx = Int32(length(recs) + 1) + p = _fattach!(recs, pstack, lastchild, idx) + push!(recs, _FlatRec(Text, p, Int32(0), Int32(0), Int32(0), Int32(0), off, len, Int32(0), Int32(0))) + push!(spans, (off, off + abs(len))) + + elseif k === K.OPEN_TAG + nm = tag_name(token, xml) + W !== :lenient && (isempty(nm) || !_is_name_start(first(nm))) && + error("not well-formed: invalid element name \"$nm\"") + noff, nlen = _frng(nm) + idx = Int32(length(recs) + 1) + p = _fattach!(recs, pstack, lastchild, idx) + push!(recs, _FlatRec(Element, p, Int32(0), Int32(0), noff, nlen, Int32(-1), Int32(0), Int32(0), Int32(0))) + push!(spans, (Int32(token.offset), Int32(0))) + push!(pstack, idx); push!(lastchild, Int32(0)) + cur = idx + + elseif k === K.SELF_CLOSE + ci = @inbounds pstack[end] + @inbounds spans[ci] = (spans[ci][1], tok_end(token)) + pop!(pstack); pop!(lastchild) + + elseif k === K.CLOSE_TAG + close_name = tag_name(token, xml) + length(pstack) > 1 || error("Closing tag with no matching open tag.") + ci = @inbounds pstack[end] + open_rec = @inbounds recs[ci] + t = _fsub_or_empty(xml, open_rec.name_offset, open_rec.name_len) + t == close_name || error("Mismatched tags: expected , got .") + # the CLOSE_TAG token covers `` (ETag: '') + # is consumed silently by the tokenizer — locate it to end the element span + gt = findnext(==('>'), xml, Int(tok_end(token)) + 1) + @inbounds spans[ci] = (spans[ci][1], gt === nothing ? tok_end(token) : Int32(gt)) + pop!(pstack); pop!(lastchild) + + elseif k === K.ATTR_NAME + pan_off, pan_len = _frng(raw(token, xml)) + + elseif k === K.ATTR_VALUE + rawval = attr_value(token, xml) + W !== :lenient && occursin('<', rawval) && error("not well-formed: '<' in attribute value (XML 1.0 §3.1)") + W === :strict && _check_chars_strict(rawval) + W === :strict && token.has_entities && _check_charrefs_strict(rawval) + name = _fsub_or_empty(xml, pan_off, pan_len) + r = @inbounds recs[cur] + ai = r.attr_first + for _ in 1:r.attr_count + a = @inbounds attrs[ai] + _fsub_or_empty(xml, a.name_offset, a.name_len) == name && error("Duplicate attribute: $name") + ai += Int32(1) + end + voff, vlen = _frng(rawval) + token.has_entities && (vlen = -vlen) + push!(attrs, _FlatAttr(pan_off, pan_len, voff, vlen)) + @inbounds recs[cur] = _fadd_attr(recs[cur], Int32(length(attrs))) + + elseif k === K.XML_DECL_OPEN + W !== :lenient && length(pstack) > 1 && + error("not well-formed: XML declaration inside element content") + idx = Int32(length(recs) + 1) + p = _fattach!(recs, pstack, lastchild, idx) + push!(recs, _FlatRec(Declaration, p, Int32(0), Int32(0), Int32(0), Int32(0), Int32(-1), Int32(0), Int32(0), Int32(0))) + push!(spans, (Int32(token.offset), Int32(0))) + cur = idx + pend = idx + + elseif k === K.XML_DECL_CLOSE + @inbounds spans[pend] = (spans[pend][1], tok_end(token)) + + elseif k === K.COMMENT_OPEN || k === K.CDATA_OPEN || k === K.DOCTYPE_OPEN + open_off = Int32(token.offset) + + elseif k === K.COMMENT_CLOSE || k === K.CDATA_CLOSE || k === K.DOCTYPE_CLOSE || k === K.PI_CLOSE + @inbounds spans[pend] = (spans[pend][1], tok_end(token)) + + elseif k === K.COMMENT_CONTENT + cmt = raw(token, xml) + W === :strict && _check_chars_strict(cmt) + W === :strict && occursin("--", cmt) && error("not well-formed: \"--\" within a comment") + W === :strict && endswith(cmt, '-') && error("not well-formed: \"-\" immediately before \"-->\" in a comment (XML 1.0 §2.5)") + off, len = _frng(cmt) + idx = Int32(length(recs) + 1) + p = _fattach!(recs, pstack, lastchild, idx) + push!(recs, _FlatRec(Comment, p, Int32(0), Int32(0), Int32(0), Int32(0), off, len, Int32(0), Int32(0))) + push!(spans, (open_off, Int32(0))) + pend = idx + + elseif k === K.CDATA_CONTENT + cdata = raw(token, xml) + W === :strict && _check_chars_strict(cdata) + off, len = _frng(cdata) + idx = Int32(length(recs) + 1) + p = _fattach!(recs, pstack, lastchild, idx) + push!(recs, _FlatRec(CData, p, Int32(0), Int32(0), Int32(0), Int32(0), off, len, Int32(0), Int32(0))) + push!(spans, (open_off, Int32(0))) + pend = idx + + elseif k === K.DOCTYPE_CONTENT + W !== :lenient && length(pstack) > 1 && + error("not well-formed: DOCTYPE declaration inside element content") + off, len = _frng(lstrip(raw(token, xml))) # mirror _parse: lstrip'd DTD value + idx = Int32(length(recs) + 1) + p = _fattach!(recs, pstack, lastchild, idx) + push!(recs, _FlatRec(DTD, p, Int32(0), Int32(0), Int32(0), Int32(0), off, len, Int32(0), Int32(0))) + push!(spans, (open_off, Int32(0))) + pend = idx + + elseif k === K.PI_OPEN + target = pi_target(token, xml) + W === :strict && (isempty(target) || !_is_name_start(first(target))) && + error("not well-formed: invalid processing-instruction target \"$target\"") + noff, nlen = _frng(target) + idx = Int32(length(recs) + 1) + p = _fattach!(recs, pstack, lastchild, idx) + push!(recs, _FlatRec(ProcessingInstruction, p, Int32(0), Int32(0), noff, nlen, Int32(-1), Int32(0), Int32(0), Int32(0))) + push!(spans, (Int32(token.offset), Int32(0))) + pend = idx + + elseif k === K.PI_CONTENT + content = lstrip(raw(token, xml)) # mirror _parse: lstrip'd, empty → none + W === :strict && _check_chars_strict(content) + if !isempty(content) + off, len = _frng(content) + @inbounds recs[pend] = _fset_value(recs[pend], off, len) + end + end + # TAG_CLOSE: no tree action (element spans close at CLOSE_TAG/SELF_CLOSE) + end + + if length(pstack) > 1 + open_names = [_fsub_or_empty(xml, recs[i].name_offset, recs[i].name_len) for i in pstack[2:end]] + error("Unclosed tags: $(join(open_names, ", "))") + end + store = FlatStore(xml, recs, attrs, spans) + W !== :lenient && _check_document_wellformed(children(FlatNode(store, Int32(1)))) + FlatNode(store, Int32(1)) +end + +@inline _fsub_or_empty(source::String, off::Int32, len::Int32) = + len > 0 ? (@inbounds SubString(source, off + 1, prevind(source, off + len + 1))) : SubString(source, 1, 0) + +#-----------------------------------------------------------------------------# parse / read entry points +Base.parse(xml::AbstractString, ::Type{FlatNode}; wellformed::Symbol=:structural) = + _flat_parse(_drop_bom(String(xml)), Val(wellformed)) +Base.parse(::Type{FlatNode}, xml::AbstractString; wellformed::Symbol=:structural) = + parse(xml, FlatNode; wellformed) +Base.read(filename::AbstractString, ::Type{FlatNode}; wellformed::Symbol=:structural) = + parse(String(_normalize_bom(read(filename))), FlatNode; wellformed) +Base.read(io::IO, ::Type{FlatNode}; wellformed::Symbol=:structural) = + parse(String(_normalize_bom(read(io))), FlatNode; wellformed) + +#-----------------------------------------------------------------------------# accessors +@inline nodetype(n::FlatNode) = _rec(n).kind + +@inline function tag(n::FlatNode) + r = _rec(n) + r.name_len > 0 ? _fsub(n.store, r.name_offset, r.name_len) : nothing +end + +@inline function value(n::FlatNode) + r = _rec(n) + r.value_offset < 0 && return nothing # absent (empty content is offset ≥ 0, len 0) + raw = _fsub(n.store, r.value_offset, abs(r.value_len)) + r.value_len < 0 ? unescape(raw) : raw +end + +function attributes(n::FlatNode) + r = _rec(n) + r.attr_count == 0 && return nothing + out = Pair{SubString{String}, SubString{String}}[] + sizehint!(out, r.attr_count) + ai = r.attr_first + for _ in 1:r.attr_count + a = @inbounds n.store.attrs[ai] + name = _fsub(n.store, a.name_offset, a.name_len) + rawv = _fsub(n.store, a.value_offset, abs(a.value_len)) + val = a.value_len < 0 ? _as_substring(unescape(rawv)) : rawv + push!(out, name => val) + ai += Int32(1) + end + out +end + +""" + sourcetext(n::FlatNode) -> SubString + +The node's exact raw source slice, markup included — `…` for an element, +`` for a comment, and the whole document for the Document node. Zero-copy: the +store keeps per-record source spans. +""" +function sourcetext(n::FlatNode) + so, se = @inbounds n.store.spans[n.i] + _fsub(n.store, so, se - so) +end + +""" + parent(n::FlatNode) -> FlatNode or nothing + +O(1) parent lookup (the flat store keeps parent links). Returns `nothing` for the +Document node. +""" +function Base.parent(n::FlatNode) + p = _rec(n).parent + p == 0 ? nothing : FlatNode(n.store, p) +end + +""" + depth(n::FlatNode) -> Int + +Depth of the node in its document: the Document node is 1, the root element 2, … +O(depth), via the stored parent links. +""" +function depth(n::FlatNode) + d = 1 + p = _rec(n).parent + while p != 0 + d += 1 + p = @inbounds n.store.recs[p].parent + end + d +end + +#-----------------------------------------------------------------------------# child iteration +struct FlatChildIterator + store::FlatStore + first::Int32 +end + +Base.IteratorSize(::Type{FlatChildIterator}) = Base.SizeUnknown() +Base.eltype(::Type{FlatChildIterator}) = FlatNode + +function Base.iterate(it::FlatChildIterator, i::Int32 = it.first) + i == 0 && return nothing + (FlatNode(it.store, i), (@inbounds it.store.recs[i].next_sibling)) +end + +""" + eachchildnode(n::FlatNode) + +Lazy iterator over the children of `n`, one `FlatNode` handle at a time (no vector +materialized). See also [`children`](@ref) and [`eachelement`](@ref). +""" +eachchildnode(n::FlatNode) = FlatChildIterator(n.store, _rec(n).first_child) + +function children(n::FlatNode) + nt = _rec(n).kind + (nt === Document || nt === Element) || return () + out = FlatNode[] + for c in eachchildnode(n) + push!(out, c) + end + out +end + +# eachelement's generic definition filters children(node); build on the lazy child +# iterator instead so no intermediate Vector is materialized (same move as LazyNode's). +eachelement(node::FlatNode) = Iterators.filter(n -> nodetype(n) === Element, eachchildnode(node)) + +function Base.length(n::FlatNode) + len = 0 + for _ in eachchildnode(n) + len += 1 + end + len +end + +function Base.getindex(n::FlatNode, i::Integer) + k = 0 + for c in eachchildnode(n) + (k += 1) == i && return c + end + throw(BoundsError(n, i)) +end + +is_simple(n::FlatNode) = (r = _rec(n); + r.kind === Element && r.attr_count == 0 && r.first_child != 0 && + (c = @inbounds n.store.recs[r.first_child]; c.next_sibling == 0 && (c.kind === Text || c.kind === CData))) + +is_simple_value(n::FlatNode) = is_simple(n) ? value(FlatNode(n.store, _rec(n).first_child)) : nothing + +function simple_value(n::FlatNode) + is_simple(n) || error("`simple_value` requires a simple node: an Element with no attributes and exactly one Text or CData child. See `is_simple`.") + value(FlatNode(n.store, _rec(n).first_child)) +end + +#-----------------------------------------------------------------------------# conversion / write / show +""" + Node(n::FlatNode) -> Node{String} + +Materialize a flat handle (and its whole subtree) as an ordinary mutable `Node`, with +decoded text and attribute values — the bridge from the read-only store to the mutable +DOM (e.g. to edit a subtree or compare content with `==`). +""" +function Node(n::FlatNode) + r = _rec(n) + a = attributes(n) + attrs = a === nothing ? nothing : Pair{String,String}[String(k) => String(v) for (k, v) in a] + t = tag(n) + v = value(n) + ch = (r.kind === Document || r.kind === Element) ? + Node{String}[Node(c) for c in eachchildnode(n)] : nothing + Node{String}(r.kind, t === nothing ? nothing : String(t), attrs, + v === nothing ? nothing : String(v), + ch === nothing || isempty(ch) ? nothing : ch) +end + +write(n::FlatNode; kw...) = write(Node(n); kw...) +write(io::IO, n::FlatNode; kw...) = write(io, Node(n); kw...) +write(filename::AbstractString, n::FlatNode; kw...) = write(filename, Node(n); kw...) + +function Base.show(io::IO, n::FlatNode) + r = _rec(n) + print(io, "FlatNode ", r.kind) + if r.kind === Element + print(io, " <", tag(n)) + a = attributes(n) + if a !== nothing + for (k, v) in a + print(io, ' ', k, "=\"", v, '"') + end + end + print(io, '>') + elseif r.kind === ProcessingInstruction + print(io, " ") + elseif r.value_offset >= 0 + s = something(value(n), "") + print(io, ' ', repr(first(s, 40)), ncodeunits(s) > 40 ? "…" : "") + end + nc = r.kind === Document || r.kind === Element ? length(n) : 0 + nc > 0 && print(io, " (", nc, " children)") +end diff --git a/test/runtests.jl b/test/runtests.jl index bc4d2d8..9f8dbc4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3746,6 +3746,7 @@ end @testset "test_w3c" begin include("test_w3c.jl") end @testset "test_tokenizer" begin include("test_tokenizer.jl") end @testset "test_cursor" begin include("test_cursor.jl") end +@testset "test_flatnode" begin include("test_flatnode.jl") end Test.pop_testset() Test.finish(_ROOT_TS) # prints the aggregated summary and throws (exit 1) if anything failed diff --git a/test/test_flatnode.jl b/test/test_flatnode.jl new file mode 100644 index 0000000..f07da71 --- /dev/null +++ b/test/test_flatnode.jl @@ -0,0 +1,199 @@ +# FlatNode — the read-only columnar reader: decoded equivalence with Node, accessor +# surface, positional identity, well-formedness parity with the Node parser. + +# Recursive decoded comparison: FlatNode must agree with Node{String} on kind, tag, +# decoded value, decoded attributes, and tree shape. +function flat_agrees_with_node(a, b) + nodetype(a) === nodetype(b) || return false + isequal(tag(a), tag(b)) || return false + isequal(value(a), value(b)) || return false + aa, ba = attributes(a), attributes(b) + (aa === nothing) === (ba === nothing) || return false + if aa !== nothing + [String(k) => String(v) for (k, v) in aa] == [String(k) => String(v) for (k, v) in ba] || return false + end + ca, cb = children(a), children(b) + length(ca) == length(cb) || return false + all(flat_agrees_with_node(x, y) for (x, y) in zip(ca, cb)) +end + +const RICH_XML = """]> + + + plain + enti&ty + + + + mixed text A +""" + +@testset "decoded equivalence with Node" begin + f = parse(RICH_XML, FlatNode) + n = parse(RICH_XML, Node) + @test flat_agrees_with_node(f, n) + @test XML.write(f) == XML.write(n) + + path = joinpath(@__DIR__, "data", "books.xml") + if isfile(path) + @test flat_agrees_with_node(read(path, FlatNode), read(path, Node)) + end + + # Empty content is a VALUE ("" — value_offset ≥ 0), distinct from no value (nothing, + # value_offset == -1). Caught by W3C parity (sun/invalid/empty.xml, oasis p15/p18). + empties = "" + fe, ne = parse(empties, FlatNode), parse(empties, Node) + @test flat_agrees_with_node(fe, ne) + rootels = children(only(eachelement(fe))) + @test value(rootels[1]) == "" && nodetype(rootels[1]) === XML.Comment + @test value(rootels[2]) == "" && nodetype(rootels[2]) === XML.CData + @test value(rootels[3]) === nothing && nodetype(rootels[3]) === XML.ProcessingInstruction + @test attributes(only(eachelement(fe))) == ["x" => ""] +end + +@testset "accessor surface" begin + f = parse(RICH_XML, FlatNode) + n = parse(RICH_XML, Node) + root = only(eachelement(f)) + rootn = only(eachelement(n)) + + @test nodetype(f) === XML.Document && nodetype(root) === XML.Element + @test tag(root) == "root" + @test length(root) == length(children(rootn)) + @test attributes(root) == ["a" => "1", "b" => "two & three"] # decoded at access + + els = collect(eachelement(root)) + @test [tag(e) for e in els] == ["item", "item", "empty"] + @test value(children(els[2])[1]) == "enti&ty" # decoded at access + @test attributes(els[2]) == ["attr" => "v: no attributes + + # parent is O(1) and defined; depth counts from the Document node + @test parent(f) === nothing + @test parent(root) == f && parent(els[1]) == root + @test depth(f) == 1 && depth(root) == 2 && depth(els[1]) == 3 + + # indexing walks children (whitespace Text nodes included, as everywhere in v0.4) + @test root[2] == els[1] + @test_throws BoundsError root[length(root) + 1] + + # is_simple/simple_value/is_simple_value parity with Node + elsn = filter(c -> nodetype(c) === XML.Element, children(rootn)) + @test [is_simple(e) for e in els] == [is_simple(e) for e in elsn] + @test simple_value(els[1]) == simple_value(elsn[1]) == "plain" + @test [is_simple_value(e) for e in els] == [is_simple_value(e) for e in elsn] + + @test occursin("Element", repr(root)) # show smoke +end + +@testset "positional identity (== and hash)" begin + f = parse(RICH_XML, FlatNode) + root = only(eachelement(f)) + els = collect(eachelement(root)) + @test els[1] == els[1] && els[1] != els[2] + @test hash(els[1]) != hash(els[2]) + @test length(unique([els[1], els[1], els[2]])) == 2 # the #55 behavior, fixed here + # two parses of the same document are DIFFERENT stores: handles compare unequal; + # compare content by materializing. + f2 = parse(RICH_XML, FlatNode) + @test f != f2 + @test Node(f) == Node(f2) +end + +@testset "Node materialization" begin + f = parse(RICH_XML, FlatNode) + n = parse(RICH_XML, Node) + @test Node(f) == n + els = collect(eachelement(only(eachelement(f)))) + elsn = filter(c -> nodetype(c) === XML.Element, children(only(eachelement(n)))) + @test Node(els[2]) == elsn[2] +end + +@testset "well-formedness parity with the Node parser" begin + cases = [ + ("", NamedTuple()), # mismatched tags (ungated) + ("", NamedTuple()), # close without open (ungated) + ("", NamedTuple()), # unclosed tag (ungated) + ("", NamedTuple()), # duplicate attribute (ungated) + ("", NamedTuple()), # multiple roots (:structural) + ("top", NamedTuple()), # top-level text (:structural) + ("", NamedTuple()), # '<' in attribute (:structural) + ("<1a/>", NamedTuple()), # invalid name start (:structural) + ("�", (wellformed = :strict,)), # illegal charref (:strict) + ("", (wellformed = :strict,)), # "--" in comment (:strict) + ] + for (bad, kw) in cases + err_node = try parse(bad, Node; kw...); nothing catch e sprint(showerror, e) end + err_flat = try parse(bad, FlatNode; kw...); nothing catch e sprint(showerror, e) end + @test err_node !== nothing + @test err_node == err_flat + end + # :lenient accepts what Node's :lenient accepts + @test parse("", FlatNode; wellformed = :lenient) isa FlatNode + @test XML.write(parse("", FlatNode; wellformed = :lenient)) == + XML.write(parse("", Node; wellformed = :lenient)) +end + +@testset "sourcetext (source spans)" begin + xml = """]> + plain +""" + f = parse(xml, FlatNode; wellformed = :lenient) + lz = parse(xml, LazyNode) + + # exact-slice parity with LazyNode's sourcetext, top-level and inside the root + for (a, b) in zip(children(f), children(lz)) + @test isequal(sourcetext(a), sourcetext(b)) + end + root = only(eachelement(f)) + rootlz = first(Iterators.filter(c -> nodetype(c) === XML.Element, children(lz))) + for (a, b) in zip(children(root), children(rootlz)) + @test isequal(sourcetext(a), sourcetext(b)) + end + + @test sourcetext(f) == xml # Document = whole source + it = first(eachelement(root)) + @test sourcetext(it) == "plain" + # re-parsing an element's slice reproduces the subtree + @test Node(only(eachelement(parse(String(sourcetext(it)), FlatNode; wellformed = :lenient)))) == Node(it) +end + +@testset "BOM handling" begin + bom = "x" + @test flat_agrees_with_node(parse(bom, FlatNode), parse(bom, Node)) + utf8_bom = vcat([0xEF, 0xBB, 0xBF], Vector{UInt8}("x")) + @test simple_value(only(eachelement(read(IOBuffer(utf8_bom), FlatNode)))) == "x" + utf16le = vcat([0xFF, 0xFE], reinterpret(UInt8, Vector{UInt16}(transcode(UInt16, "x")))) + @test simple_value(only(eachelement(read(IOBuffer(utf16le), FlatNode)))) == "x" +end + +# W3C conformance parity: FlatNode must agree with the Node parser on every document of +# the pinned suite — decoded-equivalent trees on the well-formed corpus, and the exact +# same accept/reject verdict on the not-well-formed corpus. (`valid_tests`/`notwf_tests` +# are globals from test_w3c.jl, which runs earlier in the suite; guard for standalone runs.) +if @isdefined(valid_tests) + @testset "W3C parity with the Node parser" begin + n_cmp = 0 + for test in valid_tests + isfile(test.uri) || continue + n = try read(test.uri, Node; wellformed = :strict) catch; continue end + f = read(test.uri, FlatNode; wellformed = :strict) + @test flat_agrees_with_node(f, n) + n_cmp += 1 + end + @info "W3C valid: FlatNode ≡ Node on $n_cmp documents" + + n_agree = 0 + disagreements = String[] + for test in notwf_tests + isfile(test.uri) || continue + rejects_node = try read(test.uri, Node; wellformed = :strict); false catch; true end + rejects_flat = try read(test.uri, FlatNode; wellformed = :strict); false catch; true end + rejects_node == rejects_flat ? (n_agree += 1) : push!(disagreements, test.id) + end + isempty(disagreements) || @warn "verdict disagreements" disagreements=first(disagreements, 20) + @test isempty(disagreements) + @info "W3C not-wf: identical verdicts on $n_agree documents" + end +end