diff --git a/CHANGELOG.md b/CHANGELOG.md index f519568..c56e541 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,14 @@ > ⚠️ We discourage the use of `process(input).first` / `process(input)[0]` because it silently drops potential additional documents > Please use `process_one` if you are expecting only one JSON doc, e.g. in API payloads, because it emits on_warning if it finds multiple docs. +## 1.2.5 (2026-07-01) + +RSpec tests: 1,282 → 1,308 + +### New Features + + - **RFC 7464 JSON Text Sequences are now read natively.** The record separator (`0x1E`) that frames each record is a first-class document separator, so an RS-framed stream parses into its documents without a warning, and bare-scalar records (a number / keyword / string on its own) are read correctly. + ## 1.2.4 (2026-07-01) RSpec tests: 1,268 → 1,282 diff --git a/ext/smarter_json/smarter_json.c b/ext/smarter_json/smarter_json.c index faf4176..ec11695 100644 --- a/ext/smarter_json/smarter_json.c +++ b/ext/smarter_json/smarter_json.c @@ -1624,14 +1624,17 @@ static int fj_implicit_root_ahead(fj_state *st) { return result; } -/* Between top-level documents, whitespace, comments, AND commas all separate - * (commas collapse like the in-container lenient-comma rule). A space alone never - * separates — that is handled inside the document by the quoteless run. Mirrors - * the Ruby Parser#skip_document_separators. */ +/* Between top-level documents, whitespace, comments, commas, AND the RFC 7464 + * record separator (0x1E) all separate (commas collapse like the in-container + * lenient-comma rule; 0x1E frames each JSON Text Sequence record). A space alone + * never separates — that is handled inside the document by the quoteless run. + * Mirrors the Ruby Parser#skip_document_separators. */ static void fj_skip_document_separators(fj_state *st) { for (;;) { + int b; fj_skip_ws_comments(st); - if (fj_byte(st) != ',') break; + b = fj_byte(st); + if (b != ',' && b != 0x1E) break; fj_advance(st, 1); } } @@ -1640,8 +1643,9 @@ static int fj_is_hws(int b) { return b == ' ' || b == '\t' || b == 0x0B || b == /* After a top-level value: a self-delimiting value (object / array / string) may be * followed by anything, but a bare scalar (number / keyword) must be followed by a - * real separator — a newline, ',', a comment, or EOF. A space is NOT a separator, so - * `1 2 3` and `42 "x" true` raise. Mirrors the Ruby Parser#enforce_scalar_boundary. */ + * real separator — a newline, ',', the RFC 7464 record separator (0x1E), a comment, + * or EOF. A space is NOT a separator, so `1 2 3` and `42 "x" true` raise. Mirrors the + * Ruby Parser#enforce_scalar_boundary. */ static void fj_enforce_scalar_boundary(fj_state *st, VALUE value) { int b, nx; if (RB_TYPE_P(value, T_STRING) || RB_TYPE_P(value, T_HASH) || RB_TYPE_P(value, T_ARRAY)) return; @@ -1655,7 +1659,7 @@ static void fj_enforce_scalar_boundary(fj_state *st, VALUE value) { break; } b = fj_byte(st); - if (b == -1 || b == 0x0A || b == 0x0D || b == ',') return; + if (b == -1 || b == 0x0A || b == 0x0D || b == ',' || b == 0x1E) return; if (b == '#') return; if (b == '/') { nx = fj_byte_at(st, 1); if (nx == '/' || nx == '*') return; } fj_error(st, "a top-level number or keyword must be followed by a newline, ',', or end of input"); diff --git a/lib/smarter_json/parser.rb b/lib/smarter_json/parser.rb index 07260e9..bbb8d77 100644 --- a/lib/smarter_json/parser.rb +++ b/lib/smarter_json/parser.rb @@ -384,6 +384,7 @@ module Bytes TAB = 0x09 LF = 0x0A CR = 0x0D + RS = 0x1E # RFC 7464 record separator — frames each JSON Text Sequence record end module Framer @@ -1052,13 +1053,14 @@ def parse_document parse_iter(implicit_root_object_ahead?) end - # Between top-level documents, whitespace, comments, AND commas all separate - # (commas collapse like the in-container lenient-comma rule). A space alone never - # separates — that is handled inside the document by the quoteless run, so + # Between top-level documents, whitespace, comments, commas, AND the RFC 7464 + # record separator (0x1E) all separate (commas collapse like the in-container + # lenient-comma rule; 0x1E frames each JSON Text Sequence record). A space alone + # never separates — that is handled inside the document by the quoteless run, so # `1 2 3` is one document (the string "1 2 3") while `1, 2, 3` is three. def skip_document_separators skip_whitespace_and_comments - while byte == COMMA + while (b = byte) == COMMA || b == RS advance(1) skip_whitespace_and_comments end @@ -1066,15 +1068,16 @@ def skip_document_separators # After a top-level value: a self-delimiting value (object / array / quoted string) # may be followed by anything (the next document self-delimits), but a bare scalar - # (number / keyword) must be followed by a real separator — a newline, ',', a - # comment, or EOF. A space is NOT a separator, so `1 2 3` and `42 "x" true` raise - # rather than silently splitting; bare top-level words raise in parse_value itself. + # (number / keyword) must be followed by a real separator — a newline, ',', the RFC + # 7464 record separator (0x1E), a comment, or EOF. A space is NOT a separator, so + # `1 2 3` and `42 "x" true` raise rather than silently splitting; bare top-level + # words raise in parse_value itself. def enforce_scalar_boundary(value) return if value.is_a?(String) || value.is_a?(Hash) || value.is_a?(Array) skip_horizontal_whitespace b = byte - return if b.nil? || b == LF || b == CR || b == COMMA + return if b.nil? || b == LF || b == CR || b == COMMA || b == RS return if b == HASH || (b == SLASH && ((c = byte_at(1)) == SLASH || c == STAR)) raise error("a top-level number or keyword must be followed by a newline, ',', or end of input") diff --git a/lib/smarter_json/version.rb b/lib/smarter_json/version.rb index 8bf57ea..221e032 100644 --- a/lib/smarter_json/version.rb +++ b/lib/smarter_json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SmarterJSON - VERSION = "1.2.4" + VERSION = "1.2.5" end diff --git a/spec/rfc7464_spec.rb b/spec/rfc7464_spec.rb new file mode 100644 index 0000000..f00e338 --- /dev/null +++ b/spec/rfc7464_spec.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +require "smarter_json" + +# RFC 7464 — JSON Text Sequences. +# +# A sequence frames each record as: RS (0x1E) LF (0x0A) +# +# The contract we assert here: the record separator 0x1E is a FIRST-CLASS, SILENT +# top-level document separator — like the newline / comma that already separate +# concatenated documents. Two consequences: +# +# 1. It never raises and never fires an on_warning (it is a real separator, not +# "non-JSON prefix text" that the recovery layer strips). +# 2. A bare-scalar record (a number / keyword / string on its own) is valid — a +# scalar followed by 0x1E is a record boundary, exactly like a scalar followed +# by a newline. +# +# 0x1E is only a separator BETWEEN top-level records; inside a quoted string it stays +# content. +RSpec.describe "RFC 7464 JSON Text Sequences (0x1E record separator)" do + [true, false].each do |acceleration| + context "acceleration: #{acceleration}" do + # RS-frame each record the way RFC 7464 does: 0x1E 0x0A + def framed(*records) + records.map { |r| "\x1E" + r + "\n" }.join + end + + # Collect every lenient-fix warning the parser reports for `input`. + def warnings_for(input, acceleration:) + collected = [] + SmarterJSON.process(input, acceleration: acceleration, on_warning: ->(w) { collected << w }) + collected + end + + describe "object records" do + it "parses a canonical RS/LF-framed object sequence into every document" do + input = framed('{"id":1}', '{"id":2}', '{"id":3}') + expect(SmarterJSON.process(input, acceleration: acceleration)) + .to eq([{ "id" => 1 }, { "id" => 2 }, { "id" => 3 }]) + end + + it "fires no warning — 0x1E is a separator, not stripped prefix text" do + input = framed('{"id":1}', '{"id":2}') + expect(warnings_for(input, acceleration: acceleration)).to eq([]) + end + + it "treats a single leading 0x1E as a silent separator, not prefix noise" do + input = "\x1E" + '{"a":1}' + expect(SmarterJSON.process(input, acceleration: acceleration)).to eq([{ "a" => 1 }]) + expect(warnings_for(input, acceleration: acceleration)).to eq([]) + end + + it "accepts 0x1E as the only separator, with no trailing LF" do + input = "\x1E" + '{"a":1}' + "\x1E" + '{"b":2}' + expect(SmarterJSON.process(input, acceleration: acceleration)) + .to eq([{ "a" => 1 }, { "b" => 2 }]) + end + end + + describe "scalar records (the case that used to raise)" do + it "parses number records" do + expect(SmarterJSON.process(framed("1", "2", "3"), acceleration: acceleration)) + .to eq([1, 2, 3]) + end + + it "parses number records separated by 0x1E alone, no trailing LF" do + expect(SmarterJSON.process("\x1E1\x1E2\x1E3", acceleration: acceleration)) + .to eq([1, 2, 3]) + end + + it "parses keyword records true / false / null" do + expect(SmarterJSON.process(framed("true", "false", "null"), acceleration: acceleration)) + .to eq([true, false, nil]) + end + + it "parses string records" do + expect(SmarterJSON.process("\x1E\"hi\"\x1E\"yo\"", acceleration: acceleration)) + .to eq(%w[hi yo]) + end + + it "fires no warning on scalar records" do + expect(warnings_for(framed("1", "2"), acceleration: acceleration)).to eq([]) + end + end + + describe "mixed-type records" do + it "parses objects, arrays, and scalars in one sequence" do + input = framed('{"a":1}', "[1,2,3]", "42", '"txt"', "true", "null") + expect(SmarterJSON.process(input, acceleration: acceleration)) + .to eq([{ "a" => 1 }, [1, 2, 3], 42, "txt", true, nil]) + end + end + + describe "0x1E inside a quoted string stays content" do + it "does not treat an in-string 0x1E as a record separator" do + value = SmarterJSON.process_one("{\"a\":\"x\x1Ey\"}", acceleration: acceleration) + expect(value["a"].bytes).to include(0x1E) + expect(value["a"]).to eq("x\x1Ey") + end + end + + describe "single-document and streaming access" do + it "returns the one value via process_one for a single RS-framed record" do + expect(SmarterJSON.process_one("\x1E" + '{"a":1}' + "\n", acceleration: acceleration)) + .to eq({ "a" => 1 }) + end + + it "streams each RS-framed record to a block" do + collected = [] + SmarterJSON.process(framed('{"id":1}', '{"id":2}', '{"id":3}'), acceleration: acceleration) do |doc| + collected << doc + end + expect(collected).to eq([{ "id" => 1 }, { "id" => 2 }, { "id" => 3 }]) + end + end + end + end +end