Version 1.2.5

CHANGELOG.md

@@ -13,6 +13,14 @@
 > ⚠️ We discourage the use of `process(input).first` / `process(input)[0]` because it silently drops potential additional documents
 >    Please use `process_one` if you are expecting only one JSON doc, e.g. in API payloads, because it emits on_warning if it finds multiple docs.
 
+## 1.2.5 (2026-07-01)
+
+RSpec tests: 1,282 → 1,308
+
+### New Features
+
+  - **RFC 7464 JSON Text Sequences are now read natively.** The record separator (`0x1E`) that frames each record is a first-class document separator, so an RS-framed stream parses into its documents without a warning, and bare-scalar records (a number / keyword / string on its own) are read correctly.
+
 ## 1.2.4 (2026-07-01)
 
 RSpec tests: 1,268 → 1,282

ext/smarter_json/smarter_json.c

@@ -1624,14 +1624,17 @@ static int fj_implicit_root_ahead(fj_state *st) {
   return result;
 }
 
-/* Between top-level documents, whitespace, comments, AND commas all separate
- * (commas collapse like the in-container lenient-comma rule). A space alone never
- * separates — that is handled inside the document by the quoteless run. Mirrors
- * the Ruby Parser#skip_document_separators. */
+/* Between top-level documents, whitespace, comments, commas, AND the RFC 7464
+ * record separator (0x1E) all separate (commas collapse like the in-container
+ * lenient-comma rule; 0x1E frames each JSON Text Sequence record). A space alone
+ * never separates — that is handled inside the document by the quoteless run.
+ * Mirrors the Ruby Parser#skip_document_separators. */
 static void fj_skip_document_separators(fj_state *st) {
   for (;;) {
+    int b;
     fj_skip_ws_comments(st);
-    if (fj_byte(st) != ',') break;
+    b = fj_byte(st);
+    if (b != ',' && b != 0x1E) break;
     fj_advance(st, 1);
   }
 }
@@ -1640,8 +1643,9 @@ static int fj_is_hws(int b) { return b == ' ' || b == '\t' || b == 0x0B || b ==
 
 /* After a top-level value: a self-delimiting value (object / array / string) may be
  * followed by anything, but a bare scalar (number / keyword) must be followed by a
- * real separator — a newline, ',', a comment, or EOF. A space is NOT a separator, so
- * `1 2 3` and `42 "x" true` raise. Mirrors the Ruby Parser#enforce_scalar_boundary. */
+ * real separator — a newline, ',', the RFC 7464 record separator (0x1E), a comment,
+ * or EOF. A space is NOT a separator, so `1 2 3` and `42 "x" true` raise. Mirrors the
+ * Ruby Parser#enforce_scalar_boundary. */
 static void fj_enforce_scalar_boundary(fj_state *st, VALUE value) {
   int b, nx;
   if (RB_TYPE_P(value, T_STRING) || RB_TYPE_P(value, T_HASH) || RB_TYPE_P(value, T_ARRAY)) return;
@@ -1655,7 +1659,7 @@ static void fj_enforce_scalar_boundary(fj_state *st, VALUE value) {
     break;
   }
   b = fj_byte(st);
-  if (b == -1 || b == 0x0A || b == 0x0D || b == ',') return;
+  if (b == -1 || b == 0x0A || b == 0x0D || b == ',' || b == 0x1E) return;
   if (b == '#') return;
   if (b == '/') { nx = fj_byte_at(st, 1); if (nx == '/' || nx == '*') return; }
   fj_error(st, "a top-level number or keyword must be followed by a newline, ',', or end of input");

lib/smarter_json/parser.rb

@@ -384,6 +384,7 @@ module Bytes
     TAB        = 0x09
     LF         = 0x0A
     CR         = 0x0D
+    RS         = 0x1E # RFC 7464 record separator — frames each JSON Text Sequence record
   end
 
   module Framer
@@ -1052,29 +1053,31 @@ def parse_document
       parse_iter(implicit_root_object_ahead?)
     end
 
-    # Between top-level documents, whitespace, comments, AND commas all separate
-    # (commas collapse like the in-container lenient-comma rule). A space alone never
-    # separates — that is handled inside the document by the quoteless run, so
+    # Between top-level documents, whitespace, comments, commas, AND the RFC 7464
+    # record separator (0x1E) all separate (commas collapse like the in-container
+    # lenient-comma rule; 0x1E frames each JSON Text Sequence record). A space alone
+    # never separates — that is handled inside the document by the quoteless run, so
     # `1 2 3` is one document (the string "1 2 3") while `1, 2, 3` is three.
     def skip_document_separators
       skip_whitespace_and_comments
-      while byte == COMMA
+      while (b = byte) == COMMA || b == RS
         advance(1)
         skip_whitespace_and_comments
       end
     end
 
     # After a top-level value: a self-delimiting value (object / array / quoted string)
     # may be followed by anything (the next document self-delimits), but a bare scalar
-    # (number / keyword) must be followed by a real separator — a newline, ',', a
-    # comment, or EOF. A space is NOT a separator, so `1 2 3` and `42 "x" true` raise
-    # rather than silently splitting; bare top-level words raise in parse_value itself.
+    # (number / keyword) must be followed by a real separator — a newline, ',', the RFC
+    # 7464 record separator (0x1E), a comment, or EOF. A space is NOT a separator, so
+    # `1 2 3` and `42 "x" true` raise rather than silently splitting; bare top-level
+    # words raise in parse_value itself.
     def enforce_scalar_boundary(value)
       return if value.is_a?(String) || value.is_a?(Hash) || value.is_a?(Array)
 
       skip_horizontal_whitespace
       b = byte
-      return if b.nil? || b == LF || b == CR || b == COMMA
+      return if b.nil? || b == LF || b == CR || b == COMMA || b == RS
       return if b == HASH || (b == SLASH && ((c = byte_at(1)) == SLASH || c == STAR))
 
       raise error("a top-level number or keyword must be followed by a newline, ',', or end of input")

lib/smarter_json/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SmarterJSON
-  VERSION = "1.2.4"
+  VERSION = "1.2.5"
 end

spec/rfc7464_spec.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require "smarter_json"
+
+# RFC 7464 — JSON Text Sequences.
+#
+# A sequence frames each record as:  RS (0x1E)  <JSON-text>  LF (0x0A)
+#
+# The contract we assert here: the record separator 0x1E is a FIRST-CLASS, SILENT
+# top-level document separator — like the newline / comma that already separate
+# concatenated documents. Two consequences:
+#
+#   1. It never raises and never fires an on_warning (it is a real separator, not
+#      "non-JSON prefix text" that the recovery layer strips).
+#   2. A bare-scalar record (a number / keyword / string on its own) is valid — a
+#      scalar followed by 0x1E is a record boundary, exactly like a scalar followed
+#      by a newline.
+#
+# 0x1E is only a separator BETWEEN top-level records; inside a quoted string it stays
+# content.
+RSpec.describe "RFC 7464 JSON Text Sequences (0x1E record separator)" do
+  [true, false].each do |acceleration|
+    context "acceleration: #{acceleration}" do
+      # RS-frame each record the way RFC 7464 does: 0x1E <text> 0x0A
+      def framed(*records)
+        records.map { |r| "\x1E" + r + "\n" }.join
+      end
+
+      # Collect every lenient-fix warning the parser reports for `input`.
+      def warnings_for(input, acceleration:)
+        collected = []
+        SmarterJSON.process(input, acceleration: acceleration, on_warning: ->(w) { collected << w })
+        collected
+      end
+
+      describe "object records" do
+        it "parses a canonical RS/LF-framed object sequence into every document" do
+          input = framed('{"id":1}', '{"id":2}', '{"id":3}')
+          expect(SmarterJSON.process(input, acceleration: acceleration))
+            .to eq([{ "id" => 1 }, { "id" => 2 }, { "id" => 3 }])
+        end
+
+        it "fires no warning — 0x1E is a separator, not stripped prefix text" do
+          input = framed('{"id":1}', '{"id":2}')
+          expect(warnings_for(input, acceleration: acceleration)).to eq([])
+        end
+
+        it "treats a single leading 0x1E as a silent separator, not prefix noise" do
+          input = "\x1E" + '{"a":1}'
+          expect(SmarterJSON.process(input, acceleration: acceleration)).to eq([{ "a" => 1 }])
+          expect(warnings_for(input, acceleration: acceleration)).to eq([])
+        end
+
+        it "accepts 0x1E as the only separator, with no trailing LF" do
+          input = "\x1E" + '{"a":1}' + "\x1E" + '{"b":2}'
+          expect(SmarterJSON.process(input, acceleration: acceleration))
+            .to eq([{ "a" => 1 }, { "b" => 2 }])
+        end
+      end
+
+      describe "scalar records (the case that used to raise)" do
+        it "parses number records" do
+          expect(SmarterJSON.process(framed("1", "2", "3"), acceleration: acceleration))
+            .to eq([1, 2, 3])
+        end
+
+        it "parses number records separated by 0x1E alone, no trailing LF" do
+          expect(SmarterJSON.process("\x1E1\x1E2\x1E3", acceleration: acceleration))
+            .to eq([1, 2, 3])
+        end
+
+        it "parses keyword records true / false / null" do
+          expect(SmarterJSON.process(framed("true", "false", "null"), acceleration: acceleration))
+            .to eq([true, false, nil])
+        end
+
+        it "parses string records" do
+          expect(SmarterJSON.process("\x1E\"hi\"\x1E\"yo\"", acceleration: acceleration))
+            .to eq(%w[hi yo])
+        end
+
+        it "fires no warning on scalar records" do
+          expect(warnings_for(framed("1", "2"), acceleration: acceleration)).to eq([])
+        end
+      end
+
+      describe "mixed-type records" do
+        it "parses objects, arrays, and scalars in one sequence" do
+          input = framed('{"a":1}', "[1,2,3]", "42", '"txt"', "true", "null")
+          expect(SmarterJSON.process(input, acceleration: acceleration))
+            .to eq([{ "a" => 1 }, [1, 2, 3], 42, "txt", true, nil])
+        end
+      end
+
+      describe "0x1E inside a quoted string stays content" do
+        it "does not treat an in-string 0x1E as a record separator" do
+          value = SmarterJSON.process_one("{\"a\":\"x\x1Ey\"}", acceleration: acceleration)
+          expect(value["a"].bytes).to include(0x1E)
+          expect(value["a"]).to eq("x\x1Ey")
+        end
+      end
+
+      describe "single-document and streaming access" do
+        it "returns the one value via process_one for a single RS-framed record" do
+          expect(SmarterJSON.process_one("\x1E" + '{"a":1}' + "\n", acceleration: acceleration))
+            .to eq({ "a" => 1 })
+        end
+
+        it "streams each RS-framed record to a block" do
+          collected = []
+          SmarterJSON.process(framed('{"id":1}', '{"id":2}', '{"id":3}'), acceleration: acceleration) do |doc|
+            collected << doc
+          end
+          expect(collected).to eq([{ "id" => 1 }, { "id" => 2 }, { "id" => 3 }])
+        end
+      end
+    end
+  end
+end