From ab0db617bc099b97f53bf0269d0585562b7a6446 Mon Sep 17 00:00:00 2001 From: Tilo Sloboda Date: Thu, 25 Jun 2026 22:54:31 -0700 Subject: [PATCH 1/9] version 1.2.3 : tests for known bugs --- lib/smarter_json/parser.rb | 9 ++- spec/fixtures/encoding_sjis.json | 1 + spec/fixtures/encoding_utf16le.json | Bin 0 -> 18 bytes spec/fixtures/encoding_utf8.json | 1 + spec/parser_spec.rb | 119 ++++++++++++++++++++++++++++ 5 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 spec/fixtures/encoding_sjis.json create mode 100644 spec/fixtures/encoding_utf16le.json create mode 100644 spec/fixtures/encoding_utf8.json diff --git a/lib/smarter_json/parser.rb b/lib/smarter_json/parser.rb index a0c9628..fb457bd 100644 --- a/lib/smarter_json/parser.rb +++ b/lib/smarter_json/parser.rb @@ -41,9 +41,12 @@ def process(input, options = {}, &block) # SmarterJSON.process_file(path, options = {}) โ€” open a file and process it. # - # The :encoding option labels the file's encoding (default "UTF-8"); it does NOT - # trigger a transcoding pass โ€” the parser works on the bytes in their native - # encoding and emits string values with the same encoding tag. With a block, + # The :encoding option labels the file's encoding (default "UTF-8"). SmarterJSON + # never normalizes the input to a different encoding on its own (we are not + # Python) โ€” it parses the bytes in whatever encoding they arrive in and emits + # string values with that same encoding tag. The caller is free to transcode the + # input themselves (e.g. open the file with a "r:ext:int" mode); however the bytes + # arrive, we parse them and preserve their encoding. With a block, # streams document-by-document straight from disk in bounded memory (never # loading the whole file); the documents are read as newline-delimited # (NDJSON / JSONL), one per line. diff --git a/spec/fixtures/encoding_sjis.json b/spec/fixtures/encoding_sjis.json new file mode 100644 index 0000000..45808cb --- /dev/null +++ b/spec/fixtures/encoding_sjis.json @@ -0,0 +1 @@ +{"k":"ƒ\"} \ No newline at end of file diff --git a/spec/fixtures/encoding_utf16le.json b/spec/fixtures/encoding_utf16le.json new file mode 100644 index 0000000000000000000000000000000000000000..60bbc86ffe068c3d083cde597d5cc72949ea04b9 GIT binary patch literal 18 Wcmb 1, "b" => [true, nil] }) + expect(result.keys.first.encoding).to eq(input.encoding) + end + + it "parses UTF-16BE input, preserving the input encoding" do + input = '{"a":"x"}'.encode("UTF-16BE") + result = SmarterJSON.process_one(input, acceleration: acceleration) + expect(result.values.first.encode("UTF-8")).to eq("x") + expect(result.values.first.encoding).to eq(input.encoding) + end + + it "parses UTF-32LE input, preserving the input encoding" do + input = '{"a":"x"}'.encode("UTF-32LE") + result = SmarterJSON.process_one(input, acceleration: acceleration) + expect(result.values.first.encode("UTF-8")).to eq("x") + expect(result.values.first.encoding).to eq(input.encoding) + end + + it "parses Shift_JIS string content with a 0x5C trail byte (ใ‚ฝ = 83 5C), preserving the input encoding" do + input = '{"k":"ใ‚ฝ"}'.encode("Shift_JIS") + result = SmarterJSON.process_one(input, acceleration: acceleration) + expect(result.values.first.encode("UTF-8")).to eq("ใ‚ฝ") + expect(result.values.first.encoding).to eq(input.encoding) + end + + it "parses Shift_JIS string content with a 0x5C trail byte (่กจ = 95 5C), preserving the input encoding" do + input = '{"k":"่กจ"}'.encode("Shift_JIS") + result = SmarterJSON.process_one(input, acceleration: acceleration) + expect(result.values.first.encode("UTF-8")).to eq("่กจ") + expect(result.values.first.encoding).to eq(input.encoding) + end + end + + # File input: parse correctly and PRESERVE whatever encoding the bytes arrive in. + # Covers a raw file already in a problematic encoding, AND the caller opening the + # file WITH TRANSCODING to a problematic encoding (UTF-16 / UTF-32 / Shift_JIS). + # These go through the whole-buffer read path (File.read / read), which actually + # transcodes; the streaming Framer's readpartial does NOT honor the IO encoding and + # is tracked separately. Fixtures all hold {"k":"ใ‚ฝ"} (ใ‚ฝ = 83 5C in Shift_JIS). + describe "file / IO input + open-with-transcoding (preserve the arriving encoding)" do + let(:utf8_file) { File.join(fixtures_dir, "encoding_utf8.json") } # {"k":"ใ‚ฝ"} UTF-8 + let(:utf16le_file) { File.join(fixtures_dir, "encoding_utf16le.json") } # {"k":"ใ‚ฝ"} UTF-16LE + let(:sjis_file) { File.join(fixtures_dir, "encoding_sjis.json") } # {"k":"ใ‚ฝ"} Shift_JIS + + # raw file already in a problematic encoding (process_file :encoding label) + it "raw UTF-16LE file preserves UTF-16LE" do + result = SmarterJSON.process_file(utf16le_file, encoding: "UTF-16LE", acceleration: acceleration) + expect(result.first.values.first.encode("UTF-8")).to eq("ใ‚ฝ") + expect(result.first.values.first.encoding).to eq(Encoding.find("UTF-16LE")) + end + + it "raw Shift_JIS file (ใ‚ฝ = 83 5C) preserves Shift_JIS" do + result = SmarterJSON.process_file(sjis_file, encoding: "Shift_JIS", acceleration: acceleration) + expect(result.first.values.first.encode("UTF-8")).to eq("ใ‚ฝ") + expect(result.first.values.first.encoding).to eq(Encoding.find("Shift_JIS")) + end + + # caller OPENS the file WITH TRANSCODING and hands the IO straight to SmarterJSON, + # exactly as a user would. process(io) without a block reads the whole stream via + # io.read (which honors the transcoding); output must preserve the IO's encoding. + it "open-with-transcoding TO UTF-16LE preserves UTF-16LE" do + File.open(utf8_file, "r:UTF-8:UTF-16LE") do |io| + result = SmarterJSON.process(io, acceleration: acceleration) + expect(result.first.values.first.encode("UTF-8")).to eq("ใ‚ฝ") + expect(result.first.values.first.encoding).to eq(Encoding.find("UTF-16LE")) + end + end + + it "open-with-transcoding TO UTF-32LE preserves UTF-32LE" do + File.open(utf8_file, "r:UTF-8:UTF-32LE") do |io| + result = SmarterJSON.process(io, acceleration: acceleration) + expect(result.first.values.first.encode("UTF-8")).to eq("ใ‚ฝ") + expect(result.first.values.first.encoding).to eq(Encoding.find("UTF-32LE")) + end + end + + it "open-with-transcoding TO Shift_JIS (ใ‚ฝ -> 83 5C) preserves Shift_JIS" do + File.open(utf8_file, "r:UTF-8:Shift_JIS") do |io| + result = SmarterJSON.process(io, acceleration: acceleration) + expect(result.first.values.first.encode("UTF-8")).to eq("ใ‚ฝ") + expect(result.first.values.first.encoding).to eq(Encoding.find("Shift_JIS")) + end + end + + # transcode-on-open via the :encoding option's "ext:int" form: File.read transcodes, + # but the option string is then reused downstream as a force_encoding label. + it "process_file(encoding: 'UTF-8:UTF-16LE') transcodes on open and preserves UTF-16LE" do + result = SmarterJSON.process_file(utf8_file, encoding: "UTF-8:UTF-16LE", acceleration: acceleration) + expect(result.first.values.first.encode("UTF-8")).to eq("ใ‚ฝ") + expect(result.first.values.first.encoding).to eq(Encoding.find("UTF-16LE")) + end + + # green: transcode FROM a problematic encoding TO UTF-8 โ€” arrives as UTF-8, we keep it. + it "open-with-transcoding FROM UTF-16LE to UTF-8 arrives as UTF-8 (green)" do + File.open(utf16le_file, "r:UTF-16LE:UTF-8") do |io| + result = SmarterJSON.process(io, acceleration: acceleration) + expect(result.first.values.first).to eq("ใ‚ฝ") + expect(result.first.values.first.encoding).to eq(Encoding::UTF_8) + end + end + end + describe "default encoding โ€” ASCII-8BIT input treated as UTF-8 (the HTTP-body case)" do it "relabels a valid-UTF-8 body tagged ASCII-8BIT to UTF-8, so equality works" do # How Net::HTTP and many HTTP libraries hand you response.body: correct From b09a234ac843069657aa9dc5db6148cf3dfe350c Mon Sep 17 00:00:00 2001 From: Tilo Sloboda Date: Thu, 25 Jun 2026 22:58:02 -0700 Subject: [PATCH 2/9] update --- lib/smarter_json/parser.rb | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/lib/smarter_json/parser.rb b/lib/smarter_json/parser.rb index fb457bd..19b3b11 100644 --- a/lib/smarter_json/parser.rb +++ b/lib/smarter_json/parser.rb @@ -41,15 +41,19 @@ def process(input, options = {}, &block) # SmarterJSON.process_file(path, options = {}) โ€” open a file and process it. # - # The :encoding option labels the file's encoding (default "UTF-8"). SmarterJSON - # never normalizes the input to a different encoding on its own (we are not - # Python) โ€” it parses the bytes in whatever encoding they arrive in and emits - # string values with that same encoding tag. The caller is free to transcode the - # input themselves (e.g. open the file with a "r:ext:int" mode); however the bytes - # arrive, we parse them and preserve their encoding. With a block, - # streams document-by-document straight from disk in bounded memory (never - # loading the whole file); the documents are read as newline-delimited - # (NDJSON / JSONL), one per line. + # The :encoding option labels the file's encoding (default "UTF-8"). + # + # The user can send any encoding to SmarterJSON - we make zero assumptions about encoding. + # We also do not "normalize" the input to a different encoding on our own (this is not Python). + # + # We parse the bytes in whatever encoding they arrive in and emit string values + # with that same encoding tag. + # + # The caller is free to transcode the input themselves (e.g. open the file with a "r:ext:int" mode); + # however the bytes arrive, we parse them and preserve their encoding. With a block, + # streams document-by-document straight from disk in bounded memory (neverloading the whole file); + # the documents are read as newline-delimited (NDJSON / JSONL), one per line. + # def process_file(path, options = {}, &block) options = Options.process_options(options) encoding = options[:encoding] || "UTF-8" From 953c20e707617d5dc476a910dfeb3073afe0ce32 Mon Sep 17 00:00:00 2001 From: Tilo Sloboda Date: Fri, 26 Jun 2026 01:21:15 -0700 Subject: [PATCH 3/9] fixing bugs --- CHANGELOG.md | 13 +++++-- lib/smarter_json/parser.rb | 73 ++++++++++++++++++++++++++++++++++++-- spec/parser_spec.rb | 57 +++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b261743..ae2a181 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,18 @@ > โš ๏ธ We discourage the use of `process(input).first` / `process(input)[0]` because it silently drops potential additional documents > Please use `process_one` if you are expecting only one JSON doc, e.g. in API payloads, because it emits on_warning if it finds multiple docs. -## 1.2.2 (unreleased) +## 1.2.3 (unreleased) -RSpec tests: 1,167 +RSpec tests: 1,167 โ†’ 1,205 + +- Input in a non-UTF-8 encoding now parses correctly and keeps its **own** encoding in the result โ€” SmarterJSON never transcodes it to UTF-8 on your behalf: + - **UTF-16 / UTF-32** and **Shift_JIS** (and other CJK double-byte encodings such as Big5 / GBK / GB18030) previously raised or mis-parsed; they now parse, with string values tagged in the input's encoding. + - Applies to String, file (`process_file`), and IO / streaming (`foreach`) input โ€” including a file the caller opened with transcoding (e.g. `File.open(path, "r:UTF-8:UTF-16LE")`), where the output is the encoding the bytes arrive in. + - Streaming a **Latin-1 / Windows-1252** (or other single-byte) file or IO now preserves that encoding too, instead of mislabelling or raising. + +## 1.2.2 (2026-06-19) + +RSpec tests: 1,165 โ†’ 1,167 - The Eisel-Lemire fast path for `decimal_precision: :float` now covers decimals with **up to 19 significant digits** (was 18). 19 digits is the most that fits exactly in a `uint64` (max 19-digit โ‰ˆ 1.0e19 < `UINT64_MAX` โ‰ˆ 1.8e19), so these no longer fall back to the slower `strtod`. Still correctly rounded, bit-for-bit identical to the stdlib โ€” verified across 18/19-digit round-to-even tie shapes. diff --git a/lib/smarter_json/parser.rb b/lib/smarter_json/parser.rb index 19b3b11..512ebf5 100644 --- a/lib/smarter_json/parser.rb +++ b/lib/smarter_json/parser.rb @@ -58,9 +58,9 @@ def process_file(path, options = {}, &block) options = Options.process_options(options) encoding = options[:encoding] || "UTF-8" if block - File.open(path, "r:#{encoding}") { |io| stream_io(io, options, &block) } + File.open(path, "rb:#{encoding}") { |io| stream_io(io, options, &block) } else - process(File.read(path, encoding: encoding), options) + process(File.read(path, mode: "rb:#{encoding}"), options) end end @@ -170,11 +170,66 @@ def normalize_default_encoding(input, options) raise EncodingError, "input is tagged ASCII-8BIT and is not valid UTF-8 โ€” pass encoding: to declare its encoding" end + # Legacy CJK double-byte encodings whose trail bytes can fall in the ASCII range, so a + # 0x5C trail byte looks like a string escape, a 0x7B like a brace, etc. โ€” i.e. they are + # ascii_compatible? yet still NOT safe to byte-scan for JSON structure. (EUC-* and + # single-byte encodings keep their non-ASCII bytes above 0x7F, so they ARE safe.) + UNSCANNABLE_ASCII_COMPATIBLE = %w[Shift_JIS Windows-31J Big5 GBK GB18030].each_with_object({}) do |name, h| + h[Encoding.find(name)] = true + rescue ArgumentError + # encoding not built into this Ruby โ€” skip it + end.freeze + + # True when an Encoding cannot be scanned directly for JSON structure โ€” the non + # ASCII-compatible ones (UTF-16 / UTF-32, where structure is in code units) and the CJK + # double-byte ones above. For these we parse a UTF-8 copy and emit the values back in the + # original encoding. (Over-including a safe encoding only costs a transcode round-trip; the + # result is still correct.) + def unscannable_enc?(enc) + return true unless enc.ascii_compatible? + + UNSCANNABLE_ASCII_COMPATIBLE.key?(enc) + end + + # The encoding the bytes arrived in when they must be parsed via a UTF-8 copy (see + # unscannable_enc?); nil when the bytes are directly byte-scannable. + def unscannable_encoding(input) + enc = input.encoding + unscannable_enc?(enc) ? enc : nil + end + + # Re-tag a parsed value's strings (Hash keys/values, Array elements, nested) into `enc`, + # so we emit values in the encoding the bytes arrived in after parsing a UTF-8 copy. + # Non-strings (numbers, true/false/nil, symbols) pass through unchanged. + def deep_encode(obj, enc) + case obj + when String then obj.encode(enc) + when Array then obj.map { |e| deep_encode(e, enc) } + when Hash then obj.each_with_object({}) { |(k, v), h| h[deep_encode(k, enc)] = deep_encode(v, enc) } + else obj + end + end + # Stream documents from an IO incrementally, yielding each recovered top-level # document without slurping the whole input into memory first. def stream_io(io, options, &block) + ext = io.respond_to?(:external_encoding) ? io.external_encoding : nil + int = io.respond_to?(:internal_encoding) ? io.internal_encoding : nil + logical = int || ext # what io.read would hand back โ€” the encoding the bytes arrive in + + # The Framer reads via readpartial, which returns ASCII-8BIT โ€” it drops the IO's encoding + # and ignores transcoding. When that matters (a transcoding IO, or a logical encoding we + # can't byte-scan: UTF-16 / UTF-32 / Shift_JIS / ...), read the whole stream via io.read + # (which honors both) and parse it as a unit; the whole-buffer path preserves the encoding. + if int || (logical && SmarterJSON.send(:unscannable_enc?, logical)) + return process(io.read, options, &block) + end + count = 0 Framer.each_document(io) do |doc| + # readpartial dropped the IO's encoding tag; restore it so a Latin-1 / Windows-1252 / + # etc. stream is parsed and emitted in its own encoding, not mislabelled. + doc = doc.dup.force_encoding(logical) if logical && doc.encoding != logical # Recovery.process_string yields each value and returns how many it yielded; # blank / comment-only framed segments yield none, so count tracks actual # documents (== values yielded), not raw framed segments. @@ -473,6 +528,20 @@ module Recovery def process_string(input, options, &block) input = SmarterJSON.send(:normalize_default_encoding, input, options) + + # UTF-16 / UTF-32 / Shift_JIS / ... cannot be byte-scanned for JSON structure. Parse + # a UTF-8 copy and emit each document's strings back in the encoding the bytes arrived + # in โ€” the caller always gets values in the encoding they handed us, never UTF-8. + if (target_enc = SmarterJSON.send(:unscannable_encoding, input)) + opts = options.merge(encoding: nil) # the working copy is UTF-8; don't re-label it downstream + utf8 = input.encode(Encoding::UTF_8) + if block + return process_string(utf8, opts) { |doc| block.call(SmarterJSON.send(:deep_encode, doc, target_enc)) } + end + + return process_string(utf8, opts).map { |doc| SmarterJSON.send(:deep_encode, doc, target_enc) } + end + return SmarterJSON.send(:process_content, input, options, &block) unless input.valid_encoding? # Recovery is REACTIVE: parse first, and only fall back to wrapper extraction when diff --git a/spec/parser_spec.rb b/spec/parser_spec.rb index 0d2530b..36e3148 100644 --- a/spec/parser_spec.rb +++ b/spec/parser_spec.rb @@ -1476,6 +1476,63 @@ end end + # Streaming path (foreach(IO) / process_file-with-block -> Framer). The Framer reads + # via readpartial, which drops the IO's encoding and ignores transcoding, so the + # streaming path must handle these encodings too โ€” output preserves the encoding the + # stream arrives in, exactly like the whole-buffer path. + describe "streaming / Framer input preserves the arriving encoding" do + require "stringio" + + let(:utf8_file) { File.join(fixtures_dir, "encoding_utf8.json") } # {"k":"ใ‚ฝ"} UTF-8 + let(:utf16le_file) { File.join(fixtures_dir, "encoding_utf16le.json") } # {"k":"ใ‚ฝ"} UTF-16LE + let(:sjis_file) { File.join(fixtures_dir, "encoding_sjis.json") } # {"k":"ใ‚ฝ"} Shift_JIS + + it "foreach(IO) on a raw UTF-16LE stream preserves UTF-16LE" do + File.open(utf16le_file, "rb:UTF-16LE") do |io| + docs = SmarterJSON.foreach(io, acceleration: acceleration).to_a + expect(docs.first.values.first.encode("UTF-8")).to eq("ใ‚ฝ") + expect(docs.first.values.first.encoding).to eq(Encoding.find("UTF-16LE")) + end + end + + it "foreach(IO) on a raw Shift_JIS stream (ใ‚ฝ = 83 5C) preserves Shift_JIS" do + File.open(sjis_file, "rb:Shift_JIS") do |io| + docs = SmarterJSON.foreach(io, acceleration: acceleration).to_a + expect(docs.first.values.first.encode("UTF-8")).to eq("ใ‚ฝ") + expect(docs.first.values.first.encoding).to eq(Encoding.find("Shift_JIS")) + end + end + + it "foreach(IO) opened with transcoding TO UTF-16LE preserves UTF-16LE" do + File.open(utf8_file, "rb:UTF-8:UTF-16LE") do |io| + docs = SmarterJSON.foreach(io, acceleration: acceleration).to_a + expect(docs.first.values.first.encode("UTF-8")).to eq("ใ‚ฝ") + expect(docs.first.values.first.encoding).to eq(Encoding.find("UTF-16LE")) + end + end + + it "process_file(path) { block } on a raw UTF-16LE file preserves UTF-16LE" do + docs = [] + SmarterJSON.process_file(utf16le_file, encoding: "UTF-16LE", acceleration: acceleration) { |d| docs << d } + expect(docs.first.values.first.encode("UTF-8")).to eq("ใ‚ฝ") + expect(docs.first.values.first.encoding).to eq(Encoding.find("UTF-16LE")) + end + + it "foreach(IO) on a raw Latin-1 stream preserves Latin-1" do + io = StringIO.new("{\"name\":\"caf\xE9\"}".b.force_encoding("ISO-8859-1")) + docs = SmarterJSON.foreach(io, acceleration: acceleration).to_a + expect(docs.first["name"].bytes).to eq([0x63, 0x61, 0x66, 0xE9]) + expect(docs.first["name"].encoding).to eq(Encoding::ISO_8859_1) + end + + it "foreach(IO) on a plain UTF-8 stream still works (green)" do + io = StringIO.new('{"k":"x"}') + docs = SmarterJSON.foreach(io, acceleration: acceleration).to_a + expect(docs.first).to eq({ "k" => "x" }) + expect(docs.first.values.first.encoding).to eq(Encoding::UTF_8) + end + end + describe "default encoding โ€” ASCII-8BIT input treated as UTF-8 (the HTTP-body case)" do it "relabels a valid-UTF-8 body tagged ASCII-8BIT to UTF-8, so equality works" do # How Net::HTTP and many HTTP libraries hand you response.body: correct From c42b96b5427efbdf35094c6c02ae5e4add705991 Mon Sep 17 00:00:00 2001 From: Tilo Sloboda Date: Fri, 26 Jun 2026 01:26:21 -0700 Subject: [PATCH 4/9] update --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae2a181..6fdbdc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ RSpec tests: 1,167 โ†’ 1,205 -- Input in a non-UTF-8 encoding now parses correctly and keeps its **own** encoding in the result โ€” SmarterJSON never transcodes it to UTF-8 on your behalf: +Fixing some encoding corner cases: - **UTF-16 / UTF-32** and **Shift_JIS** (and other CJK double-byte encodings such as Big5 / GBK / GB18030) previously raised or mis-parsed; they now parse, with string values tagged in the input's encoding. - Applies to String, file (`process_file`), and IO / streaming (`foreach`) input โ€” including a file the caller opened with transcoding (e.g. `File.open(path, "r:UTF-8:UTF-16LE")`), where the output is the encoding the bytes arrive in. - Streaming a **Latin-1 / Windows-1252** (or other single-byte) file or IO now preserves that encoding too, instead of mislabelling or raising. From 78a404f6d3f6638de47e5b6c18c4d42e6c9183ec Mon Sep 17 00:00:00 2001 From: Tilo Sloboda Date: Fri, 26 Jun 2026 09:34:59 -0700 Subject: [PATCH 5/9] update --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fdbdc2..e89de1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ > โš ๏ธ We discourage the use of `process(input).first` / `process(input)[0]` because it silently drops potential additional documents > Please use `process_one` if you are expecting only one JSON doc, e.g. in API payloads, because it emits on_warning if it finds multiple docs. -## 1.2.3 (unreleased) +## 1.2.3 (2026-06-26) RSpec tests: 1,167 โ†’ 1,205 From d1edf77e7f0eb2c1c25d1ed753d35ac9598e5e0d Mon Sep 17 00:00:00 2001 From: Tilo Sloboda Date: Sat, 27 Jun 2026 02:49:56 -0700 Subject: [PATCH 6/9] update --- .github/workflows/main.yml | 8 ++++ CHANGELOG.md | 2 +- lib/smarter_json/parser.rb | 88 +++++++++++++++++++++++++++++++++----- spec/parser_spec.rb | 77 +++++++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 15530de..ca9159a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,6 +35,14 @@ jobs: - "3.4" - "4.0" - head + # Windows: one OS is enough to catch the platform-specific corner cases (newline + # translation, encoding defaults, file read modes). A couple of modern Rubies โ€” not + # the whole grid: old Rubies on Windows have flaky toolchains and add little signal. + include: + - os: windows + ruby: "3.3" + - os: windows + ruby: "3.4" steps: - uses: actions/checkout@v6 diff --git a/CHANGELOG.md b/CHANGELOG.md index e89de1b..b2a64d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ ## 1.2.3 (2026-06-26) -RSpec tests: 1,167 โ†’ 1,205 +RSpec tests: 1,167 โ†’ 1,219 Fixing some encoding corner cases: - **UTF-16 / UTF-32** and **Shift_JIS** (and other CJK double-byte encodings such as Big5 / GBK / GB18030) previously raised or mis-parsed; they now parse, with string values tagged in the input's encoding. diff --git a/lib/smarter_json/parser.rb b/lib/smarter_json/parser.rb index 512ebf5..5048632 100644 --- a/lib/smarter_json/parser.rb +++ b/lib/smarter_json/parser.rb @@ -57,13 +57,26 @@ def process(input, options = {}, &block) def process_file(path, options = {}, &block) options = Options.process_options(options) encoding = options[:encoding] || "UTF-8" + mode = file_read_mode(encoding) if block - File.open(path, "rb:#{encoding}") { |io| stream_io(io, options, &block) } + File.open(path, mode) { |io| stream_io(io, options, &block) } else - process(File.read(path, mode: "rb:#{encoding}"), options) + process(File.read(path, mode: mode), options) end end + # Read mode for process_file. Binary mode is required for ASCII-incompatible encodings + # (UTF-16 / UTF-32) โ€” text mode refuses them ("ASCII incompatible encoding needs binmode"). + # ASCII-compatible encodings keep TEXT mode, so newline translation (e.g. \r\n on Windows) + # is unchanged โ€” binmode only applies where text mode is impossible anyway. + def file_read_mode(encoding) + incompatible = encoding.to_s.split(":").any? do |name| + enc = Encoding.find(name) rescue nil + enc && !enc.ascii_compatible? + end + incompatible ? "rb:#{encoding}" : "r:#{encoding}" + end + # SmarterJSON.foreach(source, options = {}) โ€” the streaming, composable sibling of # process_file, mirroring the stdlib convention (CSV.foreach / File.foreach): a # plain Enumerator (NOT Enumerator::Lazy), so .map / .select behave the normal way @@ -174,7 +187,10 @@ def normalize_default_encoding(input, options) # 0x5C trail byte looks like a string escape, a 0x7B like a brace, etc. โ€” i.e. they are # ascii_compatible? yet still NOT safe to byte-scan for JSON structure. (EUC-* and # single-byte encodings keep their non-ASCII bytes above 0x7F, so they ARE safe.) - UNSCANNABLE_ASCII_COMPATIBLE = %w[Shift_JIS Windows-31J Big5 GBK GB18030].each_with_object({}) do |name, h| + UNSCANNABLE_ASCII_COMPATIBLE = %w[ + Shift_JIS Windows-31J MacJapanese SHIFT_JISX0213 SJIS-DoCoMo SJIS-KDDI SJIS-SoftBank + Big5 Big5-HKSCS Big5-UAO CP950 GBK GB18030 GB12345 + ].each_with_object({}) do |name, h| h[Encoding.find(name)] = true rescue ArgumentError # encoding not built into this Ruby โ€” skip it @@ -198,16 +214,65 @@ def unscannable_encoding(input) unscannable_enc?(enc) ? enc : nil end + # Generic UTF-16 / UTF-32 prepend a byte-order mark to EVERY string when you encode TO them. + # Map the generic encoding to the concrete endianness (read from the input's own BOM) so the + # re-tagged values are BOM-free and usable. Concrete and non-Unicode encodings pass through. + def concrete_unicode_encoding(input, enc) + return enc unless enc == Encoding::UTF_16 || enc == Encoding::UTF_32 + + head = input.byteslice(0, 4).to_s.b + if enc == Encoding::UTF_16 + head.start_with?("\xFF\xFE".b) ? Encoding::UTF_16LE : Encoding::UTF_16BE + else + head.start_with?("\xFF\xFE\x00\x00".b) ? Encoding::UTF_32LE : Encoding::UTF_32BE + end + end + + # Transcode the input to a UTF-8 working copy for scanning. Invalid bytes raise the gem's + # own SmarterJSON::EncodingError (not a bare Ruby Encoding error), matching the rest. + def to_utf8_copy(input) + input.encode(Encoding::UTF_8) + rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError + raise EncodingError, "invalid byte sequence for #{input.encoding.name}" + end + + # Re-tag one scalar into `enc`. A character not representable in `enc` (e.g. an emoji from a + # `\u` escape inside a Shift_JIS document) keeps its UTF-8 value โ€” lossless, never raises. + def reencode_scalar(obj, enc) + return obj unless obj.is_a?(String) + + obj.encode(enc) + rescue Encoding::UndefinedConversionError + obj + end + # Re-tag a parsed value's strings (Hash keys/values, Array elements, nested) into `enc`, # so we emit values in the encoding the bytes arrived in after parsing a UTF-8 copy. - # Non-strings (numbers, true/false/nil, symbols) pass through unchanged. - def deep_encode(obj, enc) - case obj - when String then obj.encode(enc) - when Array then obj.map { |e| deep_encode(e, enc) } - when Hash then obj.each_with_object({}) { |(k, v), h| h[deep_encode(k, enc)] = deep_encode(v, enc) } - else obj + # ITERATIVE (an explicit work stack, not recursion) so a deeply nested document is + # depth-safe โ€” like the parser itself โ€” and can't raise SystemStackError. + def deep_encode(root, enc) + return reencode_scalar(root, enc) unless root.is_a?(Array) || root.is_a?(Hash) + + out = root.is_a?(Array) ? [] : {} + stack = [[root, out]] + until stack.empty? + src, dst = stack.pop + if src.is_a?(Array) + src.each do |v| + child = (v.is_a?(Array) ? [] : {}) if v.is_a?(Array) || v.is_a?(Hash) + dst << (child || reencode_scalar(v, enc)) + stack.push([v, child]) if child + end + else + src.each do |k, v| + key = reencode_scalar(k, enc) + child = (v.is_a?(Array) ? [] : {}) if v.is_a?(Array) || v.is_a?(Hash) + dst[key] = child || reencode_scalar(v, enc) + stack.push([v, child]) if child + end + end end + out end # Stream documents from an IO incrementally, yielding each recovered top-level @@ -533,8 +598,9 @@ def process_string(input, options, &block) # a UTF-8 copy and emit each document's strings back in the encoding the bytes arrived # in โ€” the caller always gets values in the encoding they handed us, never UTF-8. if (target_enc = SmarterJSON.send(:unscannable_encoding, input)) + target_enc = SmarterJSON.send(:concrete_unicode_encoding, input, target_enc) # avoid per-string BOMs opts = options.merge(encoding: nil) # the working copy is UTF-8; don't re-label it downstream - utf8 = input.encode(Encoding::UTF_8) + utf8 = SmarterJSON.send(:to_utf8_copy, input) # invalid bytes -> SmarterJSON::EncodingError if block return process_string(utf8, opts) { |doc| block.call(SmarterJSON.send(:deep_encode, doc, target_enc)) } end diff --git a/spec/parser_spec.rb b/spec/parser_spec.rb index 36e3148..e2f505f 100644 --- a/spec/parser_spec.rb +++ b/spec/parser_spec.rb @@ -1533,6 +1533,83 @@ end end + # 1.2.3 hardening โ€” corner cases of the transcode-and-re-tag path: a char not + # representable in the target encoding, deep nesting (depth-safety), BOM handling, + # invalid bytes, denylist variants, and the file read-mode scoping. + describe "encoding corner cases (1.2.3 hardening)" do + it "keeps a non-representable \\u-escaped char as UTF-8 instead of raising (Shift_JIS)" do + # valid Shift_JIS input (all ASCII bytes); the escape decodes to an emoji not in Shift_JIS + json = "{\"k\":\"\\uD83D\\uDE00\"}".dup.force_encoding("Shift_JIS") + result = SmarterJSON.process_one(json, acceleration: acceleration) + expect(result.values.first).to eq("๐Ÿ˜€") + expect(result.values.first.encoding).to eq(Encoding::UTF_8) # not representable in SJIS -> kept UTF-8 + expect(result.keys.first.encoding).to eq(Encoding.find("Shift_JIS")) # representable key still SJIS + end + + it "re-encodes deeply nested UTF-16 without SystemStackError (depth-safe)" do + depth = 12_000 + json = (("[" * depth) + "1" + ("]" * depth)).encode("UTF-16LE") + result = SmarterJSON.process_one(json, acceleration: acceleration) + node = result + node = node.first while node.is_a?(Array) + expect(node).to eq(1) + end + + it "parses generic UTF-16 (with BOM) and emits BOM-free values in a concrete endianness" do + json = '{"k":"x"}'.encode("UTF-16") # generic UTF-16: #encode auto-prepends a BOM + result = SmarterJSON.process_one(json, acceleration: acceleration) + expect(result.values.first.encode("UTF-8")).to eq("x") + expect([Encoding::UTF_16LE, Encoding::UTF_16BE]).to include(result.values.first.encoding) + expect(result.values.first.bytes.first(2)).not_to eq([0xFE, 0xFF]) # no BOM bytes + expect(result.values.first.bytes.first(2)).not_to eq([0xFF, 0xFE]) + end + + it "skips a leading BOM in concrete UTF-16LE and preserves UTF-16LE" do + json = "๏ปฟ{\"k\":\"x\"}".encode("UTF-16LE") + result = SmarterJSON.process_one(json, acceleration: acceleration) + expect(result.values.first.encode("UTF-8")).to eq("x") + expect(result.values.first.encoding).to eq(Encoding.find("UTF-16LE")) + end + + it "parses generic UTF-32 (with BOM) and emits BOM-free values in a concrete endianness" do + json = '{"k":"x"}'.encode("UTF-32") # generic UTF-32: #encode auto-prepends a BOM + result = SmarterJSON.process_one(json, acceleration: acceleration) + expect(result.values.first.encode("UTF-8")).to eq("x") + expect([Encoding::UTF_32LE, Encoding::UTF_32BE]).to include(result.values.first.encoding) + expect(result.values.first.bytes.first(4)).not_to eq([0x00, 0x00, 0xFE, 0xFF]) # no BOM (BE) + expect(result.values.first.bytes.first(4)).not_to eq([0xFF, 0xFE, 0x00, 0x00]) # no BOM (LE) + end + + it "parses a generic UTF-16 stream with a little-endian BOM, preserving UTF-16LE" do + # hand-built generic UTF-16 with an LE BOM (FF FE), so concrete_unicode_encoding picks LE + bytes = "\xFF\xFE".b + '{"k":"x"}'.encode("UTF-16LE").b + json = bytes.force_encoding("UTF-16") + result = SmarterJSON.process_one(json, acceleration: acceleration) + expect(result.values.first.encode("UTF-8")).to eq("x") + expect(result.values.first.encoding).to eq(Encoding.find("UTF-16LE")) + expect(result.values.first.bytes.first(2)).not_to eq([0xFF, 0xFE]) # no BOM in the value + end + + it "raises SmarterJSON::EncodingError (not a bare Ruby error) on invalid unscannable bytes" do + bad = "\x00\xD8".dup.force_encoding("UTF-16LE") # lone high surrogate -> invalid UTF-16 + expect { SmarterJSON.process_one(bad, acceleration: acceleration) }.to raise_error(SmarterJSON::EncodingError) + end + + it "handles Windows-31J (a Shift_JIS variant in the denylist), preserving it" do + json = '{"k":"ใ‚ฝ"}'.encode("Windows-31J") + result = SmarterJSON.process_one(json, acceleration: acceleration) + expect(result.values.first.encode("UTF-8")).to eq("ใ‚ฝ") + expect(result.values.first.encoding).to eq(Encoding.find("Windows-31J")) + end + + it "reads ASCII-compatible file encodings in text mode and UTF-16/32 in binary mode" do + expect(SmarterJSON.send(:file_read_mode, "UTF-8")).to eq("r:UTF-8") + expect(SmarterJSON.send(:file_read_mode, "ISO-8859-1")).to eq("r:ISO-8859-1") + expect(SmarterJSON.send(:file_read_mode, "UTF-16LE")).to eq("rb:UTF-16LE") + expect(SmarterJSON.send(:file_read_mode, "UTF-8:UTF-16LE")).to eq("rb:UTF-8:UTF-16LE") + end + end + describe "default encoding โ€” ASCII-8BIT input treated as UTF-8 (the HTTP-body case)" do it "relabels a valid-UTF-8 body tagged ASCII-8BIT to UTF-8, so equality works" do # How Net::HTTP and many HTTP libraries hand you response.body: correct From 5d549a825e1d75c24d0f722835e72d7f2ea594fa Mon Sep 17 00:00:00 2001 From: Tilo Sloboda Date: Sat, 27 Jun 2026 11:00:50 -0700 Subject: [PATCH 7/9] updates --- CHANGELOG.md | 5 +++-- README.md | 3 +++ docs/options.md | 7 ++++++- lib/smarter_json/options.rb | 4 ++++ lib/smarter_json/parser.rb | 30 +++++++++++++++++------------- lib/smarter_json/version.rb | 2 +- spec/options_spec.rb | 1 + spec/parser_spec.rb | 21 +++++++++++++++++---- 8 files changed, 52 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2a64d1..46fb820 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,14 +13,15 @@ > โš ๏ธ We discourage the use of `process(input).first` / `process(input)[0]` because it silently drops potential additional documents > Please use `process_one` if you are expecting only one JSON doc, e.g. in API payloads, because it emits on_warning if it finds multiple docs. -## 1.2.3 (2026-06-26) +## 1.2.3 (2026-06-27) -RSpec tests: 1,167 โ†’ 1,219 +RSpec tests: 1,167 โ†’ 1,279 Fixing some encoding corner cases: - **UTF-16 / UTF-32** and **Shift_JIS** (and other CJK double-byte encodings such as Big5 / GBK / GB18030) previously raised or mis-parsed; they now parse, with string values tagged in the input's encoding. - Applies to String, file (`process_file`), and IO / streaming (`foreach`) input โ€” including a file the caller opened with transcoding (e.g. `File.open(path, "r:UTF-8:UTF-16LE")`), where the output is the encoding the bytes arrive in. - Streaming a **Latin-1 / Windows-1252** (or other single-byte) file or IO now preserves that encoding too, instead of mislabelling or raising. + - New `:replace_char` option (default `"?"`): when a `\uXXXX` escape decodes to a character the input's encoding can't represent (e.g. an emoji inside a Shift_JIS document), that character is replaced rather than raising. `replace_char: ""` drops it. ## 1.2.2 (2026-06-19) diff --git a/README.md b/README.md index 2cc9563..2bd42bc 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,7 @@ In short: **SmarterJSON's C path matches or beats Oj/strict on every file** (app | `decimal_precision` | `:auto` | `:auto` keeps high-precision decimals as `BigDecimal`; `:float` forces `Float`; `:bigdecimal` forces `BigDecimal` | | `acceleration` | `true` | `true` uses the C extension when compiled and loadable; `false` forces pure Ruby (identical results) | | `encoding` | `nil` | labels the input's encoding; `nil` keeps the input's own (no transcoding pass; see below) | +| `replace_char` | `"?"` | replacement for a char a `\uXXXX` escape decodes to that the input's encoding can't represent (e.g. an emoji inside Shift_JIS); `""` drops it | | `on_warning` | `nil` | a callable invoked once per lenient fix applied (`:empty_slot`, `:empty_value`, `:duplicate_key`, `:number_overflow`), passed a `SmarterJSON::Warning`; the return value is never changed. See below. | ## Examples @@ -358,6 +359,8 @@ TEXT `encoding:` (default `nil`) labels what the input is โ€” it does **not** transcode. With `nil`, SmarterJSON keeps the input's own encoding tag and emits string values with that same tag, the way `smarter_csv` does โ€” **with one smart default:** input tagged `ASCII-8BIT` (BINARY) whose bytes are valid UTF-8 is treated as UTF-8. That is exactly how `Net::HTTP` and many HTTP libraries hand you a `response.body` (correct UTF-8 bytes, BINARY tag); without this, string values would come back tagged `ASCII-8BIT` and compare unequal to UTF-8 literals. If such `ASCII-8BIT` input is *not* valid UTF-8, it raises `SmarterJSON::EncodingError` rather than guess a legacy encoding โ€” pass an explicit `encoding:` (e.g. `"ISO-8859-1"`) for that. Bytes invalid for an explicitly claimed encoding also raise `SmarterJSON::EncodingError` (a kind of `SmarterJSON::ParseError`). +UTF-16 / UTF-32, Shift_JIS, and other CJK double-byte encodings parse too: SmarterJSON works on a UTF-8 copy internally and re-tags the result back into the input's own encoding, so values come back in the encoding the bytes arrived in (a UTF-16 / UTF-32 BOM is consumed on the way in). The one edge case โ€” a `\uXXXX` escape that decodes to a character that encoding can't represent (e.g. an emoji inside a Shift_JIS document) โ€” is replaced by `replace_char` (default `"?"`, or `""` to drop it) rather than raising. + ## Nesting & untrusted input Both the C extension and the pure-Ruby engine are **iterative, not recursive** โ€” they track nesting on an explicit, heap-allocated stack rather than the call stack. So deeply nested input **cannot overflow the call stack or segfault**: nesting is bounded only by available memory, the same posture as Oj (which also ships no nesting limit; the stdlib `json` caps at 100). The `deeply_nested.json` benchmark (212 MB of nesting) is handled without issue. **`generate` is iterative too**, so serializing a deeply nested Ruby structure can't overflow the stack either โ€” reading *and* writing are both depth-safe. diff --git a/docs/options.md b/docs/options.md index c85f823..5b76c28 100644 --- a/docs/options.md +++ b/docs/options.md @@ -22,6 +22,7 @@ These options are passed to [`SmarterJSON.process`](./basic_read_api.md), `Smart | `:duplicate_key` | `:last_wins` | How to handle a key that repeats within one object: `:last_wins` or `:first_wins`. (Every repeat is also reported through `:on_warning` โ€” see below.) | | `:encoding` | `nil` | Labels the input's encoding (e.g. `"UTF-8"`). It does **not** trigger a transcoding pass โ€” see below. | | `:on_warning` | `nil` | A callable invoked once per lenient fix applied, passed a `SmarterJSON::Warning`. Never changes the return value. See below. | +| `:replace_char` | `"?"` | Replacement for a character a `\uXXXX` escape decodes to that the input's encoding can't represent (e.g. an emoji inside Shift_JIS). `""` drops it. See below. | | `:symbolize_keys` | `false` | Return object keys as Symbols instead of Strings. | ```ruby @@ -47,7 +48,11 @@ The warning types are `:empty_slot` (a collapsed empty comma slot, e.g. `[1,,2]` ### A note on `:encoding` -`:encoding` labels what the input *is* โ€” it does not transcode. With the default `nil`, SmarterJSON keeps the input's own encoding tag and emits string values with that tag, the same way `smarter_csv` handles encodings โ€” **with one smart default:** input tagged `ASCII-8BIT` (BINARY) that is valid UTF-8 is treated as UTF-8. This is how `Net::HTTP` returns a `response.body`; without it, those string values would compare unequal to UTF-8 literals. `ASCII-8BIT` input that is *not* valid UTF-8 raises `SmarterJSON::EncodingError` โ€” pass an explicit `:encoding` (e.g. `"ISO-8859-1"`) for genuinely-legacy bytes. Bytes invalid for an explicitly claimed encoding also raise `SmarterJSON::EncodingError` (a kind of `SmarterJSON::ParseError`). A UTF-8 BOM is handled automatically; UTF-16 / UTF-32 input is out of scope. +`:encoding` labels what the input *is* โ€” it does not transcode. With the default `nil`, SmarterJSON keeps the input's own encoding tag and emits string values with that tag, the same way `smarter_csv` handles encodings โ€” **with one smart default:** input tagged `ASCII-8BIT` (BINARY) that is valid UTF-8 is treated as UTF-8. This is how `Net::HTTP` returns a `response.body`; without it, those string values would compare unequal to UTF-8 literals. `ASCII-8BIT` input that is *not* valid UTF-8 raises `SmarterJSON::EncodingError` โ€” pass an explicit `:encoding` (e.g. `"ISO-8859-1"`) for genuinely-legacy bytes. Bytes invalid for an explicitly claimed encoding also raise `SmarterJSON::EncodingError` (a kind of `SmarterJSON::ParseError`). A UTF-8 BOM is handled automatically. UTF-16 / UTF-32, Shift_JIS, and other CJK double-byte encodings are now supported as well: the document parses and string values come back tagged in the input's own encoding (a UTF-16 / UTF-32 BOM is consumed on the way in). The one wrinkle โ€” a `\uXXXX` escape that decodes to a character the input's encoding can't represent โ€” is handled by `:replace_char` (above). + +### A note on `:replace_char` + +For an input in an encoding that can't be byte-scanned directly (UTF-16 / UTF-32, Shift_JIS, and other CJK double-byte encodings), SmarterJSON parses a UTF-8 copy and re-tags the result back into the input's encoding, so you get values in the encoding the bytes arrived in. A `\uXXXX` escape can decode to a character that encoding can't represent โ€” e.g. an emoji inside a Shift_JIS document. Rather than raise, that single character is replaced by `:replace_char` (default `"?"`). Set `replace_char: ""` to drop it, or pass any string your target encoding can hold (e.g. the geta mark `"ใ€“"` for Shift_JIS). It applies only on this transcode-and-re-tag path; for plain UTF-8 / single-byte input it never comes into play. ### A note on `:decimal_precision` diff --git a/lib/smarter_json/options.rb b/lib/smarter_json/options.rb index a3844c1..6b709c5 100644 --- a/lib/smarter_json/options.rb +++ b/lib/smarter_json/options.rb @@ -12,6 +12,7 @@ module Options duplicate_key: :last_wins, # :last_wins | :first_wins (repeats are also reported via on_warning) decimal_precision: :auto, # :auto | :float | :bigdecimal (Oj-compatible decimal handling) on_warning: nil, # a callable invoked once per non-fatal lenient fix (a SmarterJSON::Warning) + replace_char: "?", # replacement for a char not representable in the input's encoding (undef: :replace); "" drops it }.freeze module_function @@ -56,6 +57,9 @@ def validate_options!(options) unless encoding.nil? || encoding.is_a?(String) errors << "encoding must be nil or a String (got #{encoding.class})" end + unless options[:replace_char].is_a?(String) + errors << "replace_char must be a String (got #{options[:replace_char].class})" + end raise ArgumentError, "SmarterJSON: invalid options โ€” #{errors.join('; ')}" if errors.any? diff --git a/lib/smarter_json/parser.rb b/lib/smarter_json/parser.rb index 5048632..be173ad 100644 --- a/lib/smarter_json/parser.rb +++ b/lib/smarter_json/parser.rb @@ -237,21 +237,21 @@ def to_utf8_copy(input) end # Re-tag one scalar into `enc`. A character not representable in `enc` (e.g. an emoji from a - # `\u` escape inside a Shift_JIS document) keeps its UTF-8 value โ€” lossless, never raises. - def reencode_scalar(obj, enc) + # `\u` escape inside a Shift_JIS document) is replaced by `replace` (the :replace_char option, + # default "?") โ€” uniform encoding, never raises. (`invalid:` can't trigger here: the value + # came from a valid UTF-8 parse.) + def reencode_scalar(obj, enc, replace) return obj unless obj.is_a?(String) - obj.encode(enc) - rescue Encoding::UndefinedConversionError - obj + obj.encode(enc, invalid: :replace, undef: :replace, replace: replace) end # Re-tag a parsed value's strings (Hash keys/values, Array elements, nested) into `enc`, # so we emit values in the encoding the bytes arrived in after parsing a UTF-8 copy. # ITERATIVE (an explicit work stack, not recursion) so a deeply nested document is # depth-safe โ€” like the parser itself โ€” and can't raise SystemStackError. - def deep_encode(root, enc) - return reencode_scalar(root, enc) unless root.is_a?(Array) || root.is_a?(Hash) + def deep_encode(root, enc, replace) + return reencode_scalar(root, enc, replace) unless root.is_a?(Array) || root.is_a?(Hash) out = root.is_a?(Array) ? [] : {} stack = [[root, out]] @@ -260,14 +260,14 @@ def deep_encode(root, enc) if src.is_a?(Array) src.each do |v| child = (v.is_a?(Array) ? [] : {}) if v.is_a?(Array) || v.is_a?(Hash) - dst << (child || reencode_scalar(v, enc)) + dst << (child || reencode_scalar(v, enc, replace)) stack.push([v, child]) if child end else src.each do |k, v| - key = reencode_scalar(k, enc) + key = reencode_scalar(k, enc, replace) child = (v.is_a?(Array) ? [] : {}) if v.is_a?(Array) || v.is_a?(Hash) - dst[key] = child || reencode_scalar(v, enc) + dst[key] = child || reencode_scalar(v, enc, replace) stack.push([v, child]) if child end end @@ -319,7 +319,10 @@ def warn_extra_documents(options) end end - private_class_method :process_content, :stream_io, :warn_extra_documents + private_class_method :process_content, :stream_io, :warn_extra_documents, + :file_read_mode, :normalize_default_encoding, :unscannable_enc?, + :unscannable_encoding, :concrete_unicode_encoding, :to_utf8_copy, + :reencode_scalar, :deep_encode # Named byte values, shared by the Parser FSM and the Framer / Recovery byte # scanners so none of them spell out raw hex. Included where needed. @@ -601,11 +604,12 @@ def process_string(input, options, &block) target_enc = SmarterJSON.send(:concrete_unicode_encoding, input, target_enc) # avoid per-string BOMs opts = options.merge(encoding: nil) # the working copy is UTF-8; don't re-label it downstream utf8 = SmarterJSON.send(:to_utf8_copy, input) # invalid bytes -> SmarterJSON::EncodingError + replace = options[:replace_char] if block - return process_string(utf8, opts) { |doc| block.call(SmarterJSON.send(:deep_encode, doc, target_enc)) } + return process_string(utf8, opts) { |doc| block.call(SmarterJSON.send(:deep_encode, doc, target_enc, replace)) } end - return process_string(utf8, opts).map { |doc| SmarterJSON.send(:deep_encode, doc, target_enc) } + return process_string(utf8, opts).map { |doc| SmarterJSON.send(:deep_encode, doc, target_enc, replace) } end return SmarterJSON.send(:process_content, input, options, &block) unless input.valid_encoding? diff --git a/lib/smarter_json/version.rb b/lib/smarter_json/version.rb index 20632d8..64997b2 100644 --- a/lib/smarter_json/version.rb +++ b/lib/smarter_json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SmarterJSON - VERSION = "1.2.2" + VERSION = "1.2.3" end diff --git a/spec/options_spec.rb b/spec/options_spec.rb index 4fba948..8d769c3 100644 --- a/spec/options_spec.rb +++ b/spec/options_spec.rb @@ -44,6 +44,7 @@ duplicate_key: { valid: %i[last_wins first_wins], invalid: [:raise, "last_wins", nil] }, encoding: { valid: [nil, "UTF-8", "ASCII-8BIT"], invalid: [123, :utf8] }, on_warning: { valid: [nil, ->(w) { w }], invalid: ["nope", 42] }, + replace_char: { valid: ["?", "", "_", "??"], invalid: [nil, 1, :q] }, } label = ->(v) { v.is_a?(Proc) ? "a callable" : v.inspect } diff --git a/spec/parser_spec.rb b/spec/parser_spec.rb index e2f505f..22c8cdb 100644 --- a/spec/parser_spec.rb +++ b/spec/parser_spec.rb @@ -1537,13 +1537,26 @@ # representable in the target encoding, deep nesting (depth-safety), BOM handling, # invalid bytes, denylist variants, and the file read-mode scoping. describe "encoding corner cases (1.2.3 hardening)" do - it "keeps a non-representable \\u-escaped char as UTF-8 instead of raising (Shift_JIS)" do + it "replaces a non-representable \\u-escaped char (default '?') instead of raising (Shift_JIS)" do # valid Shift_JIS input (all ASCII bytes); the escape decodes to an emoji not in Shift_JIS json = "{\"k\":\"\\uD83D\\uDE00\"}".dup.force_encoding("Shift_JIS") result = SmarterJSON.process_one(json, acceleration: acceleration) - expect(result.values.first).to eq("๐Ÿ˜€") - expect(result.values.first.encoding).to eq(Encoding::UTF_8) # not representable in SJIS -> kept UTF-8 - expect(result.keys.first.encoding).to eq(Encoding.find("Shift_JIS")) # representable key still SJIS + expect(result.values.first.encode("UTF-8")).to eq("?") # emoji -> ? (default replacement), no raise + expect(result.values.first.encoding).to eq(Encoding.find("Shift_JIS")) # uniform encoding, no mixing + end + + it "honors the replace_char: option for non-representable chars" do + json = "{\"k\":\"\\uD83D\\uDE00\"}".dup.force_encoding("Shift_JIS") + custom = SmarterJSON.process_one(json, replace_char: "_", acceleration: acceleration) + expect(custom.values.first.encode("UTF-8")).to eq("_") + expect(custom.values.first.encoding).to eq(Encoding.find("Shift_JIS")) + + dropped = SmarterJSON.process_one(json, replace_char: "", acceleration: acceleration) + expect(dropped.values.first.encode("UTF-8")).to eq("") # "" drops the char + end + + it "rejects a non-String replace_char: option" do + expect { SmarterJSON.process_one('{}', replace_char: 5, acceleration: acceleration) }.to raise_error(ArgumentError, /replace_char must be a String/) end it "re-encodes deeply nested UTF-16 without SystemStackError (depth-safe)" do From 2e439feccbd560bde9ccfdeb5b6721dff2b43e9f Mon Sep 17 00:00:00 2001 From: Tilo Sloboda Date: Sat, 27 Jun 2026 18:40:05 -0700 Subject: [PATCH 8/9] fix streaming for UTF-16 / UTF-32 / Shift_JIS (or transcoding) --- CHANGELOG.md | 5 ++- lib/smarter_json/parser.rb | 91 ++++++++++++++++++++++++++++++++++---- spec/parser_spec.rb | 62 ++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46fb820..7e5d102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,14 +13,15 @@ > โš ๏ธ We discourage the use of `process(input).first` / `process(input)[0]` because it silently drops potential additional documents > Please use `process_one` if you are expecting only one JSON doc, e.g. in API payloads, because it emits on_warning if it finds multiple docs. -## 1.2.3 (2026-06-27) +## 1.2.3 (2026-06-28) -RSpec tests: 1,167 โ†’ 1,279 +RSpec tests: 1,167 โ†’ 1,264 Fixing some encoding corner cases: - **UTF-16 / UTF-32** and **Shift_JIS** (and other CJK double-byte encodings such as Big5 / GBK / GB18030) previously raised or mis-parsed; they now parse, with string values tagged in the input's encoding. - Applies to String, file (`process_file`), and IO / streaming (`foreach`) input โ€” including a file the caller opened with transcoding (e.g. `File.open(path, "r:UTF-8:UTF-16LE")`), where the output is the encoding the bytes arrive in. - Streaming a **Latin-1 / Windows-1252** (or other single-byte) file or IO now preserves that encoding too, instead of mislabelling or raising. + - Streaming a UTF-16 / UTF-32 / Shift_JIS (or transcoding) source via `foreach` / `process_file` is now **bounded-memory** โ€” it frames and parses one document at a time instead of reading the whole input into memory. - New `:replace_char` option (default `"?"`): when a `\uXXXX` escape decodes to a character the input's encoding can't represent (e.g. an emoji inside a Shift_JIS document), that character is replaced rather than raising. `replace_char: ""` drops it. ## 1.2.2 (2026-06-19) diff --git a/lib/smarter_json/parser.rb b/lib/smarter_json/parser.rb index be173ad..07260e9 100644 --- a/lib/smarter_json/parser.rb +++ b/lib/smarter_json/parser.rb @@ -280,21 +280,24 @@ def deep_encode(root, enc, replace) def stream_io(io, options, &block) ext = io.respond_to?(:external_encoding) ? io.external_encoding : nil int = io.respond_to?(:internal_encoding) ? io.internal_encoding : nil - logical = int || ext # what io.read would hand back โ€” the encoding the bytes arrive in + out_enc = int || ext # the encoding the caller expects in the output + source = ext # the encoding readpartial's raw bytes are actually in # The Framer reads via readpartial, which returns ASCII-8BIT โ€” it drops the IO's encoding - # and ignores transcoding. When that matters (a transcoding IO, or a logical encoding we - # can't byte-scan: UTF-16 / UTF-32 / Shift_JIS / ...), read the whole stream via io.read - # (which honors both) and parse it as a unit; the whole-buffer path preserves the encoding. - if int || (logical && SmarterJSON.send(:unscannable_enc?, logical)) - return process(io.read, options, &block) + # and ignores transcoding. When the byte-scanner can't frame the raw bytes directly โ€” they + # are in an unscannable encoding (UTF-16 / UTF-32 / Shift_JIS / ...), or the IO transcodes + # (the wanted output encoding differs from the on-wire bytes) โ€” transcode each chunk to a + # UTF-8 view, frame documents there one at a time, and emit each in `out_enc`. Bounded + # memory: only one document is buffered, never the whole stream. + if source && out_enc && (unscannable_enc?(source) || out_enc != source) + return stream_transcoded(io, source, out_enc, options, &block) end count = 0 Framer.each_document(io) do |doc| # readpartial dropped the IO's encoding tag; restore it so a Latin-1 / Windows-1252 / # etc. stream is parsed and emitted in its own encoding, not mislabelled. - doc = doc.dup.force_encoding(logical) if logical && doc.encoding != logical + doc = doc.dup.force_encoding(out_enc) if out_enc && doc.encoding != out_enc # Recovery.process_string yields each value and returns how many it yielded; # blank / comment-only framed segments yield none, so count tracks actual # documents (== values yielded), not raw framed segments. @@ -303,6 +306,25 @@ def stream_io(io, options, &block) count end + # Bounded-memory streaming for an unscannable or transcoding IO (see stream_io). Each chunk + # is transcoded to a UTF-8 view and framed there one document at a time; each framed document + # is parsed and emitted in `out_enc` โ€” the same parse-then-re-tag path as the whole-buffer + # case, but per document, so peak memory is bounded by one document, not the whole stream. + def stream_transcoded(io, source, out_enc, options, &block) + first = Framer.read_chunk(io) + out_enc = concrete_unicode_encoding(first.to_s, out_enc) # generic UTF-16/32 -> concrete via BOM + # No converter when the raw bytes are already UTF-8 (e.g. a UTF-8 -> UTF-16 transcoding IO): + # the bytes need no transcoding to be byte-scanned, only the OUTPUT is re-tagged (deep_encode). + conv = [Encoding::UTF_8, Encoding::US_ASCII].include?(source) ? nil : Encoding::Converter.new(source, Encoding::UTF_8) + opts = options.merge(encoding: nil) + replace = options[:replace_char] + count = 0 + Framer.each_document_transcoded(io, conv, first) do |utf8_doc| + count += Recovery.process_string(utf8_doc, opts) { |v| block.call(deep_encode(v, out_enc, replace)) } + end + count + end + # process_one's "more than one document" notice โ€” routed to on_warning if the caller # gave one, else Rails.logger when Rails is loaded, else Kernel#warn. Never silent, # never raised. @@ -319,7 +341,7 @@ def warn_extra_documents(options) end end - private_class_method :process_content, :stream_io, :warn_extra_documents, + private_class_method :process_content, :stream_io, :stream_transcoded, :warn_extra_documents, :file_read_mode, :normalize_default_encoding, :unscannable_enc?, :unscannable_encoding, :concrete_unicode_encoding, :to_utf8_copy, :reencode_scalar, :deep_encode @@ -391,6 +413,59 @@ def each_document(io) yield buffer unless separators_only?(buffer) end + # Like each_document, but the IO's raw bytes are in `conv`'s source encoding (UTF-16 / + # UTF-32 / Shift_JIS / ...): each chunk is transcoded to a UTF-8 view and framed there, so + # the byte-level splitter works. `first_chunk` is the already-read first raw chunk (the + # caller sniffs a BOM from it). Memory stays bounded by one document, like each_document. + def each_document_transcoded(io, conv, first_chunk) + buffer = +"" + scan = 0 + doc_start = nil + stack = [] + mode = nil + + raw = first_chunk + while raw + chunk = transcode_chunk(conv, raw) + unless chunk.empty? + buffer << chunk + loop do + emitted, buffer, scan, doc_start, stack, mode = scan_buffer(buffer, scan, doc_start, stack, mode) + break unless emitted + + yield emitted + end + end + raw = read_chunk(io) + end + + finish_transcode(conv) # truncated / invalid trailing bytes -> SmarterJSON::EncodingError + + yield buffer unless separators_only?(buffer) + end + + # Push one raw chunk through the converter, returning the UTF-8 produced so far. An + # incomplete trailing multibyte sequence is held inside the converter until the next chunk; + # invalid bytes raise SmarterJSON::EncodingError (matching the whole-buffer to_utf8_copy). + def transcode_chunk(conv, raw) + return raw.dup.force_encoding(Encoding::UTF_8) if conv.nil? # raw bytes are already UTF-8 + + out = +"" + status = conv.primitive_convert(raw.dup, out, nil, nil, partial_input: true) + raise SmarterJSON::EncodingError, "invalid byte sequence in stream" if status == :invalid_byte_sequence + + out + end + + # Flush the converter at end of stream. A held incomplete multibyte sequence means the input + # was truncated mid-character โ€” surface it the same way an invalid encoding is surfaced. + def finish_transcode(conv) + return if conv.nil? + + status = conv.primitive_convert("".b, +"") + raise SmarterJSON::EncodingError, "invalid byte sequence in stream" unless status == :finished + end + def read_chunk(io) if io.respond_to?(:readpartial) io.readpartial(CHUNK_SIZE) diff --git a/spec/parser_spec.rb b/spec/parser_spec.rb index 22c8cdb..a71c60c 100644 --- a/spec/parser_spec.rb +++ b/spec/parser_spec.rb @@ -1531,6 +1531,68 @@ expect(docs.first).to eq({ "k" => "x" }) expect(docs.first.values.first.encoding).to eq(Encoding::UTF_8) end + + it "streams every document of a multi-doc UTF-16LE NDJSON source, preserving UTF-16LE" do + io = StringIO.new(%({"a":1}\n{"b":2}\n{"c":3}\n).encode("UTF-16LE")) + docs = SmarterJSON.foreach(io, acceleration: acceleration).to_a + expect(docs.map { |d| d.transform_keys { |k| k.encode("UTF-8") } }).to eq([{ "a" => 1 }, { "b" => 2 }, { "c" => 3 }]) + expect(docs.all? { |d| d.keys.first.encoding == Encoding.find("UTF-16LE") }).to be true + end + + it "streams a UTF-16LE source in BOUNDED memory โ€” does not read the whole input" do + # > CHUNK_SIZE (16 KiB) of UTF-16LE NDJSON, so a real chunked read is exercised + big = StringIO.new(%({"n":1}\n).encode("UTF-16LE") * 6_000) + first = SmarterJSON.foreach(big, acceleration: acceleration).first(1) + expect(first.first.values.first).to eq(1) + expect(first.first.keys.first.encoding).to eq(Encoding.find("UTF-16LE")) + expect(big.pos).to be < big.size # stopped after ~one chunk, didn't slurp the whole stream + end + + # Every encoding the bounded-streaming path is meant to handle, each streamed through + # foreach(IO) and checked for correct value + preserved encoding. + { + "UTF-16LE" => "x", "UTF-16BE" => "x", "UTF-32LE" => "x", "UTF-32BE" => "x", + "Shift_JIS" => "ใ‚ฝ", "Windows-31J" => "ใ‚ฝ", "Big5" => "x", "GBK" => "x" + }.each do |enc_name, ch| + it "streams a #{enc_name} source via foreach(IO), preserving #{enc_name}" do + io = StringIO.new(%({"k":"#{ch}"}).encode(enc_name)) + doc = SmarterJSON.foreach(io, acceleration: acceleration).to_a.first + expect(doc.values.first.encode("UTF-8")).to eq(ch) + expect(doc.values.first.encoding).to eq(Encoding.find(enc_name)) + end + end + + # Generic UTF-16 / UTF-32 (BOM-tagged) โ€” output is BOM-free concrete endianness. + { "UTF-16" => [Encoding::UTF_16LE, Encoding::UTF_16BE], "UTF-32" => [Encoding::UTF_32LE, Encoding::UTF_32BE] }.each do |enc_name, concretes| + it "streams a generic #{enc_name} (BOM) source, emitting BOM-free concrete output" do + io = StringIO.new(%({"k":"x"}).encode(enc_name)) # #encode auto-prepends a BOM + doc = SmarterJSON.foreach(io, acceleration: acceleration).to_a.first + expect(doc.values.first.encode("UTF-8")).to eq("x") + expect(concretes).to include(doc.values.first.encoding) + expect(doc.values.first.bytes.first(2)).not_to eq([0xFE, 0xFF]) # no BOM in the value + end + end + + # NEGATIVE: byte-scannable encodings must NOT route through the transcoding path. + it "marks only the right encodings as needing the transcoding path (routing predicate)" do + %w[UTF-8 US-ASCII ISO-8859-1 Windows-1252 EUC-JP].each do |e| + expect(SmarterJSON.send(:unscannable_enc?, Encoding.find(e))).to be(false), "#{e} should be byte-scannable" + end + %w[UTF-16LE UTF-16BE UTF-32LE UTF-32BE UTF-16 UTF-32 Shift_JIS Windows-31J Big5 GBK GB18030].each do |e| + expect(SmarterJSON.send(:unscannable_enc?, Encoding.find(e))).to be(true), "#{e} should need transcoding" + end + end + + it "does NOT invoke stream_transcoded for a scannable stream (UTF-8 / Latin-1)" do + expect(SmarterJSON).not_to receive(:stream_transcoded) + SmarterJSON.foreach(StringIO.new('{"k":"x"}'), acceleration: acceleration).to_a + SmarterJSON.foreach(StringIO.new("{\"k\":\"caf\xE9\"}".b.force_encoding("ISO-8859-1")), acceleration: acceleration).to_a + end + + it "DOES invoke stream_transcoded for an unscannable stream (UTF-16LE)" do + expect(SmarterJSON).to receive(:stream_transcoded).and_call_original + SmarterJSON.foreach(StringIO.new('{"k":"x"}'.encode("UTF-16LE")), acceleration: acceleration).to_a + end end # 1.2.3 hardening โ€” corner cases of the transcode-and-re-tag path: a char not From 69e8bed802ca7c4b9a042b94ba97c584f812cfc7 Mon Sep 17 00:00:00 2001 From: Tilo Sloboda Date: Sun, 28 Jun 2026 00:18:01 -0700 Subject: [PATCH 9/9] adding tests --- CHANGELOG.md | 2 +- spec/parser_spec.rb | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e5d102..e5189d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ ## 1.2.3 (2026-06-28) -RSpec tests: 1,167 โ†’ 1,264 +RSpec tests: 1,167 โ†’ 1,268 Fixing some encoding corner cases: - **UTF-16 / UTF-32** and **Shift_JIS** (and other CJK double-byte encodings such as Big5 / GBK / GB18030) previously raised or mis-parsed; they now parse, with string values tagged in the input's encoding. diff --git a/spec/parser_spec.rb b/spec/parser_spec.rb index a71c60c..474852e 100644 --- a/spec/parser_spec.rb +++ b/spec/parser_spec.rb @@ -1405,6 +1405,31 @@ expect(result.values.first.encode("UTF-8")).to eq("่กจ") expect(result.values.first.encoding).to eq(input.encoding) end + + # process(unscannable_string) WITH A BLOCK takes the block branch of + # Recovery.process_string (parser.rb:684): each parsed document is deep_encode'd + # back to the input encoding and yielded. process_one / process WITHOUT a block + # take the .map branch (line 687) instead, so this is the only call that reaches 684. + it "yields each document in the input encoding when process is given a block (UTF-16LE)" do + input = %({"a":1}\n{"b":"x"}).encode("UTF-16LE") + yielded = [] + SmarterJSON.process(input, acceleration: acceleration) { |doc| yielded << doc } + decoded = yielded.map do |doc| + doc.each_with_object({}) { |(k, v), h| h[k.encode("UTF-8")] = v.is_a?(String) ? v.encode("UTF-8") : v } + end + expect(decoded).to eq([{ "a" => 1 }, { "b" => "x" }]) + expect(yielded.first.keys.first.encoding).to eq(input.encoding) + expect(yielded.last.values.first.encoding).to eq(input.encoding) + end + + it "yields each document in the input encoding when process is given a block (Shift_JIS, ใ‚ฝ = 83 5C)" do + input = '{"k":"ใ‚ฝ"}'.encode("Shift_JIS") + yielded = [] + SmarterJSON.process(input, acceleration: acceleration) { |doc| yielded << doc } + expect(yielded.length).to eq(1) + expect(yielded.first.values.first.encode("UTF-8")).to eq("ใ‚ฝ") + expect(yielded.first.values.first.encoding).to eq(input.encoding) + end end # File input: parse correctly and PRESERVE whatever encoding the bytes arrive in.