diff --git a/CHANGELOG.md b/CHANGELOG.md index c56e541..9b41b9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,14 @@ > ⚠️ We discourage the use of `process(input).first` / `process(input)[0]` because it silently drops potential additional documents > Please use `process_one` if you are expecting only one JSON doc, e.g. in API payloads, because it emits on_warning if it finds multiple docs. +## 1.2.6 (2026-07-02) + +RSpec tests: 1,308 → 1,325 + +### Bug Fixes + + - **The `:number_overflow` warning now reports the sign of the collapsed value.** A negative number literal beyond Float range (e.g. `-1e400`) collapses to `-Infinity`, but the warning said "collapsed to Infinity" — always on the C path, and for top-level numbers on the pure-Ruby path. Both paths now report the actual collapsed value (`Infinity` or `-Infinity`) in every position, and the warning-parity suite locks them together. + ## 1.2.5 (2026-07-01) RSpec tests: 1,282 → 1,308 diff --git a/ext/smarter_json/smarter_json.c b/ext/smarter_json/smarter_json.c index ec11695..7a11da2 100644 --- a/ext/smarter_json/smarter_json.c +++ b/ext/smarter_json/smarter_json.c @@ -649,7 +649,11 @@ static FJ_ALWAYS_INLINE VALUE fj_float_from_parts(fj_state *st, uint64_t m10, in still returned). The Infinity/NaN keywords take separate paths and never get here. Gate isinf on a listening handler (matches the Ruby float_or_warn): no handler -> no point detecting, and it keeps the test off the hot number path. */ - if (st->on_warning != Qnil && isinf(d)) fj_warn(st, fj_sym_number_overflow, "number literal out of Float range — collapsed to Infinity"); + if (st->on_warning != Qnil && isinf(d)) { + fj_warn(st, fj_sym_number_overflow, + d > 0 ? "number literal out of Float range — collapsed to Infinity" + : "number literal out of Float range — collapsed to -Infinity"); + } return rb_float_new(d); } diff --git a/lib/smarter_json/parser.rb b/lib/smarter_json/parser.rb index bbb8d77..67d5245 100644 --- a/lib/smarter_json/parser.rb +++ b/lib/smarter_json/parser.rb @@ -1925,6 +1925,7 @@ def decode_unicode_escape(i) # --- numbers (top-level / strict positions) --- def parse_number + token_start = @pos negative = false signed = false if byte == MINUS @@ -1945,7 +1946,6 @@ def parse_number return Float::NAN end - int_start = @pos had_leading_zero = false if byte == ZERO @@ -2001,9 +2001,13 @@ def parse_number raise error("invalid number with a leading zero") end - slice = @input.byteslice(int_start, @pos - int_start).delete("_") - value = is_float ? decimal_value(slice) : slice.to_i - negative ? -value : value + # token_start sits BEFORE the sign, so the slice keeps it ("-1e400", not "1e400"): + # decimal_value -> float_or_warn sees the signed token, and a negative overflow + # warns "collapsed to -Infinity" — the same warning numeric_value produces for a + # member-position number. String#to_i, String#to_f, and BigDecimal() all parse + # the leading sign, so there is no negation step here. + slice = @input.byteslice(token_start, @pos - token_start).delete("_") + is_float ? decimal_value(slice) : slice.to_i end def hex_digit?(b) diff --git a/lib/smarter_json/version.rb b/lib/smarter_json/version.rb index 221e032..809d646 100644 --- a/lib/smarter_json/version.rb +++ b/lib/smarter_json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SmarterJSON - VERSION = "1.2.5" + VERSION = "1.2.6" end diff --git a/spec/coverage_spec.rb b/spec/coverage_spec.rb index 7c36b40..dbfa775 100644 --- a/spec/coverage_spec.rb +++ b/spec/coverage_spec.rb @@ -53,6 +53,38 @@ def read(length = nil) it "treats a trailing block comment after the last document as separators only" do expect(stream(%({"a":1}\n/* trailing */\n))).to eq([{ "a" => 1 }]) end + + # readpartial returning ONE byte per call forces every multi-byte marker + # (''' // /* */) to straddle a read boundary — the worst case for + # Framer#defer_for_split_marker?, which must pause the scan until the whole + # marker has arrived instead of guessing from its lead byte. + def one_byte_at_a_time(string) + Class.new do + def initialize(string) + @io = StringIO.new(string) + end + + def read(length = nil) + @io.read(length) + end + + def readpartial(_max_len) + @io.readpartial(1) + end + end.new(string) + end + + it "frames multi-byte markers that straddle read boundaries (1-byte chunks)" do + content = %({"a": '''text with } and { braces''', // comment with }\n"b": /* block } */ 2}\n{"c": 3}) + expect(stream(one_byte_at_a_time(content))) + .to eq([{ "a" => "text with } and { braces", "b" => 2 }, { "c" => 3 }]) + end + + it "does not treat '#' or '//' inside string values as comment starts when framing (1-byte chunks)" do + content = %({"url": "http://x/#a", "note": "semi // not comment"}\n# real comment\n{"d": 4}) + expect(stream(one_byte_at_a_time(content))) + .to eq([{ "url" => "http://x/#a", "note" => "semi // not comment" }, { "d" => 4 }]) + end end describe "wrapper-recovery byte FSM" do diff --git a/spec/foreach_spec.rb b/spec/foreach_spec.rb index 82bd6bc..c3bd735 100644 --- a/spec/foreach_spec.rb +++ b/spec/foreach_spec.rb @@ -137,6 +137,16 @@ expect(enum.to_a).to eq([{ "id" => 1 }, { "id" => 2 }]) # first pass drains the IO expect(enum.to_a).to eq([]) # IO now at EOF — nothing left end + + it "streams scalar and mixed NDJSON documents from an IO (non-container documents)" do + io = StringIO.new(%(1\n{"a":1}\ntrue\n"str"\n)) + expect(SmarterJSON.foreach(io, acceleration: acceleration).to_a).to eq([1, { "a" => 1 }, true, "str"]) + end + + it "streams comma-separated concatenated documents from an IO" do + io = StringIO.new(%({"a":1},{"b":2})) + expect(SmarterJSON.foreach(io, acceleration: acceleration).to_a).to eq([{ "a" => 1 }, { "b" => 2 }]) + end end end end diff --git a/spec/generator_spec.rb b/spec/generator_spec.rb index 96662bc..f35874e 100644 --- a/spec/generator_spec.rb +++ b/spec/generator_spec.rb @@ -364,6 +364,32 @@ def inner.as_json(*) end end + # The gem-wide contract — respect and PRESERVE the input String's encoding, never + # transcode — holds on the way OUT too: a value keeps its own encoding tag in the + # generated JSON, so a non-UTF-8 document round-trips unchanged. + describe "string value encodings (preserved, never transcoded)" do + it "emits a non-UTF-8 value in its own encoding (Latin-1 in, Latin-1 out)" do + latin1 = "caf\xE9".dup.force_encoding(Encoding::ISO_8859_1) + out = SmarterJSON.generate({ "k" => latin1 }) + expect(out).to eq(%({"k":"caf\xE9"}).dup.force_encoding(Encoding::ISO_8859_1)) + expect(out.encoding).to eq(Encoding::ISO_8859_1) + end + + it "round-trips a Latin-1 document: process -> generate -> process" do + input = %({"k": "caf\xE9"}).dup.force_encoding(Encoding::ISO_8859_1) + value = SmarterJSON.process_one(input) + expect(value["k"].encoding).to eq(Encoding::ISO_8859_1) + regenerated = SmarterJSON.generate(value) + expect(SmarterJSON.process_one(regenerated)).to eq(value) + expect(SmarterJSON.process_one(regenerated)["k"].encoding).to eq(Encoding::ISO_8859_1) + end + + it "ascii_only: true escapes a non-UTF-8 value into pure ASCII (safe to embed anywhere)" do + latin1 = "caf\xE9".dup.force_encoding(Encoding::ISO_8859_1) + expect(SmarterJSON.generate(latin1, ascii_only: true)).to eq("\"caf\\u00e9\"") + end + end + # The parser is iterative (handles 212 MB of nesting); the generator must match — # deeply nested Ruby structures must not blow the call stack with SystemStackError. describe "deep nesting (iterative generator — no stack overflow)" do diff --git a/spec/parser_spec.rb b/spec/parser_spec.rb index 474852e..a90da0d 100644 --- a/spec/parser_spec.rb +++ b/spec/parser_spec.rb @@ -70,6 +70,18 @@ expect(SmarterJSON.process("1e500", acceleration: acceleration)).to eq([Float::INFINITY]) expect(SmarterJSON.process_one("1e500", acceleration: acceleration)).to eq(Float::INFINITY) end + + it "parses integers at and just past the 64-bit boundary exactly (Integer, not Float)" do + expect(SmarterJSON.process("9223372036854775807", acceleration: acceleration)).to eq([9_223_372_036_854_775_807]) + expect(SmarterJSON.process_one("-9223372036854775808", acceleration: acceleration)).to eq(-9_223_372_036_854_775_808) + expect(SmarterJSON.process_one("9223372036854775808", acceleration: acceleration)).to eq(9_223_372_036_854_775_808) + expect(SmarterJSON.process_one("18446744073709551616", acceleration: acceleration)).to eq(18_446_744_073_709_551_616) + end + + it "parses very long integers exactly in top-level and member position (no precision loss)" do + expect(SmarterJSON.process_one("99999999999999999999999999", acceleration: acceleration)).to eq(99_999_999_999_999_999_999_999_999) + expect(SmarterJSON.process_one('{"n": [99999999999999999999999999]}', acceleration: acceleration)).to eq({ "n" => [99_999_999_999_999_999_999_999_999] }) + end end describe "strings" do @@ -300,6 +312,11 @@ expect(SmarterJSON.process("-0x10", acceleration: acceleration)).to eq([-16]) expect(SmarterJSON.process_one("-0x10", acceleration: acceleration)).to eq(-16) end + + it "parses hex numbers beyond 64 bits exactly (Bignum)" do + expect(SmarterJSON.process("0xFFFFFFFFFFFFFFFFFF", acceleration: acceleration)).to eq([0xFFFFFFFFFFFFFFFFFF]) + expect(SmarterJSON.process_one("0xFFFFFFFFFFFFFFFFFF", acceleration: acceleration)).to eq(0xFFFFFFFFFFFFFFFFFF) + end end describe "leading/trailing decimal points" do diff --git a/spec/warning_parity_spec.rb b/spec/warning_parity_spec.rb index 463ad17..7ba6217 100644 --- a/spec/warning_parity_spec.rb +++ b/spec/warning_parity_spec.rb @@ -79,6 +79,8 @@ def first_warning(input, **opts) "duplicate_key" => [:process, '{"a":1,"a":2}'], "duplicate_key (multiline)" => [:process, "{\n \"a\": 1,\n \"a\": 2\n}"], "number_overflow" => [:process, "1e400"], + "number_overflow (negative)" => [:process, "-1e400"], + "number_overflow (negative, member position)" => [:process, "[-1e400]"], "code_fence_stripped" => [:process, "```json\n{\"a\":1}\n```"], "prefix_text_ignored" => [:process, 'Here is the json: {"a":1}'], "suffix_text_ignored" => [:process, '{"a":1} thanks!'],