Skip to content

perf(json): recover UTF-16 parser hot paths #1000

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
frostney merged 2 commits into from
Jul 22, 2026
+84 −23
Merged

perf(json): recover UTF-16 parser hot paths #1000

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3b8597d
perf(json): fast-path unescaped string parsing
frostney Jul 21, 2026
7ca9df3
perf(json): avoid Unicode set conversion in number parsing
frostney Jul 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions source/shared/JSONParser.Test.pas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ TJSONParserTests = class(TTestSuite)
procedure TestParseStringBackslashEscape;
procedure TestParseStringForwardSlashEscape;
procedure TestParseStringQuoteEscape;
procedure TestParseStringUnicodeText;
procedure TestParseStringUnicodeEscape;
procedure TestParseStringSurrogatePair;
procedure TestParseEmptyString;
Expand All @@ -71,6 +72,7 @@ TJSONParserTests = class(TTestSuite)
// Error cases: strict mode
procedure TestErrorEmptyInput;
procedure TestErrorTrailingGarbage;
procedure TestErrorUnescapedControlCharacter;
procedure TestErrorUnterminatedString;
procedure TestErrorUnterminatedObject;
procedure TestErrorUnterminatedArray;
Expand Down Expand Up @@ -205,6 +207,7 @@ procedure TJSONParserTests.SetupTests;
Test('Parse string with backslash escape', TestParseStringBackslashEscape);
Test('Parse string with forward slash escape', TestParseStringForwardSlashEscape);
Test('Parse string with quote escape', TestParseStringQuoteEscape);
Test('Parse unescaped Unicode string text', TestParseStringUnicodeText);
Test('Parse string with unicode escape \u0041', TestParseStringUnicodeEscape);
Test('Parse string with surrogate pair', TestParseStringSurrogatePair);
Test('Parse empty string', TestParseEmptyString);
Expand All @@ -221,6 +224,7 @@ procedure TJSONParserTests.SetupTests;
// Error cases: strict mode
Test('Error on empty input', TestErrorEmptyInput);
Test('Error on trailing garbage', TestErrorTrailingGarbage);
Test('Error on unescaped control character', TestErrorUnescapedControlCharacter);
Test('Error on unterminated string', TestErrorUnterminatedString);
Test('Error on unterminated object', TestErrorUnterminatedObject);
Test('Error on unterminated array', TestErrorUnterminatedArray);
Expand Down Expand Up @@ -490,6 +494,22 @@ procedure TJSONParserTests.TestParseStringQuoteEscape;
end;
end;

procedure TJSONParserTests.TestParseStringUnicodeText;
var
Expected: string;
P: TRecordingJSONParser;
begin
Expected := #$03BB#$D83D#$DE00;
P := TRecordingJSONParser.Create;
try
P.Parse('"' + Expected + '"');
Expect<Integer>(P.Events.Count).ToBe(1);
Expect<string>(P.Events[0]).ToBe('string:' + Expected);
finally
P.Free;
end;
end;

procedure TJSONParserTests.TestParseStringUnicodeEscape;
var
P: TRecordingJSONParser;
Expand Down Expand Up @@ -711,6 +731,26 @@ procedure TJSONParserTests.TestErrorTrailingGarbage;
end;
end;

procedure TJSONParserTests.TestErrorUnescapedControlCharacter;
var
P: TRecordingJSONParser;
Raised: Boolean;
begin
P := TRecordingJSONParser.Create;
try
Raised := False;
try
P.Parse('"a' + #1 + 'b"');
except
on E: EJSONParseError do
Raised := True;
end;
Expect<Boolean>(Raised).ToBe(True);
finally
P.Free;
end;
end;

procedure TJSONParserTests.TestErrorUnterminatedString;
var
P: TRecordingJSONParser;
Expand Down
67 changes: 44 additions & 23 deletions source/shared/JSONParser.pas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ implementation

uses
NumericText,
StringBuffer,
TextEncoding,
TextSemantics;

Expand Down Expand Up @@ -176,7 +177,8 @@ class function TAbstractJSONParser.HexDigitValue(const AChar: Char): Integer;

class function TAbstractJSONParser.IsASCIIDigit(const AChar: Char): Boolean;
begin
Result := AChar in ['0'..'9'];
// FPC 3.2.2 lowers WideChar set membership through an ANSI conversion.
Result := (AChar >= '0') and (AChar <= '9');
end;

class function TAbstractJSONParser.IsASCIIHexDigit(const AChar: Char): Boolean;
Expand Down Expand Up @@ -514,7 +516,8 @@ procedure TAbstractJSONParser.DoParseNumber;
DecimalPointSeen := False;
ExponentSeen := False;

if PeekChar in ['-', '+'] then
// Keep ASCII WideChar checks direct for the same reason as IsASCIIDigit.
if (PeekChar = '-') or (PeekChar = '+') then
begin
SignText := ReadChar;
if (SignText = '+') and not Supports(jpcAllowLeadingPlusSign) then
Expand Down Expand Up @@ -546,7 +549,7 @@ procedure TAbstractJSONParser.DoParseNumber;
if not IsAtEnd and IsASCIIDigit(PeekChar) then
RaiseParseError('Invalid number format');
end
else if not IsAtEnd and (PeekChar in ['1'..'9']) then
else if not IsAtEnd and (PeekChar >= '1') and (PeekChar <= '9') then
begin
while not IsAtEnd and IsASCIIDigit(PeekChar) do
NumStr := NumStr + ReadChar;
Expand All @@ -568,11 +571,11 @@ procedure TAbstractJSONParser.DoParseNumber;
NumStr := NumStr + ReadChar;
end;

if not IsAtEnd and (PeekChar in ['e', 'E']) then
if not IsAtEnd and ((PeekChar = 'e') or (PeekChar = 'E')) then
begin
ExponentSeen := True;
NumStr := NumStr + ReadChar;
if not IsAtEnd and (PeekChar in ['+', '-']) then
if not IsAtEnd and ((PeekChar = '+') or (PeekChar = '-')) then
NumStr := NumStr + ReadChar;
if IsAtEnd or not IsASCIIDigit(PeekChar) then
RaiseParseError('Invalid number format in exponent');
Expand Down Expand Up @@ -694,18 +697,36 @@ function TAbstractJSONParser.TryParseStringContinuation: Boolean;

function TAbstractJSONParser.ParseString(const AQuote: Char): string;
var
Text: string;
Buffer: TStringBuffer;
Ch: Char;
StartPosition: Integer;
begin
ExpectChar(AQuote);
Text := '';

StartPosition := FPosition;
while FPosition <= FLength do
begin
Ch := FText[FPosition];
if Ch = AQuote then
begin
Result := Copy(FText, StartPosition, FPosition - StartPosition);
Inc(FPosition);
Exit;
end;
if (Ch = '\') or (Ord(Ch) < 32) then
Break;
Inc(FPosition);
end;
FPosition := StartPosition;

Buffer := TStringBuffer.Create;

while not IsAtEnd do
begin
Ch := ReadChar;
if Ch = AQuote then
begin
Result := Text;
Result := Buffer.ToString;
Exit;
end;

Expand All @@ -720,36 +741,36 @@ function TAbstractJSONParser.ParseString(const AQuote: Char): string;
Ch := ReadChar;
case Ch of
'"':
Text := Text + '"';
Buffer.AppendChar('"');
'''':
if Supports(jpcAllowExtendedStringEscapes) or (AQuote = '''') then
Text := Text + ''''
Buffer.AppendChar('''')
else
RaiseParseError('Invalid escape character: \' + Ch);
'\':
Text := Text + '\';
Buffer.AppendChar('\');
'/':
Text := Text + '/';
Buffer.AppendChar('/');
'b':
Text := Text + #8;
Buffer.AppendChar(#8);
'f':
Text := Text + #12;
Buffer.AppendChar(#12);
'n':
Text := Text + #10;
Buffer.AppendChar(#10);
'r':
Text := Text + #13;
Buffer.AppendChar(#13);
't':
Text := Text + #9;
Buffer.AppendChar(#9);
'u':
Text := Text + ParseUnicodeEscape;
Buffer.Append(ParseUnicodeEscape);
'v':
if Supports(jpcAllowExtendedStringEscapes) then
Text := Text + #11
Buffer.AppendChar(#11)
else
RaiseParseError('Invalid escape character: \' + Ch);
'x':
if Supports(jpcAllowExtendedStringEscapes) then
Text := Text + ParseHexEscape(2)
Buffer.Append(ParseHexEscape(2))
else
RaiseParseError('Invalid escape character: \' + Ch);
'0':
Expand All @@ -758,7 +779,7 @@ function TAbstractJSONParser.ParseString(const AQuote: Char): string;
begin
if not IsAtEnd and IsASCIIDigit(PeekChar) then
RaiseParseError('Invalid escape character: \' + PeekChar);
Text := Text + #0;
Buffer.AppendChar(#0);
end
else
RaiseParseError('Invalid escape character: \' + Ch);
Expand All @@ -770,7 +791,7 @@ function TAbstractJSONParser.ParseString(const AQuote: Char): string;
RaiseParseError('Invalid escape character: \' + Ch);
else
if Supports(jpcAllowExtendedStringEscapes) then
Text := Text + Ch
Buffer.AppendChar(Ch)
else
RaiseParseError('Invalid escape character: \' + Ch);
end;
Expand All @@ -780,7 +801,7 @@ function TAbstractJSONParser.ParseString(const AQuote: Char): string;
else if Ord(Ch) < 32 then
RaiseParseError('Unescaped control character in string')
else
Text := Text + Ch;
Buffer.AppendChar(Ch);
end;

RaiseParseError('Unterminated string');
Expand Down
Loading