From b6688433adc78b90328794b21e91e5ce0eeaf457 Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Fri, 13 Feb 2026 14:19:15 +0100 Subject: [PATCH 01/17] LL: add class InvalidEncodingException and flag VALIDATE_UTF8_ENCODING to call validateUTF8Encoding() function. --- .../JsonLint/InvalidEncodingException.php | 47 ++++++ src/Seld/JsonLint/JsonParser.php | 134 ++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 src/Seld/JsonLint/InvalidEncodingException.php diff --git a/src/Seld/JsonLint/InvalidEncodingException.php b/src/Seld/JsonLint/InvalidEncodingException.php new file mode 100644 index 0000000..e07e4ea --- /dev/null +++ b/src/Seld/JsonLint/InvalidEncodingException.php @@ -0,0 +1,47 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Seld\JsonLint; + +class InvalidEncodingException extends ParsingException +{ + /** + * @var array{key: string, line: int} + */ + protected $details; + + /** + * @param string $message + * @param string $key + * @phpstan-param array{line: int} $details + */ + public function __construct($message, $key, array $details) + { + $details['key'] = $key; + parent::__construct($message, $details); + } + + /** + * @return string + */ + public function getKey() + { + return $this->details['key']; + } + + /** + * @phpstan-return array{key: string, line: int} + */ + public function getDetails() + { + return $this->details; + } +} diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index cec4a87..f097217 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -32,6 +32,7 @@ class JsonParser const PARSE_TO_ASSOC = 4; const ALLOW_COMMENTS = 8; const ALLOW_DUPLICATE_KEYS_TO_ARRAY = 16; + const VALIDATE_UTF8_ENCODING = 32; /** @var Lexer */ private $lexer; @@ -184,6 +185,13 @@ class JsonParser */ public function lint($input, $flags = 0) { + if ($flags & self::VALIDATE_UTF8_ENCODING) { + try { + $this->validateUTF8Encoding($input); + } catch (InvalidEncodingException $e) { + return $e; + } + } try { $this->parse($input, $flags); } catch (ParsingException $e) { @@ -605,4 +613,130 @@ private function failOnBOM($input) $this->parseError("BOM detected, make sure your input does not include a Unicode Byte-Order-Mark"); } } + + /** + * @param string $input + * @return void + */ + private function validateUTF8Encoding($input) + { + $iContinuationOctetNeeded = 0; + $iCharacterStartPosition = 0; + $iCurrentLineNumber = 1; + $iOffsetInOctetsFromLineStart = -1; + $iOffsetInCharactersFromLineStart = -1; + for ($i = 0, $iMax = strlen($input); $i < $iMax; ++$i) { + $iCurrentOctet = ord($input[$i]); + $iOffsetInOctetsFromLineStart += 1; + if ($iContinuationOctetNeeded > 0) { + if ($iCurrentOctet < 128 || $iCurrentOctet >= 192) { + throw new InvalidEncodingException( + "Non-UTF8 character found on line " + .$iCurrentLineNumber + ."; the octet " + .($iOffsetInOctetssFromLineStart + 1) + .", part of the character " + .($iOffsetInCharactersFromLineStart + 1) + .", has value " + .$iCurrentOctet + ." which is not a continuation octet." + ." (Sequential positions in octets without line splitting:" + ." character start position " + .$iCharacterStartPosition + .", octet position " + .$i + .")", + $iCurrentOctet, + array( + 'line' => $iCurrentLineNumber, + 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, + 'octet_position' => $i, + 'character_start_position' => $iCharacterStartPosition, + ) + ); + } + --$iContinuationOctetNeeded; + continue; + } + + $iCharacterStartPosition = $i; + $iOffsetInCharactersFromLineStart += 1; + if ($iCurrentOctet < 128) { // 0xxxxxxx ASCII + if ($input[$i] === "\n") { + $iCurrentLineNumber += 1; + $iOffsetInOctetsFromLineStart = -1; + $iOffsetInCharactersFromLineStart = -1; + } + continue; + } + if ($iCurrentOctet >= 128 && $iCurrentOctet < 192) { + throw new InvalidEncodingException( + "Non-UTF8 character found on line " + .$iCurrentLineNumber + ."; the octet " + .($iOffsetInOctetssFromLineStart + 1) + .", part of the character " + .($iOffsetInCharactersFromLineStart + 1) + .", has value " + .$iCurrentOctet + ." which is a continuation octet." + ." (Sequential positions in octets without line splitting:" + ." character start position " + .$iCharacterStartPosition + .", octet position " + .$i + .")", + $iCurrentOctet, + array( + 'line' => $iCurrentLineNumber, + 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, + 'octet_position' => $i, + 'character_start_position' => $iCharacterStartPosition, + ) + ); + } + if ($iCurrentOctet >= 192 && $iCurrentOctet < 224) { + // 110xxxxx 10xxxxxx + $iContinuationOctetNeeded = 1; + continue; + } + if ($iCurrentOctet >= 224 && $iCurrentOctet < 240) { + // 1110xxxx 10xxxxxx 10xxxxxx + $iContinuationOctetNeeded = 2; + continue; + } + if ($iCurrentOctet >= 240 && $iCurrentOctet < 248) { + // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + $iContinuationOctetNeeded = 3; + continue; + } + throw new InvalidEncodingException( + "Non-UTF8 character found on line " + .$iCurrentLineNumber + ."; the octet " + .($iOffsetInOctetssFromLineStart + 1) + .", part of the character " + .($iOffsetInCharactersFromLineStart + 1) + .", has value " + .$iCurrentOctet + ." which is invalid." + ." (Sequential positions in octets without line splitting:" + ." character start position " + .$iCharacterStartPosition + .", octet position " + .$i + .")", + $iCurrentOctet, + array( + 'line' => $iCurrentLineNumber, + 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, + 'octet_position' => $i, + 'character_start_position' => $iCharacterStartPosition, + ) + ); + } + } } From dd6e82e20cf10f28dbb0e343fef8ad0aad3b8e7d Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Wed, 18 Mar 2026 13:30:29 +0100 Subject: [PATCH 02/17] JsonParser.php correction Octetss -> Octets --- src/Seld/JsonLint/JsonParser.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index f097217..4bc74d0 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -634,7 +634,7 @@ private function validateUTF8Encoding($input) "Non-UTF8 character found on line " .$iCurrentLineNumber ."; the octet " - .($iOffsetInOctetssFromLineStart + 1) + .($iOffsetInOctetsFromLineStart + 1) .", part of the character " .($iOffsetInCharactersFromLineStart + 1) .", has value " @@ -675,7 +675,7 @@ private function validateUTF8Encoding($input) "Non-UTF8 character found on line " .$iCurrentLineNumber ."; the octet " - .($iOffsetInOctetssFromLineStart + 1) + .($iOffsetInOctetsFromLineStart + 1) .", part of the character " .($iOffsetInCharactersFromLineStart + 1) .", has value " @@ -716,7 +716,7 @@ private function validateUTF8Encoding($input) "Non-UTF8 character found on line " .$iCurrentLineNumber ."; the octet " - .($iOffsetInOctetssFromLineStart + 1) + .($iOffsetInOctetsFromLineStart + 1) .", part of the character " .($iOffsetInCharactersFromLineStart + 1) .", has value " From a9c2a52104210aced6a8db7b58042ec0705cebf4 Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Thu, 4 Jun 2026 19:39:26 +0200 Subject: [PATCH 03/17] Keep same copyright as other files, and add author tag in PHPDoc-string for the class InvalidEncodingException. --- src/Seld/JsonLint/InvalidEncodingException.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Seld/JsonLint/InvalidEncodingException.php b/src/Seld/JsonLint/InvalidEncodingException.php index e07e4ea..ee5e7d5 100644 --- a/src/Seld/JsonLint/InvalidEncodingException.php +++ b/src/Seld/JsonLint/InvalidEncodingException.php @@ -3,7 +3,7 @@ /* * This file is part of the JSON Lint package. * - * (c) Laurent Lyaudet + * (c) Jordi Boggiano * * For the full copyright and license information, please view the LICENSE * file that was distributed with this source code. @@ -11,6 +11,9 @@ namespace Seld\JsonLint; +/** + * @author Laurent Lyaudet + */ class InvalidEncodingException extends ParsingException { /** From b62f4a05a6e5e3bf74ede17e2baf24eb1d78211c Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Sat, 6 Jun 2026 19:12:13 +0200 Subject: [PATCH 04/17] add corrections to warnings detected by phpstan. --- src/Seld/JsonLint/JsonParser.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index 4bc74d0..0cbd134 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -646,7 +646,7 @@ private function validateUTF8Encoding($input) .", octet position " .$i .")", - $iCurrentOctet, + (string) $iCurrentOctet, array( 'line' => $iCurrentLineNumber, 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, @@ -670,7 +670,7 @@ private function validateUTF8Encoding($input) } continue; } - if ($iCurrentOctet >= 128 && $iCurrentOctet < 192) { + if (/*$iCurrentOctet >= 128 &&*/$iCurrentOctet < 192) { throw new InvalidEncodingException( "Non-UTF8 character found on line " .$iCurrentLineNumber @@ -687,7 +687,7 @@ private function validateUTF8Encoding($input) .", octet position " .$i .")", - $iCurrentOctet, + (string) $iCurrentOctet, array( 'line' => $iCurrentLineNumber, 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, @@ -697,17 +697,17 @@ private function validateUTF8Encoding($input) ) ); } - if ($iCurrentOctet >= 192 && $iCurrentOctet < 224) { + if (/*$iCurrentOctet >= 192 &&*/$iCurrentOctet < 224) { // 110xxxxx 10xxxxxx $iContinuationOctetNeeded = 1; continue; } - if ($iCurrentOctet >= 224 && $iCurrentOctet < 240) { + if (/*$iCurrentOctet >= 224 &&*/$iCurrentOctet < 240) { // 1110xxxx 10xxxxxx 10xxxxxx $iContinuationOctetNeeded = 2; continue; } - if ($iCurrentOctet >= 240 && $iCurrentOctet < 248) { + if (/*$iCurrentOctet >= 240 &&*/$iCurrentOctet < 248) { // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx $iContinuationOctetNeeded = 3; continue; @@ -728,7 +728,7 @@ private function validateUTF8Encoding($input) .", octet position " .$i .")", - $iCurrentOctet, + (string)$iCurrentOctet, array( 'line' => $iCurrentLineNumber, 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, From 4b27710384ac5fca3e66bd8f33234abb5fac1ebc Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Wed, 10 Jun 2026 12:14:40 +0200 Subject: [PATCH 05/17] activate long option --validate-utf8-encodin in bin/jsonlint (remove TODOs). --- bin/jsonlint | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/jsonlint b/bin/jsonlint index e3c4626..f8c4578 100755 --- a/bin/jsonlint +++ b/bin/jsonlint @@ -88,7 +88,7 @@ function getFlags( $flags = 0; if($allowComments){ $flags |= JsonParser::ALLOW_COMMENTS; } if($detectKeyConflicts){ $flags |= JsonParser::DETECT_KEY_CONFLICTS; } - // TODO if($validateUTF8Encoding){ $flags |= JsonParser::VALIDATE_UTF8_ENCODING; } + if($validateUTF8Encoding){ $flags |= JsonParser::VALIDATE_UTF8_ENCODING; } return $flags; } @@ -142,6 +142,6 @@ function showUsage($programPath) echo ' -h, --help Show this message'.PHP_EOL; echo ' --allow-comments Cause jsonlint to enable syntax for comments similar to JSON5.'.PHP_EOL; echo ' --detect-key-conflicts Cause jsonlint to detect key conflicts in objects.'.PHP_EOL; - // TODO echo ' --validate-utf8-encoding Cause jsonlint to validate that the whole file is valid UTF-8 encoding.'.PHP_EOL; + echo ' --validate-utf8-encoding Cause jsonlint to validate that the whole file is valid UTF-8 encoding.'.PHP_EOL; exit(0); } From 485836e17cda97565883e3b72a0d073d48307554 Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Wed, 10 Jun 2026 12:20:55 +0200 Subject: [PATCH 06/17] move UTF-8 validation into parse. --- src/Seld/JsonLint/JsonParser.php | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index 0cbd134..e2e19a1 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -185,13 +185,6 @@ class JsonParser */ public function lint($input, $flags = 0) { - if ($flags & self::VALIDATE_UTF8_ENCODING) { - try { - $this->validateUTF8Encoding($input); - } catch (InvalidEncodingException $e) { - return $e; - } - } try { $this->parse($input, $flags); } catch (ParsingException $e) { @@ -213,6 +206,9 @@ public function parse($input, $flags = 0) if (($flags & self::ALLOW_DUPLICATE_KEYS_TO_ARRAY) && ($flags & self::ALLOW_DUPLICATE_KEYS)) { throw new \InvalidArgumentException('Only one of ALLOW_DUPLICATE_KEYS and ALLOW_DUPLICATE_KEYS_TO_ARRAY can be used, you passed in both.'); } + if ($flags & self::VALIDATE_UTF8_ENCODING) { + $this->validateUTF8Encoding($input); + } $this->failOnBOM($input); From 086fb2f53ac7679911bc3a72631685e4a2d2ecf8 Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Wed, 10 Jun 2026 12:44:32 +0200 Subject: [PATCH 07/17] improve data for exceptions in validateUTF8Encoding similar to my own repository, and re?correct bug when some continuation octet is still needed at end of string as suggested by Jordi Boggiano. --- src/Seld/JsonLint/JsonParser.php | 130 +++++++++++++++++++++++-------- 1 file changed, 98 insertions(+), 32 deletions(-) diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index e2e19a1..166998f 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -616,14 +616,19 @@ private function failOnBOM($input) */ private function validateUTF8Encoding($input) { + $iCurrentOctet = null; $iContinuationOctetNeeded = 0; - $iCharacterStartPosition = 0; + $iOffsetInOctetsFromStringStart = 0; + $iOffsetInCharactersFromStringStart = -1; + $iCharacterStartPositionFromStringStart = 0; $iCurrentLineNumber = 1; $iOffsetInOctetsFromLineStart = -1; $iOffsetInCharactersFromLineStart = -1; + $iCharacterStartPositionFromLineStart = -1; for ($i = 0, $iMax = strlen($input); $i < $iMax; ++$i) { $iCurrentOctet = ord($input[$i]); - $iOffsetInOctetsFromLineStart += 1; + $iOffsetInOctetsFromStringStart = $i; + ++$iOffsetInOctetsFromLineStart; if ($iContinuationOctetNeeded > 0) { if ($iCurrentOctet < 128 || $iCurrentOctet >= 192) { throw new InvalidEncodingException( @@ -636,20 +641,29 @@ private function validateUTF8Encoding($input) .", has value " .$iCurrentOctet ." which is not a continuation octet." - ." (Sequential positions in octets without line splitting:" - ." character start position " - .$iCharacterStartPosition - .", octet position " - .$i - .")", + ." This character starts at octet " + .($iCharacterStartPositionFromLineStart + 1) + ." of the current line." + ." (Sequential positions without line splitting:" + ." This is at character " + .($iOffsetInCharactersFromStringStart + 1) + ." and octet " + .($iOffsetInOctetsFromStringStart + 1) + ."." + ." This character starts at octet " + .($iCharacterStartPositionFromStringStart + 1) + .".)", (string) $iCurrentOctet, array( - 'line' => $iCurrentLineNumber, - 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, - 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, - 'octet_position' => $i, - 'character_start_position' => $iCharacterStartPosition, - ) + 'line' => $iCurrentLineNumber, + 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, + 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, + 'current_octet' => $iCurrentOctet, + 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, + 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, + 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, + ) ); } --$iContinuationOctetNeeded; @@ -677,19 +691,28 @@ private function validateUTF8Encoding($input) .", has value " .$iCurrentOctet ." which is a continuation octet." - ." (Sequential positions in octets without line splitting:" - ." character start position " - .$iCharacterStartPosition - .", octet position " - .$i - .")", + ." This character starts at octet " + .($iCharacterStartPositionFromLineStart + 1) + ." of the current line." + ." (Sequential positions without line splitting:" + ." This is at character " + .($iOffsetInCharactersFromStringStart + 1) + ." and octet " + .($iOffsetInOctetsFromStringStart + 1) + ."." + ." This character starts at octet " + .($iCharacterStartPositionFromStringStart + 1) + .".)", (string) $iCurrentOctet, array( 'line' => $iCurrentLineNumber, 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, - 'octet_position' => $i, - 'character_start_position' => $iCharacterStartPosition, + 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, + 'current_octet' => $iCurrentOctet, + 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, + 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, + 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, ) ); } @@ -709,7 +732,7 @@ private function validateUTF8Encoding($input) continue; } throw new InvalidEncodingException( - "Non-UTF8 character found on line " + "Non-UTF8 character found on line " .$iCurrentLineNumber ."; the octet " .($iOffsetInOctetsFromLineStart + 1) @@ -718,19 +741,62 @@ private function validateUTF8Encoding($input) .", has value " .$iCurrentOctet ." which is invalid." - ." (Sequential positions in octets without line splitting:" - ." character start position " - .$iCharacterStartPosition - .", octet position " - .$i - .")", - (string)$iCurrentOctet, + ." This character starts at octet " + .($iCharacterStartPositionFromLineStart + 1) + ." of the current line." + ." (Sequential positions without line splitting:" + ." This is at character " + .($iOffsetInCharactersFromStringStart + 1) + ." and octet " + .($iOffsetInOctetsFromStringStart + 1) + ."." + ." This character starts at octet " + .($iCharacterStartPositionFromStringStart + 1) + .".)", + (string) $iCurrentOctet, + array( + 'line' => $iCurrentLineNumber, + 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, + 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, + 'current_octet' => $iCurrentOctet, + 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, + 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, + 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, + ) + ); + } + if ($iContinuationOctetNeeded > 0) { + throw new InvalidEncodingException( + "Non-UTF8 character found on line " + .$iCurrentLineNumber + ."; at octet " + .($iOffsetInOctetsFromLineStart + 1) + .", part of the character " + .($iOffsetInCharactersFromLineStart + 1) + .", end of string was found instead of a continuation octet." + ." This character starts at octet " + .($iCharacterStartPositionFromLineStart + 1) + ." of the current line." + ." (Sequential positions without line splitting:" + ." This is at character " + .($iOffsetInCharactersFromStringStart + 1) + ." and octet " + .($iOffsetInOctetsFromStringStart + 1) + ."." + ." This character starts at octet " + .($iCharacterStartPositionFromStringStart + 1) + .".)", + "0", array( 'line' => $iCurrentLineNumber, 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, - 'octet_position' => $i, - 'character_start_position' => $iCharacterStartPosition, + 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, + 'current_octet' => $iCurrentOctet, + 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, + 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, + 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, ) ); } From f69273efe2028bca9a1aebf9f65cfdd84968d8cb Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Wed, 10 Jun 2026 13:09:37 +0200 Subject: [PATCH 08/17] add tests in JsonParserTest.php with files nonvalidutf8.json and validutf8.json. --- tests/JsonParserTest.php | 27 +++++++++++++++++++++++++++ tests/nonvalidutf8.json | 1 + tests/validutf8.json | 1 + 3 files changed, 29 insertions(+) create mode 100644 tests/nonvalidutf8.json create mode 100644 tests/validutf8.json diff --git a/tests/JsonParserTest.php b/tests/JsonParserTest.php index 9b903bf..66dda3c 100644 --- a/tests/JsonParserTest.php +++ b/tests/JsonParserTest.php @@ -286,6 +286,33 @@ public function testParseToArray() $this->assertSame(json_decode($json, true), $result); } + public function testFileValidUTF8() + { + try { + $parser = new JsonParser(); + $parser->parse + ((string) file_get_contents(dirname(__FILE__) .'/validutf8.json'), + JsonParser::VALIDATE_UTF8_ENCODING + ); + } catch (ParsingException $e) { + $this->fail('validutf8.json file should pass validation.'); + } + } + + public function testFileNonValidUTF8() + { + try { + $parser = new JsonParser(); + $parser->parse + ((string) file_get_contents(dirname(__FILE__) .'/nonvalidutf8.json'), + JsonParser::VALIDATE_UTF8_ENCODING + ); + $this->fail('nonvalidutf8.json file should not pass validation.'); + } catch (ParsingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + } + } + public function testFileWithBOM() { try { diff --git a/tests/nonvalidutf8.json b/tests/nonvalidutf8.json new file mode 100644 index 0000000..6a7cbb6 --- /dev/null +++ b/tests/nonvalidutf8.json @@ -0,0 +1 @@ +"abcdé" diff --git a/tests/validutf8.json b/tests/validutf8.json new file mode 100644 index 0000000..148fada --- /dev/null +++ b/tests/validutf8.json @@ -0,0 +1 @@ +"abcdĂ©" From 07b84fc5acd6fa11cfdbc1471c4e59f4af2ba46f Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Wed, 10 Jun 2026 13:12:52 +0200 Subject: [PATCH 09/17] add fast-path for mainly valid-UTF8 files using preg_match. --- src/Seld/JsonLint/JsonParser.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index 166998f..681c587 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -616,6 +616,17 @@ private function failOnBOM($input) */ private function validateUTF8Encoding($input) { + //Fast-path + if (function_exists("mb_check_encoding")) { + if(mb_check_encoding($input, 'UTF-8')){ + return; + } + } else { + if (preg_match('//u', $input) === 1) { + return; + } + } + $iCurrentOctet = null; $iContinuationOctetNeeded = 0; $iOffsetInOctetsFromStringStart = 0; From fbadc8548078c5b3cd671d3e7acaf4dcb6b0f193 Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Wed, 10 Jun 2026 13:33:54 +0200 Subject: [PATCH 10/17] do not allow octets C0, C1, F5, FF in UTF8 validation. --- src/Seld/JsonLint/JsonParser.php | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index 681c587..2eda968 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -640,6 +640,49 @@ private function validateUTF8Encoding($input) $iCurrentOctet = ord($input[$i]); $iOffsetInOctetsFromStringStart = $i; ++$iOffsetInOctetsFromLineStart; + + if ( + $iCurrentOctet === 192 + || $iCurrentOctet === 193 + || $iCurrentOctet === 245 + || $iCurrentOctet === 255 + ) { + throw new InvalidEncodingException( + "Non-UTF8 character found on line " + .$iCurrentLineNumber + ."; the octet " + .($iOffsetInOctetsFromLineStart + 1) + .", part of the character " + .($iOffsetInCharactersFromLineStart + 1) + .", has value " + .$iCurrentOctet + ." which is one of the four forbidden values (C0, C1, F5, FF)." + ." This character starts at octet " + .($iCharacterStartPositionFromLineStart + 1) + ." of the current line." + ." (Sequential positions without line splitting:" + ." This is at character " + .($iOffsetInCharactersFromStringStart + 1) + ." and octet " + .($iOffsetInOctetsFromStringStart + 1) + ."." + ." This character starts at octet " + .($iCharacterStartPositionFromStringStart + 1) + .".)", + (string) $iCurrentOctet, + array( + 'line' => $iCurrentLineNumber, + 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, + 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, + 'current_octet' => $iCurrentOctet, + 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, + 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, + 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, + ) + ); + } + if ($iContinuationOctetNeeded > 0) { if ($iCurrentOctet < 128 || $iCurrentOctet >= 192) { throw new InvalidEncodingException( @@ -691,6 +734,7 @@ private function validateUTF8Encoding($input) } continue; } + if (/*$iCurrentOctet >= 128 &&*/$iCurrentOctet < 192) { throw new InvalidEncodingException( "Non-UTF8 character found on line " From 4d6fd5c1b298ac922443965397704c3182ae9be8 Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Wed, 10 Jun 2026 14:02:22 +0200 Subject: [PATCH 11/17] UTF8 validation must forbid range D800 to DFFF --- src/Seld/JsonLint/JsonParser.php | 174 +++++++++++++++++++++++++++---- 1 file changed, 152 insertions(+), 22 deletions(-) diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index 2eda968..88b6b46 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -636,11 +636,26 @@ private function validateUTF8Encoding($input) $iOffsetInOctetsFromLineStart = -1; $iOffsetInCharactersFromLineStart = -1; $iCharacterStartPositionFromLineStart = -1; + + // For the second octet of a character, hence first continuation octet, + // further restriction may apply. + $I_CONTINUATION_OCTET_MINIMUM = 128; + $I_CONTINUATION_OCTET_MAXIMUM = 191; + $iCurrentContinuationOctetMinimum = $I_CONTINUATION_OCTET_MINIMUM; + $iCurrentContinuationOctetMaximum = $I_CONTINUATION_OCTET_MAXIMUM; + for ($i = 0, $iMax = strlen($input); $i < $iMax; ++$i) { $iCurrentOctet = ord($input[$i]); $iOffsetInOctetsFromStringStart = $i; ++$iOffsetInOctetsFromLineStart; + /* + The octet values C0, C1, F5 to FF never appear. + C0 = 12*16 = 192 = 11000000 + C1 = 12*16 + 1 = 193 = 11000001 + F5 = 15*16 + 5 = 245 = 11110101 + FF = 15*16 + 15 = 255 = 11111111 + */ if ( $iCurrentOctet === 192 || $iCurrentOctet === 193 @@ -671,20 +686,42 @@ private function validateUTF8Encoding($input) .".)", (string) $iCurrentOctet, array( - 'line' => $iCurrentLineNumber, - 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, - 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, - 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, 'current_octet' => $iCurrentOctet, + 'continuation_octet_needed' => $iContinuationOctetNeeded, 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, + 'line' => $iCurrentLineNumber, + 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, + 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, + 'current_continuation_octet_minimum' => $iCurrentContinuationOctetMinimum, + 'current_continuation_octet_maximum' => $iCurrentContinuationOctetMaximum, ) ); } + /* + UTF8-octets = *( UTF8-char ) + UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4 + UTF8-1 = %x00-7F + UTF8-2 = %xC2-DF UTF8-tail + Notice that values C0 and C1 are forbidden, hence %xC2 + UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / + %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) + Notice that E0 = 224 adds a restriction on second octet. + Notice that ED = 237 adds a restriction on second octet. + UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / + %xF4 %x80-8F 2( UTF8-tail ) + Notice that F0 = 240 adds a restriction on second octet. + Notice that F4 = 244 adds a restriction on second octet. + UTF8-tail = %x80-BF + */ if ($iContinuationOctetNeeded > 0) { - if ($iCurrentOctet < 128 || $iCurrentOctet >= 192) { + if( + $iCurrentOctet < $iCurrentContinuationOctetMinimum + || $iCurrentOctet > $iCurrentContinuationOctetMaximum + ){ throw new InvalidEncodingException( "Non-UTF8 character found on line " .$iCurrentLineNumber @@ -709,26 +746,35 @@ private function validateUTF8Encoding($input) .".)", (string) $iCurrentOctet, array( - 'line' => $iCurrentLineNumber, - 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, - 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, - 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, 'current_octet' => $iCurrentOctet, + 'continuation_octet_needed' => $iContinuationOctetNeeded, 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, + 'line' => $iCurrentLineNumber, + 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, + 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, + 'current_continuation_octet_minimum' => $iCurrentContinuationOctetMinimum, + 'current_continuation_octet_maximum' => $iCurrentContinuationOctetMaximum, ) ); } --$iContinuationOctetNeeded; + $iCurrentContinuationOctetMinimum = $I_CONTINUATION_OCTET_MINIMUM; + $iCurrentContinuationOctetMaximum = $I_CONTINUATION_OCTET_MAXIMUM; continue; } - $iCharacterStartPosition = $i; - $iOffsetInCharactersFromLineStart += 1; + ++$iOffsetInCharactersFromStringStart; + ++$iOffsetInCharactersFromLineStart; + $iCharacterStartPositionFromStringStart = $iOffsetInOctetsFromStringStart; + $iCharacterStartPositionFromLineStart = $iOffsetInOctetsFromLineStart; + if ($iCurrentOctet < 128) { // 0xxxxxxx ASCII - if ($input[$i] === "\n") { - $iCurrentLineNumber += 1; + // if ($input[$i] === "\n") { + if ($iCurrentOctet === 10) { + ++$iCurrentLineNumber; $iOffsetInOctetsFromLineStart = -1; $iOffsetInCharactersFromLineStart = -1; } @@ -760,17 +806,67 @@ private function validateUTF8Encoding($input) .".)", (string) $iCurrentOctet, array( + 'current_octet' => $iCurrentOctet, + 'continuation_octet_needed' => $iContinuationOctetNeeded, + 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, + 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, + 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, 'line' => $iCurrentLineNumber, 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, + 'current_continuation_octet_minimum' => $iCurrentContinuationOctetMinimum, + 'current_continuation_octet_maximum' => $iCurrentContinuationOctetMaximum, + ) + ); + } + + /* + The definition of UTF-8 prohibits encoding character numbers between + U+D800 and U+DFFF. + D8 = 13*16 + 8 = 216 = 11010100 + DF = 13*16 + 15 = 223 = 11010101 + */ + if ($iCurrentOctet >= 216 && $iCurrentOctet <= 223) { + throw new InvalidEncodingException( + "Non-UTF8 character found on line " + .$iCurrentLineNumber + ."; the octet " + .($iOffsetInOctetsFromLineStart + 1) + .", part of the character " + .($iOffsetInCharactersFromLineStart + 1) + .", has value " + .$iCurrentOctet + ." which is into a forbidden range of values for first octet of character." + ." This character starts at octet " + .($iCharacterStartPositionFromLineStart + 1) + ." of the current line." + ." (Sequential positions without line splitting:" + ." This is at character " + .($iOffsetInCharactersFromStringStart + 1) + ." and octet " + .($iOffsetInOctetsFromStringStart + 1) + ."." + ." This character starts at octet " + .($iCharacterStartPositionFromStringStart + 1) + .".)", + (string) $iCurrentOctet, + array( 'current_octet' => $iCurrentOctet, + 'continuation_octet_needed' => $iContinuationOctetNeeded, 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, + 'line' => $iCurrentLineNumber, + 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, + 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, + 'current_continuation_octet_minimum' => $iCurrentContinuationOctetMinimum, + 'current_continuation_octet_maximum' => $iCurrentContinuationOctetMaximum, ) ); } + if (/*$iCurrentOctet >= 192 &&*/$iCurrentOctet < 224) { // 110xxxxx 10xxxxxx $iContinuationOctetNeeded = 1; @@ -779,11 +875,39 @@ private function validateUTF8Encoding($input) if (/*$iCurrentOctet >= 224 &&*/$iCurrentOctet < 240) { // 1110xxxx 10xxxxxx 10xxxxxx $iContinuationOctetNeeded = 2; + /* + UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / + %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) + Notice that E0 = 224 adds a restriction on second octet. + Notice that ED = 237 adds a restriction on second octet. + */ + if($iCurrentOctet === 224){ + $iCurrentContinuationOctetMinimum = 160; + $iCurrentContinuationOctetMaximum = 191; // Normal value + } + if($iCurrentOctet === 237){ + $iCurrentContinuationOctetMinimum = 128; // Normal value + $iCurrentContinuationOctetMaximum = 159; + } continue; } if (/*$iCurrentOctet >= 240 &&*/$iCurrentOctet < 248) { // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx $iContinuationOctetNeeded = 3; + /* + UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / + %xF4 %x80-8F 2( UTF8-tail ) + Notice that F0 = 240 adds a restriction on second octet. + Notice that F4 = 244 adds a restriction on second octet. + */ + if($iCurrentOctet === 240){ + $iCurrentContinuationOctetMinimum = 144; + $iCurrentContinuationOctetMaximum = 191; // Normal value + } + if($iCurrentOctet === 244){ + $iCurrentContinuationOctetMinimum = 128; // Normal value + $iCurrentContinuationOctetMaximum = 143; + } continue; } throw new InvalidEncodingException( @@ -810,20 +934,23 @@ private function validateUTF8Encoding($input) .".)", (string) $iCurrentOctet, array( - 'line' => $iCurrentLineNumber, - 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, - 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, - 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, 'current_octet' => $iCurrentOctet, + 'continuation_octet_needed' => $iContinuationOctetNeeded, 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, + 'line' => $iCurrentLineNumber, + 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, + 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, + 'current_continuation_octet_minimum' => $iCurrentContinuationOctetMinimum, + 'current_continuation_octet_maximum' => $iCurrentContinuationOctetMaximum, ) ); } if ($iContinuationOctetNeeded > 0) { throw new InvalidEncodingException( - "Non-UTF8 character found on line " + "Non-UTF8 character found on line " .$iCurrentLineNumber ."; at octet " .($iOffsetInOctetsFromLineStart + 1) @@ -844,14 +971,17 @@ private function validateUTF8Encoding($input) .".)", "0", array( - 'line' => $iCurrentLineNumber, - 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, - 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, - 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, 'current_octet' => $iCurrentOctet, + 'continuation_octet_needed' => $iContinuationOctetNeeded, 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, + 'line' => $iCurrentLineNumber, + 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, + 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, + 'current_continuation_octet_minimum' => $iCurrentContinuationOctetMinimum, + 'current_continuation_octet_maximum' => $iCurrentContinuationOctetMaximum, ) ); } From 04efacdb84137384bb8ec4e55a7e269d151ac2b7 Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Wed, 10 Jun 2026 16:38:26 +0200 Subject: [PATCH 12/17] It should be the last non-UTF-8 correction, F5 to F7 are too high values for UTF-8 first octet of sequences of 4 octets. --- src/Seld/JsonLint/JsonParser.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index 88b6b46..44ab8fb 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -891,7 +891,7 @@ private function validateUTF8Encoding($input) } continue; } - if (/*$iCurrentOctet >= 240 &&*/$iCurrentOctet < 248) { + if (/*$iCurrentOctet >= 240 &&*/$iCurrentOctet < /*248*/ 245) { // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx $iContinuationOctetNeeded = 3; /* From d2a194b1fc0dfde921d0815d0f01c52db7c1fdc4 Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Thu, 11 Jun 2026 10:24:07 +0200 Subject: [PATCH 13/17] add further tests to check all InvalidEncodingException messages. --- tests/JsonParserTest.php | 172 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 166 insertions(+), 6 deletions(-) diff --git a/tests/JsonParserTest.php b/tests/JsonParserTest.php index 66dda3c..6a80400 100644 --- a/tests/JsonParserTest.php +++ b/tests/JsonParserTest.php @@ -13,6 +13,7 @@ use Seld\JsonLint\JsonParser; use Seld\JsonLint\ParsingException; use Seld\JsonLint\DuplicateKeyException; +use Seld\JsonLint\InvalidEncodingException; class JsonParserTest extends TestCase { @@ -290,11 +291,12 @@ public function testFileValidUTF8() { try { $parser = new JsonParser(); - $parser->parse - ((string) file_get_contents(dirname(__FILE__) .'/validutf8.json'), + $string = $parser->parse( + (string) file_get_contents(dirname(__FILE__) .'/validutf8.json'), JsonParser::VALIDATE_UTF8_ENCODING ); - } catch (ParsingException $e) { + $this->assertEquals($string, "abcdĂ©"); + } catch (InvalidEncodingException $e) { $this->fail('validutf8.json file should pass validation.'); } } @@ -303,13 +305,171 @@ public function testFileNonValidUTF8() { try { $parser = new JsonParser(); - $parser->parse - ((string) file_get_contents(dirname(__FILE__) .'/nonvalidutf8.json'), + $parser->parse( + (string) file_get_contents(dirname(__FILE__) .'/nonvalidutf8.json'), JsonParser::VALIDATE_UTF8_ENCODING ); $this->fail('nonvalidutf8.json file should not pass validation.'); - } catch (ParsingException $e) { + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + } + } + + public function testAllErrorTypesNonValidUTF8() + { + $parser = new JsonParser(); + try { + $parser->parse( + '"abcd'.chr(233).'"', + JsonParser::VALIDATE_UTF8_ENCODING + ); + $this->fail('ISO 8859-15 "abcdĂ©" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + } + try { + $parser->parse( + '"abcd'.chr(233).'"', + JsonParser::VALIDATE_UTF8_ENCODING + ); + $this->fail('ISO 8859-15 "abcdĂ©" should not pass validation.'); + } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains(' which is not a continuation octet.', $e->getMessage()); + } + try { + $parser->parse( + '"abcd'.chr(233), + JsonParser::VALIDATE_UTF8_ENCODING + ); + $this->fail('ISO 8859-15 "abcdĂ© should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains( + ', end of string was found instead of a continuation octet.', + $e->getMessage() + ); + } + $forbiddenOctets = array( + 192, + 193, + 245, + 255 + ); + foreach ($forbiddenOctets as $forbiddenOctet) { + try { + $parser->parse( + '"abcd'.chr(233).chr($forbiddenOctet).'"', + JsonParser::VALIDATE_UTF8_ENCODING + ); + $this->fail('"abcd\d233\d'.$forbiddenOctet.'" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains( + ' which is one of the four forbidden values (C0, C1, F5, FF).', + $e->getMessage() + ); + } + try { + $parser->parse( + '"abcd'.chr($forbiddenOctet).'"', + JsonParser::VALIDATE_UTF8_ENCODING + ); + $this->fail('"abcd\d'.$forbiddenOctet.'" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains( + ' which is one of the four forbidden values (C0, C1, F5, FF).', + $e->getMessage() + ); + } + } + try { + $parser->parse( + '"abcd'.chr(129).'"', + JsonParser::VALIDATE_UTF8_ENCODING + ); + $this->fail('"abcd\d129" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains( + ' which is a continuation octet.', + $e->getMessage() + ); + } + try { + $parser->parse( + '"abcd'.chr(129).'"', + JsonParser::VALIDATE_UTF8_ENCODING + ); + $this->fail('"abcd\d129" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains( + ' which is a continuation octet.', + $e->getMessage() + ); + } + for ($i = 216; $i <= 223; ++$i) { + try { + $parser->parse( + '"abcd'.chr($i).chr(129).'"', + JsonParser::VALIDATE_UTF8_ENCODING + ); + $this->fail('"abcd\d'.$i.'\d129" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains( + ' which is into a forbidden range of values for first octet of character.', + $e->getMessage() + ); + } + try { + $parser->parse( + '"abcd'.chr($i).'"', + JsonParser::VALIDATE_UTF8_ENCODING + ); + $this->fail('"abcd\d'.$i.'" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains( + ' which is into a forbidden range of values for first octet of character.', + $e->getMessage() + ); + } + } + for ($i = 245; $i <= 255; ++$i) { + try { + $parser->parse( + '"abcd'.chr($i).chr(129).chr(129).chr(129).'"', + JsonParser::VALIDATE_UTF8_ENCODING + ); + $this->fail('"abcd\d'.$i.'\d129\d129\d129" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains(' which is invalid.', $e->getMessage()); + } + try { + $parser->parse( + '"abcd'.chr($i).'"', + JsonParser::VALIDATE_UTF8_ENCODING + ); + $this->fail('"abcd\d'.$i.'" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains(' which is invalid.', $e->getMessage()); + } + try { + $parser->parse( + '"abcd'.chr(195).chr($i).'"', + JsonParser::VALIDATE_UTF8_ENCODING + ); + $this->fail('"abcd\d195\d'.$i.'" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains(' which is not a continuation octet.', $e->getMessage()); + } } } From 7170180ebc1e5d27eb6455a05701ed95a5a194bb Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Thu, 11 Jun 2026 10:48:07 +0200 Subject: [PATCH 14/17] disable fast path for UTF8-validation since it doesn't check for RFC3629 : The definition of UTF-8 prohibits encoding character numbers between U+D800 and U+DFFF, which are reserved for use with the UTF-16 encoding form (as surrogate pairs) and do not directly represent characters. --- src/Seld/JsonLint/JsonParser.php | 2 ++ tests/JsonParserTest.php | 25 +------------------------ 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index 44ab8fb..dfc6a74 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -617,6 +617,7 @@ private function failOnBOM($input) private function validateUTF8Encoding($input) { //Fast-path + /* Buggy if (function_exists("mb_check_encoding")) { if(mb_check_encoding($input, 'UTF-8')){ return; @@ -626,6 +627,7 @@ private function validateUTF8Encoding($input) return; } } + */ $iCurrentOctet = null; $iContinuationOctetNeeded = 0; diff --git a/tests/JsonParserTest.php b/tests/JsonParserTest.php index 6a80400..745b0dd 100644 --- a/tests/JsonParserTest.php +++ b/tests/JsonParserTest.php @@ -318,16 +318,6 @@ public function testFileNonValidUTF8() public function testAllErrorTypesNonValidUTF8() { $parser = new JsonParser(); - try { - $parser->parse( - '"abcd'.chr(233).'"', - JsonParser::VALIDATE_UTF8_ENCODING - ); - $this->fail('ISO 8859-15 "abcdĂ©" should not pass validation.'); - } catch (InvalidEncodingException $e) { - $this->assertContains('Non-UTF8 character found', $e->getMessage()); - $this->assertContains('Non-UTF8 character found', $e->getMessage()); - } try { $parser->parse( '"abcd'.chr(233).'"', @@ -398,19 +388,6 @@ public function testAllErrorTypesNonValidUTF8() $e->getMessage() ); } - try { - $parser->parse( - '"abcd'.chr(129).'"', - JsonParser::VALIDATE_UTF8_ENCODING - ); - $this->fail('"abcd\d129" should not pass validation.'); - } catch (InvalidEncodingException $e) { - $this->assertContains('Non-UTF8 character found', $e->getMessage()); - $this->assertContains( - ' which is a continuation octet.', - $e->getMessage() - ); - } for ($i = 216; $i <= 223; ++$i) { try { $parser->parse( @@ -439,7 +416,7 @@ public function testAllErrorTypesNonValidUTF8() ); } } - for ($i = 245; $i <= 255; ++$i) { + for ($i = 246; $i < 255; ++$i) { // 245 and 255 already forbidden try { $parser->parse( '"abcd'.chr($i).chr(129).chr(129).chr(129).'"', From 38be61dd1b7f0410bc18537d4cd28674e0b94fdb Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Thu, 11 Jun 2026 10:57:34 +0200 Subject: [PATCH 15/17] add PHP issue link explaining why it is buggy. --- src/Seld/JsonLint/JsonParser.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index dfc6a74..34c66c7 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -617,7 +617,7 @@ private function failOnBOM($input) private function validateUTF8Encoding($input) { //Fast-path - /* Buggy + /* Buggy https://github.com/php/php-src/issues/22279 if (function_exists("mb_check_encoding")) { if(mb_check_encoding($input, 'UTF-8')){ return; From 728b4eba42fbe534dd72cff5db9dff3c83bac0ab Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Thu, 11 Jun 2026 11:52:31 +0200 Subject: [PATCH 16/17] correction of my bug on surrogate pairs that were checked in another range of characters. --- src/Seld/JsonLint/JsonParser.php | 77 ++++++++++++-------------------- tests/JsonParserTest.php | 14 +++--- 2 files changed, 35 insertions(+), 56 deletions(-) diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index 34c66c7..0ae8551 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -617,7 +617,6 @@ private function failOnBOM($input) private function validateUTF8Encoding($input) { //Fast-path - /* Buggy https://github.com/php/php-src/issues/22279 if (function_exists("mb_check_encoding")) { if(mb_check_encoding($input, 'UTF-8')){ return; @@ -627,7 +626,6 @@ private function validateUTF8Encoding($input) return; } } - */ $iCurrentOctet = null; $iContinuationOctetNeeded = 0; @@ -645,6 +643,7 @@ private function validateUTF8Encoding($input) $I_CONTINUATION_OCTET_MAXIMUM = 191; $iCurrentContinuationOctetMinimum = $I_CONTINUATION_OCTET_MINIMUM; $iCurrentContinuationOctetMaximum = $I_CONTINUATION_OCTET_MAXIMUM; + $sMessageForContinuationOctetAboveMaximum = null; for ($i = 0, $iMax = strlen($input); $i < $iMax; ++$i) { $iCurrentOctet = ord($input[$i]); @@ -733,7 +732,12 @@ private function validateUTF8Encoding($input) .($iOffsetInCharactersFromLineStart + 1) .", has value " .$iCurrentOctet - ." which is not a continuation octet." + .( + $sMessageForContinuationOctetAboveMaximum !== null + && $iCurrentOctet > $iCurrentContinuationOctetMaximum + ? $sMessageForContinuationOctetAboveMaximum + : " which is not a continuation octet." + ) ." This character starts at octet " .($iCharacterStartPositionFromLineStart + 1) ." of the current line." @@ -765,6 +769,7 @@ private function validateUTF8Encoding($input) --$iContinuationOctetNeeded; $iCurrentContinuationOctetMinimum = $I_CONTINUATION_OCTET_MINIMUM; $iCurrentContinuationOctetMaximum = $I_CONTINUATION_OCTET_MAXIMUM; + $sMessageForContinuationOctetAboveMaximum = null; continue; } @@ -823,52 +828,6 @@ private function validateUTF8Encoding($input) ); } - /* - The definition of UTF-8 prohibits encoding character numbers between - U+D800 and U+DFFF. - D8 = 13*16 + 8 = 216 = 11010100 - DF = 13*16 + 15 = 223 = 11010101 - */ - if ($iCurrentOctet >= 216 && $iCurrentOctet <= 223) { - throw new InvalidEncodingException( - "Non-UTF8 character found on line " - .$iCurrentLineNumber - ."; the octet " - .($iOffsetInOctetsFromLineStart + 1) - .", part of the character " - .($iOffsetInCharactersFromLineStart + 1) - .", has value " - .$iCurrentOctet - ." which is into a forbidden range of values for first octet of character." - ." This character starts at octet " - .($iCharacterStartPositionFromLineStart + 1) - ." of the current line." - ." (Sequential positions without line splitting:" - ." This is at character " - .($iOffsetInCharactersFromStringStart + 1) - ." and octet " - .($iOffsetInOctetsFromStringStart + 1) - ."." - ." This character starts at octet " - .($iCharacterStartPositionFromStringStart + 1) - .".)", - (string) $iCurrentOctet, - array( - 'current_octet' => $iCurrentOctet, - 'continuation_octet_needed' => $iContinuationOctetNeeded, - 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, - 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, - 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, - 'line' => $iCurrentLineNumber, - 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, - 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, - 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, - 'current_continuation_octet_minimum' => $iCurrentContinuationOctetMinimum, - 'current_continuation_octet_maximum' => $iCurrentContinuationOctetMaximum, - ) - ); - } - if (/*$iCurrentOctet >= 192 &&*/$iCurrentOctet < 224) { // 110xxxxx 10xxxxxx $iContinuationOctetNeeded = 1; @@ -882,6 +841,23 @@ private function validateUTF8Encoding($input) %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) Notice that E0 = 224 adds a restriction on second octet. Notice that ED = 237 adds a restriction on second octet. + + The definition of UTF-8 prohibits encoding character numbers between + U+D800 and U+DFFF. + D8 = 13*16 + 8 = 216 = 11010100 + DF = 13*16 + 15 = 223 = 11010101 + 216*256 = 55296 = 1101 1000 0000 0000 = D800 + to + 223*256 + 255 = 57343 = 1101 1111 1111 1111 = DFFF + + 1110xxxx + 11101101 = 237 = ED + + 237,160,128 + 237,191,191 + Thus, the whole "continuation range" is forbidden if start + is 237 and second octet, first continuation octet, + is >= 160. */ if($iCurrentOctet === 224){ $iCurrentContinuationOctetMinimum = 160; @@ -890,6 +866,9 @@ private function validateUTF8Encoding($input) if($iCurrentOctet === 237){ $iCurrentContinuationOctetMinimum = 128; // Normal value $iCurrentContinuationOctetMaximum = 159; + $sMessageForContinuationOctetAboveMaximum = ( + " which is into the forbidden range of surrogate pairs." + ); } continue; } diff --git a/tests/JsonParserTest.php b/tests/JsonParserTest.php index 745b0dd..370c838 100644 --- a/tests/JsonParserTest.php +++ b/tests/JsonParserTest.php @@ -388,30 +388,30 @@ public function testAllErrorTypesNonValidUTF8() $e->getMessage() ); } - for ($i = 216; $i <= 223; ++$i) { + for ($i = 160; $i <= 191; ++$i) { try { $parser->parse( - '"abcd'.chr($i).chr(129).'"', + '"abcd'.chr(237).chr($i).chr(129).'"', JsonParser::VALIDATE_UTF8_ENCODING ); - $this->fail('"abcd\d'.$i.'\d129" should not pass validation.'); + $this->fail('"abcd\d237\d'.$i.'\d129" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); $this->assertContains( - ' which is into a forbidden range of values for first octet of character.', + ' which is into the forbidden range of surrogate pairs.', $e->getMessage() ); } try { $parser->parse( - '"abcd'.chr($i).'"', + '"abcd'.chr(237).chr($i).'"', JsonParser::VALIDATE_UTF8_ENCODING ); - $this->fail('"abcd\d'.$i.'" should not pass validation.'); + $this->fail('"abcd\d237\d'.$i.'" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); $this->assertContains( - ' which is into a forbidden range of values for first octet of character.', + ' which is into the forbidden range of surrogate pairs.', $e->getMessage() ); } From 202ff2918f13135d9c7ff6b3231f9c8f6c072291 Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Thu, 11 Jun 2026 12:52:58 +0200 Subject: [PATCH 17/17] deal with bugs in Unicode support of PHP before 5.4. --- src/Seld/JsonLint/JsonParser.php | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index 0ae8551..64c3f04 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -616,14 +616,17 @@ private function failOnBOM($input) */ private function validateUTF8Encoding($input) { - //Fast-path - if (function_exists("mb_check_encoding")) { - if(mb_check_encoding($input, 'UTF-8')){ - return; - } - } else { - if (preg_match('//u', $input) === 1) { - return; + // Fast-path + // But before PHP 5.4 Unicode support has bugs. + if (PHP_VERSION_ID >= 50400) { + if (function_exists("mb_check_encoding")) { + if(mb_check_encoding($input, 'UTF-8')){ + return; + } + } else { + if (preg_match('//u', $input) === 1) { + return; + } } }