From 30dda49c6d87cd92bb28611cb9e60d60ef061e59 Mon Sep 17 00:00:00 2001 From: Laurent Lyaudet Date: Thu, 11 Jun 2026 17:37:55 +0200 Subject: [PATCH 1/2] correction possible bug maybe if fast-path detects an UTF-8 violation but my code doesn't. --- src/Seld/JsonLint/JsonParser.php | 2 +- src/Seld/JsonLint/Utf8Validator.php | 44 +++++++++++++++++++++++++++-- tests/Utf8ValidatorTest.php | 35 ++++++++++++----------- 3 files changed, 61 insertions(+), 20 deletions(-) diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index cb2c4bc..d699c49 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -208,7 +208,7 @@ public function parse($input, $flags = 0) throw new \InvalidArgumentException('Only one of ALLOW_DUPLICATE_KEYS and ALLOW_DUPLICATE_KEYS_TO_ARRAY can be used, you passed in both.'); } if ($flags & self::VALIDATE_UTF8_ENCODING) { - Utf8Validator::validate($input); + Utf8Validator::validate($input, Utf8Validator::getUseFastPath()); } $this->failOnBOM($input); diff --git a/src/Seld/JsonLint/Utf8Validator.php b/src/Seld/JsonLint/Utf8Validator.php index e785049..3abd0ae 100644 --- a/src/Seld/JsonLint/Utf8Validator.php +++ b/src/Seld/JsonLint/Utf8Validator.php @@ -32,18 +32,34 @@ class Utf8Validator */ const CONTINUATION_OCTET_MAXIMUM = 191; + /** + * Checks that the version of PHP allows using the fast-path. + * + * @return bool + */ + public static function getUseFastPath() + { + // Before PHP 5.4, Unicode support has bugs. + return PHP_VERSION_ID >= 50400; + } + /** * Validates that the whole input string is valid UTF-8. * * @param string $input + * @param bool $useFastPath * @return void * @throws InvalidEncodingException if the input is not valid UTF-8 */ - public static function validate($input) + public static function validate($input, $useFastPath = true) { // Fast-path - // But before PHP 5.4 Unicode support has bugs. - if (PHP_VERSION_ID >= 50400) { + if ($useFastPath && !Utf8Validator::getUseFastPath()) { + throw new \Exception( + 'Fast-path is not available with version '.PHP_VERSION_ID.' of PHP.' + ); + } + if ($useFastPath) { if (function_exists("mb_check_encoding")) { if (mb_check_encoding($input, 'UTF-8')) { return; @@ -294,6 +310,28 @@ public static function validate($input) true ); } + if ($useFastPath) { + throw new InvalidEncodingException( + "Fast-path detected an error that this function couldn't found." + .'Please create an issue at ' + .'"https://github.com/Seldaek/jsonlint/issues" and if possible at' + .'"https://github.com/LLyaudet/DevOrSysAdminScripts/issues".', + "0", + array( + 'current_octet' => $currentOctet, + 'continuation_octet_needed' => $continuationOctetNeeded, + 'offset_in_octets_from_string_start' => $offsetInOctetsFromStringStart, + 'offset_in_characters_from_string_start' => $offsetInCharactersFromStringStart, + 'character_start_position_from_string_start' => $characterStartPositionFromStringStart, + 'line' => $currentLineNumber, + 'offset_in_octets_from_line_start' => $offsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $offsetInCharactersFromLineStart, + 'character_start_position_from_line_start' => $characterStartPositionFromLineStart, + 'current_continuation_octet_minimum' => $currentContinuationOctetMinimum, + 'current_continuation_octet_maximum' => $currentContinuationOctetMaximum, + ) + ); + } } /** diff --git a/tests/Utf8ValidatorTest.php b/tests/Utf8ValidatorTest.php index ed432a8..9775745 100644 --- a/tests/Utf8ValidatorTest.php +++ b/tests/Utf8ValidatorTest.php @@ -20,12 +20,13 @@ class Utf8ValidatorTest extends TestCase { public function testValidUtf8() { + $useFastPath = Utf8Validator::getUseFastPath(); try { - Utf8Validator::validate(''); - Utf8Validator::validate('abcdé'); // 2-octet character - Utf8Validator::validate('euro sign: € and more'); // 3-octet character - Utf8Validator::validate('musical clef: 𝄞'); // 4-octet character - Utf8Validator::validate("line 1\nline 2\nline 3"); // newline handling + Utf8Validator::validate('', $useFastPath); + Utf8Validator::validate('abcdé', $useFastPath); // 2-octet character + Utf8Validator::validate('euro sign: € and more', $useFastPath); // 3-octet character + Utf8Validator::validate('musical clef: 𝄞', $useFastPath); // 4-octet character + Utf8Validator::validate("line 1\nline 2\nline 3", $useFastPath); // newline handling $this->addToAssertionCount(1); } catch (InvalidEncodingException $e) { $this->fail('Valid UTF-8 should pass validation: '.$e->getMessage()); @@ -34,15 +35,16 @@ public function testValidUtf8() public function testAllErrorTypes() { + $useFastPath = Utf8Validator::getUseFastPath(); try { - Utf8Validator::validate('"abcd'.chr(233).'"'); + Utf8Validator::validate('"abcd'.chr(233).'"', $useFastPath); $this->fail('ISO 8859-15 "abcdé" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); $this->assertContains(' which is not a continuation octet.', $e->getMessage()); } try { - Utf8Validator::validate('"abcd'.chr(233)); + Utf8Validator::validate('"abcd'.chr(233), $useFastPath); $this->fail('ISO 8859-15 "abcdé should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); @@ -59,7 +61,7 @@ public function testAllErrorTypes() ); foreach ($forbiddenOctets as $forbiddenOctet) { try { - Utf8Validator::validate('"abcd'.chr(233).chr($forbiddenOctet).'"'); + Utf8Validator::validate('"abcd'.chr(233).chr($forbiddenOctet).'"', $useFastPath); $this->fail('"abcd\d233\d'.$forbiddenOctet.'" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); @@ -69,7 +71,7 @@ public function testAllErrorTypes() ); } try { - Utf8Validator::validate('"abcd'.chr($forbiddenOctet).'"'); + Utf8Validator::validate('"abcd'.chr($forbiddenOctet).'"', $useFastPath); $this->fail('"abcd\d'.$forbiddenOctet.'" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); @@ -80,7 +82,7 @@ public function testAllErrorTypes() } } try { - Utf8Validator::validate('"abcd'.chr(129).'"'); + Utf8Validator::validate('"abcd'.chr(129).'"', $useFastPath); $this->fail('"abcd\d129" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); @@ -91,7 +93,7 @@ public function testAllErrorTypes() } for ($i = 160; $i <= 191; ++$i) { try { - Utf8Validator::validate('"abcd'.chr(237).chr($i).chr(129).'"'); + Utf8Validator::validate('"abcd'.chr(237).chr($i).chr(129).'"', $useFastPath); $this->fail('"abcd\d237\d'.$i.'\d129" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); @@ -101,7 +103,7 @@ public function testAllErrorTypes() ); } try { - Utf8Validator::validate('"abcd'.chr(237).chr($i).'"'); + Utf8Validator::validate('"abcd'.chr(237).chr($i).'"', $useFastPath); $this->fail('"abcd\d237\d'.$i.'" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); @@ -113,21 +115,21 @@ public function testAllErrorTypes() } for ($i = 246; $i < 255; ++$i) { // 245 and 255 already forbidden try { - Utf8Validator::validate('"abcd'.chr($i).chr(129).chr(129).chr(129).'"'); + Utf8Validator::validate('"abcd'.chr($i).chr(129).chr(129).chr(129).'"', $useFastPath); $this->fail('"abcd\d'.$i.'\d129\d129\d129" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); $this->assertContains(' which is invalid.', $e->getMessage()); } try { - Utf8Validator::validate('"abcd'.chr($i).'"'); + Utf8Validator::validate('"abcd'.chr($i).'"', $useFastPath); $this->fail('"abcd\d'.$i.'" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); $this->assertContains(' which is invalid.', $e->getMessage()); } try { - Utf8Validator::validate('"abcd'.chr(195).chr($i).'"'); + Utf8Validator::validate('"abcd'.chr(195).chr($i).'"', $useFastPath); $this->fail('"abcd\d195\d'.$i.'" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); @@ -138,10 +140,11 @@ public function testAllErrorTypes() public function testExceptionExposesPositionDetails() { + $useFastPath = Utf8Validator::getUseFastPath(); try { // 255 (0xFF) is one of the always-forbidden octets, so the failure // is reported on the octet itself rather than a following one. - Utf8Validator::validate('ab'.chr(255)); + Utf8Validator::validate('ab'.chr(255), $useFastPath); $this->fail('Invalid UTF-8 should not pass validation.'); } catch (InvalidEncodingException $e) { $details = $e->getDetails(); From 4a6fad72857b2634a62f3a05f5f261861c6007de Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Fri, 12 Jun 2026 11:12:28 +0200 Subject: [PATCH 2/2] Remove getFastPAth and arg complications --- src/Seld/JsonLint/JsonParser.php | 2 +- src/Seld/JsonLint/Utf8Validator.php | 31 ++++++++----------------- tests/Utf8ValidatorTest.php | 35 +++++++++++++---------------- 3 files changed, 26 insertions(+), 42 deletions(-) diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index d699c49..cb2c4bc 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -208,7 +208,7 @@ public function parse($input, $flags = 0) throw new \InvalidArgumentException('Only one of ALLOW_DUPLICATE_KEYS and ALLOW_DUPLICATE_KEYS_TO_ARRAY can be used, you passed in both.'); } if ($flags & self::VALIDATE_UTF8_ENCODING) { - Utf8Validator::validate($input, Utf8Validator::getUseFastPath()); + Utf8Validator::validate($input); } $this->failOnBOM($input); diff --git a/src/Seld/JsonLint/Utf8Validator.php b/src/Seld/JsonLint/Utf8Validator.php index 3abd0ae..0d19c7a 100644 --- a/src/Seld/JsonLint/Utf8Validator.php +++ b/src/Seld/JsonLint/Utf8Validator.php @@ -32,33 +32,18 @@ class Utf8Validator */ const CONTINUATION_OCTET_MAXIMUM = 191; - /** - * Checks that the version of PHP allows using the fast-path. - * - * @return bool - */ - public static function getUseFastPath() - { - // Before PHP 5.4, Unicode support has bugs. - return PHP_VERSION_ID >= 50400; - } - /** * Validates that the whole input string is valid UTF-8. * * @param string $input - * @param bool $useFastPath * @return void * @throws InvalidEncodingException if the input is not valid UTF-8 */ - public static function validate($input, $useFastPath = true) + public static function validate($input) { // Fast-path - if ($useFastPath && !Utf8Validator::getUseFastPath()) { - throw new \Exception( - 'Fast-path is not available with version '.PHP_VERSION_ID.' of PHP.' - ); - } + // But before PHP 5.4 Unicode support has bugs. + $useFastPath = PHP_VERSION_ID >= 50400; if ($useFastPath) { if (function_exists("mb_check_encoding")) { if (mb_check_encoding($input, 'UTF-8')) { @@ -310,12 +295,14 @@ public static function validate($input, $useFastPath = true) true ); } + // The fast-path flagged this input as invalid UTF-8, yet the manual RFC 3629 + // scan above found no fault. The two must always agree, so reaching here means + // the fast-path caught something the scan missed - surface it rather than + // silently accepting the input as valid. if ($useFastPath) { throw new InvalidEncodingException( - "Fast-path detected an error that this function couldn't found." - .'Please create an issue at ' - .'"https://github.com/Seldaek/jsonlint/issues" and if possible at' - .'"https://github.com/LLyaudet/DevOrSysAdminScripts/issues".', + "Fast-path detected an error that the manual scan could not find. " + ."Please report it at https://github.com/Seldaek/jsonlint/issues.", "0", array( 'current_octet' => $currentOctet, diff --git a/tests/Utf8ValidatorTest.php b/tests/Utf8ValidatorTest.php index 9775745..ed432a8 100644 --- a/tests/Utf8ValidatorTest.php +++ b/tests/Utf8ValidatorTest.php @@ -20,13 +20,12 @@ class Utf8ValidatorTest extends TestCase { public function testValidUtf8() { - $useFastPath = Utf8Validator::getUseFastPath(); try { - Utf8Validator::validate('', $useFastPath); - Utf8Validator::validate('abcdé', $useFastPath); // 2-octet character - Utf8Validator::validate('euro sign: € and more', $useFastPath); // 3-octet character - Utf8Validator::validate('musical clef: 𝄞', $useFastPath); // 4-octet character - Utf8Validator::validate("line 1\nline 2\nline 3", $useFastPath); // newline handling + Utf8Validator::validate(''); + Utf8Validator::validate('abcdé'); // 2-octet character + Utf8Validator::validate('euro sign: € and more'); // 3-octet character + Utf8Validator::validate('musical clef: 𝄞'); // 4-octet character + Utf8Validator::validate("line 1\nline 2\nline 3"); // newline handling $this->addToAssertionCount(1); } catch (InvalidEncodingException $e) { $this->fail('Valid UTF-8 should pass validation: '.$e->getMessage()); @@ -35,16 +34,15 @@ public function testValidUtf8() public function testAllErrorTypes() { - $useFastPath = Utf8Validator::getUseFastPath(); try { - Utf8Validator::validate('"abcd'.chr(233).'"', $useFastPath); + Utf8Validator::validate('"abcd'.chr(233).'"'); $this->fail('ISO 8859-15 "abcdé" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); $this->assertContains(' which is not a continuation octet.', $e->getMessage()); } try { - Utf8Validator::validate('"abcd'.chr(233), $useFastPath); + Utf8Validator::validate('"abcd'.chr(233)); $this->fail('ISO 8859-15 "abcdé should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); @@ -61,7 +59,7 @@ public function testAllErrorTypes() ); foreach ($forbiddenOctets as $forbiddenOctet) { try { - Utf8Validator::validate('"abcd'.chr(233).chr($forbiddenOctet).'"', $useFastPath); + Utf8Validator::validate('"abcd'.chr(233).chr($forbiddenOctet).'"'); $this->fail('"abcd\d233\d'.$forbiddenOctet.'" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); @@ -71,7 +69,7 @@ public function testAllErrorTypes() ); } try { - Utf8Validator::validate('"abcd'.chr($forbiddenOctet).'"', $useFastPath); + Utf8Validator::validate('"abcd'.chr($forbiddenOctet).'"'); $this->fail('"abcd\d'.$forbiddenOctet.'" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); @@ -82,7 +80,7 @@ public function testAllErrorTypes() } } try { - Utf8Validator::validate('"abcd'.chr(129).'"', $useFastPath); + Utf8Validator::validate('"abcd'.chr(129).'"'); $this->fail('"abcd\d129" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); @@ -93,7 +91,7 @@ public function testAllErrorTypes() } for ($i = 160; $i <= 191; ++$i) { try { - Utf8Validator::validate('"abcd'.chr(237).chr($i).chr(129).'"', $useFastPath); + Utf8Validator::validate('"abcd'.chr(237).chr($i).chr(129).'"'); $this->fail('"abcd\d237\d'.$i.'\d129" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); @@ -103,7 +101,7 @@ public function testAllErrorTypes() ); } try { - Utf8Validator::validate('"abcd'.chr(237).chr($i).'"', $useFastPath); + Utf8Validator::validate('"abcd'.chr(237).chr($i).'"'); $this->fail('"abcd\d237\d'.$i.'" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); @@ -115,21 +113,21 @@ public function testAllErrorTypes() } for ($i = 246; $i < 255; ++$i) { // 245 and 255 already forbidden try { - Utf8Validator::validate('"abcd'.chr($i).chr(129).chr(129).chr(129).'"', $useFastPath); + Utf8Validator::validate('"abcd'.chr($i).chr(129).chr(129).chr(129).'"'); $this->fail('"abcd\d'.$i.'\d129\d129\d129" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); $this->assertContains(' which is invalid.', $e->getMessage()); } try { - Utf8Validator::validate('"abcd'.chr($i).'"', $useFastPath); + Utf8Validator::validate('"abcd'.chr($i).'"'); $this->fail('"abcd\d'.$i.'" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); $this->assertContains(' which is invalid.', $e->getMessage()); } try { - Utf8Validator::validate('"abcd'.chr(195).chr($i).'"', $useFastPath); + Utf8Validator::validate('"abcd'.chr(195).chr($i).'"'); $this->fail('"abcd\d195\d'.$i.'" should not pass validation.'); } catch (InvalidEncodingException $e) { $this->assertContains('Non-UTF8 character found', $e->getMessage()); @@ -140,11 +138,10 @@ public function testAllErrorTypes() public function testExceptionExposesPositionDetails() { - $useFastPath = Utf8Validator::getUseFastPath(); try { // 255 (0xFF) is one of the always-forbidden octets, so the failure // is reported on the octet itself rather than a following one. - Utf8Validator::validate('ab'.chr(255), $useFastPath); + Utf8Validator::validate('ab'.chr(255)); $this->fail('Invalid UTF-8 should not pass validation.'); } catch (InvalidEncodingException $e) { $details = $e->getDetails();