From 284cdf19368227d68f52102d3d407cc808fc7386 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Thu, 11 Jun 2026 14:42:52 +0200 Subject: [PATCH 1/3] CS fixes, php5.3 support fixes --- bin/jsonlint | 27 ++++++++++++----------- src/Seld/JsonLint/JsonParser.php | 37 ++++++++++++++++---------------- src/Seld/JsonLint/Lexer.php | 6 +++--- tests/JsonParserTest.php | 8 ++++--- 4 files changed, 41 insertions(+), 37 deletions(-) diff --git a/bin/jsonlint b/bin/jsonlint index f8c4578..32316be 100755 --- a/bin/jsonlint +++ b/bin/jsonlint @@ -54,11 +54,7 @@ if (isset($_SERVER['argc']) && $_SERVER['argc'] > 1) { } } -$flags = getFlags( - $allowComments, - $detectKeyConflicts, - $validateUTF8Encoding, -); +$flags = getFlags($allowComments, $detectKeyConflicts, $validateUTF8Encoding); if (!empty($files)) { // file linting @@ -80,15 +76,19 @@ if (!empty($files)) { } } -function getFlags( - $allowComments = false, - $detectKeyConflicts = false, - $validateUTF8Encoding = false, -){ +function getFlags($allowComments = false, $detectKeyConflicts = false, $validateUTF8Encoding = false) +{ $flags = 0; - if($allowComments){ $flags |= JsonParser::ALLOW_COMMENTS; } - if($detectKeyConflicts){ $flags |= JsonParser::DETECT_KEY_CONFLICTS; } - if($validateUTF8Encoding){ $flags |= JsonParser::VALIDATE_UTF8_ENCODING; } + if ($allowComments) { + $flags |= JsonParser::ALLOW_COMMENTS; + } + if ($detectKeyConflicts) { + $flags |= JsonParser::DETECT_KEY_CONFLICTS; + } + if ($validateUTF8Encoding) { + $flags |= JsonParser::VALIDATE_UTF8_ENCODING; + } + return $flags; } @@ -129,6 +129,7 @@ function lintFile($file, $quiet = false, $flags = 0) if (!$quiet) { echo 'Valid JSON (' . $file . ')' . PHP_EOL; } + return true; } diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index 64c3f04..bb546b8 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -10,6 +10,7 @@ */ namespace Seld\JsonLint; + use stdClass; /** @@ -244,7 +245,7 @@ public function parse($input, $flags = 0) while (true) { // retrieve state number from top of stack - $state = $this->stack[\count($this->stack)-1]; + $state = $this->stack[\count($this->stack) - 1]; // use default actions if available if (isset($this->defaultActions[$state])) { @@ -282,7 +283,7 @@ public function parse($input, $flags = 0) } } - $errStr = 'Parse error on line ' . ($yylineno+1) . ":\n"; + $errStr = 'Parse error on line ' . ($yylineno + 1) . ":\n"; $errStr .= $this->lexer->showPosition() . "\n"; if ($message) { $errStr .= $message; @@ -328,12 +329,12 @@ public function parse($input, $flags = 0) throw new ParsingException($errStr ?: 'Parsing halted.'); } $this->popStack(1); - $state = $this->stack[\count($this->stack)-1]; + $state = $this->stack[\count($this->stack) - 1]; } $preErrorSymbol = $symbol; // save the lookahead token $symbol = Lexer::T_ERROR; // insert generic error symbol as new lookahead - $state = $this->stack[\count($this->stack)-1]; + $state = $this->stack[\count($this->stack) - 1]; /** @var array|false */ $action = isset($this->table[$state][Lexer::T_ERROR]) ? $this->table[$state][Lexer::T_ERROR] : false; if ($action === false) { @@ -395,7 +396,7 @@ public function parse($input, $flags = 0) $this->vstack[] = $newToken; $this->lstack[] = $position; /** @var int */ - $newState = $this->table[$this->stack[\count($this->stack)-2]][$this->stack[\count($this->stack)-1]]; + $newState = $this->table[$this->stack[\count($this->stack) - 2]][$this->stack[\count($this->stack) - 1]]; $this->stack[] = $newState; break; @@ -620,7 +621,7 @@ private function validateUTF8Encoding($input) // But before PHP 5.4 Unicode support has bugs. if (PHP_VERSION_ID >= 50400) { if (function_exists("mb_check_encoding")) { - if(mb_check_encoding($input, 'UTF-8')){ + if (mb_check_encoding($input, 'UTF-8')) { return; } } else { @@ -722,10 +723,10 @@ private function validateUTF8Encoding($input) UTF8-tail = %x80-BF */ if ($iContinuationOctetNeeded > 0) { - if( + if ( $iCurrentOctet < $iCurrentContinuationOctetMinimum || $iCurrentOctet > $iCurrentContinuationOctetMaximum - ){ + ) { throw new InvalidEncodingException( "Non-UTF8 character found on line " .$iCurrentLineNumber @@ -736,7 +737,7 @@ private function validateUTF8Encoding($input) .", has value " .$iCurrentOctet .( - $sMessageForContinuationOctetAboveMaximum !== null + $sMessageForContinuationOctetAboveMaximum !== null && $iCurrentOctet > $iCurrentContinuationOctetMaximum ? $sMessageForContinuationOctetAboveMaximum : " which is not a continuation octet." @@ -862,15 +863,15 @@ private function validateUTF8Encoding($input) is 237 and second octet, first continuation octet, is >= 160. */ - if($iCurrentOctet === 224){ + if ($iCurrentOctet === 224) { $iCurrentContinuationOctetMinimum = 160; $iCurrentContinuationOctetMaximum = 191; // Normal value } - if($iCurrentOctet === 237){ + if ($iCurrentOctet === 237) { $iCurrentContinuationOctetMinimum = 128; // Normal value $iCurrentContinuationOctetMaximum = 159; $sMessageForContinuationOctetAboveMaximum = ( - " which is into the forbidden range of surrogate pairs." + " which is into the forbidden range of surrogate pairs." ); } continue; @@ -884,13 +885,13 @@ private function validateUTF8Encoding($input) Notice that F0 = 240 adds a restriction on second octet. Notice that F4 = 244 adds a restriction on second octet. */ - if($iCurrentOctet === 240){ - $iCurrentContinuationOctetMinimum = 144; - $iCurrentContinuationOctetMaximum = 191; // Normal value + if ($iCurrentOctet === 240) { + $iCurrentContinuationOctetMinimum = 144; + $iCurrentContinuationOctetMaximum = 191; // Normal value } - if($iCurrentOctet === 244){ - $iCurrentContinuationOctetMinimum = 128; // Normal value - $iCurrentContinuationOctetMaximum = 143; + if ($iCurrentOctet === 244) { + $iCurrentContinuationOctetMinimum = 128; // Normal value + $iCurrentContinuationOctetMaximum = 143; } continue; } diff --git a/src/Seld/JsonLint/Lexer.php b/src/Seld/JsonLint/Lexer.php index 6db5e1e..7a2e140 100644 --- a/src/Seld/JsonLint/Lexer.php +++ b/src/Seld/JsonLint/Lexer.php @@ -102,7 +102,7 @@ public function lex() case self::T_COMMENT: case self::T_OPEN_COMMENT: if (!($this->flags & JsonParser::ALLOW_COMMENTS)) { - $this->parseError('Lexical error on line ' . ($this->yylineno+1) . ". Comments are not allowed.\n" . $this->showPosition()); + $this->parseError('Lexical error on line ' . ($this->yylineno + 1) . ". Comments are not allowed.\n" . $this->showPosition()); } $this->skipUntil($symbol === self::T_COMMENT ? self::T_BREAK_LINE : self::T_CLOSE_COMMENT); if ($this->done) { @@ -247,7 +247,7 @@ private function next() $this->yylineno += $lineCount; $this->yylloc = array( 'first_line' => $this->yylloc['last_line'], - 'last_line' => $this->yylineno+1, + 'last_line' => $this->yylineno + 1, 'first_column' => $this->yylloc['last_column'], 'last_column' => $lineCount > 0 ? \strlen($lines[$lineCount - 1]) : $this->yylloc['last_column'] + \strlen($match[0]), ); @@ -265,7 +265,7 @@ private function next() } $this->parseError( - 'Lexical error on line ' . ($this->yylineno+1) . ". Unrecognized text.\n" . $this->showPosition() + 'Lexical error on line ' . ($this->yylineno + 1) . ". Unrecognized text.\n" . $this->showPosition() ); } diff --git a/tests/JsonParserTest.php b/tests/JsonParserTest.php index 370c838..d3dd44a 100644 --- a/tests/JsonParserTest.php +++ b/tests/JsonParserTest.php @@ -235,7 +235,8 @@ public function testDuplicateKeys() $str = '{"a":"b", "a":"c", "a":"d"}'; $result = $parser->parse($str, JsonParser::ALLOW_DUPLICATE_KEYS); - $this->assertThat($result, + $this->assertThat( + $result, $this->logicalAnd( $this->objectHasAttribute('a'), $this->objectHasAttribute('a.1'), @@ -270,7 +271,8 @@ public function testDuplicateKeysWithEmpty() $this->markTestSkipped('Only for PHP < 7.1'); } $result = $parser->parse('{"":"a", "_empty_":"b"}', JsonParser::ALLOW_DUPLICATE_KEYS); - $this->assertThat($result, + $this->assertThat( + $result, $this->logicalAnd( $this->objectHasAttribute('_empty_'), $this->objectHasAttribute('_empty_.1') @@ -465,7 +467,7 @@ public function testLongString() { $parser = new JsonParser(); - $json = '{"k":"' . str_repeat("a\\n",10000) . '"}'; + $json = '{"k":"' . str_repeat("a\\n", 10000) . '"}'; $this->assertEquals(json_decode($json), $parser->parse($json)); } From 19f6f858ff442a3023b9c1b8e3170c5fd03f42eb Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Thu, 11 Jun 2026 15:01:33 +0200 Subject: [PATCH 2/3] Deduplicate exception throwing and fix InvalidEncodingException data types --- .github/workflows/lint.yml | 2 +- .../JsonLint/InvalidEncodingException.php | 6 +- src/Seld/JsonLint/JsonParser.php | 185 +++++++----------- 3 files changed, 75 insertions(+), 118 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d60726e..7c72de7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -36,7 +36,7 @@ jobs: - name: "Lint PHP files" run: | hasErrors=0 - for f in $(find src/ tests/ -type f -name '*.php' ! -path '*/vendor/*' ! -path '*/Fixtures/*') + for f in bin/jsonlint $(find src/ tests/ -type f -name '*.php' ! -path '*/vendor/*' ! -path '*/Fixtures/*') do { error="$(php -derror_reporting=-1 -ddisplay_errors=1 -l -f $f 2>&1 1>&3 3>&-)"; } 3>&1; if [ "$error" != "" ]; then diff --git a/src/Seld/JsonLint/InvalidEncodingException.php b/src/Seld/JsonLint/InvalidEncodingException.php index ee5e7d5..70ca494 100644 --- a/src/Seld/JsonLint/InvalidEncodingException.php +++ b/src/Seld/JsonLint/InvalidEncodingException.php @@ -17,14 +17,14 @@ class InvalidEncodingException extends ParsingException { /** - * @var array{key: string, line: int} + * @var array{key: string, current_octet: int|null, continuation_octet_needed: int, offset_in_octets_from_string_start: int, offset_in_characters_from_string_start: int, character_start_position_from_string_start: int, line: int, offset_in_octets_from_line_start: int, offset_in_characters_from_line_start: int, character_start_position_from_line_start: int, current_continuation_octet_minimum: int, current_continuation_octet_maximum: int} */ protected $details; /** * @param string $message * @param string $key - * @phpstan-param array{line: int} $details + * @phpstan-param array{current_octet: int|null, continuation_octet_needed: int, offset_in_octets_from_string_start: int, offset_in_characters_from_string_start: int, character_start_position_from_string_start: int, line: int, offset_in_octets_from_line_start: int, offset_in_characters_from_line_start: int, character_start_position_from_line_start: int, current_continuation_octet_minimum: int, current_continuation_octet_maximum: int} $details */ public function __construct($message, $key, array $details) { @@ -41,7 +41,7 @@ public function getKey() } /** - * @phpstan-return array{key: string, line: int} + * @phpstan-return array{key: string, current_octet: int|null, continuation_octet_needed: int, offset_in_octets_from_string_start: int, offset_in_characters_from_string_start: int, character_start_position_from_string_start: int, line: int, offset_in_octets_from_line_start: int, offset_in_characters_from_line_start: int, character_start_position_from_line_start: int, current_continuation_octet_minimum: int, current_continuation_octet_maximum: int} */ public function getDetails() { diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index bb546b8..d4eb6d8 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -667,28 +667,8 @@ private function validateUTF8Encoding($input) || $iCurrentOctet === 245 || $iCurrentOctet === 255 ) { - throw new InvalidEncodingException( - "Non-UTF8 character found on line " - .$iCurrentLineNumber - ."; the octet " - .($iOffsetInOctetsFromLineStart + 1) - .", part of the character " - .($iOffsetInCharactersFromLineStart + 1) - .", has value " - .$iCurrentOctet - ." which is one of the four forbidden values (C0, C1, F5, FF)." - ." This character starts at octet " - .($iCharacterStartPositionFromLineStart + 1) - ." of the current line." - ." (Sequential positions without line splitting:" - ." This is at character " - .($iOffsetInCharactersFromStringStart + 1) - ." and octet " - .($iOffsetInOctetsFromStringStart + 1) - ."." - ." This character starts at octet " - .($iCharacterStartPositionFromStringStart + 1) - .".)", + throw $this->createUTF8EncodingException( + " which is one of the four forbidden values (C0, C1, F5, FF).", (string) $iCurrentOctet, array( 'current_octet' => $iCurrentOctet, @@ -727,33 +707,13 @@ private function validateUTF8Encoding($input) $iCurrentOctet < $iCurrentContinuationOctetMinimum || $iCurrentOctet > $iCurrentContinuationOctetMaximum ) { - throw new InvalidEncodingException( - "Non-UTF8 character found on line " - .$iCurrentLineNumber - ."; the octet " - .($iOffsetInOctetsFromLineStart + 1) - .", part of the character " - .($iOffsetInCharactersFromLineStart + 1) - .", has value " - .$iCurrentOctet - .( - $sMessageForContinuationOctetAboveMaximum !== null - && $iCurrentOctet > $iCurrentContinuationOctetMaximum - ? $sMessageForContinuationOctetAboveMaximum - : " which is not a continuation octet." - ) - ." This character starts at octet " - .($iCharacterStartPositionFromLineStart + 1) - ." of the current line." - ." (Sequential positions without line splitting:" - ." This is at character " - .($iOffsetInCharactersFromStringStart + 1) - ." and octet " - .($iOffsetInOctetsFromStringStart + 1) - ."." - ." This character starts at octet " - .($iCharacterStartPositionFromStringStart + 1) - .".)", + $reason = + $sMessageForContinuationOctetAboveMaximum !== null + && $iCurrentOctet > $iCurrentContinuationOctetMaximum + ? $sMessageForContinuationOctetAboveMaximum + : " which is not a continuation octet."; + throw $this->createUTF8EncodingException( + $reason, (string) $iCurrentOctet, array( 'current_octet' => $iCurrentOctet, @@ -793,28 +753,8 @@ private function validateUTF8Encoding($input) } if (/*$iCurrentOctet >= 128 &&*/$iCurrentOctet < 192) { - throw new InvalidEncodingException( - "Non-UTF8 character found on line " - .$iCurrentLineNumber - ."; the octet " - .($iOffsetInOctetsFromLineStart + 1) - .", part of the character " - .($iOffsetInCharactersFromLineStart + 1) - .", has value " - .$iCurrentOctet - ." which is a continuation octet." - ." This character starts at octet " - .($iCharacterStartPositionFromLineStart + 1) - ." of the current line." - ." (Sequential positions without line splitting:" - ." This is at character " - .($iOffsetInCharactersFromStringStart + 1) - ." and octet " - .($iOffsetInOctetsFromStringStart + 1) - ."." - ." This character starts at octet " - .($iCharacterStartPositionFromStringStart + 1) - .".)", + throw $this->createUTF8EncodingException( + " which is a continuation octet.", (string) $iCurrentOctet, array( 'current_octet' => $iCurrentOctet, @@ -895,28 +835,8 @@ private function validateUTF8Encoding($input) } continue; } - throw new InvalidEncodingException( - "Non-UTF8 character found on line " - .$iCurrentLineNumber - ."; the octet " - .($iOffsetInOctetsFromLineStart + 1) - .", part of the character " - .($iOffsetInCharactersFromLineStart + 1) - .", has value " - .$iCurrentOctet - ." which is invalid." - ." This character starts at octet " - .($iCharacterStartPositionFromLineStart + 1) - ." of the current line." - ." (Sequential positions without line splitting:" - ." This is at character " - .($iOffsetInCharactersFromStringStart + 1) - ." and octet " - .($iOffsetInOctetsFromStringStart + 1) - ."." - ." This character starts at octet " - .($iCharacterStartPositionFromStringStart + 1) - .".)", + throw $this->createUTF8EncodingException( + " which is invalid.", (string) $iCurrentOctet, array( 'current_octet' => $iCurrentOctet, @@ -934,26 +854,8 @@ private function validateUTF8Encoding($input) ); } if ($iContinuationOctetNeeded > 0) { - throw new InvalidEncodingException( - "Non-UTF8 character found on line " - .$iCurrentLineNumber - ."; at octet " - .($iOffsetInOctetsFromLineStart + 1) - .", part of the character " - .($iOffsetInCharactersFromLineStart + 1) - .", end of string was found instead of a continuation octet." - ." This character starts at octet " - .($iCharacterStartPositionFromLineStart + 1) - ." of the current line." - ." (Sequential positions without line splitting:" - ." This is at character " - .($iOffsetInCharactersFromStringStart + 1) - ." and octet " - .($iOffsetInOctetsFromStringStart + 1) - ."." - ." This character starts at octet " - .($iCharacterStartPositionFromStringStart + 1) - .".)", + throw $this->createUTF8EncodingException( + "", "0", array( 'current_octet' => $iCurrentOctet, @@ -967,8 +869,63 @@ private function validateUTF8Encoding($input) 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, 'current_continuation_octet_minimum' => $iCurrentContinuationOctetMinimum, 'current_continuation_octet_maximum' => $iCurrentContinuationOctetMaximum, - ) + ), + true ); } } + + /** + * Builds the InvalidEncodingException thrown by validateUTF8Encoding(). + * + * The message scaffold is shared by every detection branch; only the + * "$reason" fragment (appended after "has value N") and the end-of-string + * wording differ between call sites. + * + * @param string $reason explanation appended after the octet value (ignored when $endOfInput is true) + * @param string $key the offending octet value, as a string + * @param bool $endOfInput whether the input ended in the middle of a multi-octet character + * @return InvalidEncodingException + * + * @phpstan-param array{current_octet: int|null, continuation_octet_needed: int, offset_in_octets_from_string_start: int, offset_in_characters_from_string_start: int, character_start_position_from_string_start: int, line: int, offset_in_octets_from_line_start: int, offset_in_characters_from_line_start: int, character_start_position_from_line_start: int, current_continuation_octet_minimum: int, current_continuation_octet_maximum: int} $details + */ + private function createUTF8EncodingException($reason, $key, array $details, $endOfInput = false) + { + if ($endOfInput) { + $middle = + "; at octet " + .($details['offset_in_octets_from_line_start'] + 1) + .", part of the character " + .($details['offset_in_characters_from_line_start'] + 1) + .", end of string was found instead of a continuation octet."; + } else { + $middle = + "; the octet " + .($details['offset_in_octets_from_line_start'] + 1) + .", part of the character " + .($details['offset_in_characters_from_line_start'] + 1) + .", has value " + .$details['current_octet'] + .$reason; + } + + $message = + "Non-UTF8 character found on line " + .$details['line'] + .$middle + ." This character starts at octet " + .($details['character_start_position_from_line_start'] + 1) + ." of the current line." + ." (Sequential positions without line splitting:" + ." This is at character " + .($details['offset_in_characters_from_string_start'] + 1) + ." and octet " + .($details['offset_in_octets_from_string_start'] + 1) + ."." + ." This character starts at octet " + .($details['character_start_position_from_string_start'] + 1) + .".)"; + + return new InvalidEncodingException($message, $key, $details); + } } From 794ec6a2a4b8788ac299029e5ab9978e1ced8aef Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Thu, 11 Jun 2026 15:23:25 +0200 Subject: [PATCH 3/3] Extract Utf8Validator into its own class --- README.md | 23 ++ phpstan.neon.dist | 2 +- src/Seld/JsonLint/JsonParser.php | 320 +------------------------ src/Seld/JsonLint/Utf8Validator.php | 352 ++++++++++++++++++++++++++++ tests/JsonParserTest.php | 135 ----------- tests/Utf8ValidatorTest.php | 150 ++++++++++++ 6 files changed, 527 insertions(+), 455 deletions(-) create mode 100644 src/Seld/JsonLint/Utf8Validator.php create mode 100644 tests/Utf8ValidatorTest.php diff --git a/README.md b/README.md index e658f72..987f0e1 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ You can also pass additional flags to `JsonParser::lint/parse` that tweak the fu - `JsonParser::PARSE_TO_ASSOC` parses to associative arrays instead of stdClass objects. - `JsonParser::ALLOW_COMMENTS` parses while allowing (and ignoring) inline `//` and multiline `/* */` comments in the JSON document. - `JsonParser::ALLOW_DUPLICATE_KEYS_TO_ARRAY` collects duplicate keys. e.g. if you have two `foo` keys the `foo` key will become an object (or array in assoc mode) with all `foo` values accessible as an array in `$result->foo->__duplicates__` (or `$result['foo']['__duplicates__']` in assoc mode). +- `JsonParser::VALIDATE_UTF8_ENCODING` validates that the whole input is well-formed UTF-8 (as defined by [RFC 3629](https://www.rfc-editor.org/rfc/rfc3629)) before parsing, throwing an `InvalidEncodingException` that points at the first invalid byte otherwise. Example: @@ -49,6 +50,28 @@ try { } ``` +Validating UTF-8 encoding +------------------------- + +The UTF-8 validation behind the `VALIDATE_UTF8_ENCODING` flag is also available as a +standalone, reusable utility through the `Seld\JsonLint\Utf8Validator` class, should you +need to validate UTF-8 encoding outside of JSON parsing: + +```php +use Seld\JsonLint\Utf8Validator; +use Seld\JsonLint\InvalidEncodingException; + +try { + // returns void on success, throws on the first invalid byte + Utf8Validator::validate($string); +} catch (InvalidEncodingException $e) { + // getMessage() describes the offending byte and its position + echo $e->getMessage(); + // getDetails() returns that position information as an array + $details = $e->getDetails(); +} +``` + > **Note:** This library is meant to parse JSON while providing good error messages on failure. There is no way it can be as fast as php native `json_decode()`. > > It is recommended to parse with `json_decode`, and when it fails parse again with seld/jsonlint to get a proper error message back to the user. See for example [how Composer uses this library](https://github.com/composer/composer/blob/56edd53046fd697d32b2fd2fbaf45af5d7951671/src/Composer/Json/JsonFile.php#L283-L318): diff --git a/phpstan.neon.dist b/phpstan.neon.dist index bd3a0ec..59849bb 100644 --- a/phpstan.neon.dist +++ b/phpstan.neon.dist @@ -12,4 +12,4 @@ parameters: - tests/ ignoreErrors: - - '/Method JsonParserTest::[a-zA-Z0-9]+\(\) has no return type/' + - '/Method [a-zA-Z0-9]+Test::[a-zA-Z0-9]+\(\) has no return type/' diff --git a/src/Seld/JsonLint/JsonParser.php b/src/Seld/JsonLint/JsonParser.php index d4eb6d8..cb2c4bc 100644 --- a/src/Seld/JsonLint/JsonParser.php +++ b/src/Seld/JsonLint/JsonParser.php @@ -208,7 +208,7 @@ public function parse($input, $flags = 0) throw new \InvalidArgumentException('Only one of ALLOW_DUPLICATE_KEYS and ALLOW_DUPLICATE_KEYS_TO_ARRAY can be used, you passed in both.'); } if ($flags & self::VALIDATE_UTF8_ENCODING) { - $this->validateUTF8Encoding($input); + Utf8Validator::validate($input); } $this->failOnBOM($input); @@ -610,322 +610,4 @@ private function failOnBOM($input) $this->parseError("BOM detected, make sure your input does not include a Unicode Byte-Order-Mark"); } } - - /** - * @param string $input - * @return void - */ - private function validateUTF8Encoding($input) - { - // Fast-path - // But before PHP 5.4 Unicode support has bugs. - if (PHP_VERSION_ID >= 50400) { - if (function_exists("mb_check_encoding")) { - if (mb_check_encoding($input, 'UTF-8')) { - return; - } - } else { - if (preg_match('//u', $input) === 1) { - return; - } - } - } - - $iCurrentOctet = null; - $iContinuationOctetNeeded = 0; - $iOffsetInOctetsFromStringStart = 0; - $iOffsetInCharactersFromStringStart = -1; - $iCharacterStartPositionFromStringStart = 0; - $iCurrentLineNumber = 1; - $iOffsetInOctetsFromLineStart = -1; - $iOffsetInCharactersFromLineStart = -1; - $iCharacterStartPositionFromLineStart = -1; - - // For the second octet of a character, hence first continuation octet, - // further restriction may apply. - $I_CONTINUATION_OCTET_MINIMUM = 128; - $I_CONTINUATION_OCTET_MAXIMUM = 191; - $iCurrentContinuationOctetMinimum = $I_CONTINUATION_OCTET_MINIMUM; - $iCurrentContinuationOctetMaximum = $I_CONTINUATION_OCTET_MAXIMUM; - $sMessageForContinuationOctetAboveMaximum = null; - - for ($i = 0, $iMax = strlen($input); $i < $iMax; ++$i) { - $iCurrentOctet = ord($input[$i]); - $iOffsetInOctetsFromStringStart = $i; - ++$iOffsetInOctetsFromLineStart; - - /* - The octet values C0, C1, F5 to FF never appear. - C0 = 12*16 = 192 = 11000000 - C1 = 12*16 + 1 = 193 = 11000001 - F5 = 15*16 + 5 = 245 = 11110101 - FF = 15*16 + 15 = 255 = 11111111 - */ - if ( - $iCurrentOctet === 192 - || $iCurrentOctet === 193 - || $iCurrentOctet === 245 - || $iCurrentOctet === 255 - ) { - throw $this->createUTF8EncodingException( - " which is one of the four forbidden values (C0, C1, F5, FF).", - (string) $iCurrentOctet, - array( - 'current_octet' => $iCurrentOctet, - 'continuation_octet_needed' => $iContinuationOctetNeeded, - 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, - 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, - 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, - 'line' => $iCurrentLineNumber, - 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, - 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, - 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, - 'current_continuation_octet_minimum' => $iCurrentContinuationOctetMinimum, - 'current_continuation_octet_maximum' => $iCurrentContinuationOctetMaximum, - ) - ); - } - - /* - UTF8-octets = *( UTF8-char ) - UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4 - UTF8-1 = %x00-7F - UTF8-2 = %xC2-DF UTF8-tail - Notice that values C0 and C1 are forbidden, hence %xC2 - UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / - %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) - Notice that E0 = 224 adds a restriction on second octet. - Notice that ED = 237 adds a restriction on second octet. - UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / - %xF4 %x80-8F 2( UTF8-tail ) - Notice that F0 = 240 adds a restriction on second octet. - Notice that F4 = 244 adds a restriction on second octet. - UTF8-tail = %x80-BF - */ - if ($iContinuationOctetNeeded > 0) { - if ( - $iCurrentOctet < $iCurrentContinuationOctetMinimum - || $iCurrentOctet > $iCurrentContinuationOctetMaximum - ) { - $reason = - $sMessageForContinuationOctetAboveMaximum !== null - && $iCurrentOctet > $iCurrentContinuationOctetMaximum - ? $sMessageForContinuationOctetAboveMaximum - : " which is not a continuation octet."; - throw $this->createUTF8EncodingException( - $reason, - (string) $iCurrentOctet, - array( - 'current_octet' => $iCurrentOctet, - 'continuation_octet_needed' => $iContinuationOctetNeeded, - 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, - 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, - 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, - 'line' => $iCurrentLineNumber, - 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, - 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, - 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, - 'current_continuation_octet_minimum' => $iCurrentContinuationOctetMinimum, - 'current_continuation_octet_maximum' => $iCurrentContinuationOctetMaximum, - ) - ); - } - --$iContinuationOctetNeeded; - $iCurrentContinuationOctetMinimum = $I_CONTINUATION_OCTET_MINIMUM; - $iCurrentContinuationOctetMaximum = $I_CONTINUATION_OCTET_MAXIMUM; - $sMessageForContinuationOctetAboveMaximum = null; - continue; - } - - ++$iOffsetInCharactersFromStringStart; - ++$iOffsetInCharactersFromLineStart; - $iCharacterStartPositionFromStringStart = $iOffsetInOctetsFromStringStart; - $iCharacterStartPositionFromLineStart = $iOffsetInOctetsFromLineStart; - - if ($iCurrentOctet < 128) { // 0xxxxxxx ASCII - // if ($input[$i] === "\n") { - if ($iCurrentOctet === 10) { - ++$iCurrentLineNumber; - $iOffsetInOctetsFromLineStart = -1; - $iOffsetInCharactersFromLineStart = -1; - } - continue; - } - - if (/*$iCurrentOctet >= 128 &&*/$iCurrentOctet < 192) { - throw $this->createUTF8EncodingException( - " which is a continuation octet.", - (string) $iCurrentOctet, - array( - 'current_octet' => $iCurrentOctet, - 'continuation_octet_needed' => $iContinuationOctetNeeded, - 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, - 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, - 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, - 'line' => $iCurrentLineNumber, - 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, - 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, - 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, - 'current_continuation_octet_minimum' => $iCurrentContinuationOctetMinimum, - 'current_continuation_octet_maximum' => $iCurrentContinuationOctetMaximum, - ) - ); - } - - if (/*$iCurrentOctet >= 192 &&*/$iCurrentOctet < 224) { - // 110xxxxx 10xxxxxx - $iContinuationOctetNeeded = 1; - continue; - } - if (/*$iCurrentOctet >= 224 &&*/$iCurrentOctet < 240) { - // 1110xxxx 10xxxxxx 10xxxxxx - $iContinuationOctetNeeded = 2; - /* - UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / - %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) - Notice that E0 = 224 adds a restriction on second octet. - Notice that ED = 237 adds a restriction on second octet. - - The definition of UTF-8 prohibits encoding character numbers between - U+D800 and U+DFFF. - D8 = 13*16 + 8 = 216 = 11010100 - DF = 13*16 + 15 = 223 = 11010101 - 216*256 = 55296 = 1101 1000 0000 0000 = D800 - to - 223*256 + 255 = 57343 = 1101 1111 1111 1111 = DFFF - - 1110xxxx - 11101101 = 237 = ED - - 237,160,128 - 237,191,191 - Thus, the whole "continuation range" is forbidden if start - is 237 and second octet, first continuation octet, - is >= 160. - */ - if ($iCurrentOctet === 224) { - $iCurrentContinuationOctetMinimum = 160; - $iCurrentContinuationOctetMaximum = 191; // Normal value - } - if ($iCurrentOctet === 237) { - $iCurrentContinuationOctetMinimum = 128; // Normal value - $iCurrentContinuationOctetMaximum = 159; - $sMessageForContinuationOctetAboveMaximum = ( - " which is into the forbidden range of surrogate pairs." - ); - } - continue; - } - if (/*$iCurrentOctet >= 240 &&*/$iCurrentOctet < /*248*/ 245) { - // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - $iContinuationOctetNeeded = 3; - /* - UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / - %xF4 %x80-8F 2( UTF8-tail ) - Notice that F0 = 240 adds a restriction on second octet. - Notice that F4 = 244 adds a restriction on second octet. - */ - if ($iCurrentOctet === 240) { - $iCurrentContinuationOctetMinimum = 144; - $iCurrentContinuationOctetMaximum = 191; // Normal value - } - if ($iCurrentOctet === 244) { - $iCurrentContinuationOctetMinimum = 128; // Normal value - $iCurrentContinuationOctetMaximum = 143; - } - continue; - } - throw $this->createUTF8EncodingException( - " which is invalid.", - (string) $iCurrentOctet, - array( - 'current_octet' => $iCurrentOctet, - 'continuation_octet_needed' => $iContinuationOctetNeeded, - 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, - 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, - 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, - 'line' => $iCurrentLineNumber, - 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, - 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, - 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, - 'current_continuation_octet_minimum' => $iCurrentContinuationOctetMinimum, - 'current_continuation_octet_maximum' => $iCurrentContinuationOctetMaximum, - ) - ); - } - if ($iContinuationOctetNeeded > 0) { - throw $this->createUTF8EncodingException( - "", - "0", - array( - 'current_octet' => $iCurrentOctet, - 'continuation_octet_needed' => $iContinuationOctetNeeded, - 'offset_in_octets_from_string_start' => $iOffsetInOctetsFromStringStart, - 'offset_in_characters_from_string_start' => $iOffsetInCharactersFromStringStart, - 'character_start_position_from_string_start' => $iCharacterStartPositionFromStringStart, - 'line' => $iCurrentLineNumber, - 'offset_in_octets_from_line_start' => $iOffsetInOctetsFromLineStart, - 'offset_in_characters_from_line_start' => $iOffsetInCharactersFromLineStart, - 'character_start_position_from_line_start' => $iCharacterStartPositionFromLineStart, - 'current_continuation_octet_minimum' => $iCurrentContinuationOctetMinimum, - 'current_continuation_octet_maximum' => $iCurrentContinuationOctetMaximum, - ), - true - ); - } - } - - /** - * Builds the InvalidEncodingException thrown by validateUTF8Encoding(). - * - * The message scaffold is shared by every detection branch; only the - * "$reason" fragment (appended after "has value N") and the end-of-string - * wording differ between call sites. - * - * @param string $reason explanation appended after the octet value (ignored when $endOfInput is true) - * @param string $key the offending octet value, as a string - * @param bool $endOfInput whether the input ended in the middle of a multi-octet character - * @return InvalidEncodingException - * - * @phpstan-param array{current_octet: int|null, continuation_octet_needed: int, offset_in_octets_from_string_start: int, offset_in_characters_from_string_start: int, character_start_position_from_string_start: int, line: int, offset_in_octets_from_line_start: int, offset_in_characters_from_line_start: int, character_start_position_from_line_start: int, current_continuation_octet_minimum: int, current_continuation_octet_maximum: int} $details - */ - private function createUTF8EncodingException($reason, $key, array $details, $endOfInput = false) - { - if ($endOfInput) { - $middle = - "; at octet " - .($details['offset_in_octets_from_line_start'] + 1) - .", part of the character " - .($details['offset_in_characters_from_line_start'] + 1) - .", end of string was found instead of a continuation octet."; - } else { - $middle = - "; the octet " - .($details['offset_in_octets_from_line_start'] + 1) - .", part of the character " - .($details['offset_in_characters_from_line_start'] + 1) - .", has value " - .$details['current_octet'] - .$reason; - } - - $message = - "Non-UTF8 character found on line " - .$details['line'] - .$middle - ." This character starts at octet " - .($details['character_start_position_from_line_start'] + 1) - ." of the current line." - ." (Sequential positions without line splitting:" - ." This is at character " - .($details['offset_in_characters_from_string_start'] + 1) - ." and octet " - .($details['offset_in_octets_from_string_start'] + 1) - ."." - ." This character starts at octet " - .($details['character_start_position_from_string_start'] + 1) - .".)"; - - return new InvalidEncodingException($message, $key, $details); - } } diff --git a/src/Seld/JsonLint/Utf8Validator.php b/src/Seld/JsonLint/Utf8Validator.php new file mode 100644 index 0000000..e785049 --- /dev/null +++ b/src/Seld/JsonLint/Utf8Validator.php @@ -0,0 +1,352 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Seld\JsonLint; + +/** + * Validates that a string is well-formed UTF-8 as defined by RFC 3629. + * + * @author Laurent Lyaudet + */ +class Utf8Validator +{ + /** + * Lowest value a continuation octet (10xxxxxx) may take. + * + * @private + */ + const CONTINUATION_OCTET_MINIMUM = 128; + + /** + * Highest value a continuation octet (10xxxxxx) may take. + * + * @private + */ + const CONTINUATION_OCTET_MAXIMUM = 191; + + /** + * Validates that the whole input string is valid UTF-8. + * + * @param string $input + * @return void + * @throws InvalidEncodingException if the input is not valid UTF-8 + */ + public static function validate($input) + { + // Fast-path + // But before PHP 5.4 Unicode support has bugs. + if (PHP_VERSION_ID >= 50400) { + if (function_exists("mb_check_encoding")) { + if (mb_check_encoding($input, 'UTF-8')) { + return; + } + } else { + if (preg_match('//u', $input) === 1) { + return; + } + } + } + + $currentOctet = null; + $continuationOctetNeeded = 0; + $offsetInOctetsFromStringStart = 0; + $offsetInCharactersFromStringStart = -1; + $characterStartPositionFromStringStart = 0; + $currentLineNumber = 1; + $offsetInOctetsFromLineStart = -1; + $offsetInCharactersFromLineStart = -1; + $characterStartPositionFromLineStart = -1; + + // For the second octet of a character, hence first continuation octet, + // further restriction may apply. + $currentContinuationOctetMinimum = self::CONTINUATION_OCTET_MINIMUM; + $currentContinuationOctetMaximum = self::CONTINUATION_OCTET_MAXIMUM; + $messageForContinuationOctetAboveMaximum = null; + + for ($i = 0, $max = strlen($input); $i < $max; ++$i) { + $currentOctet = ord($input[$i]); + $offsetInOctetsFromStringStart = $i; + ++$offsetInOctetsFromLineStart; + + /* + The octet values C0, C1, F5 to FF never appear. + C0 = 12*16 = 192 = 11000000 + C1 = 12*16 + 1 = 193 = 11000001 + F5 = 15*16 + 5 = 245 = 11110101 + FF = 15*16 + 15 = 255 = 11111111 + */ + if ( + $currentOctet === 192 + || $currentOctet === 193 + || $currentOctet === 245 + || $currentOctet === 255 + ) { + throw self::createException( + " which is one of the four forbidden values (C0, C1, F5, FF).", + (string) $currentOctet, + array( + 'current_octet' => $currentOctet, + 'continuation_octet_needed' => $continuationOctetNeeded, + 'offset_in_octets_from_string_start' => $offsetInOctetsFromStringStart, + 'offset_in_characters_from_string_start' => $offsetInCharactersFromStringStart, + 'character_start_position_from_string_start' => $characterStartPositionFromStringStart, + 'line' => $currentLineNumber, + 'offset_in_octets_from_line_start' => $offsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $offsetInCharactersFromLineStart, + 'character_start_position_from_line_start' => $characterStartPositionFromLineStart, + 'current_continuation_octet_minimum' => $currentContinuationOctetMinimum, + 'current_continuation_octet_maximum' => $currentContinuationOctetMaximum, + ) + ); + } + + /* + UTF8-octets = *( UTF8-char ) + UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4 + UTF8-1 = %x00-7F + UTF8-2 = %xC2-DF UTF8-tail + Notice that values C0 and C1 are forbidden, hence %xC2 + UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / + %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) + Notice that E0 = 224 adds a restriction on second octet. + Notice that ED = 237 adds a restriction on second octet. + UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / + %xF4 %x80-8F 2( UTF8-tail ) + Notice that F0 = 240 adds a restriction on second octet. + Notice that F4 = 244 adds a restriction on second octet. + UTF8-tail = %x80-BF + */ + if ($continuationOctetNeeded > 0) { + if ( + $currentOctet < $currentContinuationOctetMinimum + || $currentOctet > $currentContinuationOctetMaximum + ) { + $reason = + $messageForContinuationOctetAboveMaximum !== null + && $currentOctet > $currentContinuationOctetMaximum + ? $messageForContinuationOctetAboveMaximum + : " which is not a continuation octet."; + throw self::createException( + $reason, + (string) $currentOctet, + array( + 'current_octet' => $currentOctet, + 'continuation_octet_needed' => $continuationOctetNeeded, + 'offset_in_octets_from_string_start' => $offsetInOctetsFromStringStart, + 'offset_in_characters_from_string_start' => $offsetInCharactersFromStringStart, + 'character_start_position_from_string_start' => $characterStartPositionFromStringStart, + 'line' => $currentLineNumber, + 'offset_in_octets_from_line_start' => $offsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $offsetInCharactersFromLineStart, + 'character_start_position_from_line_start' => $characterStartPositionFromLineStart, + 'current_continuation_octet_minimum' => $currentContinuationOctetMinimum, + 'current_continuation_octet_maximum' => $currentContinuationOctetMaximum, + ) + ); + } + --$continuationOctetNeeded; + $currentContinuationOctetMinimum = self::CONTINUATION_OCTET_MINIMUM; + $currentContinuationOctetMaximum = self::CONTINUATION_OCTET_MAXIMUM; + $messageForContinuationOctetAboveMaximum = null; + continue; + } + + ++$offsetInCharactersFromStringStart; + ++$offsetInCharactersFromLineStart; + $characterStartPositionFromStringStart = $offsetInOctetsFromStringStart; + $characterStartPositionFromLineStart = $offsetInOctetsFromLineStart; + + if ($currentOctet < 128) { // 0xxxxxxx ASCII + // if ($input[$i] === "\n") { + if ($currentOctet === 10) { + ++$currentLineNumber; + $offsetInOctetsFromLineStart = -1; + $offsetInCharactersFromLineStart = -1; + } + continue; + } + + if (/*$currentOctet >= 128 &&*/$currentOctet < 192) { + throw self::createException( + " which is a continuation octet.", + (string) $currentOctet, + array( + 'current_octet' => $currentOctet, + 'continuation_octet_needed' => $continuationOctetNeeded, + 'offset_in_octets_from_string_start' => $offsetInOctetsFromStringStart, + 'offset_in_characters_from_string_start' => $offsetInCharactersFromStringStart, + 'character_start_position_from_string_start' => $characterStartPositionFromStringStart, + 'line' => $currentLineNumber, + 'offset_in_octets_from_line_start' => $offsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $offsetInCharactersFromLineStart, + 'character_start_position_from_line_start' => $characterStartPositionFromLineStart, + 'current_continuation_octet_minimum' => $currentContinuationOctetMinimum, + 'current_continuation_octet_maximum' => $currentContinuationOctetMaximum, + ) + ); + } + + if (/*$currentOctet >= 192 &&*/$currentOctet < 224) { + // 110xxxxx 10xxxxxx + $continuationOctetNeeded = 1; + continue; + } + if (/*$currentOctet >= 224 &&*/$currentOctet < 240) { + // 1110xxxx 10xxxxxx 10xxxxxx + $continuationOctetNeeded = 2; + /* + UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / + %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) + Notice that E0 = 224 adds a restriction on second octet. + Notice that ED = 237 adds a restriction on second octet. + + The definition of UTF-8 prohibits encoding character numbers between + U+D800 and U+DFFF. + D8 = 13*16 + 8 = 216 = 11010100 + DF = 13*16 + 15 = 223 = 11010101 + 216*256 = 55296 = 1101 1000 0000 0000 = D800 + to + 223*256 + 255 = 57343 = 1101 1111 1111 1111 = DFFF + + 1110xxxx + 11101101 = 237 = ED + + 237,160,128 + 237,191,191 + Thus, the whole "continuation range" is forbidden if start + is 237 and second octet, first continuation octet, + is >= 160. + */ + if ($currentOctet === 224) { + $currentContinuationOctetMinimum = 160; + $currentContinuationOctetMaximum = 191; // Normal value + } + if ($currentOctet === 237) { + $currentContinuationOctetMinimum = 128; // Normal value + $currentContinuationOctetMaximum = 159; + $messageForContinuationOctetAboveMaximum = + " which is into the forbidden range of surrogate pairs."; + } + continue; + } + if (/*$currentOctet >= 240 &&*/$currentOctet < /*248*/ 245) { + // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + $continuationOctetNeeded = 3; + /* + UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / + %xF4 %x80-8F 2( UTF8-tail ) + Notice that F0 = 240 adds a restriction on second octet. + Notice that F4 = 244 adds a restriction on second octet. + */ + if ($currentOctet === 240) { + $currentContinuationOctetMinimum = 144; + $currentContinuationOctetMaximum = 191; // Normal value + } + if ($currentOctet === 244) { + $currentContinuationOctetMinimum = 128; // Normal value + $currentContinuationOctetMaximum = 143; + } + continue; + } + throw self::createException( + " which is invalid.", + (string) $currentOctet, + array( + 'current_octet' => $currentOctet, + 'continuation_octet_needed' => $continuationOctetNeeded, + 'offset_in_octets_from_string_start' => $offsetInOctetsFromStringStart, + 'offset_in_characters_from_string_start' => $offsetInCharactersFromStringStart, + 'character_start_position_from_string_start' => $characterStartPositionFromStringStart, + 'line' => $currentLineNumber, + 'offset_in_octets_from_line_start' => $offsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $offsetInCharactersFromLineStart, + 'character_start_position_from_line_start' => $characterStartPositionFromLineStart, + 'current_continuation_octet_minimum' => $currentContinuationOctetMinimum, + 'current_continuation_octet_maximum' => $currentContinuationOctetMaximum, + ) + ); + } + if ($continuationOctetNeeded > 0) { + throw self::createException( + "", + "0", + array( + 'current_octet' => $currentOctet, + 'continuation_octet_needed' => $continuationOctetNeeded, + 'offset_in_octets_from_string_start' => $offsetInOctetsFromStringStart, + 'offset_in_characters_from_string_start' => $offsetInCharactersFromStringStart, + 'character_start_position_from_string_start' => $characterStartPositionFromStringStart, + 'line' => $currentLineNumber, + 'offset_in_octets_from_line_start' => $offsetInOctetsFromLineStart, + 'offset_in_characters_from_line_start' => $offsetInCharactersFromLineStart, + 'character_start_position_from_line_start' => $characterStartPositionFromLineStart, + 'current_continuation_octet_minimum' => $currentContinuationOctetMinimum, + 'current_continuation_octet_maximum' => $currentContinuationOctetMaximum, + ), + true + ); + } + } + + /** + * Builds the InvalidEncodingException thrown by validate(). + * + * The message scaffold is shared by every detection branch; only the + * "$reason" fragment (appended after "has value N") and the end-of-string + * wording differ between call sites. + * + * @param string $reason explanation appended after the octet value (ignored when $endOfInput is true) + * @param string $key the offending octet value, as a string + * @param bool $endOfInput whether the input ended in the middle of a multi-octet character + * @return InvalidEncodingException + * + * @phpstan-param array{current_octet: int|null, continuation_octet_needed: int, offset_in_octets_from_string_start: int, offset_in_characters_from_string_start: int, character_start_position_from_string_start: int, line: int, offset_in_octets_from_line_start: int, offset_in_characters_from_line_start: int, character_start_position_from_line_start: int, current_continuation_octet_minimum: int, current_continuation_octet_maximum: int} $details + */ + private static function createException($reason, $key, array $details, $endOfInput = false) + { + if ($endOfInput) { + $middle = + "; at octet " + .($details['offset_in_octets_from_line_start'] + 1) + .", part of the character " + .($details['offset_in_characters_from_line_start'] + 1) + .", end of string was found instead of a continuation octet."; + } else { + $middle = + "; the octet " + .($details['offset_in_octets_from_line_start'] + 1) + .", part of the character " + .($details['offset_in_characters_from_line_start'] + 1) + .", has value " + .$details['current_octet'] + .$reason; + } + + $message = + "Non-UTF8 character found on line " + .$details['line'] + .$middle + ." This character starts at octet " + .($details['character_start_position_from_line_start'] + 1) + ." of the current line." + ." (Sequential positions without line splitting:" + ." This is at character " + .($details['offset_in_characters_from_string_start'] + 1) + ." and octet " + .($details['offset_in_octets_from_string_start'] + 1) + ."." + ." This character starts at octet " + .($details['character_start_position_from_string_start'] + 1) + .".)"; + + return new InvalidEncodingException($message, $key, $details); + } +} diff --git a/tests/JsonParserTest.php b/tests/JsonParserTest.php index d3dd44a..51252a6 100644 --- a/tests/JsonParserTest.php +++ b/tests/JsonParserTest.php @@ -317,141 +317,6 @@ public function testFileNonValidUTF8() } } - public function testAllErrorTypesNonValidUTF8() - { - $parser = new JsonParser(); - try { - $parser->parse( - '"abcd'.chr(233).'"', - JsonParser::VALIDATE_UTF8_ENCODING - ); - $this->fail('ISO 8859-15 "abcdé" should not pass validation.'); - } catch (InvalidEncodingException $e) { - $this->assertContains('Non-UTF8 character found', $e->getMessage()); - $this->assertContains(' which is not a continuation octet.', $e->getMessage()); - } - try { - $parser->parse( - '"abcd'.chr(233), - JsonParser::VALIDATE_UTF8_ENCODING - ); - $this->fail('ISO 8859-15 "abcdé should not pass validation.'); - } catch (InvalidEncodingException $e) { - $this->assertContains('Non-UTF8 character found', $e->getMessage()); - $this->assertContains( - ', end of string was found instead of a continuation octet.', - $e->getMessage() - ); - } - $forbiddenOctets = array( - 192, - 193, - 245, - 255 - ); - foreach ($forbiddenOctets as $forbiddenOctet) { - try { - $parser->parse( - '"abcd'.chr(233).chr($forbiddenOctet).'"', - JsonParser::VALIDATE_UTF8_ENCODING - ); - $this->fail('"abcd\d233\d'.$forbiddenOctet.'" should not pass validation.'); - } catch (InvalidEncodingException $e) { - $this->assertContains('Non-UTF8 character found', $e->getMessage()); - $this->assertContains( - ' which is one of the four forbidden values (C0, C1, F5, FF).', - $e->getMessage() - ); - } - try { - $parser->parse( - '"abcd'.chr($forbiddenOctet).'"', - JsonParser::VALIDATE_UTF8_ENCODING - ); - $this->fail('"abcd\d'.$forbiddenOctet.'" should not pass validation.'); - } catch (InvalidEncodingException $e) { - $this->assertContains('Non-UTF8 character found', $e->getMessage()); - $this->assertContains( - ' which is one of the four forbidden values (C0, C1, F5, FF).', - $e->getMessage() - ); - } - } - try { - $parser->parse( - '"abcd'.chr(129).'"', - JsonParser::VALIDATE_UTF8_ENCODING - ); - $this->fail('"abcd\d129" should not pass validation.'); - } catch (InvalidEncodingException $e) { - $this->assertContains('Non-UTF8 character found', $e->getMessage()); - $this->assertContains( - ' which is a continuation octet.', - $e->getMessage() - ); - } - for ($i = 160; $i <= 191; ++$i) { - try { - $parser->parse( - '"abcd'.chr(237).chr($i).chr(129).'"', - JsonParser::VALIDATE_UTF8_ENCODING - ); - $this->fail('"abcd\d237\d'.$i.'\d129" should not pass validation.'); - } catch (InvalidEncodingException $e) { - $this->assertContains('Non-UTF8 character found', $e->getMessage()); - $this->assertContains( - ' which is into the forbidden range of surrogate pairs.', - $e->getMessage() - ); - } - try { - $parser->parse( - '"abcd'.chr(237).chr($i).'"', - JsonParser::VALIDATE_UTF8_ENCODING - ); - $this->fail('"abcd\d237\d'.$i.'" should not pass validation.'); - } catch (InvalidEncodingException $e) { - $this->assertContains('Non-UTF8 character found', $e->getMessage()); - $this->assertContains( - ' which is into the forbidden range of surrogate pairs.', - $e->getMessage() - ); - } - } - for ($i = 246; $i < 255; ++$i) { // 245 and 255 already forbidden - try { - $parser->parse( - '"abcd'.chr($i).chr(129).chr(129).chr(129).'"', - JsonParser::VALIDATE_UTF8_ENCODING - ); - $this->fail('"abcd\d'.$i.'\d129\d129\d129" should not pass validation.'); - } catch (InvalidEncodingException $e) { - $this->assertContains('Non-UTF8 character found', $e->getMessage()); - $this->assertContains(' which is invalid.', $e->getMessage()); - } - try { - $parser->parse( - '"abcd'.chr($i).'"', - JsonParser::VALIDATE_UTF8_ENCODING - ); - $this->fail('"abcd\d'.$i.'" should not pass validation.'); - } catch (InvalidEncodingException $e) { - $this->assertContains('Non-UTF8 character found', $e->getMessage()); - $this->assertContains(' which is invalid.', $e->getMessage()); - } - try { - $parser->parse( - '"abcd'.chr(195).chr($i).'"', - JsonParser::VALIDATE_UTF8_ENCODING - ); - $this->fail('"abcd\d195\d'.$i.'" should not pass validation.'); - } catch (InvalidEncodingException $e) { - $this->assertContains('Non-UTF8 character found', $e->getMessage()); - $this->assertContains(' which is not a continuation octet.', $e->getMessage()); - } - } - } - public function testFileWithBOM() { try { diff --git a/tests/Utf8ValidatorTest.php b/tests/Utf8ValidatorTest.php new file mode 100644 index 0000000..1e142c1 --- /dev/null +++ b/tests/Utf8ValidatorTest.php @@ -0,0 +1,150 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +use PHPUnit\Framework\TestCase; +use Seld\JsonLint\Utf8Validator; +use Seld\JsonLint\InvalidEncodingException; + +class Utf8ValidatorTest extends TestCase +{ + public function testValidUtf8() + { + try { + Utf8Validator::validate(''); + Utf8Validator::validate('abcdé'); // 2-octet character + Utf8Validator::validate('euro sign: € and more'); // 3-octet character + Utf8Validator::validate('musical clef: 𝄞'); // 4-octet character + Utf8Validator::validate("line 1\nline 2\nline 3"); // newline handling + $this->addToAssertionCount(1); + } catch (InvalidEncodingException $e) { + $this->fail('Valid UTF-8 should pass validation: '.$e->getMessage()); + } + } + + public function testAllErrorTypes() + { + try { + Utf8Validator::validate('"abcd'.chr(233).'"'); + $this->fail('ISO 8859-15 "abcdé" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains(' which is not a continuation octet.', $e->getMessage()); + } + try { + Utf8Validator::validate('"abcd'.chr(233)); + $this->fail('ISO 8859-15 "abcdé should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains( + ', end of string was found instead of a continuation octet.', + $e->getMessage() + ); + } + $forbiddenOctets = array( + 192, + 193, + 245, + 255 + ); + foreach ($forbiddenOctets as $forbiddenOctet) { + try { + Utf8Validator::validate('"abcd'.chr(233).chr($forbiddenOctet).'"'); + $this->fail('"abcd\d233\d'.$forbiddenOctet.'" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains( + ' which is one of the four forbidden values (C0, C1, F5, FF).', + $e->getMessage() + ); + } + try { + Utf8Validator::validate('"abcd'.chr($forbiddenOctet).'"'); + $this->fail('"abcd\d'.$forbiddenOctet.'" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains( + ' which is one of the four forbidden values (C0, C1, F5, FF).', + $e->getMessage() + ); + } + } + try { + Utf8Validator::validate('"abcd'.chr(129).'"'); + $this->fail('"abcd\d129" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains( + ' which is a continuation octet.', + $e->getMessage() + ); + } + for ($i = 160; $i <= 191; ++$i) { + try { + Utf8Validator::validate('"abcd'.chr(237).chr($i).chr(129).'"'); + $this->fail('"abcd\d237\d'.$i.'\d129" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains( + ' which is into the forbidden range of surrogate pairs.', + $e->getMessage() + ); + } + try { + Utf8Validator::validate('"abcd'.chr(237).chr($i).'"'); + $this->fail('"abcd\d237\d'.$i.'" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains( + ' which is into the forbidden range of surrogate pairs.', + $e->getMessage() + ); + } + } + for ($i = 246; $i < 255; ++$i) { // 245 and 255 already forbidden + try { + Utf8Validator::validate('"abcd'.chr($i).chr(129).chr(129).chr(129).'"'); + $this->fail('"abcd\d'.$i.'\d129\d129\d129" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains(' which is invalid.', $e->getMessage()); + } + try { + Utf8Validator::validate('"abcd'.chr($i).'"'); + $this->fail('"abcd\d'.$i.'" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains(' which is invalid.', $e->getMessage()); + } + try { + Utf8Validator::validate('"abcd'.chr(195).chr($i).'"'); + $this->fail('"abcd\d195\d'.$i.'" should not pass validation.'); + } catch (InvalidEncodingException $e) { + $this->assertContains('Non-UTF8 character found', $e->getMessage()); + $this->assertContains(' which is not a continuation octet.', $e->getMessage()); + } + } + } + + public function testExceptionExposesPositionDetails() + { + try { + // 255 (0xFF) is one of the always-forbidden octets, so the failure + // is reported on the octet itself rather than a following one. + Utf8Validator::validate('ab'.chr(255)); + $this->fail('Invalid UTF-8 should not pass validation.'); + } catch (InvalidEncodingException $e) { + $details = $e->getDetails(); + $this->assertSame(255, $details['current_octet']); + $this->assertSame(1, $details['line']); + $this->assertSame('255', $e->getKey()); + } + } +}