From 716f14d42bc0cfc9d37f99a84b285e8ba79d902d Mon Sep 17 00:00:00 2001
From: Timm Friebe <thekid@thekid.de>
Date: Sun, 26 Feb 2023 10:54:41 +0100
Subject: [PATCH 1/7] Implement JSON streaming

---
 src/main/php/util/address/Address.class.php   |   4 +-
 .../php/util/address/ByAddresses.class.php    |   2 +-
 src/main/php/util/address/Iteration.class.php |   4 +-
 .../php/util/address/JsonIterator.class.php   | 130 ++++++++++++++++++
 .../php/util/address/JsonStreaming.class.php  |  13 ++
 .../php/util/address/StreamIterator.class.php |  90 ++++++++++++
 src/main/php/util/address/Streaming.class.php |   4 +-
 .../php/util/address/XmlIterator.class.php    |  77 +----------
 .../php/util/address/XmlStreaming.class.php   |   4 +-
 .../util/address/unittest/Composer.class.php  |  31 +++++
 .../address/unittest/JsonInputTest.class.php  |  39 ++++++
 .../unittest/JsonIteratorTest.class.php       | 101 ++++++++++++++
 .../unittest/StreamIteratorTest.class.php     |  21 +++
 .../address/unittest/UsingNextTest.class.php  |   2 +-
 .../unittest/XmlIteratorTest.class.php        |  16 +--
 .../util/address/unittest/composer.json       |  20 +++
 16 files changed, 458 insertions(+), 100 deletions(-)
 create mode 100755 src/main/php/util/address/JsonIterator.class.php
 create mode 100755 src/main/php/util/address/JsonStreaming.class.php
 create mode 100755 src/main/php/util/address/StreamIterator.class.php
 create mode 100755 src/test/php/util/address/unittest/Composer.class.php
 create mode 100755 src/test/php/util/address/unittest/JsonInputTest.class.php
 create mode 100755 src/test/php/util/address/unittest/JsonIteratorTest.class.php
 create mode 100755 src/test/php/util/address/unittest/StreamIteratorTest.class.php
 create mode 100755 src/test/resources/util/address/unittest/composer.json

diff --git a/src/main/php/util/address/Address.class.php b/src/main/php/util/address/Address.class.php
index 242203c..9bf13db 100755
--- a/src/main/php/util/address/Address.class.php
+++ b/src/main/php/util/address/Address.class.php
@@ -1,7 +1,5 @@
 <?php namespace util\address;
 
-use Iterator;
-
 /**
  * Base class for all XML inputs
  *
@@ -20,6 +18,6 @@ protected function stream() { return $this->stream; }
    * Creates an iterator. Default implementation is to return an
    * `XmlIterator` instance for BC reasons.
    */
-  public function iterator(): Iterator { return new XmlIterator($this->stream()); }
+  public function iterator(): StreamIterator { return new XmlIterator($this->stream()); }
 
 }
\ No newline at end of file
diff --git a/src/main/php/util/address/ByAddresses.class.php b/src/main/php/util/address/ByAddresses.class.php
index 02209e5..fe986ed 100755
--- a/src/main/php/util/address/ByAddresses.class.php
+++ b/src/main/php/util/address/ByAddresses.class.php
@@ -72,7 +72,7 @@ protected function next($iteration, $result) {
 
     // Select attributes and children
     while (null !== ($path= $iteration->path()) && 0 === strncmp($path, $base, $offset)) {
-      $relative= substr($iteration->path(), $offset);
+      $relative= strtr(substr($iteration->path(), $offset), "\x1D", '/');
       if ('@' === $relative[0]) {
         $address= $this->addresses[$relative] ?? $this->addresses['@*'] ?? null;
         $relative= substr($relative, 1);
diff --git a/src/main/php/util/address/Iteration.class.php b/src/main/php/util/address/Iteration.class.php
index 2967e9b..80a6157 100755
--- a/src/main/php/util/address/Iteration.class.php
+++ b/src/main/php/util/address/Iteration.class.php
@@ -7,14 +7,14 @@ class Iteration {
   /**
    * Creates an iteration
    *
-   * @param  util.address.XmlIterator $it
+   * @param  util.address.StreamIterator $it
    * @param  string $base
    */
   public function __construct($it) {
     $this->it= $it;
   }
 
-  /** @return util.address.XmlIterator */
+  /** @return util.address.StreamIterator */
   public function iterator() { return $this->it; }
 
   /** @return bool */
diff --git a/src/main/php/util/address/JsonIterator.class.php b/src/main/php/util/address/JsonIterator.class.php
new file mode 100755
index 0000000..f887306
--- /dev/null
+++ b/src/main/php/util/address/JsonIterator.class.php
@@ -0,0 +1,130 @@
+<?php namespace util\address;
+
+use Iterator, ReturnTypeWillChange;
+use io\streams\InputStream;
+use lang\FormatException;
+use text\StreamTokenizer;
+
+/**
+ * XML stream iterator
+ *
+ * @test  util.address.unittest.JsonIteratorTest
+ */
+class JsonIterator extends StreamIterator {
+  private $it;
+
+  /**
+   * Creates a new JSON iterator on a given stream
+   *
+   * @param  io.streams.InputStream $input If seekable, this iterator will be rewindable.
+   */
+  public function __construct(InputStream $input) {
+    parent::__construct(new StreamTokenizer($input, ":{}[]\"\r\n\t ", true));
+  }
+
+  /**
+   * Fetches next token, skipping over whitespace
+   * 
+   * @param  ?string $delimiters
+   * @return ?string
+   */
+  protected function token($delimiters= null) {
+    do {
+      $token= $this->input->nextToken($delimiters ?? $this->input->delimiters);
+    } while (null !== $token && 0 === strcspn($token, "\r\n\t "));
+    return $token;
+  }
+
+  /**
+   * Reads a string, handling escape sequences
+   *
+   * @return string
+   */
+  protected function string() {
+    $n= $this->input->nextToken('"');
+    if ('"' === $n) return '';
+
+    $s= '';
+    do {
+      $s.= $n;
+      if ('\\' !== $n[strlen($n) - 1]) break;
+
+      $s.= $this->input->nextToken('"');
+      $n= $this->input->nextToken('"');
+    } while ($this->input->hasMoreTokens());
+
+    $this->input->nextToken('"');
+    return json_decode('"'.$s.'"');
+  }
+
+  /**
+   * Yields values based on a given token, destructuring lists and maps
+   * into their components.
+   * 
+   * @param  string $token
+   * @return iterable
+   */
+  protected function iterate($token) {
+    if ('{' === $token) {
+      yield $this->path => null;
+
+      $next= $this->token();
+      while ('}' !== $next && $this->input->hasMoreTokens()) {
+        $this->path.= self::SEPARATOR.strtr($this->string(), self::SEPARATOR, "\x1D");
+        $this->token(':');
+
+        foreach ($this->iterate($this->token()) as $value) {
+          yield $this->path => $value;
+        }
+
+        $this->path= substr($this->path, 0, strrpos($this->path, self::SEPARATOR));
+        if (',' === ($next= $this->token(',}'))) {
+          $next= $this->token();
+        }
+      }
+    } else if ('[' === $token) {
+      yield $this->path => null;
+
+      $next= $this->token();
+      $i= 0;
+      while (']' !== $next && $this->input->hasMoreTokens()) {
+        $this->path.= self::SEPARATOR.'[]';
+
+        foreach ($this->iterate($next) as $value) {
+          yield $this->path => $value;
+        }
+
+        $this->path= substr($this->path, 0, strrpos($this->path, self::SEPARATOR));
+        if (',' === ($next= $this->token(',]'))) {
+          $next= $this->token();
+        }
+      }
+    } else if ('"' === $token) {
+      yield $this->string();
+    } else if ('null' === $token) {
+      yield null;
+    } else if ('false' === $token) {
+      yield false;
+    } else if ('true' === $token) {
+      yield true;
+    } else if (0 === strcspn($token, '.0123456789eE+-')) {
+      yield strlen($token) === strcspn($token, '.eE') ? (int)$token : (float)$token;
+    } else {
+      throw new FormatException('Unexpected token `'.$token.'`');
+    }
+  }
+
+  /** @return ?util.address.Token */
+  protected function nextToken() {
+    if ($this->tokens) return array_shift($this->tokens);
+
+    if (null === $this->it) {
+      $this->path= self::SEPARATOR;
+      $this->it= $this->iterate($this->token());
+    } else {
+      $this->it->next();
+    }
+
+    return $this->it->valid() ? new Token($this->it->key(), $this->it->current()) : null;
+  }
+}
\ No newline at end of file
diff --git a/src/main/php/util/address/JsonStreaming.class.php b/src/main/php/util/address/JsonStreaming.class.php
new file mode 100755
index 0000000..3c8ebbb
--- /dev/null
+++ b/src/main/php/util/address/JsonStreaming.class.php
@@ -0,0 +1,13 @@
+<?php namespace util\address;
+
+/**
+ * JSON streaming input
+ *
+ * @test  util.address.unittest.JsonInputTest
+ */
+class JsonStreaming extends Streaming {
+
+  /** Iterator implementation */
+  public function iterator(): StreamIterator { return new JsonIterator($this->stream); }
+
+}
\ No newline at end of file
diff --git a/src/main/php/util/address/StreamIterator.class.php b/src/main/php/util/address/StreamIterator.class.php
new file mode 100755
index 0000000..7e5aea9
--- /dev/null
+++ b/src/main/php/util/address/StreamIterator.class.php
@@ -0,0 +1,90 @@
+<?php namespace util\address;
+
+use Iterator, ReturnTypeWillChange;
+
+/**
+ * Abstract base class for format-based iterators. Subclasses implement
+ * the `nextToken()` method to return the next token.
+ *
+ * @see  util.address.JsonIterator
+ * @see  util.address.XmlIterator
+ */
+abstract class StreamIterator implements Iterator {
+  const SEPARATOR= '/';
+
+  public $token;
+  protected $input;
+  protected $path= null;
+  protected $tokens= [];
+
+  /** @param text.Tokenizer $input */
+  public function __construct($input) {
+    $this->input= $input;
+  }
+
+  /** @return ?util.address.Token */
+  protected abstract function nextToken();
+
+  /**
+   * Creates value from definition.
+   *
+   * @param  util.address.Definition $definition
+   * @param  bool $source
+   * @return var
+   */
+  public function value($definition, $source) {
+    if (null === $this->token->source) {
+      $token= $this->token;
+
+      // Create value, storing tokens during the iteration
+      $iteration= new Iteration($this);
+      $value= $definition->create($iteration);
+
+      // Unless we are at the end of the stream, push back last token.
+      $this->token && array_unshift($this->tokens, $this->token);
+      $this->token= $source ? $token->from($iteration->tokens) : $token;
+      return $value;
+    } else {
+
+      // Restore tokens consumed by previous iteration
+      $this->tokens= array_merge($this->token->source, [$this->token], $this->tokens);
+      $this->token= array_shift($this->tokens);
+      return $definition->create(new Iteration($this));
+    }
+  }
+
+  /** @return void */
+  #[ReturnTypeWillChange]
+  public function rewind() {
+    if (null !== $this->path) {
+      $this->input->reset();
+    }
+
+    $this->path= '';
+    $this->token= $this->nextToken();
+  }
+
+  /** @return string */
+  #[ReturnTypeWillChange]
+  public function current() {
+    return $this->token->content;
+  }
+
+  /** @return string */
+  #[ReturnTypeWillChange]
+  public function key() {
+    return $this->token->path;
+  }
+
+  /** @return void */
+  #[ReturnTypeWillChange]
+  public function next() {
+    $this->token= $this->nextToken();
+  }
+
+  /** @return bool */
+  #[ReturnTypeWillChange]
+  public function valid() {
+    return null !== $this->token;
+  }
+}
\ No newline at end of file
diff --git a/src/main/php/util/address/Streaming.class.php b/src/main/php/util/address/Streaming.class.php
index dc59b89..f58fd4d 100755
--- a/src/main/php/util/address/Streaming.class.php
+++ b/src/main/php/util/address/Streaming.class.php
@@ -1,6 +1,6 @@
 <?php namespace util\address;
 
-use Iterator, IteratorAggregate, Traversable;
+use IteratorAggregate, Traversable;
 use io\Channel;
 use io\streams\{InputStream, MemoryInputStream};
 use lang\{Closeable, Value};
@@ -27,7 +27,7 @@ public function __construct($source) {
   }
 
   /** Returns the iterator implementation */ 
-  public abstract function iterator(): Iterator;
+  public abstract function iterator(): StreamIterator;
 
   /**
    * Gets iterator
diff --git a/src/main/php/util/address/XmlIterator.class.php b/src/main/php/util/address/XmlIterator.class.php
index fdc04b9..583eaa4 100755
--- a/src/main/php/util/address/XmlIterator.class.php
+++ b/src/main/php/util/address/XmlIterator.class.php
@@ -10,14 +10,10 @@
  *
  * @test  xp://util.address.unittest.XmlIteratorTest
  */
-class XmlIterator implements Iterator {
-  const SEPARATOR= '/';
-
-  private $input, $path, $valid, $node;
+class XmlIterator extends StreamIterator {
+  private $valid, $node;
   private $encoding= 'utf-8';
-  private $tokens= [];
   private $entities= ['amp' => '&', 'apos' => "'", 'quot' => '"', 'gt' => '>', 'lt' => '<'];
-  public $token;
 
   /**
    * Creates a new XML iterator on a given stream
@@ -26,7 +22,6 @@ class XmlIterator implements Iterator {
    */
   public function __construct(InputStream $input) {
     $this->input= new StreamTokenizer($input, '<>', true);
-    $this->path= null;
   }
 
   /**
@@ -222,8 +217,8 @@ protected function attributesIn($string) {
     return $attributes;
   }
 
-  /** @return util.address.Token */
-  protected function token() {
+  /** @return ?util.address.Token */
+  protected function nextToken() {
     if (empty($this->tokens)) {
       $this->valid= false;
       while (null !== ($token= $this->input->nextToken())) {
@@ -269,68 +264,4 @@ protected function token() {
     // echo "<<< ", $token ? "token<{$token->path}= {$token->content}>" : "(null)", "\n";
     return $token;
   }
-
-  /**
-   * Creates value from definition.
-   *
-   * @param  util.address.Definition $definition
-   * @param  bool $source
-   * @return var
-   */
-  public function value($definition, $source) {
-    if (null === $this->token->source) {
-      $token= $this->token;
-
-      // Create value, storing tokens during the iteration
-      $iteration= new Iteration($this);
-      $value= $definition->create($iteration);
-
-      // Unless we are at the end of the stream, push back last token.
-      $this->valid= true;
-      $this->token && array_unshift($this->tokens, $this->token);
-      $this->token= $source ? $token->from($iteration->tokens) : $token;
-      return $value;
-    } else {
-
-      // Restore tokens consumed by previous iteration
-      $this->tokens= array_merge($this->token->source, [$this->token], $this->tokens);
-      $this->token= array_shift($this->tokens);
-      return $definition->create(new Iteration($this));
-    }
-  }
-
-  /** @return void */
-  #[ReturnTypeWillChange]
-  public function rewind() {
-    if (null !== $this->path) {
-      $this->input->reset();
-    }
-
-    $this->path= '';
-    $this->token= $this->token();
-  }
-
-  /** @return string */
-  #[ReturnTypeWillChange]
-  public function current() {
-    return $this->token->content;
-  }
-
-  /** @return string */
-  #[ReturnTypeWillChange]
-  public function key() {
-    return $this->token->path;
-  }
-
-  /** @return void */
-  #[ReturnTypeWillChange]
-  public function next() {
-    $this->token= $this->token();
-  }
-
-  /** @return bool */
-  #[ReturnTypeWillChange]
-  public function valid() {
-    return $this->valid;
-  }
 }
\ No newline at end of file
diff --git a/src/main/php/util/address/XmlStreaming.class.php b/src/main/php/util/address/XmlStreaming.class.php
index 21565d0..4f23496 100755
--- a/src/main/php/util/address/XmlStreaming.class.php
+++ b/src/main/php/util/address/XmlStreaming.class.php
@@ -1,7 +1,5 @@
 <?php namespace util\address;
 
-use Iterator;
-
 /**
  * XML streaming input
  *
@@ -11,6 +9,6 @@
 class XmlStreaming extends Streaming {
 
   /** Iterator implementation */
-  public function iterator(): Iterator { return new XmlIterator($this->stream); }
+  public function iterator(): StreamIterator { return new XmlIterator($this->stream); }
 
 }
\ No newline at end of file
diff --git a/src/test/php/util/address/unittest/Composer.class.php b/src/test/php/util/address/unittest/Composer.class.php
new file mode 100755
index 0000000..abf3761
--- /dev/null
+++ b/src/test/php/util/address/unittest/Composer.class.php
@@ -0,0 +1,31 @@
+<?php namespace util\address\unittest;
+
+use lang\Value;
+use util\Objects;
+
+class Composer implements Value {
+  private $name, $type, $keywords, $requirements;
+
+  public function __construct($name, $type, $keywords= [], $requirements= []) {
+    $this->name= $name;
+    $this->type= $type;
+    $this->keywords= $keywords;
+    $this->requirements= $requirements;
+  }
+
+  /** @return string */
+  public function hashCode() { return 'C'.Objects::hashOf((array)$this); }
+
+  /** @return string */
+  public function toString() { return nameof($this).'@'.Objects::stringOf(get_object_vars($this)); }
+
+  /**
+   * Compares this
+   *
+   * @param  var $value
+   * @return int
+   */
+  public function compareTo($value) {
+    return $value instanceof self ? Objects::compare((array)$this, (array)$value) : 1;
+  }
+}
\ No newline at end of file
diff --git a/src/test/php/util/address/unittest/JsonInputTest.class.php b/src/test/php/util/address/unittest/JsonInputTest.class.php
new file mode 100755
index 0000000..a3b174e
--- /dev/null
+++ b/src/test/php/util/address/unittest/JsonInputTest.class.php
@@ -0,0 +1,39 @@
+<?php namespace util\address\unittest;
+
+use test\{Assert, Test, Values};
+use util\address\{JsonStreaming, ObjectOf, ValueOf};
+
+class JsonInputTest {
+
+  /** @return var[][] */
+  protected function inputs() {
+    $package= typeof($this)->getPackage();
+    return [
+      [new JsonStreaming($package->getResource('composer.json'))],
+      [new JsonStreaming($package->getResourceAsStream('composer.json')->in())],
+      [new JsonStreaming($package->getResourceAsStream('composer.json'))]
+    ];
+  }
+
+  #[Test, Values(from: 'inputs')]
+  public function feed($input) {
+    $composer= $input->next(new ObjectOf(Composer::class, [
+      'name'        => function($self) { $self->name= yield; },
+      'type'        => function($self) { $self->type= yield; },
+      'keywords/[]' => function($self) { $self->keywords[]= yield; },
+      'require'     => function($self) { $self->requirements= yield new ValueOf([], [
+        '*' => function(&$self, $path) { $self[$path]= yield; }
+      ]); }
+    ]));
+
+    Assert::equals(
+      new Composer('xp-forge/address', 'library', ['module', 'xp'], [
+        'xp-framework/core'       => '^11.0 | ^10.0',
+        'xp-framework/reflection' => '^2.0 | ^1.9',
+        'xp-framework/tokenize'   => '^9.0 | ^8.1',
+        'php'                     => '>=7.0.0',
+      ]),
+      $composer
+    );
+  }
+}
\ No newline at end of file
diff --git a/src/test/php/util/address/unittest/JsonIteratorTest.class.php b/src/test/php/util/address/unittest/JsonIteratorTest.class.php
new file mode 100755
index 0000000..b18c635
--- /dev/null
+++ b/src/test/php/util/address/unittest/JsonIteratorTest.class.php
@@ -0,0 +1,101 @@
+<?php namespace util\address\unittest;
+
+use io\streams\{InputStream, MemoryInputStream};
+use lang\{IllegalStateException, FormatException};
+use test\{Assert, Test, Values};
+use util\address\JsonIterator;
+
+class JsonIteratorTest extends StreamIteratorTest {
+
+  /** @return iterable */
+  private function scalars() {
+    yield ['"Test"', 'Test'];
+    yield ['""', ''];
+    yield ['" "', ' '];
+    yield ['"\""', '"'];
+    yield ['"A \"quote\"."', 'A "quote".'];
+    yield ['"A\nB"', "A\nB"];
+    yield ['"1\u20ac"', '1€'];
+
+    yield ['1', 1];
+    yield ['+42', +42];
+    yield ['-6100', -6100];
+
+    yield ['1.5', 1.5];
+    yield ['1.123e3', 1.123e3];
+    yield ['1E2', 1E2];
+    yield ['1E-2', 1E-2];
+    yield ['-0.5', -0.5];
+    yield ['+6.1', +6.1];
+
+    yield ['true', true];
+    yield ['false', false];
+    yield ['null', null];
+  }
+
+  #[Test]
+  public function can_create() {
+    new JsonIterator(new MemoryInputStream('{}'));
+  }
+
+  #[Test, Values(from: 'scalars')]
+  public function scalar($input, $expected) {
+    $this->assertIterated([[$expected]], new JsonIterator(new MemoryInputStream($input)));
+  }
+
+  #[Test]
+  public function empty_map() {
+    $this->assertIterated(
+      [['/' => null]],
+      new JsonIterator(new MemoryInputStream('{}'))
+    );
+  }
+
+  #[Test]
+  public function single_pair() {
+    $this->assertIterated(
+      [['/' => null], ['//test' => 'Test']],
+      new JsonIterator(new MemoryInputStream('{"test":"Test"}'))
+    );
+  }
+
+  #[Test, Values(['{"color":"Green","price":12.99}', '{"color": "Green", "price": 12.99}'])]
+  public function two_pairs($input) {
+    $this->assertIterated(
+      [['/' => null], ['//color' => 'Green'], ['//price' => 12.99]],
+      new JsonIterator(new MemoryInputStream($input))
+    );
+  }
+
+  #[Test]
+  public function empty_list() {
+    $this->assertIterated(
+      [['/' => null]],
+      new JsonIterator(new MemoryInputStream('[]'))
+    );
+  }
+
+  #[Test]
+  public function single_element() {
+    $this->assertIterated(
+      [['/' => null], ['//[]' => 'Test']],
+      new JsonIterator(new MemoryInputStream('["Test"]'))
+    );
+  }
+
+  #[Test, Values(['["Color","Price"]', '["Color", "Price"]'])]
+  public function two_elements($input) {
+    $this->assertIterated(
+      [['/' => null], ['//[]' => 'Color'], ['//[]' => 'Price']],
+      new JsonIterator(new MemoryInputStream($input))
+    );
+  }
+
+  #[Test]
+  public function map_containing_list() {
+    $this->assertIterated(
+      [['/' => null], ['//items' => null], ['//items/[]' => 'One'], ['//items/[]' => 'Two']],
+      new JsonIterator(new MemoryInputStream('{"items":["One","Two"]}'))
+    );
+  }
+}
\ No newline at end of file
diff --git a/src/test/php/util/address/unittest/StreamIteratorTest.class.php b/src/test/php/util/address/unittest/StreamIteratorTest.class.php
new file mode 100755
index 0000000..33778a9
--- /dev/null
+++ b/src/test/php/util/address/unittest/StreamIteratorTest.class.php
@@ -0,0 +1,21 @@
+<?php namespace util\address\unittest;
+
+use test\Assert;
+use util\address\StreamIterator;
+
+abstract class StreamIteratorTest {
+
+  /**
+   * Assert iteration result
+   *
+   * @param  [:var][] $expected
+   * @param  util.address.StreamIterator $fixture
+   */
+  protected function assertIterated($expected, StreamIterator $fixture) {
+    $actual= [];
+    foreach ($fixture as $key => $value) {
+      $actual[]= [$key => $value];
+    }
+    Assert::equals($expected, $actual);
+  }
+}
\ No newline at end of file
diff --git a/src/test/php/util/address/unittest/UsingNextTest.class.php b/src/test/php/util/address/unittest/UsingNextTest.class.php
index 8f04cce..428019c 100755
--- a/src/test/php/util/address/unittest/UsingNextTest.class.php
+++ b/src/test/php/util/address/unittest/UsingNextTest.class.php
@@ -4,7 +4,7 @@
 use test\{Action, Assert, Test};
 use util\address\{ValueOf, XmlStreaming};
 
-/** @deprecated */
+/** @deprecated Use `fn($self[, $path]) => ... yield;` instead */
 class UsingNextTest {
   const BOOK = '<book id="1"><name>Name</name><author/><date>1977-12-14</date></book>';
 
diff --git a/src/test/php/util/address/unittest/XmlIteratorTest.class.php b/src/test/php/util/address/unittest/XmlIteratorTest.class.php
index ef1fd24..1e413fd 100755
--- a/src/test/php/util/address/unittest/XmlIteratorTest.class.php
+++ b/src/test/php/util/address/unittest/XmlIteratorTest.class.php
@@ -5,21 +5,7 @@
 use test\{Assert, Expect, Test, Values};
 use util\address\XmlIterator;
 
-class XmlIteratorTest {
-
-  /**
-   * Assert iteration result
-   *
-   * @param  [:var][] $expected
-   * @param  util.data.XmlIterator $fixture
-   */
-  protected function assertIterated($expected, XmlIterator $fixture) {
-    $actual= [];
-    foreach ($fixture as $key => $value) {
-      $actual[]= [$key => $value];
-    }
-    Assert::equals($expected, $actual);
-  }
+class XmlIteratorTest extends StreamIteratorTest {
 
   #[Test]
   public function can_create() {
diff --git a/src/test/resources/util/address/unittest/composer.json b/src/test/resources/util/address/unittest/composer.json
new file mode 100755
index 0000000..94209f0
--- /dev/null
+++ b/src/test/resources/util/address/unittest/composer.json
@@ -0,0 +1,20 @@
+{
+  "name" : "xp-forge/address",
+  "type" : "library",
+  "homepage" : "http://xp-framework.net/",
+  "license" : "BSD-3-Clause",
+  "description" : "Creates objects from XML input streams while parsing them.",
+  "keywords": ["module", "xp"],
+  "require" : {
+    "xp-framework/core": "^11.0 | ^10.0",
+    "xp-framework/reflection": "^2.0 | ^1.9",
+    "xp-framework/tokenize": "^9.0 | ^8.1",
+    "php" : ">=7.0.0"
+  },
+  "require-dev" : {
+    "xp-framework/test": "^1.0"
+  },
+  "autoload" : {
+    "files" : ["src/main/php/autoload.php"]
+  }
+}

From 4c0400f59df658ff4c60a50d5cd4e36cdedae9ac Mon Sep 17 00:00:00 2001
From: Timm Friebe <thekid@thekid.de>
Date: Sun, 26 Feb 2023 12:33:23 +0100
Subject: [PATCH 2/7] Fix string parsing in escaping edge cases

---
 .../php/util/address/JsonIterator.class.php   | 29 ++++++++++---------
 .../unittest/JsonIteratorTest.class.php       | 11 +++++++
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/src/main/php/util/address/JsonIterator.class.php b/src/main/php/util/address/JsonIterator.class.php
index f887306..3de585c 100755
--- a/src/main/php/util/address/JsonIterator.class.php
+++ b/src/main/php/util/address/JsonIterator.class.php
@@ -19,7 +19,7 @@ class JsonIterator extends StreamIterator {
    * @param  io.streams.InputStream $input If seekable, this iterator will be rewindable.
    */
   public function __construct(InputStream $input) {
-    parent::__construct(new StreamTokenizer($input, ":{}[]\"\r\n\t ", true));
+    parent::__construct(new StreamTokenizer($input, ":{}[],\"\r\n\t ", true));
   }
 
   /**
@@ -36,25 +36,26 @@ protected function token($delimiters= null) {
   }
 
   /**
-   * Reads a string, handling escape sequences
+   * Reads a string, handling escape sequences and unclosed strings
    *
    * @return string
+   * @throws lang.FormatException
    */
   protected function string() {
-    $n= $this->input->nextToken('"');
-    if ('"' === $n) return '';
-
-    $s= '';
+    $s= '"';
     do {
-      $s.= $n;
-      if ('\\' !== $n[strlen($n) - 1]) break;
-
-      $s.= $this->input->nextToken('"');
-      $n= $this->input->nextToken('"');
-    } while ($this->input->hasMoreTokens());
+      $chunk= $this->input->nextToken('\\"');
+      if (null === $chunk) {
+        throw new FormatException('Unclosed string literal');
+      } else if ('\\' === $chunk) {
+        $s.= $chunk.$this->input->nextToken('\\"');
+      } else {
+        $s.= $chunk;
+      }
+    } while ('"' !== $chunk);
 
-    $this->input->nextToken('"');
-    return json_decode('"'.$s.'"');
+    // Optimize empty string case
+    return '""' === $s ? '' : json_decode($s);
   }
 
   /**
diff --git a/src/test/php/util/address/unittest/JsonIteratorTest.class.php b/src/test/php/util/address/unittest/JsonIteratorTest.class.php
index b18c635..b1cf55a 100755
--- a/src/test/php/util/address/unittest/JsonIteratorTest.class.php
+++ b/src/test/php/util/address/unittest/JsonIteratorTest.class.php
@@ -13,9 +13,12 @@ private function scalars() {
     yield ['""', ''];
     yield ['" "', ' '];
     yield ['"\""', '"'];
+    yield ['"Escape\\\\"', 'Escape\\'];
     yield ['"A \"quote\"."', 'A "quote".'];
+    yield ['"Have \"Error: A\""', 'Have "Error: A"'];
     yield ['"A\nB"', "A\nB"];
     yield ['"1\u20ac"', '1€'];
+    yield ['"测测"', '测测'];  // Chinese for "measurement"
 
     yield ['1', 1];
     yield ['+42', +42];
@@ -67,6 +70,14 @@ public function two_pairs($input) {
     );
   }
 
+  #[Test, Values(from: 'scalars')]
+  public function map_with_scalar($input, $expected) {
+    $this->assertIterated(
+      [['/' => null], ['//value' => $expected], ['//ok' => true]],
+      new JsonIterator(new MemoryInputStream('{"value":'.$input.',"ok":true}'))
+    );
+  }
+
   #[Test]
   public function empty_list() {
     $this->assertIterated(

From df77d0b6ada9a4a2983234e436d3d5fda0c2f387 Mon Sep 17 00:00:00 2001
From: Timm Friebe <thekid@thekid.de>
Date: Sun, 26 Feb 2023 19:22:24 +0100
Subject: [PATCH 3/7] Add StructureOf definition

---
 .../php/util/address/StructureOf.class.php    | 45 ++++++++++++++++
 .../unittest/StructureOfTest.class.php        | 52 +++++++++++++++++++
 2 files changed, 97 insertions(+)
 create mode 100755 src/main/php/util/address/StructureOf.class.php
 create mode 100755 src/test/php/util/address/unittest/StructureOfTest.class.php

diff --git a/src/main/php/util/address/StructureOf.class.php b/src/main/php/util/address/StructureOf.class.php
new file mode 100755
index 0000000..3dfd488
--- /dev/null
+++ b/src/main/php/util/address/StructureOf.class.php
@@ -0,0 +1,45 @@
+<?php namespace util\address;
+
+/**
+ * Creates a generic data structure from the iteration
+ *
+ * @test  util.address.unittest.StructureOfTest
+ */
+class StructureOf implements Definition {
+
+  /**
+   * Creates a value from a given iteration
+   *
+   * @param  util.address.Iteration $iteration
+   * @return var
+   */
+  public function create($iteration) {
+    $base= $iteration->path().'/';
+    $offset= strlen($base);
+    $value= $iteration->next();
+
+    while (null !== ($path= $iteration->path()) && 0 === strncmp($path, $base, $offset)) {
+      if (0 === substr_compare($path, '/[]', -3, 3)) {
+        $segments= substr($path, $offset, -3);
+        $array= true;
+      } else {
+        $segments= substr($path, $offset);
+        $array= false;
+      }
+
+      $ptr= &$value;
+      if (strlen($segments) > 0) {
+        foreach (explode('/', $segments) as $segment) {
+          $ptr= &$ptr[$segment];
+        }
+      }
+
+      if ($array) {
+        $ptr[]= $this->create($iteration);
+      } else {
+        $ptr= $iteration->next();
+      }
+    }
+    return $value;
+  }
+}
\ No newline at end of file
diff --git a/src/test/php/util/address/unittest/StructureOfTest.class.php b/src/test/php/util/address/unittest/StructureOfTest.class.php
new file mode 100755
index 0000000..8f7bf01
--- /dev/null
+++ b/src/test/php/util/address/unittest/StructureOfTest.class.php
@@ -0,0 +1,52 @@
+<?php namespace util\address\unittest;
+
+use test\{Assert, Test};
+use util\address\{StructureOf, JsonStreaming};
+
+class StructureOfTest {
+
+  #[Test]
+  public function string() {
+    $address= new JsonStreaming('"Test"');
+    Assert::equals('Test', $address->next(new StructureOf()));
+  }
+
+  #[Test]
+  public function list() {
+    $address= new JsonStreaming('["red","green","blue"]"');
+    Assert::equals(
+      ['red', 'green', 'blue'],
+      $address->next(new StructureOf())
+    );
+  }
+
+  #[Test]
+  public function object() {
+    $address= new JsonStreaming('{"name":"Test","ok":true,"undefined":null}"');
+    Assert::equals(
+      ['name' => 'Test', 'ok' => true, 'undefined' => null],
+      $address->next(new StructureOf())
+    );
+  }
+
+  #[Test]
+  public function object_containing_list() {
+    $address= new JsonStreaming('{"colors":["red","green","blue"]}');
+    Assert::equals(
+      ['colors' => ['red', 'green', 'blue']],
+      $address->next(new StructureOf())
+    );
+  }
+
+  #[Test]
+  public function containing_list_of_objects() {
+    $address= new JsonStreaming('{"colors":[{"id":"green","component":"G"},{"id":"red","component":"R"}]}');
+    Assert::equals(
+      ['colors' => [
+        ['id' => 'green', 'component' => 'G'],
+        ['id' => 'red', 'component' => 'R'],
+      ]],
+      $address->next(new StructureOf())
+    );
+  }
+}
\ No newline at end of file

From 7ff79c1c2f3fa348a0a4a8487514f2fbf79e8d80 Mon Sep 17 00:00:00 2001
From: Timm Friebe <thekid@thekid.de>
Date: Sun, 26 Feb 2023 19:40:54 +0100
Subject: [PATCH 4/7] Use StructureOf

---
 src/test/php/util/address/unittest/JsonInputTest.class.php | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/test/php/util/address/unittest/JsonInputTest.class.php b/src/test/php/util/address/unittest/JsonInputTest.class.php
index a3b174e..a09540b 100755
--- a/src/test/php/util/address/unittest/JsonInputTest.class.php
+++ b/src/test/php/util/address/unittest/JsonInputTest.class.php
@@ -1,7 +1,7 @@
 <?php namespace util\address\unittest;
 
 use test\{Assert, Test, Values};
-use util\address\{JsonStreaming, ObjectOf, ValueOf};
+use util\address\{JsonStreaming, ObjectOf, StructureOf};
 
 class JsonInputTest {
 
@@ -21,9 +21,7 @@ public function feed($input) {
       'name'        => function($self) { $self->name= yield; },
       'type'        => function($self) { $self->type= yield; },
       'keywords/[]' => function($self) { $self->keywords[]= yield; },
-      'require'     => function($self) { $self->requirements= yield new ValueOf([], [
-        '*' => function(&$self, $path) { $self[$path]= yield; }
-      ]); }
+      'require'     => function($self) { $self->requirements= yield new StructureOf(); },
     ]));
 
     Assert::equals(

From 0d2ec8fa1de7d8c99be76bb5af8e687bac30cd39 Mon Sep 17 00:00:00 2001
From: Timm Friebe <thekid@thekid.de>
Date: Sun, 26 Feb 2023 19:41:05 +0100
Subject: [PATCH 5/7] Fix paths containing "/"

---
 src/main/php/util/address/StructureOf.class.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/php/util/address/StructureOf.class.php b/src/main/php/util/address/StructureOf.class.php
index 3dfd488..26864ea 100755
--- a/src/main/php/util/address/StructureOf.class.php
+++ b/src/main/php/util/address/StructureOf.class.php
@@ -30,7 +30,7 @@ public function create($iteration) {
       $ptr= &$value;
       if (strlen($segments) > 0) {
         foreach (explode('/', $segments) as $segment) {
-          $ptr= &$ptr[$segment];
+          $ptr= &$ptr[strtr($segment, "\x1D", '/')];
         }
       }
 

From 9b0f525c6c6d6b708e901ece3ba2826c64d0c973 Mon Sep 17 00:00:00 2001
From: Timm Friebe <thekid@thekid.de>
Date: Sun, 26 Feb 2023 19:44:26 +0100
Subject: [PATCH 6/7] QA: Add reference to
 util.address.unittest.StructureOfTest [skip ci]

---
 src/main/php/util/address/JsonStreaming.class.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/php/util/address/JsonStreaming.class.php b/src/main/php/util/address/JsonStreaming.class.php
index 3c8ebbb..eac3fb5 100755
--- a/src/main/php/util/address/JsonStreaming.class.php
+++ b/src/main/php/util/address/JsonStreaming.class.php
@@ -4,6 +4,7 @@
  * JSON streaming input
  *
  * @test  util.address.unittest.JsonInputTest
+ * @test  util.address.unittest.StructureOfTest
  */
 class JsonStreaming extends Streaming {
 

From 044b91867bf6b52cfe4738a8001e4fa0a4e22df4 Mon Sep 17 00:00:00 2001
From: Timm Friebe <thekid@thekid.de>
Date: Sun, 26 Feb 2023 19:47:18 +0100
Subject: [PATCH 7/7] Include JSON in library purpose [skip ci]

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 81097e5..d37ac94 100755
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ Address
 [![Supports PHP 8.0+](https://raw.githubusercontent.com/xp-framework/web/master/static/php-8_0plus.svg)](http://php.net/)
 [![Latest Stable Version](https://poser.pugx.org/xp-forge/address/version.png)](https://packagist.org/packages/xp-forge/address)
 
-Creates objects from XML input streams while parsing them. Yes, this still happens today 😉
+Creates objects from JSON or XML input streams while parsing them. Yes, XML still happens today 😉
 
 Example
 -------