Make the value of tokens templatable

The ORM Lexer does not alter the type of token values in its getType()
method. Having a way to restrict the type of token values to string
should be very valuable from a static analysis point of view.

docs/en/simple-parser-example.rst

@@ -11,6 +11,9 @@ It tokenizes a string to ``T_UPPER``, ``T_LOWER`` and``T_NUMBER`` tokens:
 
     use Doctrine\Common\Lexer\AbstractLexer;
 
+    /**
+     * @extends AbstractLexer<CharacterTypeLexer::T_*, string>
+     */
     class CharacterTypeLexer extends AbstractLexer
     {
         const T_UPPER =  1;

phpstan.neon.dist

@@ -3,3 +3,11 @@ parameters:
   paths:
     - %rootDir%/../../../src
     - %rootDir%/../../../tests
+
+  ignoreErrors:
+    -
+      message: '#^Property Doctrine\\Common\\Lexer\\AbstractLexer\<T of int\|string\|UnitEnum,V of int\|string\>\:\:\$tokens \(array\<int, Doctrine\\Common\\Lexer\\Token\<T of int\|string\|UnitEnum, V of int\|string\>\>\) does not accept non\-empty\-array\<int, Doctrine\\Common\\Lexer\\Token\<T of int\|string\|UnitEnum, int\|string\>\>\.$#'
+      path: src/AbstractLexer.php
+    -
+      message: '#^Method Doctrine\\Tests\\Common\\Lexer\\AbstractLexerTest\:\:dataProvider\(\) should return array\<int, array\{string, array\<int, Doctrine\\Common\\Lexer\\Token\<string, int\|string\>\>\}\> but returns array\{array\{''price\=10'', array\{Doctrine\\Common\\Lexer\\Token\<string, string\>, Doctrine\\Common\\Lexer\\Token\<string, string\>, Doctrine\\Common\\Lexer\\Token\<string, int\>\}\}\}\.$#'
+      path: tests/AbstractLexerTest.php

psalm.xml

@@ -52,6 +52,12 @@
                 <file name="src/Token.php" />
             </errorLevel>
         </MixedReturnStatement>
+        <ReferenceConstraintViolation>
+            <errorLevel type="suppress">
+                <!-- https://github.com/vimeo/psalm/issues/8891 -->
+                <file name="src/AbstractLexer.php" />
+            </errorLevel>
+        </ReferenceConstraintViolation>
         <RedundantConditionGivenDocblockType>
             <errorLevel type="suppress">
                 <!-- that test checks non-obvious things guaranteed by static analysis, just in case -->

src/AbstractLexer.php

@@ -21,6 +21,7 @@
  * Base class for writing simple lexers, i.e. for creating small DSLs.
  *
  * @template T of UnitEnum|string|int
+ * @template V of string|int
  */
 abstract class AbstractLexer
 {
@@ -34,7 +35,7 @@ abstract class AbstractLexer
     /**
      * Array of scanned tokens.
      *
-     * @var list<Token<T>>
+     * @var list<Token<T, V>>
      */
     private $tokens = [];
 
@@ -56,15 +57,15 @@ abstract class AbstractLexer
      * The next token in the input.
      *
      * @var mixed[]|null
-     * @psalm-var Token<T>|null
+     * @psalm-var Token<T, V>|null
      */
     public $lookahead;
 
     /**
      * The last matched/seen token.
      *
      * @var mixed[]|null
-     * @psalm-var Token<T>|null
+     * @psalm-var Token<T, V>|null
      */
     public $token;
 
@@ -218,7 +219,7 @@ public function isA($value, $token)
      * Moves the lookahead token forward.
      *
      * @return mixed[]|null The next token or NULL if there are no more tokens ahead.
-     * @psalm-return Token<T>|null
+     * @psalm-return Token<T, V>|null
      */
     public function peek()
     {
@@ -233,7 +234,7 @@ public function peek()
      * Peeks at the next token, returns it and immediately resets the peek.
      *
      * @return mixed[]|null The next token or NULL if there are no more tokens ahead.
-     * @psalm-return Token<T>|null
+     * @psalm-return Token<T, V>|null
      */
     public function glimpse()
     {
@@ -271,10 +272,11 @@ protected function scan($input)
 
         foreach ($matches as $match) {
             // Must remain before 'value' assignment since it can change content
-            $type = $this->getType($match[0]);
+            $firstMatch = $match[0];
+            $type       = $this->getType($firstMatch);
 
             $this->tokens[] = new Token(
-                $match[0],
+                $firstMatch,
                 $type,
                 $match[1]
             );
@@ -338,6 +340,8 @@ abstract protected function getNonCatchablePatterns();
      * @param string $value
      *
      * @return T|null
+     *
+     * @param-out V $value
      */
     abstract protected function getType(&$value);
 }

src/Token.php

@@ -13,6 +13,7 @@
 
 /**
  * @template T of UnitEnum|string|int
+ * @template V of string|int
  * @implements ArrayAccess<string,mixed>
  */
 final class Token implements ArrayAccess
@@ -21,7 +22,7 @@ final class Token implements ArrayAccess
      * The string value of the token in the input string
      *
      * @readonly
-     * @var string|int
+     * @var V
      */
     public $value;
 
@@ -42,8 +43,8 @@ final class Token implements ArrayAccess
     public $position;
 
     /**
-     * @param string|int $value
-     * @param T|null     $type
+     * @param V      $value
+     * @param T|null $type
      */
     public function __construct($value, $type, int $position)
     {
@@ -83,7 +84,7 @@ public function offsetExists($offset): bool
      * @return mixed
      * @psalm-return (
      *     O is 'value'
-     *     ? string|int
+     *     ? V
      *     : (
      *         O is 'type'
      *         ? T|null

tests/AbstractLexerTest.php

@@ -30,7 +30,7 @@ public function tearDown(): void
     }
 
     /**
-     * @psalm-return list<array{string, list<Token<string>>}>
+     * @psalm-return list<array{string, list<Token<string, string|int>>}>
      */
     public function dataProvider(): array
     {
@@ -86,7 +86,7 @@ public function testResetPosition(): void
     }
 
     /**
-     * @psalm-param list<Token<string>>  $expectedTokens
+     * @psalm-param list<Token<string, string|int>>  $expectedTokens
      *
      * @dataProvider dataProvider
      */
@@ -130,7 +130,7 @@ public function testUtf8Mismatch(): void
     }
 
     /**
-     * @psalm-param list<Token<string>> $expectedTokens
+     * @psalm-param list<Token<string, string|int>> $expectedTokens
      *
      * @dataProvider dataProvider
      */
@@ -150,7 +150,7 @@ public function testPeek(string $input, array $expectedTokens): void
     }
 
     /**
-     * @psalm-param list<Token<string>> $expectedTokens
+     * @psalm-param list<Token<string, string|int>> $expectedTokens
      *
      * @dataProvider dataProvider
      */
@@ -196,7 +196,7 @@ public function testGetInputUntilPosition(
     }
 
     /**
-     * @psalm-param list<Token<string>> $expectedTokens
+     * @psalm-param list<Token<string, string|int>> $expectedTokens
      *
      * @dataProvider dataProvider
      */
@@ -213,7 +213,7 @@ public function testIsNextToken(string $input, array $expectedTokens): void
     }
 
     /**
-     * @psalm-param list<Token<string>> $expectedTokens
+     * @psalm-param list<Token<string, string|int>> $expectedTokens
      *
      * @dataProvider dataProvider
      */

tests/ConcreteLexer.php

@@ -9,7 +9,7 @@
 use function in_array;
 use function is_numeric;
 
-/** @extends AbstractLexer<string> */
+/** @extends AbstractLexer<string, string|int> */
 class ConcreteLexer extends AbstractLexer
 {
     public const INT = 'int';

tests/EnumLexer.php

@@ -9,7 +9,7 @@
 use function in_array;
 use function is_numeric;
 
-/** @extends AbstractLexer<TokenType> */
+/** @extends AbstractLexer<TokenType, string|int> */
 class EnumLexer extends AbstractLexer
 {
     /**

tests/MutableLexer.php

@@ -6,7 +6,7 @@
 
 use Doctrine\Common\Lexer\AbstractLexer;
 
-/** @extends AbstractLexer<int> */
+/** @extends AbstractLexer<int, string> */
 class MutableLexer extends AbstractLexer
 {
     /** @var string[] */

tests/TokenTest.php

@@ -14,7 +14,7 @@ final class TokenTest extends TestCase
 
     public function testIsA(): void
     {
-        /** @var Token<'string'|'int'> $token */
+        /** @var Token<'string'|'int', string> $token */
         $token = new Token('foo', 'string', 1);
 
         self::assertTrue($token->isA('string'));