Merge pull request #104 from doctrine/2.1.x-merge-up-into-3.0.x_GyzLhYcQ

Merge release 2.1.0 into 3.0.x
doctrine · Dec 14, 2022 · c391831 · c391831
2 parents c00280c + b3bd837
commit c391831
Show file tree

Hide file tree

Showing 11 changed files with 79 additions and 27 deletions.
diff --git a/.doctrine-project.json b/.doctrine-project.json
@@ -4,22 +4,36 @@
     "slug": "lexer",
     "docsSlug": "doctrine-lexer",
     "versions": [
+        {
+            "name": "3.0",
+            "branchName": "3.0.x",
+            "slug": "latest",
+            "upcoming": true
+        },
+        {
+            "name": "2.1",
+            "branchName": "2.1.x",
+            "upcoming": true
+        },
+        {
+            "name": "2.0",
+            "branchName": "2.0.x",
+            "aliases": [
+                "current",
+                "stable"
+            ]
+        },
         {
             "name": "1.3",
             "branchName": "1.3.x",
-            "slug": "latest",
             "upcoming": true
         },
         {
             "name": "1.2",
             "branchName": "1.2.x",
             "slug": "1.2",
             "current": true,
-            "maintained": true,
-            "aliases": [
-                "current",
-                "stable"
-            ]
+            "maintained": true
         },
         {
             "name": "1.1",

diff --git a/docs/en/simple-parser-example.rst b/docs/en/simple-parser-example.rst
@@ -11,6 +11,9 @@ It tokenizes a string to ``T_UPPER``, ``T_LOWER`` and``T_NUMBER`` tokens:
 
     use Doctrine\Common\Lexer\AbstractLexer;
 
+    /**
+     * @extends AbstractLexer<CharacterTypeLexer::T_*, string>
+     */
     class CharacterTypeLexer extends AbstractLexer
     {
         const T_UPPER =  1;

diff --git a/phpstan.neon.dist b/phpstan.neon.dist
@@ -3,3 +3,8 @@ parameters:
   paths:
     - %rootDir%/../../../src
     - %rootDir%/../../../tests
+
+  ignoreErrors:
+    -
+      message: '#^Method Doctrine\\Tests\\Common\\Lexer\\AbstractLexerTest\:\:dataProvider\(\) should return array\<int, array\{string, array\<int, Doctrine\\Common\\Lexer\\Token\<string, int\|string\>\>\}\> but returns array\{array\{''price\=10'', array\{Doctrine\\Common\\Lexer\\Token\<string, string\>, Doctrine\\Common\\Lexer\\Token\<string, string\>, Doctrine\\Common\\Lexer\\Token\<string, int\>\}\}\}\.$#'
+      path: tests/AbstractLexerTest.php
diff --git a/psalm.xml b/psalm.xml
@@ -35,5 +35,22 @@
                 <file name="src/AbstractLexer.php" />
             </errorLevel>
         </MixedAssignment>
+        <MixedReturnStatement>
+            <errorLevel type="suppress">
+                <file name="src/Token.php" />
+            </errorLevel>
+        </MixedReturnStatement>
+        <ReferenceConstraintViolation>
+            <errorLevel type="suppress">
+                <!-- https://github.com/vimeo/psalm/issues/8891 -->
+                <file name="src/AbstractLexer.php" />
+            </errorLevel>
+        </ReferenceConstraintViolation>
+        <RedundantConditionGivenDocblockType>
+            <errorLevel type="suppress">
+                <!-- that test checks non-obvious things guaranteed by static analysis, just in case -->
+                <file name="tests/AbstractLexerTest.php" />
+            </errorLevel>
+        </RedundantConditionGivenDocblockType>
     </issueHandlers>
 </psalm>
diff --git a/src/AbstractLexer.php b/src/AbstractLexer.php
@@ -21,6 +21,7 @@
  * Base class for writing simple lexers, i.e. for creating small DSLs.
  *
  * @template T of UnitEnum|string|int
+ * @template V of string|int
  */
 abstract class AbstractLexer
 {
@@ -34,7 +35,7 @@ abstract class AbstractLexer
     /**
      * Array of scanned tokens.
      *
-     * @var list<Token<T>>
+     * @var list<Token<T, V>>
      */
     private $tokens = [];
 
@@ -56,15 +57,15 @@ abstract class AbstractLexer
      * The next token in the input.
      *
      * @var mixed[]|null
-     * @psalm-var Token<T>|null
+     * @psalm-var Token<T, V>|null
      */
     public $lookahead;
 
     /**
      * The last matched/seen token.
      *
      * @var mixed[]|null
-     * @psalm-var Token<T>|null
+     * @psalm-var Token<T, V>|null
      */
     public $token;
 
@@ -147,6 +148,8 @@ public function getInputUntilPosition($position)
      * @param T $type
      *
      * @return bool
+     *
+     * @psalm-assert-if-true !=null $this->lookahead
      */
     public function isNextToken($type)
     {
@@ -159,6 +162,8 @@ public function isNextToken($type)
      * @param list<T> $types
      *
      * @return bool
+     *
+     * @psalm-assert-if-true !=null $this->lookahead
      */
     public function isNextTokenAny(array $types)
     {
@@ -169,6 +174,8 @@ public function isNextTokenAny(array $types)
      * Moves to the next token in the input string.
      *
      * @return bool
+     *
+     * @psalm-assert-if-true !null $this->lookahead
      */
     public function moveNext()
     {
@@ -211,7 +218,7 @@ public function isA($value, $token)
      * Moves the lookahead token forward.
      *
      * @return mixed[]|null The next token or NULL if there are no more tokens ahead.
-     * @psalm-return Token<T>|null
+     * @psalm-return Token<T, V>|null
      */
     public function peek()
     {
@@ -226,7 +233,7 @@ public function peek()
      * Peeks at the next token, returns it and immediately resets the peek.
      *
      * @return mixed[]|null The next token or NULL if there are no more tokens ahead.
-     * @psalm-return Token<T>|null
+     * @psalm-return Token<T, V>|null
      */
     public function glimpse()
     {
@@ -264,10 +271,11 @@ protected function scan($input)
 
         foreach ($matches as $match) {
             // Must remain before 'value' assignment since it can change content
-            $type = $this->getType($match[0]);
+            $firstMatch = $match[0];
+            $type       = $this->getType($firstMatch);
 
             $this->tokens[] = new Token(
-                $match[0],
+                $firstMatch,
                 $type,
                 $match[1]
             );
@@ -331,6 +339,8 @@ abstract protected function getNonCatchablePatterns();
      * @param string $value
      *
      * @return T|null
+     *
+     * @param-out V $value
      */
     abstract protected function getType(&$value);
 }
diff --git a/src/Token.php b/src/Token.php
@@ -8,14 +8,17 @@
 
 use function in_array;
 
-/** @template T of UnitEnum|string|int */
+/**
+ * @template T of UnitEnum|string|int
+ * @template V of string|int
+ */
 final class Token
 {
     /**
      * The string value of the token in the input string
      *
      * @readonly
-     * @var string|int
+     * @var V
      */
     public $value;
 
@@ -36,8 +39,8 @@ final class Token
     public $position;
 
     /**
-     * @param string|int $value
-     * @param T|null     $type
+     * @param V      $value
+     * @param T|null $type
      */
     public function __construct($value, $type, int $position)
     {

diff --git a/tests/AbstractLexerTest.php b/tests/AbstractLexerTest.php
@@ -30,7 +30,7 @@ public function tearDown(): void
     }
 
     /**
-     * @psalm-return list<array{string, list<Token<string>>}>
+     * @psalm-return list<array{string, list<Token<string, string|int>>}>
      */
     public function dataProvider(): array
     {
@@ -86,7 +86,7 @@ public function testResetPosition(): void
     }
 
     /**
-     * @psalm-param list<Token<string>>  $expectedTokens
+     * @psalm-param list<Token<string, string|int>>  $expectedTokens
      *
      * @dataProvider dataProvider
      */
@@ -130,7 +130,7 @@ public function testUtf8Mismatch(): void
     }
 
     /**
-     * @psalm-param list<Token<string>> $expectedTokens
+     * @psalm-param list<Token<string, string|int>> $expectedTokens
      *
      * @dataProvider dataProvider
      */
@@ -150,7 +150,7 @@ public function testPeek(string $input, array $expectedTokens): void
     }
 
     /**
-     * @psalm-param list<Token<string>> $expectedTokens
+     * @psalm-param list<Token<string, string|int>> $expectedTokens
      *
      * @dataProvider dataProvider
      */
@@ -196,7 +196,7 @@ public function testGetInputUntilPosition(
     }
 
     /**
-     * @psalm-param list<Token<string>> $expectedTokens
+     * @psalm-param list<Token<string, string|int>> $expectedTokens
      *
      * @dataProvider dataProvider
      */
@@ -213,7 +213,7 @@ public function testIsNextToken(string $input, array $expectedTokens): void
     }
 
     /**
-     * @psalm-param list<Token<string>> $expectedTokens
+     * @psalm-param list<Token<string, string|int>> $expectedTokens
      *
      * @dataProvider dataProvider
      */

diff --git a/tests/ConcreteLexer.php b/tests/ConcreteLexer.php
@@ -9,7 +9,7 @@
 use function in_array;
 use function is_numeric;
 
-/** @extends AbstractLexer<string> */
+/** @extends AbstractLexer<string, string|int> */
 class ConcreteLexer extends AbstractLexer
 {
     public const INT = 'int';

diff --git a/tests/EnumLexer.php b/tests/EnumLexer.php
@@ -9,7 +9,7 @@
 use function in_array;
 use function is_numeric;
 
-/** @extends AbstractLexer<TokenType> */
+/** @extends AbstractLexer<TokenType, string|int> */
 class EnumLexer extends AbstractLexer
 {
     /**

diff --git a/tests/MutableLexer.php b/tests/MutableLexer.php
@@ -6,7 +6,7 @@
 
 use Doctrine\Common\Lexer\AbstractLexer;
 
-/** @extends AbstractLexer<int> */
+/** @extends AbstractLexer<int, string> */
 class MutableLexer extends AbstractLexer
 {
     /** @var string[] */

diff --git a/tests/TokenTest.php b/tests/TokenTest.php
@@ -11,7 +11,7 @@ final class TokenTest extends TestCase
 {
     public function testIsA(): void
     {
-        /** @var Token<'string'|'int'> $token */
+        /** @var Token<'string'|'int', string> $token */
         $token = new Token('foo', 'string', 1);
 
         self::assertTrue($token->isA('string'));