Merge pull request #26 from nicolas-grekas/utf8

Work around PHP bug
doctrine · Jun 8, 2019 · 1febd6c · 1febd6c
2 parents ee614dd + f01102e
commit 1febd6c
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 1 deletion.
diff --git a/lib/Doctrine/Common/Lexer/AbstractLexer.php b/lib/Doctrine/Common/Lexer/AbstractLexer.php
@@ -258,6 +258,11 @@ protected function scan($input)
         $flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;
         $matches = preg_split($regex, $input, -1, $flags);
 
+        if (false === $matches) {
+            // Work around https://bugs.php.net/78122
+            $matches = array(array($input, 0));
+        }
+
         foreach ($matches as $match) {
             // Must remain before 'value' assignment since it can change content
             $type = $this->getType($match[0]);

diff --git a/tests/Doctrine/Common/Lexer/AbstractLexerTest.php b/tests/Doctrine/Common/Lexer/AbstractLexerTest.php
@@ -123,7 +123,6 @@ public function testMoveNext($input, $expectedTokens)
         $this->assertNull($this->concreteLexer->lookahead);
     }
 
-
     public function testSkipUntil()
     {
         $this->concreteLexer->setInput('price=10');
@@ -141,6 +140,22 @@ public function testSkipUntil()
         );
     }
 
+    public function testUtf8Mismatch()
+    {
+        $this->concreteLexer->setInput("\xE9=10");
+
+        $this->assertTrue($this->concreteLexer->moveNext());
+
+        $this->assertEquals(
+            array(
+                'value' => "\xE9=10",
+                'type' => 'string',
+                'position' => 0,
+            ),
+            $this->concreteLexer->lookahead
+        );
+    }
+
     /**
      * @dataProvider dataProvider
      *

diff --git a/tests/Doctrine/Common/Lexer/ConcreteLexer.php b/tests/Doctrine/Common/Lexer/ConcreteLexer.php
@@ -41,4 +41,9 @@ protected function getType(&$value)
 
         return;
     }
+
+    protected function getModifiers()
+    {
+        return parent::getModifiers().'u';
+    }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -41,4 +41,9 @@ protected function getType(&$value) @@
             return;
         }
+        protected function getModifiers()
+        {
+            return parent::getModifiers().'u';
+        }
     }