Comments and whitespace inside a return type now remain tokenized as-…

…is (ref #1527)
squizlabs · Apr 17, 2018 · 23134b6 · 23134b6
1 parent 35ca0c9
commit 23134b6
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 27 deletions.
diff --git a/package.xml b/package.xml
@@ -54,6 +54,7 @@ http://pear.php.net/dtd/package-2.0.xsd">
   - Function return types are now tokenized as T_RETURN_TYPE in all cases
     -- Previously, only simple types were tokenized in this way and namespaces were ignored
     -- Custom sniffs looking for T_NS_SEPERATOR inside return types will now need to look for T_RETURN_TYPE and check the value directly
+    -- Comments and whitespace inside return types remain tokenized as-is, so a return type may be split over multiple tokens
   - You can now use fully or partially qualified class names for custom reports instead of absolute file paths
     -- To support this, you must specify an autoload file in your ruleset.xml file and use it to register an autoloader
     -- Your autoloader will need to load your custom report class when requested

diff --git a/src/Files/File.php b/src/Files/File.php
@@ -1529,20 +1529,14 @@ public function getMethodProperties($stackPtr)
                     $nullableReturnType = true;
                 }
 
-                while ($this->tokens[$i]['code'] === T_RETURN_TYPE) {
+                if ($this->tokens[$i]['code'] === T_RETURN_TYPE) {
                     $returnType .= $this->tokens[$i]['content'];
-                    $i++;
                 }
             }
         }//end if
 
-        if ($returnType !== '') {
-            // Cleanup.
-            $returnType = preg_replace('/\s+/', '', $returnType);
-            $returnType = preg_replace('/\/\*.*?\*\//', '', $returnType);
-            if ($nullableReturnType === true) {
-                $returnType = '?'.$returnType;
-            }
+        if ($returnType !== '' && $nullableReturnType === true) {
+            $returnType = '?'.$returnType;
         }
 
         return [

diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
@@ -1080,17 +1080,15 @@ protected function tokenize($string)
                             T_NS_SEPARATOR => T_NS_SEPARATOR,
                         ];
 
-                        $allowed += Util\Tokens::$emptyTokens;
-
                         $typeHintStart = null;
                         $typeHintEnd   = null;
 
+                        // Find the start of the return type.
                         for ($x = ($x + 1); $x < $numTokens; $x++) {
-                            if ($typeHintStart === null
-                                && is_array($tokens[$x]) === true
+                            if (is_array($tokens[$x]) === true
                                 && isset(Util\Tokens::$emptyTokens[$tokens[$x][0]]) === true
                             ) {
-                                // Whitespace or coments before the type hint.
+                                // Whitespace or coments before the return type.
                                 continue;
                             }
 
@@ -1102,6 +1100,10 @@ protected function tokenize($string)
                                 continue;
                             }
 
+                            break;
+                        }
+
+                        do {
                             if (is_array($tokens[$x]) === true
                                 && isset($allowed[$tokens[$x][0]]) === true
                             ) {
@@ -1113,26 +1115,51 @@ protected function tokenize($string)
                                     $typeHintEnd = $x;
                                 }
 
+                                $x++;
                                 continue;
                             }
 
-                            break;
-                        }//end for
+                            // End of return type, or part of it.
+                            if ($typeHintStart !== null) {
+                                $tokens[$typeHintStart][0] = T_RETURN_TYPE;
 
-                        if ($typeHintStart !== null) {
-                            $tokens[$typeHintStart][0] = T_RETURN_TYPE;
+                                for ($i = ($typeHintStart + 1); $i <= $typeHintEnd; $i++) {
+                                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                                        $type    = Util\Tokens::tokenName($tokens[$i][0]);
+                                        $content = Util\Common::prepareForOutput($tokens[$i][1]);
+                                        echo "\t\t* token $i merged into token $typeHintStart (T_RETURN_TYPE); was: $type => $content".PHP_EOL;
+                                    }
 
-                            for ($i = ($typeHintStart + 1); $i <= $typeHintEnd; $i++) {
-                                if (PHP_CODESNIFFER_VERBOSITY > 1) {
-                                    $type    = Util\Tokens::tokenName($tokens[$i][0]);
-                                    $content = Util\Common::prepareForOutput($tokens[$i][1]);
-                                    echo "\t\t* token $i merged into token $typeHintStart (T_RETURN_TYPE); was: $type => $content".PHP_EOL;
+                                    $tokens[$typeHintStart][1] .= $tokens[$i][1];
+                                    $tokens[$i] = null;
                                 }
 
-                                $tokens[$typeHintStart][1] .= $tokens[$i][1];
-                                $tokens[$i] = null;
-                            }
-                        }
+                                $typeHintStart = null;
+                                $typeHintEnd   = null;
+
+                                $x++;
+                                continue;
+                            } else if (is_array($tokens[$x]) === true
+                                && isset(Util\Tokens::$emptyTokens[$tokens[$x][0]]) === true
+                            ) {
+                                // Empty token at the end/middle of the return type.
+                                // Find the start of the next part of it (if any).
+                                for ($x = ($x + 1); $x < $numTokens; $x++) {
+                                    if (is_array($tokens[$x]) === true
+                                        && isset(Util\Tokens::$emptyTokens[$tokens[$x][0]]) === true
+                                    ) {
+                                        // Whitespace or coments before the return type.
+                                        continue;
+                                    }
+
+                                    break;
+                                }
+
+                                continue;
+                            }//end if
+
+                            break;
+                        } while ($x < $numTokens);
                     }//end if
                 }//end if
             }//end if