Merge pull request #102 from mlocati/php-comments-extraction

Comments extraction from PHP code

src/Extractors/PhpCode.php

@@ -19,6 +19,16 @@ class PhpCode extends Extractor implements ExtractorInterface
         'p__e' => 'p__',
     );
 
+    /**
+     * Set to:
+     * - false to not extract comments
+     * - empty string to extract all comments
+     * - non-empty string to extract comments that start with that string.
+     *
+     * @var string|false
+     */
+    public static $extractComments = false;
+
     /**
      * {@inheritdoc}
      */
@@ -29,6 +39,9 @@ public static function fromString($string, Translations $translations = null, $f
         }
 
         $functions = new PhpFunctionsScanner($string);
+        if (self::$extractComments !== false) {
+            $functions->enableCommentsExtraction(self::$extractComments);
+        }
         $functions->saveGettextFunctions(self::$functions, $translations, $file);
 
         return $translations;

src/Utils/FunctionsScanner.php

@@ -71,6 +71,11 @@ public function saveGettextFunctions(array $functions, Translations $translation
 
             if (isset($translation)) {
                 $translation->addReference($file, $line);
+                if (isset($function[3])) {
+                    foreach ($function[3] as $extractedComment) {
+                        $translation->addExtractedComment($extractedComment);
+                    }
+                }
             }
         }
     }

src/Utils/ParsedFunction.php

@@ -42,6 +42,13 @@ class ParsedFunction
      */
     protected $argumentStopped;
 
+    /**
+     * Extracted comments.
+     *
+     * @var string[]|null
+     */
+    protected $comments;
+
     /**
      * Initializes the instance.
      *
@@ -55,6 +62,7 @@ public function __construct($name, $line)
         $this->arguments = array();
         $this->argumentIndex = -1;
         $this->argumentStopped = false;
+        $this->comments = null;
     }
 
     /**
@@ -101,6 +109,18 @@ public function addArgumentChunk($chunk)
         }
     }
 
+    /**
+     * Add a comment associated to this function.
+     *
+     * @param string $comment
+     */
+    public function addComment($comment)
+    {
+        if ($this->comments === null) {
+            $this->comments = array();
+        }
+        $this->comments[] = $comment;
+    }
     /**
      * A closing parenthesis was found: return the final data.
      *
@@ -109,6 +129,7 @@ public function addArgumentChunk($chunk)
      *   @var string The function name.
      *   @var int The line where the function starts.
      *   @var string[] the strings extracted from the function arguments.
+     *   @var string[] the comments associated to the function.
      * }
      */
     public function close()
@@ -122,6 +143,7 @@ public function close()
             $this->name,
             $this->line,
             $arguments,
+            $this->comments,
         );
     }
 }

src/Utils/PhpFunctionsScanner.php

@@ -6,16 +6,55 @@
 
 class PhpFunctionsScanner extends FunctionsScanner
 {
+    /**
+     * PHP tokens of the code to be parsed.
+     *
+     * @var array
+     */
     protected $tokens;
 
+    /**
+     * If not false, comments will be extracted.
+     *
+     * @var string|false
+     */
+    protected $extractComments = false;
+
+    /**
+     * Enable extracting comments that start with a tag (if $tag is empty all the comments will be extracted).
+     *
+     * @param string $tag
+     */
+    public function enableCommentsExtraction($tag = '')
+    {
+        $this->extractComments = (string) $tag;
+    }
+
+    /**
+     * Disable comments extraction.
+     */
+    public function disableCommentsExtraction()
+    {
+        $this->extractComments = false;
+    }
+
     /**
      * Constructor.
      *
      * @param string $code The php code to scan
      */
     public function __construct($code)
     {
-        $this->tokens = token_get_all($code);
+        $this->tokens = array_values(
+            array_filter(
+                token_get_all($code),
+                function ($token)
+                {
+                    return !is_array($token) || $token[0] !== T_WHITESPACE;
+                }
+            )
+        );
+        $this->extractComments = false;
     }
 
     /**
@@ -65,18 +104,30 @@ public function getFunctions()
                     //new function found
                     for ($j = $k + 1; $j < $count; ++$j) {
                         $nextToken = $this->tokens[$j];
-                        if (is_array($nextToken) && ($nextToken[0] === T_COMMENT || $nextToken[0] === T_WHITESPACE)) {
+                        if (is_array($nextToken) && $nextToken[0] === T_COMMENT) {
                             continue;
                         }
                         if ($nextToken === '(') {
-                            array_unshift($bufferFunctions, new ParsedFunction($value[1], $value[2]));
+                            $newFunction = new ParsedFunction($value[1], $value[2]);
+                            if ($k > 0 && is_array($this->tokens[$k - 1]) && $this->tokens[$k - 1][0] === T_COMMENT) {
+                                $comment = $this->parsePhpComment($this->tokens[$k - 1][1]);
+                                if ($comment !== null) {
+                                    $newFunction->addComment($comment);
+                                }
+                            }
+                            array_unshift($bufferFunctions, $newFunction);
                             $k = $j;
                         }
                         break;
                     }
                     break;
-                case T_WHITESPACE:
                 case T_COMMENT:
+                    if (isset($bufferFunctions[0])) {
+                        $comment = $this->parsePhpComment($value[1]);
+                        if ($comment !== null) {
+                            $bufferFunctions[0]->addComment($comment);
+                        }
+                    }
                     break;
                 default:
                     if (isset($bufferFunctions[0])) {
@@ -88,4 +139,25 @@ public function getFunctions()
 
         return $functions;
     }
+
+    protected function parsePhpComment($value)
+    {
+        $result = null;
+        if ($this->extractComments !== false) {
+            if ($value[0] === '#') {
+                $value = substr($value, 1);
+            }
+            elseif ($value[1] === '/') {
+                $value = substr($value, 2);
+            } else {
+                $value = substr($value, 2, -2);
+            }
+            $value = trim($value);
+            if ($value !== '' && ($this->extractComments === '' || strpos($value, $this->extractComments) === 0)) {
+                $result = $value;
+            }
+        }
+
+        return $result;
+    }
 }

tests/PhpCodeExtractorTest.php

@@ -2,6 +2,11 @@
 
 class PhpCodeExtractorTest extends PHPUnit_Framework_TestCase
 {
+    protected function tearDown()
+    {
+        Gettext\Extractors\PhpCode::$extractComments = false;
+    }
+
     public function testOne()
     {
         //Extract translations
@@ -68,9 +73,50 @@ public function testSpecialChars()
         $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, 'plain'));
         $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, 'DATE \\a\\t TIME'));
         $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, "FIELD\tFIELD"));
-        $this->assertFalse($translations->find(null, "text "));
+        $this->assertFalse($translations->find(null, 'text '));
         $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, "text concatenated with 'comments'"));
-        $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, "Stop at the variable"));
+        $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, 'Stop at the variable'));
         $this->assertCount(5, $translations);
     }
+
+    public function testExtractComments()
+    {
+        Gettext\Extractors\PhpCode::$extractComments = false;
+        $translations = Gettext\Extractors\PhpCode::fromFile(__DIR__.'/files/phpcomments.php');
+        $this->assertCount(4, $translations);
+        foreach ($translations as $translation) {
+            $this->assertEmpty($translation->getExtractedComments());
+        }
+
+        Gettext\Extractors\PhpCode::$extractComments = '';
+        $translations = Gettext\Extractors\PhpCode::fromFile(__DIR__.'/files/phpcomments.php');
+        $this->assertCount(4, $translations);
+        foreach ($translations as $translation) {
+            /* @var Gettext\Translation $translation */
+            switch ($translation->getOriginal()) {
+                case 'No comments':
+                    $this->assertEmpty($translation->getExtractedComments());
+                    break;
+                default:
+                    $this->assertCount(1, $translation->getExtractedComments());
+                    break;
+            }
+        }
+
+        Gettext\Extractors\PhpCode::$extractComments = 'i18n';
+        $translations = Gettext\Extractors\PhpCode::fromFile(__DIR__.'/files/phpcomments.php');
+        $this->assertCount(4, $translations);
+        foreach ($translations as $translation) {
+            /* @var Gettext\Translation $translation */
+            switch ($translation->getOriginal()) {
+                case 'No comments':
+                case 'All comments':
+                    $this->assertEmpty($translation->getExtractedComments());
+                    break;
+                default:
+                    $this->assertCount(1, $translation->getExtractedComments());
+                    break;
+            }
+        }
+    }
 }

tests/files/phpcomments.php

@@ -0,0 +1,15 @@
+<?php
+
+__('No comments');
+
+/* All comments */
+__('All comments');
+
+/* i18n Tagged comment */
+
+__('i18n Tagged comment');
+
+__(
+    /* i18n Tagged comment inside */
+    'i18n Tagged comment inside'
+);