From 0b6865dcc993c20a273ccc91be2129db3dd0e3a7 Mon Sep 17 00:00:00 2001 From: Michele Locati Date: Fri, 19 Feb 2016 13:26:32 +0100 Subject: [PATCH 1/3] Start implementing comments extraction from PHP code --- src/Utils/FunctionsScanner.php | 5 ++ src/Utils/ParsedFunction.php | 22 +++++++++ src/Utils/PhpFunctionsScanner.php | 80 +++++++++++++++++++++++++++++-- 3 files changed, 103 insertions(+), 4 deletions(-) diff --git a/src/Utils/FunctionsScanner.php b/src/Utils/FunctionsScanner.php index 26fbaa70..2e13010f 100644 --- a/src/Utils/FunctionsScanner.php +++ b/src/Utils/FunctionsScanner.php @@ -71,6 +71,11 @@ public function saveGettextFunctions(array $functions, Translations $translation if (isset($translation)) { $translation->addReference($file, $line); + if (isset($function[3])) { + foreach ($function[3] as $extractedComment) { + $translation->addExtractedComment($extractedComment); + } + } } } } diff --git a/src/Utils/ParsedFunction.php b/src/Utils/ParsedFunction.php index 1f6fd3c0..e2f54923 100644 --- a/src/Utils/ParsedFunction.php +++ b/src/Utils/ParsedFunction.php @@ -42,6 +42,13 @@ class ParsedFunction */ protected $argumentStopped; + /** + * Extracted comments. + * + * @var string[]|null + */ + protected $comments; + /** * Initializes the instance. * @@ -55,6 +62,7 @@ public function __construct($name, $line) $this->arguments = array(); $this->argumentIndex = -1; $this->argumentStopped = false; + $this->comments = null; } /** @@ -101,6 +109,18 @@ public function addArgumentChunk($chunk) } } + /** + * Add a comment associated to this function. + * + * @param string $comment + */ + public function addComment($comment) + { + if ($this->comments === null) { + $this->comments = array(); + } + $this->comments[] = $comment; + } /** * A closing parenthesis was found: return the final data. * @@ -109,6 +129,7 @@ public function addArgumentChunk($chunk) * @var string The function name. * @var int The line where the function starts. * @var string[] the strings extracted from the function arguments. + * @var string[] the comments associated to the function. * } */ public function close() @@ -122,6 +143,7 @@ public function close() $this->name, $this->line, $arguments, + $this->comments, ); } } diff --git a/src/Utils/PhpFunctionsScanner.php b/src/Utils/PhpFunctionsScanner.php index ecbdec9e..9c2c235f 100644 --- a/src/Utils/PhpFunctionsScanner.php +++ b/src/Utils/PhpFunctionsScanner.php @@ -6,8 +6,38 @@ class PhpFunctionsScanner extends FunctionsScanner { + /** + * PHP tokens of the code to be parsed. + * + * @var array + */ protected $tokens; + /** + * If not false, comments will be extracted. + * + * @var string|false + */ + protected $extractComments = false; + + /** + * Enable extracting comments that start with a tag (if $tag is empty all the comments will be extracted). + * + * @param string $tag + */ + public function enableCommentsExtraction($tag = '') + { + $this->extractComments = (string) $tag; + } + + /** + * Disable comments extraction. + */ + public function disableCommentsExtraction() + { + $this->extractComments = false; + } + /** * Constructor. * @@ -15,7 +45,16 @@ class PhpFunctionsScanner extends FunctionsScanner */ public function __construct($code) { - $this->tokens = token_get_all($code); + $this->tokens = array_values( + array_filter( + token_get_all($code), + function ($token) + { + return !is_array($token) || $token[0] !== T_WHITESPACE; + } + ) + ); + $this->extractComments = false; } /** @@ -65,18 +104,30 @@ public function getFunctions() //new function found for ($j = $k + 1; $j < $count; ++$j) { $nextToken = $this->tokens[$j]; - if (is_array($nextToken) && ($nextToken[0] === T_COMMENT || $nextToken[0] === T_WHITESPACE)) { + if (is_array($nextToken) && $nextToken[0] === T_COMMENT) { continue; } if ($nextToken === '(') { - array_unshift($bufferFunctions, new ParsedFunction($value[1], $value[2])); + $newFunction = new ParsedFunction($value[1], $value[2]); + if ($k > 0 && is_array($this->tokens[$k - 1]) && $this->tokens[$k - 1][0] === T_COMMENT) { + $comment = $this->parsePhpComment($this->tokens[$k - 1][1]); + if ($comment !== null) { + $newFunction->addComment($comment); + } + } + array_unshift($bufferFunctions, $newFunction); $k = $j; } break; } break; - case T_WHITESPACE: case T_COMMENT: + if (isset($bufferFunctions[0])) { + $comment = $this->parsePhpComment($value[1]); + if ($comment !== null) { + $bufferFunctions[0]->addComment($comment); + } + } break; default: if (isset($bufferFunctions[0])) { @@ -88,4 +139,25 @@ public function getFunctions() return $functions; } + + protected function parsePhpComment($value) + { + $result = null; + if ($this->extractComments !== false) { + if ($value[0] === '#') { + $value = substr($value, 1); + } + elseif ($value[1] === '/') { + $value = substr($value, 2); + } else { + $value = substr($value, 2, -2); + } + $value = trim($value); + if ($value !== '' && ($this->extractComments === '' || strpos($value, $this->extractComments) === 0)) { + $result = $value; + } + } + + return $result; + } } From 26c3b5302640be451a589e9cff8bc35e8d70b217 Mon Sep 17 00:00:00 2001 From: Michele Locati Date: Fri, 19 Feb 2016 14:13:21 +0100 Subject: [PATCH 2/3] Enable PHP comment extraction configuration --- src/Extractors/PhpCode.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Extractors/PhpCode.php b/src/Extractors/PhpCode.php index ab9dfb37..bf97b94d 100644 --- a/src/Extractors/PhpCode.php +++ b/src/Extractors/PhpCode.php @@ -19,6 +19,16 @@ class PhpCode extends Extractor implements ExtractorInterface 'p__e' => 'p__', ); + /** + * Set to: + * - false to not extract comments + * - empty string to extract all comments + * - non-empty string to extract comments that start with that string. + * + * @var string|false + */ + public static $extractComments = false; + /** * {@inheritdoc} */ @@ -29,6 +39,9 @@ public static function fromString($string, Translations $translations = null, $f } $functions = new PhpFunctionsScanner($string); + if (self::$extractComments !== false) { + $functions->enableCommentsExtraction(self::$extractComments); + } $functions->saveGettextFunctions(self::$functions, $translations, $file); return $translations; From acb881bb786041214f27449c9979123efd10bd85 Mon Sep 17 00:00:00 2001 From: Michele Locati Date: Fri, 19 Feb 2016 15:16:25 +0100 Subject: [PATCH 3/3] Add some tests for the PHP comments extraction function --- tests/PhpCodeExtractorTest.php | 50 ++++++++++++++++++++++++++++++++-- tests/files/phpcomments.php | 15 ++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 tests/files/phpcomments.php diff --git a/tests/PhpCodeExtractorTest.php b/tests/PhpCodeExtractorTest.php index 3a8a7dfa..6b57b2cf 100644 --- a/tests/PhpCodeExtractorTest.php +++ b/tests/PhpCodeExtractorTest.php @@ -2,6 +2,11 @@ class PhpCodeExtractorTest extends PHPUnit_Framework_TestCase { + protected function tearDown() + { + Gettext\Extractors\PhpCode::$extractComments = false; + } + public function testOne() { //Extract translations @@ -68,9 +73,50 @@ public function testSpecialChars() $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, 'plain')); $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, 'DATE \\a\\t TIME')); $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, "FIELD\tFIELD")); - $this->assertFalse($translations->find(null, "text ")); + $this->assertFalse($translations->find(null, 'text ')); $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, "text concatenated with 'comments'")); - $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, "Stop at the variable")); + $this->assertInstanceOf('Gettext\\Translation', $translations->find(null, 'Stop at the variable')); $this->assertCount(5, $translations); } + + public function testExtractComments() + { + Gettext\Extractors\PhpCode::$extractComments = false; + $translations = Gettext\Extractors\PhpCode::fromFile(__DIR__.'/files/phpcomments.php'); + $this->assertCount(4, $translations); + foreach ($translations as $translation) { + $this->assertEmpty($translation->getExtractedComments()); + } + + Gettext\Extractors\PhpCode::$extractComments = ''; + $translations = Gettext\Extractors\PhpCode::fromFile(__DIR__.'/files/phpcomments.php'); + $this->assertCount(4, $translations); + foreach ($translations as $translation) { + /* @var Gettext\Translation $translation */ + switch ($translation->getOriginal()) { + case 'No comments': + $this->assertEmpty($translation->getExtractedComments()); + break; + default: + $this->assertCount(1, $translation->getExtractedComments()); + break; + } + } + + Gettext\Extractors\PhpCode::$extractComments = 'i18n'; + $translations = Gettext\Extractors\PhpCode::fromFile(__DIR__.'/files/phpcomments.php'); + $this->assertCount(4, $translations); + foreach ($translations as $translation) { + /* @var Gettext\Translation $translation */ + switch ($translation->getOriginal()) { + case 'No comments': + case 'All comments': + $this->assertEmpty($translation->getExtractedComments()); + break; + default: + $this->assertCount(1, $translation->getExtractedComments()); + break; + } + } + } } diff --git a/tests/files/phpcomments.php b/tests/files/phpcomments.php new file mode 100644 index 00000000..2a61c618 --- /dev/null +++ b/tests/files/phpcomments.php @@ -0,0 +1,15 @@ +