Merge pull request #166 from swissspidy/comment-extraction-2

Further improve comment extraction

src/Utils/ParsedComment.php

@@ -15,32 +15,81 @@ class ParsedComment
     protected $comment;
 
     /**
-     * The line where the function starts.
+     * The line where the comment starts.
      *
      * @var int
      */
-    protected $line;
+    protected $firstLine;
+
+    /**
+     * The line where the comment ends.
+     *
+     * @var int
+     */
+    protected $lastLine;
 
     /**
      * Initializes the instance.
      *
      * @param string $comment The comment itself.
-     * @param int    $line The line where the comment starts.
+     * @param int    $firstLine The line where the comment starts.
+     * @param int    $lastLine The line where the comment ends.
      */
-    public function __construct($comment, $line)
+    public function __construct($comment, $firstLine, $lastLine)
     {
         $this->comment = $comment;
-        $this->line = $line;
+        $this->firstLine = $firstLine;
+        $this->lastLine = $lastLine;
+    }
+
+    /**
+     * Create new object from raw comment data.
+     *
+     * @param string $value The PHP comment string.
+     * @param int $line The line where the comment starts.
+     *
+     * @return self The parsed comment.
+     */
+    public static function create($value, $line)
+    {
+        $lastLine = $line + substr_count($value, "\n");
+
+        $lines = array_map(function ($line) {
+            if ('' === trim($line)) {
+                return null;
+            }
+
+            $line = ltrim($line, '#*/ ');
+            $line = rtrim($line, '#*/ ');
+
+            return trim($line);
+        }, explode("\n", $value));
+
+        // Remove empty lines.
+        $lines = array_filter($lines);
+        $value = implode(' ', $lines);
+
+        return new self($value, $line, $lastLine);
     }
 
     /**
-     * Return the comment's line number.
+     * Return the line where the comment starts.
      *
      * @return int Line number.
      */
-    public function getLine()
+    public function getFirstLine()
     {
-        return $this->line;
+        return $this->firstLine;
+    }
+
+    /**
+     * Return the line where the comment ends.
+     *
+     * @return int Line number.
+     */
+    public function getLastLine()
+    {
+        return $this->lastLine;
     }
 
     /**
@@ -52,4 +101,40 @@ public function getComment()
     {
         return $this->comment;
     }
+
+    /**
+     * Whether this comment is related with a given function.
+     *
+     * @param ParsedFunction $function The function to check.
+     * @return bool Whether the comment is related or not.
+     */
+    public function isRelatedWith(ParsedFunction $function)
+    {
+        return $this->getLastLine() === $function->getLine() || $this->getLastLine() === $function->getLine() - 1;
+    }
+
+    /**
+     * Whether the comment matches the required prefixes.
+     *
+     * @param array $prefixes An array of prefixes to check.
+     * @return bool Whether the comment matches the prefixes or not.
+     */
+    public function checkPrefixes(array $prefixes)
+    {
+        if ('' === $this->comment) {
+            return false;
+        }
+
+        if (empty($prefixes)) {
+            return true;
+        }
+
+        foreach ($prefixes as $prefix) {
+            if (strpos($this->comment, $prefix) === 0) {
+                return true;
+            }
+        }
+
+        return false;
+    }
 }

src/Utils/ParsedFunction.php

@@ -121,6 +121,17 @@ public function addComment($comment)
         }
         $this->comments[] = $comment;
     }
+
+    /**
+     * Return the line the function starts.
+     *
+     * @return int Line number.
+     */
+    public function getLine()
+    {
+        return $this->line;
+    }
+
     /**
      * A closing parenthesis was found: return the final data.
      * The array returned has the following values:

src/Utils/PhpFunctionsScanner.php

@@ -126,8 +126,10 @@ public function getFunctions(array $constants = [])
 
                             // add comment that was on the line before.
                             if (isset($bufferComments[0])) {
-                                if ($bufferComments[0]->getLine() === $value[2] - 1) {
-                                    $newFunction->addComment($bufferComments[0]->getComment());
+                                $comment = $bufferComments[0];
+
+                                if ($comment->isRelatedWith($newFunction)) {
+                                    $newFunction->addComment($comment->getComment());
                                 }
                             }
 
@@ -139,13 +141,14 @@ public function getFunctions(array $constants = [])
                     break;
 
                 case T_COMMENT:
-                    $comment = $this->parsePhpComment($value[1]);
+                    $comment = $this->parsePhpComment($value[1], $value[2]);
 
-                    if ($comment !== null) {
-                        array_unshift($bufferComments, new ParsedComment($comment, $value[2]));
+                    if ($comment) {
+                        array_unshift($bufferComments, $comment);
 
+                        // The comment is inside the function call.
                         if (isset($bufferFunctions[0])) {
-                            $bufferFunctions[0]->addComment($comment);
+                            $bufferFunctions[0]->addComment($comment->getComment());
                         }
                     }
                     break;
@@ -161,35 +164,31 @@ public function getFunctions(array $constants = [])
         return $functions;
     }
 
-    protected function parsePhpComment($value)
+    /**
+     * Extract the actual text from a PHP comment.
+     *
+     * If set, only returns comments that match the prefix(es).
+     *
+     * @param string $value The PHP comment.
+     * @param int $line Line number.
+     *
+     * @return null|ParsedComment Comment or null if comment extraction is disabled or if there is a prefix mismatch.
+     */
+    protected function parsePhpComment($value, $line)
     {
-        $result = null;
-
-        if ($this->extractComments !== false) {
-            if ($value[0] === '#') {
-                $value = substr($value, 1);
-            } elseif ($value[1] === '/') {
-                $value = substr($value, 2);
-            } else {
-                $value = substr($value, 2, -2);
-            }
+        if ($this->extractComments === false) {
+            return null;
+        }
 
-            $value = trim($value);
+        //this returns a comment or null
+        $comment = ParsedComment::create($value, $line);
 
-            if ($value !== '') {
-                if (is_array($this->extractComments)) {
-                    foreach ($this->extractComments as $string) {
-                        if (strpos($value, $string) === 0) {
-                            $result = $value;
-                            break;
-                        }
-                    }
-                } elseif ($this->extractComments === '' || strpos($value, $this->extractComments) === 0) {
-                    $result = $value;
-                }
-            }
+        $prefixes = array_filter((array) $this->extractComments);
+
+        if ($comment && $comment->checkPrefixes($prefixes)) {
+            return $comment;
         }
 
-        return $result;
+        return null;
     }
 }

tests/AssetsTest.php

@@ -299,7 +299,7 @@ public function testPhpCode2()
                 'CONTEXT' => 'my-context',
             ]
         ]);
-        $countTranslations = 11;
+        $countTranslations = 13;
         $countTranslated = 0;
         $countHeaders = 8;
 
@@ -337,7 +337,7 @@ public function testPhpCode3()
         $translations = static::get('phpcode3/input', 'PhpCode', [
             'extractComments' => ['allowed1', 'allowed2'],
         ]);
-        $countTranslations = 1;
+        $countTranslations = 2;
         $countTranslated = 0;
         $countHeaders = 8;
 

tests/assets/phpcode2/Csv.csv

@@ -18,3 +18,5 @@ my-context,"All comments",
 ,"i18n Tagged comment inside",
 ,"One comment",
 ,"i18n tagged %s",
+,foo,
+,bar,

tests/assets/phpcode2/CsvDictionary.csv

@@ -9,3 +9,5 @@ plain,
 "i18n Tagged comment inside",
 "One comment",
 "i18n tagged %s",
+foo,
+bar,

tests/assets/phpcode2/Jed.json

@@ -37,6 +37,12 @@
         ],
         "i18n tagged %s": [
             ""
+        ],
+        "foo": [
+            ""
+        ],
+        "bar": [
+            ""
         ]
     }
 }

tests/assets/phpcode2/Json.json

@@ -35,6 +35,12 @@
             ],
             "i18n tagged %s": [
                 ""
+            ],
+            "foo": [
+                ""
+            ],
+            "bar": [
+                ""
             ]
         },
         "my-context": {

tests/assets/phpcode2/JsonDictionary.json

@@ -9,5 +9,7 @@
     "i18n Tagged comment": "",
     "i18n Tagged comment inside": "",
     "One comment": "",
-    "i18n tagged %s": ""
+    "i18n tagged %s": "",
+    "foo": "",
+    "bar": ""
 }

tests/assets/phpcode2/PhpArray.php

@@ -57,6 +57,14 @@
       array (
         0 => '',
       ),
+      'foo' => 
+      array (
+        0 => '',
+      ),
+      'bar' => 
+      array (
+        0 => '',
+      ),
     ),
     'my-context' => 
     array (

tests/assets/phpcode2/Po.po

@@ -60,3 +60,13 @@ msgstr[0] ""
 #: ./tests/assets/phpcode2/input.php:36
 msgid "i18n tagged %s"
 msgstr ""
+
+#. Translators: This is a multi-line comment.
+#: ./tests/assets/phpcode2/input.php:42
+msgid "foo"
+msgstr ""
+
+#. translators: this should get extracted.
+#: ./tests/assets/phpcode2/input.php:44
+msgid "bar"
+msgstr ""

tests/assets/phpcode2/Xliff.xlf

@@ -127,5 +127,27 @@
         <target></target>
       </segment>
     </unit>
+    <unit id="acbd18db4cc2f85cedef654fccc4a4d8">
+      <notes>
+        <note category="context"></note>
+        <note category="extracted-comment">Translators: This is a multi-line comment.</note>
+        <note category="reference">./tests/assets/phpcode2/input.php:42</note>
+      </notes>
+      <segment>
+        <source>foo</source>
+        <target></target>
+      </segment>
+    </unit>
+    <unit id="37b51d194a7513e45b56f6524f2d51f2">
+      <notes>
+        <note category="context"></note>
+        <note category="extracted-comment">translators: this should get extracted.</note>
+        <note category="reference">./tests/assets/phpcode2/input.php:44</note>
+      </notes>
+      <segment>
+        <source>bar</source>
+        <target></target>
+      </segment>
+    </unit>
   </file>
 </xliff>

tests/assets/phpcode2/Yaml.yml

@@ -14,5 +14,7 @@ messages:
     'i18n Tagged comment inside': ''
     'One comment': ''
     'i18n tagged %s': ''
+    foo: ''
+    bar: ''
   my-context:
     'All comments': ''

tests/assets/phpcode2/YamlDictionary.yml

@@ -9,3 +9,5 @@ plain: ''
 'i18n Tagged comment inside': ''
 'One comment': ''
 'i18n tagged %s': ''
+foo: ''
+bar: ''

tests/assets/phpcode2/input.php

@@ -34,3 +34,11 @@
 
 /* i18n Tagged comment on the line before */
 sprintf( __('i18n tagged %s'), '$var');
+
+/*
+ * Translators: This is a
+ * multi-line comment.
+ */
+__( 'foo' );
+
+/* translators: this should get extracted. */ $foo = __( 'bar' );

tests/assets/phpcode3/Csv.csv

@@ -8,3 +8,4 @@ Project-Id-Version:
 Report-Msgid-Bugs-To: 
 "
 ,"Translation with comments",
+,Foo,

tests/assets/phpcode3/CsvDictionary.csv

@@ -1 +1,2 @@
 "Translation with comments",
+Foo,