Rework JsonLd extraction: ignore some objects and some names

Ignore objects with type Organization, WebSite or Person. Also maintain a list of values (only name) from these ignored objects to reject similar values in other objects (e.g. a Periodical object with the name of the WebSite). Fixes wallabag/wallabag#3878 Signed-off-by: Kevin Decherf <[email protected]>
j0k3r · Feb 12, 2019 · 879ef69 · 879ef69
1 parent c9e85d2
commit 879ef69
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 3 deletions.
diff --git a/src/Extractor/ContentExtractor.php b/src/Extractor/ContentExtractor.php
@@ -1145,9 +1145,19 @@ private function extractJsonLdInformation($html)
             return;
         }
 
+        $ignoreNames = [];
+        $candidateNames = [];
+
         foreach ($matches[1] as $matche) {
             $data = json_decode(trim($matche), true);
 
+            if (isset($data['@type']) && \in_array($data['@type'], ['Organization', 'WebSite', 'Person'], true)) {
+                if (isset($data['name'])) {
+                    $ignoreNames[] = $data['name'];
+                }
+                continue;
+            }
+
             $this->logger->info('JSON-LD data: {JsonLdData}', ['JsonLdData' => $data]);
 
             // just in case datePublished isn't defined, we use the modified one at first
@@ -1171,11 +1181,11 @@ private function extractJsonLdInformation($html)
             }
 
             if (isset($data['headline'])) {
-                $this->title = $data['headline'];
+                $candidateNames[] = $data['headline'];
             }
 
             if (isset($data['name'])) {
-                $this->title = $data['name'];
+                $candidateNames[] = $data['name'];
             }
 
             if (isset($data['author']['name'])) {
@@ -1190,5 +1200,13 @@ private function extractJsonLdInformation($html)
                 }
             }
         }
+
+        if (\is_array($candidateNames) && \count($candidateNames) > 0) {
+            foreach ($candidateNames as $name) {
+                if (!\in_array($name, $ignoreNames, true)) {
+                    $this->title = $name;
+                }
+            }
+        }
     }
 }
diff --git a/src/SiteConfig/ConfigBuilder.php b/src/SiteConfig/ConfigBuilder.php
@@ -94,7 +94,7 @@ public function getCachedVersion($key)
             $key = substr($key, 4);
         }
 
-        if (array_key_exists($key, $this->cache)) {
+        if (\array_key_exists($key, $this->cache)) {
             return $this->cache[$key];
         }
 

diff --git a/tests/Extractor/ContentExtractorTest.php b/tests/Extractor/ContentExtractorTest.php
@@ -944,6 +944,35 @@ public function testJsonLd()
         $this->assertContains('<p>hihi</p>', $content_block->ownerDocument->saveXML($content_block));
     }
 
+    public function testJsonLdIgnoreList()
+    {
+        $contentExtractor = new ContentExtractor(self::$contentExtractorConfig);
+
+        $res = $contentExtractor->process(
+            '<html><body><script type="application/ld+json">{ "@context": "http:\/\/schema.org", "@type": "NewsArticle", "publisher": { "@type": "Organization", "name": "Foobar Company" }, "description": "A method for fooling tools", "mainEntityOfPage": { "@type": "WebPage", "@id": "https:\/\/www.example.com/foobar" }, "headline": "The Foobar Company is launching globally", "datePublished": "2019-01-14T16:02:00.000+00:00", "dateModified": "2019-01-14T13:25:09.980+00:00", "author": { "@type": "Person", "name": "Foobar CEO" } }</script> <script type="application/ld+json">{ "@context": "http:\/\/schema.org", "@type": "Organization", "name": "Foobar Company", "url": "https:\/\/www.example.com" }</script><p>' . str_repeat('this is the best part of the show', 10) . '</p></body></html>',
+            'https://example.com/jsonld'
+        );
+
+        $this->assertTrue($res, 'Extraction went well');
+
+        $this->assertSame('The Foobar Company is launching globally', $contentExtractor->getTitle());
+        $this->assertContains('Foobar CEO', $contentExtractor->getAuthors());
+    }
+
+    public function testJsonLdIgnoreListWithPeriodical()
+    {
+        $contentExtractor = new ContentExtractor(self::$contentExtractorConfig);
+
+        $res = $contentExtractor->process(
+            '<html><body><script type="application/ld+json">{ "@context": "http:\/\/schema.org", "@type": "Periodical", "publisher": { "@type": "Organization", "name": "Foobar Company" }, "description": "A method for fooling tools", "mainEntityOfPage": { "@type": "WebPage", "@id": "https:\/\/www.example.com/foobar" }, "name": "Foobar Company", "datePublished": "2019-01-14T16:02:00.000+00:00", "dateModified": "2019-01-14T13:25:09.980+00:00", "author": { "@type": "Person", "name": "Foobar CEO" } }</script> <script type="application/ld+json">{ "@context": "http:\/\/schema.org", "@type": "Organization", "name": "Foobar Company", "url": "https:\/\/www.example.com" }</script><h1>Hello world, this is title</h1><p>' . str_repeat('this is the best part of the show', 10) . '</p></body></html>',
+            'https://example.com/jsonld'
+        );
+
+        $this->assertTrue($res, 'Extraction went well');
+
+        $this->assertSame('Hello world, this is title', $contentExtractor->getTitle());
+    }
+
     public function testJsonLdSkipper()
     {
         $contentExtractor = new ContentExtractor(self::$contentExtractorConfig);