Use Facebook user agent for openGraph queries

We found that some websites return opengraph information based on the user agent. Since Facebook is the creator of opengraph, using the Facebook user agent when requesting the opengraph metadata should work in the widest variety of situations. https://developers.facebook.com/docs/sharing/webmasters/#user-agent Signed-off-by: Andrew Ryan <[email protected]>
matrix-org · Feb 14, 2022 · 947bae2 · 947bae2
1 parent 55113dd
commit 947bae2
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 6 deletions.
diff --git a/changelog.d/11985.misc b/changelog.d/11985.misc
@@ -0,0 +1 @@
+Use Facebook user agent for openGraph queries.
diff --git a/synapse/res/providers.json b/synapse/res/providers.json
@@ -5,13 +5,11 @@
         "endpoints": [
             {
                 "schemes": [
-                    "https://twitter.com/*/status/*",
-                    "https://*.twitter.com/*/status/*",
                     "https://twitter.com/*/moments/*",
                     "https://*.twitter.com/*/moments/*"
                 ],
                 "url": "https://publish.twitter.com/oembed"
             }
         ]
     }
-]
+]
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
@@ -326,8 +326,9 @@ async def _do_preview(self, url: str, user: UserID, ts: int) -> bytes:
 
                 # Compile the Open Graph response by using the scraped
                 # information from the HTML and overlaying any information
-                # from the oEmbed response.
-                og = {**og_from_html, **og_from_oembed}
+                # from the oEmbed response. og tags from the original html
+                # have priority over oEmbed data.
+                og = {**og_from_oembed, **og_from_html}
 
                 await self._precache_image_url(user, media_info, og)
             else:
@@ -402,7 +403,10 @@ async def _download_url(self, url: str, output_stream: BinaryIO) -> DownloadResu
                 url,
                 output_stream=output_stream,
                 max_size=self.max_spider_size,
-                headers={"Accept-Language": self.url_preview_accept_language},
+                headers={
+                    "Accept-Language": self.url_preview_accept_language,
+                    b"User-Agent": ["Synapse (bot)"],
+                },
                 is_allowed_content_type=_is_previewable,
             )
         except SynapseError: