added the lines for the dataset/dataverse thumbnails/favicons, for th…

…e twitter/facebook/linkedin preview cards. (#5639)

doc/sphinx-guides/source/_static/util/robots.txt

@@ -13,6 +13,9 @@ Allow: /$
 Allow: /dataset.xhtml
 Allow: /dataverse/
 Allow: /sitemap/
+# The following lines are for the facebook, twitter and linkedin preview bots:
+Allow: /api/datasets/:persistentId/thumbnail
+Allow: /javax.faces.resource/images/
 # Comment out the following TWO lines if you DON'T MIND the bots crawling the search API links on dataverse pages:
 Disallow: /dataverse/*?q
 Disallow: /dataverse/*/search

src/main/webapp/robots.txt

@@ -1,23 +1,5 @@
+# Consult the guide on how to open your Dataverse installation to crawling by 
+# search engine bots: 
+# /guides/installation/config.html#letting-search-engines-crawl-your-installation 
 User-agent: *
 Disallow: /
-User-agent: Googlebot
-Disallow: /
-#Crawl-delay: 120
-#Disallow: /dvn/faces/javax.faces.resource
-#Disallow: /dvn/OAIHandler
-#Disallow: /dvn/faces/ContactUsPage.xhtml
-#Disallow: /dvn/dv/*/faces/ContactUsPage.xhtml
-#Disallow: /dvn/faces/study/TermsOfUsePage.xhtml
-#Disallow: /dvn/faces/subsetting/SubsettingPage.xhtml
-#Disallow: /dvn/dv/*/faces/subsetting/SubsettingPage.xhtml
-#Disallow: /dvn/FileDownload/
-#Disallow: /FileDownload/
-#Disallow: /dvn/dv/*/FileDownload/
-#Disallow: /dvn/resources/
-#Disallow: /dvn/api/
-#
-
-
-# Created initially using: http://www.mcanerin.com/EN/search-engine/robots-txt.asp
-# Verified using: http://tool.motoricerca.info/robots-checker.phtml
-