update robots.txt to be more explicit with exclusions

masalachai · Jul 16, 2021 · 11519b7 · 11519b7
1 parent 166f61a
commit 11519b7
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/deploy/environments/production/assets/robots.txt b/deploy/environments/production/assets/robots.txt
@@ -1,7 +1,10 @@
 User-Agent: *
-Disallow: /*tree
-Disallow: /*blob
-Disallow: /*branch
+Disallow: /ritesh/gemini-git-browser.git/tree
+Disallow: /ritesh/gemini-git-browser.git/blob
+Disallow: /ritesh/gemini-git-browser.git/branch
+Disallow: /ritesh/deploy-scripts.git/tree
+Disallow: /ritesh/deploy-scripts.git/blob
+Disallow: /ritesh/deploy-scripts.git/branch
 Allow: /ritesh/gemini-git-browser.git
 Allow: /ritesh/deploy-scripts.git
 Disallow: /chitlangi