update robots.txt to disallow old urls

ansible · Feb 21, 2024 · 9b711e9 · 9b711e9
1 parent 68684e6
commit 9b711e9
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/robots.txt b/robots.txt
@@ -1,4 +1,14 @@
 User-agent: *
 Disallow: /ansible-core/
+
+User-agent: *
+Disallow: /ansible-container/
+
+User-agent: *
+Disallow: /archive/
+
+User-agent: *
+Disallow: /v1/
+
 Sitemap: https://docs.ansible.com/ansible-sitemap.xml
 Sitemap: https://docs.ansible.com/automation-controller-sitemap.xml