ci: Exclude deploy/README.md from build (#9211)

Reverts #9175 and instead fixes local build error by not specifically excluding the new `deploy/README.md` in configuration. With this change, `upload-search-index` will again work.

Bonus:
* adds some documentation about search-index uploading
* standardizes quotes in `conf.py`
* removes no-longer-used 'examples' from exclude list (examples used to be a thing inside `/docs`, is no longer).

docs/Makefile

@@ -58,6 +58,7 @@ build: build/sp-html.stamp
 .PHONY: xml
 xml: build/sp-xml.stamp
 
+# Uploads the search index to Algolia with a version based roughly on VERSION
 .PHONY: upload-search-index
 upload-search-index: build/sp-xml.stamp
 	python3 deploy/scrape.py --upload

docs/conf.py

@@ -14,7 +14,7 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.append(os.path.abspath(os.path.dirname(__file__)))
-sys.path.append(os.path.abspath('./_extensions'))
+sys.path.append(os.path.abspath("./_extensions"))
 
 project = "Determined"
 html_title = "Determined AI Documentation"
@@ -51,11 +51,11 @@ def setup(app):
     "_build",
     "Thumbs.db",
     ".DS_Store",
-    "examples",
     "requirements.txt",
     "site",
     "README.md",
     "release-notes/README.md",
+    "deploy",
 ]
 html_baseurl = "https://docs.determined.ai"  # Base URL for sitemap.
 highlight_language = "none"
@@ -153,23 +153,23 @@ def setup(app):
 # also ignore urls that have IP-checking measures in place that block aws
 # connections based on IP.
 linkcheck_ignore = [
-    r'^#',
-    r'^http://127.0.0.1',
-    r'^\.\./',
-    'https://www.hpe.com/us/en/hpe-machine-learning-development-environment.html'
+    r"^#",
+    r"^http://127.0.0.1",
+    r"^\.\./",
+    "https://www.hpe.com/us/en/hpe-machine-learning-development-environment.html",
 ]
 
 linkcheck_timeout = 20
 
 # linkcheck gets confused by anchors sometimes
-linkcheck_anchors_ignore=[
-    'install-nvidia-device-plugin',
-    'tag/System/operation/SystemAuth',
-    'batch-size-related-parameters'
+linkcheck_anchors_ignore = [
+    "install-nvidia-device-plugin",
+    "tag/System/operation/SystemAuth",
+    "batch-size-related-parameters",
 ]
 
 # Some pages block python requests. Set user-agent to appear as a browser.
-user_agent ="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
+user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
 
 with open(".redirects/redirects.json") as f:
     redirects = json.load(f)

docs/deploy/scrape.py

@@ -1,3 +1,20 @@
+"""Build an index from the sphinx XML files and (optionally) upload it to Algolia.
+
+This script is intended to be run from the root of the docs directory.  It will scrape the XML files
+in the site/xml directory and build an index from them.
+
+If the --json flag is given, the index will be dumped to stdout as a JSON list of records.
+If the --upload flag is given, the index will be uploaded to Algolia.
+
+The version of the docs is determined by the VERSION file in the root of the docs directory.  If the
+version contains "-dev" then the index will be uploaded to a special "dev" index (shared amongst
+all dev builds). During a release, the uploaded index will have a non-suffixed version (in particular,
+the "-rc" suffix will be removed).
+
+To view Algolia's repository of indexes built for determined:
+  https://dashboard.algolia.com/apps/9H1PGK6NP7/indices
+"""
+
 import argparse
 import html
 import io