Rework documentation generation

.github/workflows/devcontainer.yml

@@ -198,7 +198,6 @@ jobs:
         rm -f doc/build/html/htmlcov/.gitignore
 
     - uses: actions/upload-artifact@v4
-      if: github.ref == 'refs/heads/main'
       with:
         name: docs
         path: doc/build/html

.vscode/settings.json

@@ -1,6 +1,18 @@
 // vim: set ft=jsonc:
 {
     "makefile.extensionOutputFolder": "./.vscode",
+    "files.exclude": {
+        ".git/": true,
+        ".mypy_cache/": true,
+        ".pytest_cache/": true,
+        "**/__pycache__/": true,
+        "**/node_modules/": true,
+        "**/*.egg-info": true,
+        "doc/source/autoapi/": true,
+        "doc/build/doctrees/": true,
+        "doc/build/html/": true,
+        "htmlcov/": true,
+    },
     // Note: this only works in WSL/Linux currently.
     "python.defaultInterpreterPath": "${env:HOME}/.conda/envs/mlos/bin/python",
     // For Windows it should be this instead:
@@ -26,6 +38,8 @@
                 "mlos_bench/mlos_bench/tests/config/environments/**/*.json",
                 "mlos_bench/mlos_bench/config/environments/**/*.jsonc",
                 "mlos_bench/mlos_bench/config/environments/**/*.json",
+                "!mlos_bench/mlos_bench/tests/config/environments/**/*-tunables.jsonc",
+                "!mlos_bench/mlos_bench/tests/config/environments/**/*-tunables.json",
                 "!mlos_bench/mlos_bench/config/environments/**/*-tunables.jsonc",
                 "!mlos_bench/mlos_bench/config/environments/**/*-tunables.json"
             ],

CONTRIBUTING.md

@@ -86,6 +86,8 @@ We expect development to follow a typical "forking" style workflow:
     make doc-test
     ```
 
+    > See the [documentation README](./doc/README.md) for more information on documentation and its testing.
+
 1. Submit changes for inclusion as a [Pull Request on Github](https://github.com/microsoft/MLOS/pulls).
 
     Some notes on organizing changes to help reviewers:

MAINTAINING.md

@@ -2,6 +2,10 @@
 
 Some notes for maintainers.
 
+## Documentation
+
+See the [documentation README](./doc/README.md) for more information on writing (and testing) documentation.
+
 ## Releasing
 
 1. Bump the version using the [`update-version.sh`](./scripts/update-version.sh) script:

Makefile

@@ -659,49 +659,32 @@ clean-doc-env:
 
 COMMON_DOC_FILES := build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp doc/source/*.rst doc/source/_templates/*.rst doc/source/conf.py
 
-doc/source/api/mlos_core/modules.rst: $(FORMAT_PREREQS) $(COMMON_DOC_FILES)
-doc/source/api/mlos_core/modules.rst: $(MLOS_CORE_PYTHON_FILES)
-	rm -rf doc/source/api/mlos_core
-	cd doc/ && conda run -n ${CONDA_ENV_NAME} sphinx-apidoc -f -e -M \
-		-o source/api/mlos_core/ \
-		../mlos_core/ \
-		../mlos_core/setup.py ../mlos_core/mlos_core/tests/
-
-doc/source/api/mlos_bench/modules.rst: $(FORMAT_PREREQS) $(COMMON_DOC_FILES)
-doc/source/api/mlos_bench/modules.rst: $(MLOS_BENCH_PYTHON_FILES)
-	rm -rf doc/source/api/mlos_bench
-	cd doc/ && conda run -n ${CONDA_ENV_NAME} sphinx-apidoc -f -e -M \
-		-o source/api/mlos_bench/ \
-		../mlos_bench/ \
-		../mlos_bench/setup.py ../mlos_bench/mlos_bench/tests/
-	# Save the help output of the mlos_bench scripts to include in the documentation.
-	# First make sure that the latest version of mlos_bench is installed (since it uses git based tagging).
-	conda run -n ${CONDA_ENV_NAME} pip install -e mlos_core -e mlos_bench -e mlos_viz
-	conda run -n ${CONDA_ENV_NAME} mlos_bench --help > doc/source/api/mlos_bench/mlos_bench.run.usage.txt
-	echo ".. literalinclude:: mlos_bench.run.usage.txt" >> doc/source/api/mlos_bench/mlos_bench.run.rst
-	echo "   :language: none" >> doc/source/api/mlos_bench/mlos_bench.run.rst
-
-doc/source/api/mlos_viz/modules.rst: $(FORMAT_PREREQS) $(COMMON_DOC_FILES)
-doc/source/api/mlos_viz/modules.rst: $(MLOS_VIZ_PYTHON_FILES)
-	rm -rf doc/source/api/mlos_viz
-	cd doc/ && conda run -n ${CONDA_ENV_NAME} sphinx-apidoc -f -e -M \
-		-o source/api/mlos_viz/ \
-		../mlos_viz/ \
-		../mlos_viz/setup.py ../mlos_viz/mlos_viz/tests/
-
-SPHINX_API_RST_FILES := doc/source/api/mlos_core/modules.rst
-SPHINX_API_RST_FILES += doc/source/api/mlos_bench/modules.rst
-SPHINX_API_RST_FILES += doc/source/api/mlos_viz/modules.rst
-
-.PHONY: sphinx-apidoc
-sphinx-apidoc: $(SPHINX_API_RST_FILES)
+SPHINX_API_RST_FILES := doc/source/index.rst doc/source/mlos_bench.run.usage.rst
 
 ifeq ($(SKIP_COVERAGE),)
 doc/build/html/index.html: build/pytest.${CONDA_ENV_NAME}.build-stamp
 doc/build/html/htmlcov/index.html: build/pytest.${CONDA_ENV_NAME}.build-stamp
 endif
 
-doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile doc/copy-source-tree-docs.sh $(MD_FILES)
+# Treat warnings as failures.
+SPHINXOPTS ?= # -v # be verbose
+SPHINXOPTS += -n -W -w $(CURDIR)/doc/build/sphinx-build.warn.log -j auto
+
+sphinx-apidoc: doc/build/html/index.html
+
+doc/source/generated/mlos_bench.run.usage.txt: build/conda-env.${CONDA_ENV_NAME}.build-stamp
+doc/source/generated/mlos_bench.run.usage.txt: $(MLOS_BENCH_PYTHON_FILES)
+	# Generate the help output from mlos_bench CLI for the docs.
+	mkdir -p doc/source/generated/
+	conda run -n ${CONDA_ENV_NAME} mlos_bench --help > doc/source/generated/mlos_bench.run.usage.txt
+
+doc/build/html/index.html: build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp
+doc/build/html/index.html: doc/source/generated/mlos_bench.run.usage.txt
+doc/build/html/index.html: $(MLOS_CORE_PYTHON_FILES)
+doc/build/html/index.html: $(MLOS_BENCH_PYTHON_FILES)
+doc/build/html/index.html: $(MLOS_VIZ_PYTHON_FILES)
+doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile doc/source/conf.py
+doc/build/html/index.html: doc/copy-source-tree-docs.sh $(MD_FILES)
 	@rm -rf doc/build
 	@mkdir -p doc/build
 	@rm -f doc/build/log.txt
@@ -715,7 +698,7 @@ doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile doc/copy-source-
 	./doc/copy-source-tree-docs.sh
 
 	# Build the rst files into html.
-	conda run -n ${CONDA_ENV_NAME} $(MAKE) -C doc/ $(MAKEFLAGS) html \
+	conda run -n ${CONDA_ENV_NAME} $(MAKE) SPHINXOPTS="$(SPHINXOPTS)" -C doc/ $(MAKEFLAGS) html \
 		>> doc/build/log.txt 2>&1 \
 		|| { cat doc/build/log.txt; exit 1; }
 	# DONE: Add some output filtering for this so we can more easily see what went wrong.
@@ -744,27 +727,21 @@ check-doc: build/check-doc.build-stamp
 build/check-doc.build-stamp: doc/build/html/index.html doc/build/html/htmlcov/index.html
 	# Check for a few files to make sure the docs got generated in a way we want.
 	test -s doc/build/html/index.html
-	test -s doc/build/html/generated/mlos_core.optimizers.optimizer.BaseOptimizer.html
-	test -s doc/build/html/generated/mlos_bench.environments.Environment.html
-	test -s doc/build/html/generated/mlos_viz.plot.html
-	test -s doc/build/html/api/mlos_core/mlos_core.html
-	test -s doc/build/html/api/mlos_bench/mlos_bench.html
-	test -s doc/build/html/api/mlos_viz/mlos_viz.html
-	test -s doc/build/html/api/mlos_viz/mlos_viz.dabl.html
-	grep -q -e '--config CONFIG' doc/build/html/api/mlos_bench/mlos_bench.run.html
+	grep -q BaseOptimizer doc/build/html/autoapi/mlos_core/optimizers/optimizer/index.html
+	grep -q Environment doc/build/html/autoapi/mlos_bench/environments/base_environment/index.html
+	grep -q plot doc/build/html/autoapi/mlos_viz/index.html
+	test -s doc/build/html/autoapi/mlos_core/index.html
+	test -s doc/build/html/autoapi/mlos_bench/index.html
+	test -s doc/build/html/autoapi/mlos_viz/index.html
+	test -s doc/build/html/autoapi/mlos_viz/dabl/index.html
+	grep -q -e '--config CONFIG' doc/build/html//mlos_bench.run.usage.html
 	# Check doc logs for errors (but skip over some known ones) ...
 	@cat doc/build/log.txt \
 		| egrep -C1 -e WARNING -e CRITICAL -e ERROR \
 		| egrep -v \
 			-e "warnings.warn\(f'\"{wd.path}\" is shallow and may cause errors'\)" \
 			-e "No such file or directory: '.*.examples'.( \[docutils\]\s*)?$$" \
-			-e 'Problems with "include" directive path:' \
-			-e 'duplicate object description' \
-			-e "document isn't included in any toctree" \
-			-e "more than one target found for cross-reference" \
 			-e "toctree contains reference to nonexisting document 'auto_examples/index'" \
-			-e "failed to import function 'create' from module '(SpaceAdapter|Optimizer)Factory'" \
-			-e "No module named '(SpaceAdapter|Optimizer)Factory'" \
 			-e '^make.*resetting jobserver mode' \
 			-e 'from cryptography.hazmat.primitives.ciphers.algorithms import' \
 		| grep -v '^\s*$$' \
@@ -798,7 +775,7 @@ build/linklint-doc.build-stamp: doc/build/html/index.html doc/build/html/htmlcov
 
 .PHONY: clean-doc
 clean-doc:
-	rm -rf doc/build/ doc/global/ doc/source/api/ doc/source/generated
+	rm -rf doc/build/ doc/global/ doc/source/api/ doc/source/generated doc/source/autoapi
 	rm -rf doc/source/source_tree_docs/*
 
 .PHONY: clean-format

README.md

@@ -188,7 +188,10 @@ Details on using a local version from git are available in [CONTRIBUTING.md](./C
 
     Working example of tuning `sqlite` with MLOS.
 
-These can be used as starting points for new autotuning projects.
+These can be used as starting points for new autotuning projects outside of the main MLOS repository if you want to keep your tuning experiment configs separate from the MLOS codebase.
+
+Alternatively, we accept PRs to add new examples to the main MLOS repository!
+See [mlos_bench/config](./mlos_bench/mlos_bench/config/) and [CONTRIBUTING.md](./CONTRIBUTING.md) for more details.
 
 ### Publications
 

doc/README.md

@@ -2,14 +2,99 @@
 
 Documentation is generated using [`sphinx`](https://www.sphinx-doc.org/).
 
+The configuration for this is in [`doc/source/conf.py`](./source/conf.py).
+
+We use the [`autoapi`](https://sphinx-autoapi.readthedocs.io/en/latest/) extension to generate documentation automatically from the docstrings in our python code.
+
+Additionally, we also use the [`copy-source-tree-docs.sh`](./copy-source-tree-docs.sh) script to copy a few Markdown files from the root of the repository to the `doc/source` build directory automatically to include them in the documentation.
+
+Those are included in the [`index.rst`](./source/index.rst) file which is the main entry point for the documentation and about the only manually maintained rst file.
+
+## Writing Documentation
+
+When writing docstrings, use the [`numpydoc`](https://numpydoc.readthedocs.io/en/latest/format.html) style.
+
+Where necessary embedded [reStructuredText (rst)](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html) markup can be used to help format the documentation.
+
+Each top level module should include a docstring that describes the module and its purpose and usage.
+
+These string should be written for consumption by both users and developers.
+
+Other function and method docstrings that aren't typically intended for users can be written for developers.
+
+### Cross Referencing
+
+You can include links between the documentation using [cross-referencing](https://www.sphinx-doc.org/en/master/usage/domains/python.html#python-xref-roles) links in the docstring.
+
+For instance:
+
+```python
+"""
+My docstring that references another module :py:mod:`fully.qualified.module.name`.
+
+Or else, a class :py:class:`fully.qualified.module.name.ClassName`.
+
+Or else, a class name :py:class:`.ClassName` that is in the same module.
+
+Or else, a class method :py:meth:`~.ClassName.method` but without the leading class name.
+"""
+```
+
+These links will be automatically resolved by `sphinx` and checked using the `nitpick` option to ensure we have well-formed links in the documentation.
+
+### Example Code
+
+Ideally, each main class should also inclue example code that demonstrates how to use the class.
+
+This code should be included in the docstring and should be runnable via [`doctest`](https://docs.python.org/3/library/doctest.html).
+
+For instance:
+
+```python
+class MyClass:
+    """
+    My class that does something.
+
+    Examples
+    --------
+    >>> from my_module import MyClass
+    >>> my_class = MyClass()
+    >>> my_class.do_something()
+    Expected output
+
+    """
+    ...
+```
+
+This code will be automatically checked with `pytest` using the `--doctest-modules` option specified in [`setup.cfg`](../setup.cfg).
+
+## Building the documentation
+
 ```sh
-make -C .. doc
+# From the root of the repository
+make SKIP_COVERAGE=true doc
 ```
 
-## Testing with Docker
+This will also run some checks on the documentation.
+
+> When running this command in a tight loop, it may be useful to run with `SKIP_COVERAGE=true` to avoid re-running the test and coverage checks each time a python file changes.
+
+## Testing
+
+### Manually with Docker
 
 ```sh
 ./nginx-docker.sh restart
 ```
 
 > Now browse to `http://localhost:8080`
+
+## Troubleshooting
+
+We use the [`intersphinx`](https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html) extension to link between external modules and the [`nitpick`](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-nitpicky) option to ensure that all references resolve correctly.
+
+Unfortunately, this process is not perfect and sometimes we need to provide [`nitpick_ignore`](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-nitpick_ignore)s in the [`doc/source/conf.py`](./source/conf.py) file.
+
+In particular, currently external `TypeVar` and `TypeAliases` are not resolved correctly and we need to ignore those.
+
+In other cases, specifying the full path to the module in the cross-reference or the `import` can help.

doc/copy-source-tree-docs.sh

@@ -24,6 +24,7 @@ for readme_file_path in README.md mlos_core/README.md mlos_bench/README.md mlos_
     cp "$readme_file_path" "doc/source/source_tree_docs/$file_dir/index.md"
 
     # Tweak source source code links.
+    # FIXME: This sed expression doesn't work in MacOS.
     sed -i -r -e "s|\]\(([^:#)]+)(#[a-zA-Z0-9_-]+)?\)|\]\(https://github.com/microsoft/MLOS/tree/main/$file_dir/\1\2\)|g" \
         "doc/source/source_tree_docs/$file_dir/index.md"
     # Tweak the lexers for local expansion by pygments instead of github's.

doc/requirements.txt

@@ -1,10 +1,11 @@
 setuptools-scm>=8.1.0
 sphinx
+sphinx-autoapi
+intersphinx_registry>=0.2410.14 # https://github.com/Quansight-Labs/intersphinx_registry/pull/41
 nbsphinx
 jupyter_core>=4.11.2    # nbsphix dependency - addresses CVE-2022-39286
 nbconvert
 mistune>=2.0.3          # Address CVE-2022-34749
-numpydoc
 sphinx-rtd-theme
 myst-parser
 

doc/source/.gitignore

@@ -1,4 +1,5 @@
 api/
+autoapi/
 generated/
 badges/
 source_tree_docs/