From e8401b68b01ac9a789f176489ea2fb4bce6a00d5 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 8 Oct 2024 22:28:48 +0000 Subject: [PATCH 001/100] black, pylint --- doc/source/conf.py | 49 ++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index a06436ba1f1..9b598884de9 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -2,6 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # +"""Sphinx configuration for MLOS documentation.""" +# pylint: disable=invalid-name # Configuration file for the Sphinx documentation builder. # @@ -21,30 +23,31 @@ from logging import warning -import sphinx_rtd_theme +import sphinx_rtd_theme # pylint: disable=unused-import -sys.path.insert(0, os.path.abspath('../../mlos_core/mlos_core')) -sys.path.insert(1, os.path.abspath('../../mlos_bench/mlos_bench')) -sys.path.insert(1, os.path.abspath('../../mlos_viz/mlos_viz')) +sys.path.insert(0, os.path.abspath("../../mlos_core/mlos_core")) +sys.path.insert(1, os.path.abspath("../../mlos_bench/mlos_bench")) +sys.path.insert(1, os.path.abspath("../../mlos_viz/mlos_viz")) # -- Project information ----------------------------------------------------- -project = 'MLOS' -copyright = '2024, Microsoft GSL' -author = 'Microsoft GSL' +project = "MLOS" +copyright = "2024, Microsoft GSL" # pylint: disable=redefined-builtin +author = "Microsoft GSL" # The full version, including alpha/beta/rc tags try: from version import VERSION except ImportError: - VERSION = '0.0.1-dev' + VERSION = "0.0.1-dev" warning(f"version.py not found, using dummy VERSION={VERSION}") try: from setuptools_scm import get_version - version = get_version(root='../..', relative_to=__file__, fallback_version=VERSION) + + version = get_version(root="../..", relative_to=__file__, fallback_version=VERSION) if version is not None: VERSION = version except ImportError: @@ -60,25 +63,25 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'nbsphinx', - 'sphinx.ext.autodoc', - 'sphinx.ext.autosummary', - 'sphinx.ext.doctest', + "nbsphinx", + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.doctest", # 'sphinx.ext.intersphinx', # 'sphinx.ext.linkcode', - 'numpydoc', - 'matplotlib.sphinxext.plot_directive', - 'myst_parser', + "numpydoc", + "matplotlib.sphinxext.plot_directive", + "myst_parser", ] source_suffix = { - '.rst': 'restructuredtext', + ".rst": "restructuredtext", # '.txt': 'markdown', - '.md': 'markdown', + ".md": "markdown", } # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # generate autosummary even if no references autosummary_generate = True @@ -100,7 +103,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', '_templates'] +exclude_patterns = ["_build", "_templates"] # -- Options for HTML output ------------------------------------------------- @@ -114,10 +117,10 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # -- nbsphinx options for rendering notebooks ------------------------------- # nbsphinx_execute = 'never' # enable to stop nbsphinx from executing notebooks -nbsphinx_kernel_name = 'python3' +nbsphinx_kernel_name = "python3" # Exclude build directory and Jupyter backup files: -exclude_patterns = ['_build', '**.ipynb_checkpoints'] +exclude_patterns = ["_build", "**.ipynb_checkpoints"] From 7a24ab148970c75e757bf40ae914203892699329 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 8 Oct 2024 22:28:59 +0000 Subject: [PATCH 002/100] enforce warnings --- doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Makefile b/doc/Makefile index c7690147ce2..1a44c456c5f 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -3,7 +3,7 @@ # You can set these variables from the command line, and also # from the environment for the first two. -SPHINXOPTS ?= +SPHINXOPTS ?= -W SPHINXBUILD ?= sphinx-build SOURCEDIR = source BUILDDIR = build From 5d4a460e358b214cc76a273d20a7e7f3ffae9deb Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 8 Oct 2024 22:29:10 +0000 Subject: [PATCH 003/100] address warning --- doc/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 9b598884de9..42459f4de92 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -111,8 +111,8 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +html_theme = "sphinx_rtd_theme" +# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, From cd70c975573aabf150ac3ebb94854bd3ea984704 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 8 Oct 2024 22:32:13 +0000 Subject: [PATCH 004/100] don't mess with vscode --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 8eb06a70f83..7cd94d1ef71 100644 --- a/Makefile +++ b/Makefile @@ -675,8 +675,6 @@ doc/source/api/mlos_bench/modules.rst: $(MLOS_BENCH_PYTHON_FILES) ../mlos_bench/ \ ../mlos_bench/setup.py ../mlos_bench/mlos_bench/tests/ # Save the help output of the mlos_bench scripts to include in the documentation. - # First make sure that the latest version of mlos_bench is installed (since it uses git based tagging). - conda run -n ${CONDA_ENV_NAME} pip install -e mlos_core -e mlos_bench -e mlos_viz conda run -n ${CONDA_ENV_NAME} mlos_bench --help > doc/source/api/mlos_bench/mlos_bench.run.usage.txt echo ".. literalinclude:: mlos_bench.run.usage.txt" >> doc/source/api/mlos_bench/mlos_bench.run.rst echo " :language: none" >> doc/source/api/mlos_bench/mlos_bench.run.rst From 6dd5aff17fb9b11932d6d7936342e924c549af66 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 9 Oct 2024 20:14:15 +0000 Subject: [PATCH 005/100] confs --- doc/source/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/conf.py b/doc/source/conf.py index 42459f4de92..9d4461f7086 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -91,6 +91,7 @@ autodoc_default_options = { "members": True, + "inherited-members": True, "undoc-members": True, # Don't generate documentation for some (non-private) functions that are more # for internal implementation use. From 4769ee6f2d5071ec4a3ae334dec388efc95cfa79 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 9 Oct 2024 20:14:30 +0000 Subject: [PATCH 006/100] remove some uses of autosummary for now --- doc/source/index.rst | 12 -- doc/source/overview.rst | 298 ---------------------------------------- 2 files changed, 310 deletions(-) delete mode 100644 doc/source/overview.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index e0302b9f78a..9019114c0a6 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -19,12 +19,6 @@ See below for additional documentation sections. source_tree_docs/mlos_bench/index source_tree_docs/mlos_viz/index -.. toctree:: - :maxdepth: 2 - :caption: API Overview - - overview - .. toctree:: :maxdepth: 3 :caption: API Reference @@ -33,12 +27,6 @@ See below for additional documentation sections. api/mlos_bench/modules api/mlos_viz/modules -.. toctree:: - :maxdepth: 2 - :caption: Examples - - auto_examples/index - .. toctree:: :maxdepth: 1 :caption: References diff --git a/doc/source/overview.rst b/doc/source/overview.rst deleted file mode 100644 index b3ca2a3fad3..00000000000 --- a/doc/source/overview.rst +++ /dev/null @@ -1,298 +0,0 @@ -########################## -MLOS Package APIs Overview -########################## - -This is a list of major functions and classes provided by the MLOS packages. - -############################# -mlos-core API -############################# - -This is a list of major functions and classes provided by `mlos_core`. - -.. currentmodule:: mlos_core - -Optimizers -========== -.. currentmodule:: mlos_core.optimizers -.. autosummary:: - :toctree: generated/ - - :template: class.rst - - OptimizerType - OptimizerFactory - - :template: function.rst - - OptimizerFactory.create - -.. currentmodule:: mlos_core.optimizers.optimizer -.. autosummary:: - :toctree: generated/ - :template: class.rst - - BaseOptimizer - -.. currentmodule:: mlos_core.optimizers.random_optimizer -.. autosummary:: - :toctree: generated/ - :template: class.rst - - RandomOptimizer - -.. currentmodule:: mlos_core.optimizers.flaml_optimizer -.. autosummary:: - :toctree: generated/ - :template: class.rst - - FlamlOptimizer - -.. currentmodule:: mlos_core.optimizers.bayesian_optimizers -.. autosummary:: - :toctree: generated/ - :template: class.rst - - BaseBayesianOptimizer - SmacOptimizer - -Spaces -====== - -Converters ----------- -.. currentmodule:: mlos_core.spaces.converters.flaml -.. autosummary:: - :toctree: generated/ - :template: function.rst - - configspace_to_flaml_space - -Space Adapters --------------- -.. currentmodule:: mlos_core.spaces.adapters -.. autosummary:: - :toctree: generated/ - - :template: class.rst - - SpaceAdapterType - SpaceAdapterFactory - - :template: function.rst - - SpaceAdapterFactory.create - -.. currentmodule:: mlos_core.spaces.adapters.adapter -.. autosummary:: - :toctree: generated/ - :template: class.rst - - BaseSpaceAdapter - -.. currentmodule:: mlos_core.spaces.adapters.identity_adapter -.. autosummary:: - :toctree: generated/ - :template: class.rst - - IdentityAdapter - -.. currentmodule:: mlos_core.spaces.adapters.llamatune -.. autosummary:: - :toctree: generated/ - :template: class.rst - - LlamaTuneAdapter - -############################# -mlos-bench API -############################# - -This is a list of major functions and classes provided by `mlos_bench`. - -.. currentmodule:: mlos_bench - -Main -==== - -:doc:`run.py ` - - The script to run the benchmarks or the optimization loop. - - Also available as `mlos_bench` command line tool. - -.. note:: - The are `json config examples `_ and `json schemas `_ on the main `source code `_ repository site. - -Benchmark Environments -====================== -.. currentmodule:: mlos_bench.environments -.. autosummary:: - :toctree: generated/ - :template: class.rst - - Status - Environment - CompositeEnv - MockEnv - -Local Environments -------------------- - -.. currentmodule:: mlos_bench.environments.local -.. autosummary:: - :toctree: generated/ - :template: class.rst - - LocalEnv - LocalFileShareEnv - -Remote Environments -------------------- - -.. currentmodule:: mlos_bench.environments.remote -.. autosummary:: - :toctree: generated/ - :template: class.rst - - RemoteEnv - OSEnv - VMEnv - HostEnv - -Tunable Parameters -================== -.. currentmodule:: mlos_bench.tunables -.. autosummary:: - :toctree: generated/ - :template: class.rst - - Tunable - TunableGroups - -Service Mix-ins -=============== -.. currentmodule:: mlos_bench.services -.. autosummary:: - :toctree: generated/ - :template: class.rst - - Service - FileShareService - -.. currentmodule:: mlos_bench.services.config_persistence -.. autosummary:: - :toctree: generated/ - :template: class.rst - - ConfigPersistenceService - -Local Services ---------------- -.. currentmodule:: mlos_bench.services.local -.. autosummary:: - :toctree: generated/ - :template: class.rst - - LocalExecService - -Remote Azure Services ---------------------- - -.. currentmodule:: mlos_bench.services.remote.azure -.. autosummary:: - :toctree: generated/ - :template: class.rst - - AzureVMService - AzureFileShareService - -Optimizer Adapters -================== -.. currentmodule:: mlos_bench.optimizers -.. autosummary:: - :toctree: generated/ - :template: class.rst - - Optimizer - MockOptimizer - MlosCoreOptimizer - -Storage -======= -Base Runtime Backends ---------------------- -.. currentmodule:: mlos_bench.storage -.. autosummary:: - :toctree: generated/ - :template: class.rst - - Storage - -.. currentmodule:: mlos_bench.storage.storage_factory -.. autosummary:: - :toctree: generated/ - :template: function.rst - - from_config - -SQL DB Storage Backend ----------------------- -.. currentmodule:: mlos_bench.storage.sql.storage -.. autosummary:: - :toctree: generated/ - :template: class.rst - - SqlStorage - -Analysis Client Access APIs ---------------------------- -.. currentmodule:: mlos_bench.storage.base_experiment_data -.. autosummary:: - :toctree: generated/ - :template: class.rst - - ExperimentData - -.. currentmodule:: mlos_bench.storage.base_trial_data -.. autosummary:: - :toctree: generated/ - :template: class.rst - - TrialData - -.. currentmodule:: mlos_bench.storage.base_tunable_config_data -.. autosummary:: - :toctree: generated/ - :template: class.rst - - TunableConfigData - -.. currentmodule:: mlos_bench.storage.base_tunable_config_trial_group_data -.. autosummary:: - :toctree: generated/ - :template: class.rst - - TunableConfigTrialGroupData - -############################# -mlos-viz API -############################# - -This is a list of major functions and classes provided by `mlos_viz`. - -.. currentmodule:: mlos_viz - -.. currentmodule:: mlos_viz -.. autosummary:: - :toctree: generated/ - :template: class.rst - - MlosVizMethod - -.. currentmodule:: mlos_viz -.. autosummary:: - :toctree: generated/ - :template: function.rst - - plot From a5cb3035b2ae7d072492904f40ed1e4297a15422 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 9 Oct 2024 21:40:28 +0000 Subject: [PATCH 007/100] Add args to sphinx-build by default --- Makefile | 8 ++++++-- doc/Makefile | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 7cd94d1ef71..b0e35ef72cf 100644 --- a/Makefile +++ b/Makefile @@ -699,7 +699,11 @@ doc/build/html/index.html: build/pytest.${CONDA_ENV_NAME}.build-stamp doc/build/html/htmlcov/index.html: build/pytest.${CONDA_ENV_NAME}.build-stamp endif -doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile doc/copy-source-tree-docs.sh $(MD_FILES) +# Treat warnings as failures. +SPHINXOPTS ?= -v +SPHINXOPTS += -W -w $(PWD)/doc/build/sphinx-build.warn.log -j auto + +doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile doc/source/conf.py doc/copy-source-tree-docs.sh $(MD_FILES) @rm -rf doc/build @mkdir -p doc/build @rm -f doc/build/log.txt @@ -713,7 +717,7 @@ doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile doc/copy-source- ./doc/copy-source-tree-docs.sh # Build the rst files into html. - conda run -n ${CONDA_ENV_NAME} $(MAKE) -C doc/ $(MAKEFLAGS) html \ + conda run -n ${CONDA_ENV_NAME} $(MAKE) SPHINXOPTS="$(SPHINXOPTS)" -C doc/ $(MAKEFLAGS) html \ >> doc/build/log.txt 2>&1 \ || { cat doc/build/log.txt; exit 1; } # DONE: Add some output filtering for this so we can more easily see what went wrong. diff --git a/doc/Makefile b/doc/Makefile index 1a44c456c5f..c7690147ce2 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -3,7 +3,7 @@ # You can set these variables from the command line, and also # from the environment for the first two. -SPHINXOPTS ?= -W +SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source BUILDDIR = build From 555d0f6045d86911cb2a107414f5f1cf8d0ada96 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 9 Oct 2024 21:40:57 +0000 Subject: [PATCH 008/100] Switch to autoapi --- Makefile | 2 +- doc/requirements.txt | 1 + doc/source/.gitignore | 1 + doc/source/conf.py | 50 ++++++++++++++++++++++++++++--------------- doc/source/index.rst | 8 +++---- 5 files changed, 40 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index b0e35ef72cf..03dbb28a924 100644 --- a/Makefile +++ b/Makefile @@ -800,7 +800,7 @@ build/linklint-doc.build-stamp: doc/build/html/index.html doc/build/html/htmlcov .PHONY: clean-doc clean-doc: - rm -rf doc/build/ doc/global/ doc/source/api/ doc/source/generated + rm -rf doc/build/ doc/global/ doc/source/api/ doc/source/generated doc/source/autoapi rm -rf doc/source/source_tree_docs/* .PHONY: clean-format diff --git a/doc/requirements.txt b/doc/requirements.txt index 9825b5b442a..4505ee58bfc 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,5 +1,6 @@ setuptools-scm>=8.1.0 sphinx +sphinx-autoapi nbsphinx jupyter_core>=4.11.2 # nbsphix dependency - addresses CVE-2022-39286 nbconvert diff --git a/doc/source/.gitignore b/doc/source/.gitignore index bf83a1d89cf..8173208e1d5 100644 --- a/doc/source/.gitignore +++ b/doc/source/.gitignore @@ -1,4 +1,5 @@ api/ +autoapi/ generated/ badges/ source_tree_docs/ diff --git a/doc/source/conf.py b/doc/source/conf.py index 9d4461f7086..f0be7f6637b 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -63,9 +63,8 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + "autoapi.extension", "nbsphinx", - "sphinx.ext.autodoc", - "sphinx.ext.autosummary", "sphinx.ext.doctest", # 'sphinx.ext.intersphinx', # 'sphinx.ext.linkcode', @@ -83,21 +82,6 @@ # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] -# generate autosummary even if no references -autosummary_generate = True -# but don't complain about missing stub files -# See Also: -numpydoc_class_members_toctree = False - -autodoc_default_options = { - "members": True, - "inherited-members": True, - "undoc-members": True, - # Don't generate documentation for some (non-private) functions that are more - # for internal implementation use. - "exclude-members": "mlos_bench.util.check_required_params", -} - # Generate the plots for the gallery # plot_gallery = True @@ -106,6 +90,38 @@ # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ["_build", "_templates"] +autoapi_dirs = [ + # Don't index setup.py or other utility scripts. + "../../mlos_core/mlos_core/", + "../../mlos_bench/mlos_bench/", + "../../mlos_viz/mlos_viz/", +] +autoapi_ignore = [ + "*/tests/*", + # Don't document internal environment scripts that aren't part of a module. + "*/mlos_bench/config/environments/*/*.py", + "*/mlos_bench/config/services/*/*.py", +] +autoapi_options = [ + "members", + # Can't document externally inherited members due to broken references. + # "inherited-members", + "undoc-members", + # Don't document private members. + # "private-members", + "show-inheritance", + # Causes issues when base class is a typing protocol. + # "show-inheritance-diagram", + "show-module-summary", + "special-members", + # Causes duplicate reference issues. For instance: + # - mlos_bench.environments.LocalEnv + # - mlos_bench.environments.local.LocalEnv + # - mlos_bench.environments.local.local_env.LocalEnv + # "imported-members", +] +autoapi_add_toctree_entry = False +# autoapi_keep_files = True # for testing # -- Options for HTML output ------------------------------------------------- diff --git a/doc/source/index.rst b/doc/source/index.rst index 9019114c0a6..f04f4244b74 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -20,12 +20,12 @@ See below for additional documentation sections. source_tree_docs/mlos_viz/index .. toctree:: - :maxdepth: 3 :caption: API Reference + :maxdepth: 2 - api/mlos_core/modules - api/mlos_bench/modules - api/mlos_viz/modules + autoapi/mlos_core/index + autoapi/mlos_bench/index + autoapi/mlos_viz/index .. toctree:: :maxdepth: 1 From f01fd6dee54f6a542917dceef2ecd3ad5d54d8d5 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 9 Oct 2024 21:43:18 +0000 Subject: [PATCH 009/100] remove use of sphinx-apidoc --- Makefile | 37 ++----------------------------------- 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/Makefile b/Makefile index 03dbb28a924..b790a6783bc 100644 --- a/Makefile +++ b/Makefile @@ -659,40 +659,7 @@ clean-doc-env: COMMON_DOC_FILES := build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp doc/source/*.rst doc/source/_templates/*.rst doc/source/conf.py -doc/source/api/mlos_core/modules.rst: $(FORMAT_PREREQS) $(COMMON_DOC_FILES) -doc/source/api/mlos_core/modules.rst: $(MLOS_CORE_PYTHON_FILES) - rm -rf doc/source/api/mlos_core - cd doc/ && conda run -n ${CONDA_ENV_NAME} sphinx-apidoc -f -e -M \ - -o source/api/mlos_core/ \ - ../mlos_core/ \ - ../mlos_core/setup.py ../mlos_core/mlos_core/tests/ - -doc/source/api/mlos_bench/modules.rst: $(FORMAT_PREREQS) $(COMMON_DOC_FILES) -doc/source/api/mlos_bench/modules.rst: $(MLOS_BENCH_PYTHON_FILES) - rm -rf doc/source/api/mlos_bench - cd doc/ && conda run -n ${CONDA_ENV_NAME} sphinx-apidoc -f -e -M \ - -o source/api/mlos_bench/ \ - ../mlos_bench/ \ - ../mlos_bench/setup.py ../mlos_bench/mlos_bench/tests/ - # Save the help output of the mlos_bench scripts to include in the documentation. - conda run -n ${CONDA_ENV_NAME} mlos_bench --help > doc/source/api/mlos_bench/mlos_bench.run.usage.txt - echo ".. literalinclude:: mlos_bench.run.usage.txt" >> doc/source/api/mlos_bench/mlos_bench.run.rst - echo " :language: none" >> doc/source/api/mlos_bench/mlos_bench.run.rst - -doc/source/api/mlos_viz/modules.rst: $(FORMAT_PREREQS) $(COMMON_DOC_FILES) -doc/source/api/mlos_viz/modules.rst: $(MLOS_VIZ_PYTHON_FILES) - rm -rf doc/source/api/mlos_viz - cd doc/ && conda run -n ${CONDA_ENV_NAME} sphinx-apidoc -f -e -M \ - -o source/api/mlos_viz/ \ - ../mlos_viz/ \ - ../mlos_viz/setup.py ../mlos_viz/mlos_viz/tests/ - -SPHINX_API_RST_FILES := doc/source/api/mlos_core/modules.rst -SPHINX_API_RST_FILES += doc/source/api/mlos_bench/modules.rst -SPHINX_API_RST_FILES += doc/source/api/mlos_viz/modules.rst - -.PHONY: sphinx-apidoc -sphinx-apidoc: $(SPHINX_API_RST_FILES) +SPHINX_API_RST_FILES := doc/source/index.rst ifeq ($(SKIP_COVERAGE),) doc/build/html/index.html: build/pytest.${CONDA_ENV_NAME}.build-stamp @@ -700,7 +667,7 @@ doc/build/html/htmlcov/index.html: build/pytest.${CONDA_ENV_NAME}.build-stamp endif # Treat warnings as failures. -SPHINXOPTS ?= -v +SPHINXOPTS ?= # -v # be verbose SPHINXOPTS += -W -w $(PWD)/doc/build/sphinx-build.warn.log -j auto doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile doc/source/conf.py doc/copy-source-tree-docs.sh $(MD_FILES) From 38174a7aba51868cff8b349f548c5a348055cafd Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 9 Oct 2024 22:24:44 +0000 Subject: [PATCH 010/100] fixup checks --- Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index b790a6783bc..cbdbc789d88 100644 --- a/Makefile +++ b/Makefile @@ -713,13 +713,13 @@ check-doc: build/check-doc.build-stamp build/check-doc.build-stamp: doc/build/html/index.html doc/build/html/htmlcov/index.html # Check for a few files to make sure the docs got generated in a way we want. test -s doc/build/html/index.html - test -s doc/build/html/generated/mlos_core.optimizers.optimizer.BaseOptimizer.html - test -s doc/build/html/generated/mlos_bench.environments.Environment.html - test -s doc/build/html/generated/mlos_viz.plot.html - test -s doc/build/html/api/mlos_core/mlos_core.html - test -s doc/build/html/api/mlos_bench/mlos_bench.html - test -s doc/build/html/api/mlos_viz/mlos_viz.html - test -s doc/build/html/api/mlos_viz/mlos_viz.dabl.html + grep -q BaseOptimizer doc/build/html/autoapi/mlos_core/optimizers/optimizer/index.html + grep -q Environment doc/build/html/autoapi/mlos_bench/environments/base_environment/index.html + grep -q plot doc/build/html/autoapi/mlos_viz/index.html + test -s doc/build/html/autoapi/mlos_core/index.html + test -s doc/build/html/autoapi/mlos_bench/index.html + test -s doc/build/html/autoapi/mlos_viz/index.html + test -s doc/build/html/autoapi/mlos_viz/dabl/index.html grep -q -e '--config CONFIG' doc/build/html/api/mlos_bench/mlos_bench.run.html # Check doc logs for errors (but skip over some known ones) ... @cat doc/build/log.txt \ From e4cd2a68051fe4f6b1df5433d3206a6340de54b9 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 9 Oct 2024 22:25:34 +0000 Subject: [PATCH 011/100] make run include the usage text dynamically --- mlos_bench/mlos_bench/launcher.py | 291 ++++++++++++++++-------------- mlos_bench/mlos_bench/run.py | 8 +- 2 files changed, 164 insertions(+), 135 deletions(-) diff --git a/mlos_bench/mlos_bench/launcher.py b/mlos_bench/mlos_bench/launcher.py index 339a11963de..0d9968fce35 100644 --- a/mlos_bench/mlos_bench/launcher.py +++ b/mlos_bench/mlos_bench/launcher.py @@ -42,10 +42,8 @@ class Launcher: # pylint: disable=too-few-public-methods,too-many-instance-attributes """Command line launcher for mlos_bench and mlos_core.""" - def __init__(self, description: str, long_text: str = "", argv: Optional[List[str]] = None): - # pylint: disable=too-many-statements - # pylint: disable=too-many-locals - _LOG.info("Launch: %s", description) + @staticmethod + def _get_parser(description: str, long_text: str) -> Tuple[argparse.ArgumentParser, List[str]]: epilog = """ Additional --key=value pairs can be specified to augment or override values listed in --globals. @@ -56,135 +54,8 @@ def __init__(self, description: str, long_text: str = "", argv: Optional[List[st the source tree: """ - parser = argparse.ArgumentParser(description=f"{description} : {long_text}", epilog=epilog) - (args, path_args, args_rest) = self._parse_args(parser, argv) - - # Bootstrap config loader: command line takes priority. - config_path = args.config_path or [] - self._config_loader = ConfigPersistenceService({"config_path": config_path}) - if args.config: - config = self._config_loader.load_config(args.config, ConfigSchema.CLI) - assert isinstance(config, Dict) - # Merge the args paths for the config loader with the paths from JSON file. - config_path += config.get("config_path", []) - self._config_loader = ConfigPersistenceService({"config_path": config_path}) - else: - config = {} - log_level = args.log_level or config.get("log_level", _LOG_LEVEL) - try: - log_level = int(log_level) - except ValueError: - # failed to parse as an int - leave it as a string and let logging - # module handle whether it's an appropriate log name or not - log_level = logging.getLevelName(log_level) - logging.root.setLevel(log_level) - log_file = args.log_file or config.get("log_file") - if log_file: - log_handler = logging.FileHandler(log_file) - log_handler.setLevel(log_level) - log_handler.setFormatter(logging.Formatter(_LOG_FORMAT)) - logging.root.addHandler(log_handler) - - self._parent_service: Service = LocalExecService(parent=self._config_loader) - - # Prepare global_config from a combination of global config files, cli - # configs, and cli args. - args_dict = vars(args) - # teardown (bool) conflicts with Environment configs that use it for shell - # commands (list), so we exclude it from copying over - excluded_cli_args = path_args + ["teardown"] - # Include (almost) any item from the cli config file that either isn't in - # the cli args at all or whose cli arg is missing. - cli_config_args = { - key: val - for (key, val) in config.items() - if (args_dict.get(key) is None) and key not in excluded_cli_args - } - - self.global_config = self._load_config( - args_globals=config.get("globals", []) + (args.globals or []), - config_path=(args.config_path or []) + config.get("config_path", []), - args_rest=args_rest, - global_config=cli_config_args, - ) - # experiment_id is generally taken from --globals files, but we also allow - # overriding it on the CLI. - # It's useful to keep it there explicitly mostly for the --help output. - if args.experiment_id: - self.global_config["experiment_id"] = args.experiment_id - # trial_config_repeat_count is a scheduler property but it's convenient to - # set it via command line - if args.trial_config_repeat_count: - self.global_config["trial_config_repeat_count"] = args.trial_config_repeat_count - # Ensure that the trial_id is present since it gets used by some other - # configs but is typically controlled by the run optimize loop. - self.global_config.setdefault("trial_id", 1) - - self.global_config = DictTemplater(self.global_config).expand_vars(use_os_env=True) - assert isinstance(self.global_config, dict) - - # --service cli args should override the config file values. - service_files: List[str] = config.get("services", []) + (args.service or []) - assert isinstance(self._parent_service, SupportsConfigLoading) - self._parent_service = self._parent_service.load_services( - service_files, - self.global_config, - self._parent_service, - ) - - env_path = args.environment or config.get("environment") - if not env_path: - _LOG.error("No environment config specified.") - parser.error( - "At least the Environment config must be specified." - + " Run `mlos_bench --help` and consult `README.md` for more info." - ) - self.root_env_config = self._config_loader.resolve_path(env_path) - - self.environment: Environment = self._config_loader.load_environment( - self.root_env_config, TunableGroups(), self.global_config, service=self._parent_service - ) - _LOG.info("Init environment: %s", self.environment) - - # NOTE: Init tunable values *after* the Environment, but *before* the Optimizer - self.tunables = self._init_tunable_values( - args.random_init or config.get("random_init", False), - config.get("random_seed") if args.random_seed is None else args.random_seed, - config.get("tunable_values", []) + (args.tunable_values or []), - ) - _LOG.info("Init tunables: %s", self.tunables) - - self.optimizer = self._load_optimizer(args.optimizer or config.get("optimizer")) - _LOG.info("Init optimizer: %s", self.optimizer) - - self.storage = self._load_storage(args.storage or config.get("storage")) - _LOG.info("Init storage: %s", self.storage) - - self.teardown: bool = ( - bool(args.teardown) - if args.teardown is not None - else bool(config.get("teardown", True)) - ) - self.scheduler = self._load_scheduler(args.scheduler or config.get("scheduler")) - _LOG.info("Init scheduler: %s", self.scheduler) - - @property - def config_loader(self) -> ConfigPersistenceService: - """Get the config loader service.""" - return self._config_loader - - @property - def service(self) -> Service: - """Get the parent service.""" - return self._parent_service - - @staticmethod - def _parse_args( - parser: argparse.ArgumentParser, - argv: Optional[List[str]], - ) -> Tuple[argparse.Namespace, List[str], List[str]]: - """Parse the command line arguments.""" + parser = argparse.ArgumentParser(description=f"{description} : {long_text}", epilog=epilog) class PathArgsTracker: """Simple class to help track which arguments are paths.""" @@ -364,14 +235,166 @@ def add_argument(self, *args: Any, **kwargs: Any) -> None: "incompatible" is not easily automatable across systems. """, ) + return (parser, path_args_tracker.path_args) + + @staticmethod + def get_help_text(description: str, long_text: str) -> str: + """Gets the help text from the argument parser. + + Parameters + ---------- + description : str + The short name of the script. + long_text : str + The long description of the script. + + Returns + ------- + str + The help text from the argument parser. + """ + (parser, _path_args) = Launcher._get_parser(description, long_text) + return parser.format_help() + + def __init__(self, description: str, long_text: str = "", argv: Optional[List[str]] = None): + # pylint: disable=too-many-statements + # pylint: disable=too-many-locals + _LOG.info("Launch: %s", description) + (parser, path_args) = self._get_parser(description, long_text) + (args, args_rest) = self._parse_args(parser, argv) + + # Bootstrap config loader: command line takes priority. + config_path = args.config_path or [] + self._config_loader = ConfigPersistenceService({"config_path": config_path}) + if args.config: + config = self._config_loader.load_config(args.config, ConfigSchema.CLI) + assert isinstance(config, Dict) + # Merge the args paths for the config loader with the paths from JSON file. + config_path += config.get("config_path", []) + self._config_loader = ConfigPersistenceService({"config_path": config_path}) + else: + config = {} + + log_level = args.log_level or config.get("log_level", _LOG_LEVEL) + try: + log_level = int(log_level) + except ValueError: + # failed to parse as an int - leave it as a string and let logging + # module handle whether it's an appropriate log name or not + log_level = logging.getLevelName(log_level) + logging.root.setLevel(log_level) + log_file = args.log_file or config.get("log_file") + if log_file: + log_handler = logging.FileHandler(log_file) + log_handler.setLevel(log_level) + log_handler.setFormatter(logging.Formatter(_LOG_FORMAT)) + logging.root.addHandler(log_handler) + + self._parent_service: Service = LocalExecService(parent=self._config_loader) + + # Prepare global_config from a combination of global config files, cli + # configs, and cli args. + args_dict = vars(args) + # teardown (bool) conflicts with Environment configs that use it for shell + # commands (list), so we exclude it from copying over + excluded_cli_args = path_args + ["teardown"] + # Include (almost) any item from the cli config file that either isn't in + # the cli args at all or whose cli arg is missing. + cli_config_args = { + key: val + for (key, val) in config.items() + if (args_dict.get(key) is None) and key not in excluded_cli_args + } + + self.global_config = self._load_config( + args_globals=config.get("globals", []) + (args.globals or []), + config_path=(args.config_path or []) + config.get("config_path", []), + args_rest=args_rest, + global_config=cli_config_args, + ) + # experiment_id is generally taken from --globals files, but we also allow + # overriding it on the CLI. + # It's useful to keep it there explicitly mostly for the --help output. + if args.experiment_id: + self.global_config["experiment_id"] = args.experiment_id + # trial_config_repeat_count is a scheduler property but it's convenient to + # set it via command line + if args.trial_config_repeat_count: + self.global_config["trial_config_repeat_count"] = args.trial_config_repeat_count + # Ensure that the trial_id is present since it gets used by some other + # configs but is typically controlled by the run optimize loop. + self.global_config.setdefault("trial_id", 1) + + self.global_config = DictTemplater(self.global_config).expand_vars(use_os_env=True) + assert isinstance(self.global_config, dict) + + # --service cli args should override the config file values. + service_files: List[str] = config.get("services", []) + (args.service or []) + assert isinstance(self._parent_service, SupportsConfigLoading) + self._parent_service = self._parent_service.load_services( + service_files, + self.global_config, + self._parent_service, + ) + + env_path = args.environment or config.get("environment") + if not env_path: + _LOG.error("No environment config specified.") + parser.error( + "At least the Environment config must be specified." + + " Run `mlos_bench --help` and consult `README.md` for more info." + ) + self.root_env_config = self._config_loader.resolve_path(env_path) + + self.environment: Environment = self._config_loader.load_environment( + self.root_env_config, TunableGroups(), self.global_config, service=self._parent_service + ) + _LOG.info("Init environment: %s", self.environment) + + # NOTE: Init tunable values *after* the Environment, but *before* the Optimizer + self.tunables = self._init_tunable_values( + args.random_init or config.get("random_init", False), + config.get("random_seed") if args.random_seed is None else args.random_seed, + config.get("tunable_values", []) + (args.tunable_values or []), + ) + _LOG.info("Init tunables: %s", self.tunables) + + self.optimizer = self._load_optimizer(args.optimizer or config.get("optimizer")) + _LOG.info("Init optimizer: %s", self.optimizer) + + self.storage = self._load_storage(args.storage or config.get("storage")) + _LOG.info("Init storage: %s", self.storage) + + self.teardown: bool = ( + bool(args.teardown) + if args.teardown is not None + else bool(config.get("teardown", True)) + ) + self.scheduler = self._load_scheduler(args.scheduler or config.get("scheduler")) + _LOG.info("Init scheduler: %s", self.scheduler) + @property + def config_loader(self) -> ConfigPersistenceService: + """Get the config loader service.""" + return self._config_loader + + @property + def service(self) -> Service: + """Get the parent service.""" + return self._parent_service + + @staticmethod + def _parse_args( + parser: argparse.ArgumentParser, + argv: Optional[List[str]], + ) -> Tuple[argparse.Namespace, List[str]]: + """Parse the command line arguments.""" # By default we use the command line arguments, but allow the caller to # provide some explicitly for testing purposes. if argv is None: argv = sys.argv[1:].copy() (args, args_rest) = parser.parse_known_args(argv) - - return (args, path_args_tracker.path_args, args_rest) + return (args, args_rest) @staticmethod def _try_parse_extra_args(cmdline: Iterable[str]) -> Dict[str, TunableValue]: diff --git a/mlos_bench/mlos_bench/run.py b/mlos_bench/mlos_bench/run.py index a554f1a803d..62ee5d123aa 100755 --- a/mlos_bench/mlos_bench/run.py +++ b/mlos_bench/mlos_bench/run.py @@ -23,6 +23,12 @@ _LOG = logging.getLogger(__name__) +_NAME = "mlos_bench" +_DESC = "Systems autotuning and benchmarking tool" +# Dynamically add the --help text to our docstring. +__doc__ += "\n" + Launcher.get_help_text(_NAME, _DESC) + + def _sanity_check_results(launcher: Launcher) -> None: """Do some sanity checking on the results and throw an exception if it looks like something went wrong. @@ -57,7 +63,7 @@ def _sanity_check_results(launcher: Launcher) -> None: def _main( argv: Optional[List[str]] = None, ) -> Tuple[Optional[Dict[str, float]], Optional[TunableGroups]]: - launcher = Launcher("mlos_bench", "Systems autotuning and benchmarking tool", argv=argv) + launcher = Launcher(_NAME, _DESC, argv=argv) with launcher.scheduler as scheduler_context: scheduler_context.start() From 00d074dc03a93ee63c459ade86df981e74b8ffd8 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 9 Oct 2024 22:26:57 +0000 Subject: [PATCH 012/100] Revert "make run include the usage text dynamically" This reverts commit e4cd2a68051fe4f6b1df5433d3206a6340de54b9. --- mlos_bench/mlos_bench/launcher.py | 291 ++++++++++++++---------------- mlos_bench/mlos_bench/run.py | 8 +- 2 files changed, 135 insertions(+), 164 deletions(-) diff --git a/mlos_bench/mlos_bench/launcher.py b/mlos_bench/mlos_bench/launcher.py index 0d9968fce35..339a11963de 100644 --- a/mlos_bench/mlos_bench/launcher.py +++ b/mlos_bench/mlos_bench/launcher.py @@ -42,8 +42,10 @@ class Launcher: # pylint: disable=too-few-public-methods,too-many-instance-attributes """Command line launcher for mlos_bench and mlos_core.""" - @staticmethod - def _get_parser(description: str, long_text: str) -> Tuple[argparse.ArgumentParser, List[str]]: + def __init__(self, description: str, long_text: str = "", argv: Optional[List[str]] = None): + # pylint: disable=too-many-statements + # pylint: disable=too-many-locals + _LOG.info("Launch: %s", description) epilog = """ Additional --key=value pairs can be specified to augment or override values listed in --globals. @@ -54,8 +56,135 @@ def _get_parser(description: str, long_text: str) -> Tuple[argparse.ArgumentPars the source tree: """ - parser = argparse.ArgumentParser(description=f"{description} : {long_text}", epilog=epilog) + (args, path_args, args_rest) = self._parse_args(parser, argv) + + # Bootstrap config loader: command line takes priority. + config_path = args.config_path or [] + self._config_loader = ConfigPersistenceService({"config_path": config_path}) + if args.config: + config = self._config_loader.load_config(args.config, ConfigSchema.CLI) + assert isinstance(config, Dict) + # Merge the args paths for the config loader with the paths from JSON file. + config_path += config.get("config_path", []) + self._config_loader = ConfigPersistenceService({"config_path": config_path}) + else: + config = {} + + log_level = args.log_level or config.get("log_level", _LOG_LEVEL) + try: + log_level = int(log_level) + except ValueError: + # failed to parse as an int - leave it as a string and let logging + # module handle whether it's an appropriate log name or not + log_level = logging.getLevelName(log_level) + logging.root.setLevel(log_level) + log_file = args.log_file or config.get("log_file") + if log_file: + log_handler = logging.FileHandler(log_file) + log_handler.setLevel(log_level) + log_handler.setFormatter(logging.Formatter(_LOG_FORMAT)) + logging.root.addHandler(log_handler) + + self._parent_service: Service = LocalExecService(parent=self._config_loader) + + # Prepare global_config from a combination of global config files, cli + # configs, and cli args. + args_dict = vars(args) + # teardown (bool) conflicts with Environment configs that use it for shell + # commands (list), so we exclude it from copying over + excluded_cli_args = path_args + ["teardown"] + # Include (almost) any item from the cli config file that either isn't in + # the cli args at all or whose cli arg is missing. + cli_config_args = { + key: val + for (key, val) in config.items() + if (args_dict.get(key) is None) and key not in excluded_cli_args + } + + self.global_config = self._load_config( + args_globals=config.get("globals", []) + (args.globals or []), + config_path=(args.config_path or []) + config.get("config_path", []), + args_rest=args_rest, + global_config=cli_config_args, + ) + # experiment_id is generally taken from --globals files, but we also allow + # overriding it on the CLI. + # It's useful to keep it there explicitly mostly for the --help output. + if args.experiment_id: + self.global_config["experiment_id"] = args.experiment_id + # trial_config_repeat_count is a scheduler property but it's convenient to + # set it via command line + if args.trial_config_repeat_count: + self.global_config["trial_config_repeat_count"] = args.trial_config_repeat_count + # Ensure that the trial_id is present since it gets used by some other + # configs but is typically controlled by the run optimize loop. + self.global_config.setdefault("trial_id", 1) + + self.global_config = DictTemplater(self.global_config).expand_vars(use_os_env=True) + assert isinstance(self.global_config, dict) + + # --service cli args should override the config file values. + service_files: List[str] = config.get("services", []) + (args.service or []) + assert isinstance(self._parent_service, SupportsConfigLoading) + self._parent_service = self._parent_service.load_services( + service_files, + self.global_config, + self._parent_service, + ) + + env_path = args.environment or config.get("environment") + if not env_path: + _LOG.error("No environment config specified.") + parser.error( + "At least the Environment config must be specified." + + " Run `mlos_bench --help` and consult `README.md` for more info." + ) + self.root_env_config = self._config_loader.resolve_path(env_path) + + self.environment: Environment = self._config_loader.load_environment( + self.root_env_config, TunableGroups(), self.global_config, service=self._parent_service + ) + _LOG.info("Init environment: %s", self.environment) + + # NOTE: Init tunable values *after* the Environment, but *before* the Optimizer + self.tunables = self._init_tunable_values( + args.random_init or config.get("random_init", False), + config.get("random_seed") if args.random_seed is None else args.random_seed, + config.get("tunable_values", []) + (args.tunable_values or []), + ) + _LOG.info("Init tunables: %s", self.tunables) + + self.optimizer = self._load_optimizer(args.optimizer or config.get("optimizer")) + _LOG.info("Init optimizer: %s", self.optimizer) + + self.storage = self._load_storage(args.storage or config.get("storage")) + _LOG.info("Init storage: %s", self.storage) + + self.teardown: bool = ( + bool(args.teardown) + if args.teardown is not None + else bool(config.get("teardown", True)) + ) + self.scheduler = self._load_scheduler(args.scheduler or config.get("scheduler")) + _LOG.info("Init scheduler: %s", self.scheduler) + + @property + def config_loader(self) -> ConfigPersistenceService: + """Get the config loader service.""" + return self._config_loader + + @property + def service(self) -> Service: + """Get the parent service.""" + return self._parent_service + + @staticmethod + def _parse_args( + parser: argparse.ArgumentParser, + argv: Optional[List[str]], + ) -> Tuple[argparse.Namespace, List[str], List[str]]: + """Parse the command line arguments.""" class PathArgsTracker: """Simple class to help track which arguments are paths.""" @@ -235,166 +364,14 @@ def add_argument(self, *args: Any, **kwargs: Any) -> None: "incompatible" is not easily automatable across systems. """, ) - return (parser, path_args_tracker.path_args) - - @staticmethod - def get_help_text(description: str, long_text: str) -> str: - """Gets the help text from the argument parser. - - Parameters - ---------- - description : str - The short name of the script. - long_text : str - The long description of the script. - - Returns - ------- - str - The help text from the argument parser. - """ - (parser, _path_args) = Launcher._get_parser(description, long_text) - return parser.format_help() - - def __init__(self, description: str, long_text: str = "", argv: Optional[List[str]] = None): - # pylint: disable=too-many-statements - # pylint: disable=too-many-locals - _LOG.info("Launch: %s", description) - (parser, path_args) = self._get_parser(description, long_text) - (args, args_rest) = self._parse_args(parser, argv) - - # Bootstrap config loader: command line takes priority. - config_path = args.config_path or [] - self._config_loader = ConfigPersistenceService({"config_path": config_path}) - if args.config: - config = self._config_loader.load_config(args.config, ConfigSchema.CLI) - assert isinstance(config, Dict) - # Merge the args paths for the config loader with the paths from JSON file. - config_path += config.get("config_path", []) - self._config_loader = ConfigPersistenceService({"config_path": config_path}) - else: - config = {} - - log_level = args.log_level or config.get("log_level", _LOG_LEVEL) - try: - log_level = int(log_level) - except ValueError: - # failed to parse as an int - leave it as a string and let logging - # module handle whether it's an appropriate log name or not - log_level = logging.getLevelName(log_level) - logging.root.setLevel(log_level) - log_file = args.log_file or config.get("log_file") - if log_file: - log_handler = logging.FileHandler(log_file) - log_handler.setLevel(log_level) - log_handler.setFormatter(logging.Formatter(_LOG_FORMAT)) - logging.root.addHandler(log_handler) - - self._parent_service: Service = LocalExecService(parent=self._config_loader) - - # Prepare global_config from a combination of global config files, cli - # configs, and cli args. - args_dict = vars(args) - # teardown (bool) conflicts with Environment configs that use it for shell - # commands (list), so we exclude it from copying over - excluded_cli_args = path_args + ["teardown"] - # Include (almost) any item from the cli config file that either isn't in - # the cli args at all or whose cli arg is missing. - cli_config_args = { - key: val - for (key, val) in config.items() - if (args_dict.get(key) is None) and key not in excluded_cli_args - } - - self.global_config = self._load_config( - args_globals=config.get("globals", []) + (args.globals or []), - config_path=(args.config_path or []) + config.get("config_path", []), - args_rest=args_rest, - global_config=cli_config_args, - ) - # experiment_id is generally taken from --globals files, but we also allow - # overriding it on the CLI. - # It's useful to keep it there explicitly mostly for the --help output. - if args.experiment_id: - self.global_config["experiment_id"] = args.experiment_id - # trial_config_repeat_count is a scheduler property but it's convenient to - # set it via command line - if args.trial_config_repeat_count: - self.global_config["trial_config_repeat_count"] = args.trial_config_repeat_count - # Ensure that the trial_id is present since it gets used by some other - # configs but is typically controlled by the run optimize loop. - self.global_config.setdefault("trial_id", 1) - - self.global_config = DictTemplater(self.global_config).expand_vars(use_os_env=True) - assert isinstance(self.global_config, dict) - - # --service cli args should override the config file values. - service_files: List[str] = config.get("services", []) + (args.service or []) - assert isinstance(self._parent_service, SupportsConfigLoading) - self._parent_service = self._parent_service.load_services( - service_files, - self.global_config, - self._parent_service, - ) - - env_path = args.environment or config.get("environment") - if not env_path: - _LOG.error("No environment config specified.") - parser.error( - "At least the Environment config must be specified." - + " Run `mlos_bench --help` and consult `README.md` for more info." - ) - self.root_env_config = self._config_loader.resolve_path(env_path) - - self.environment: Environment = self._config_loader.load_environment( - self.root_env_config, TunableGroups(), self.global_config, service=self._parent_service - ) - _LOG.info("Init environment: %s", self.environment) - - # NOTE: Init tunable values *after* the Environment, but *before* the Optimizer - self.tunables = self._init_tunable_values( - args.random_init or config.get("random_init", False), - config.get("random_seed") if args.random_seed is None else args.random_seed, - config.get("tunable_values", []) + (args.tunable_values or []), - ) - _LOG.info("Init tunables: %s", self.tunables) - - self.optimizer = self._load_optimizer(args.optimizer or config.get("optimizer")) - _LOG.info("Init optimizer: %s", self.optimizer) - - self.storage = self._load_storage(args.storage or config.get("storage")) - _LOG.info("Init storage: %s", self.storage) - - self.teardown: bool = ( - bool(args.teardown) - if args.teardown is not None - else bool(config.get("teardown", True)) - ) - self.scheduler = self._load_scheduler(args.scheduler or config.get("scheduler")) - _LOG.info("Init scheduler: %s", self.scheduler) - @property - def config_loader(self) -> ConfigPersistenceService: - """Get the config loader service.""" - return self._config_loader - - @property - def service(self) -> Service: - """Get the parent service.""" - return self._parent_service - - @staticmethod - def _parse_args( - parser: argparse.ArgumentParser, - argv: Optional[List[str]], - ) -> Tuple[argparse.Namespace, List[str]]: - """Parse the command line arguments.""" # By default we use the command line arguments, but allow the caller to # provide some explicitly for testing purposes. if argv is None: argv = sys.argv[1:].copy() (args, args_rest) = parser.parse_known_args(argv) - return (args, args_rest) + + return (args, path_args_tracker.path_args, args_rest) @staticmethod def _try_parse_extra_args(cmdline: Iterable[str]) -> Dict[str, TunableValue]: diff --git a/mlos_bench/mlos_bench/run.py b/mlos_bench/mlos_bench/run.py index 62ee5d123aa..a554f1a803d 100755 --- a/mlos_bench/mlos_bench/run.py +++ b/mlos_bench/mlos_bench/run.py @@ -23,12 +23,6 @@ _LOG = logging.getLogger(__name__) -_NAME = "mlos_bench" -_DESC = "Systems autotuning and benchmarking tool" -# Dynamically add the --help text to our docstring. -__doc__ += "\n" + Launcher.get_help_text(_NAME, _DESC) - - def _sanity_check_results(launcher: Launcher) -> None: """Do some sanity checking on the results and throw an exception if it looks like something went wrong. @@ -63,7 +57,7 @@ def _sanity_check_results(launcher: Launcher) -> None: def _main( argv: Optional[List[str]] = None, ) -> Tuple[Optional[Dict[str, float]], Optional[TunableGroups]]: - launcher = Launcher(_NAME, _DESC, argv=argv) + launcher = Launcher("mlos_bench", "Systems autotuning and benchmarking tool", argv=argv) with launcher.scheduler as scheduler_context: scheduler_context.start() From da7a9ffd418aabff6537537cbc91dac5204b0fc5 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 11 Oct 2024 12:42:57 -0500 Subject: [PATCH 013/100] restore overview prior to editing --- doc/source/overview.rst | 298 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 doc/source/overview.rst diff --git a/doc/source/overview.rst b/doc/source/overview.rst new file mode 100644 index 00000000000..800f9749c90 --- /dev/null +++ b/doc/source/overview.rst @@ -0,0 +1,298 @@ +########################## +MLOS Package APIs Overview +########################## + +This is a list of major functions and classes provided by the MLOS packages. + +############################# +mlos-core API +############################# + +This is a list of major functions and classes provided by `mlos_core`. + +.. currentmodule:: mlos_core + +Optimizers +========== +.. currentmodule:: mlos_core.optimizers +.. autosummary:: + :toctree: generated/ + + :template: class.rst + + OptimizerType + OptimizerFactory + + :template: function.rst + + OptimizerFactory.create + +.. currentmodule:: mlos_core.optimizers.optimizer +.. autosummary:: + :toctree: generated/ + :template: class.rst + + BaseOptimizer + +.. currentmodule:: mlos_core.optimizers.random_optimizer +.. autosummary:: + :toctree: generated/ + :template: class.rst + + RandomOptimizer + +.. currentmodule:: mlos_core.optimizers.flaml_optimizer +.. autosummary:: + :toctree: generated/ + :template: class.rst + + FlamlOptimizer + +.. currentmodule:: mlos_core.optimizers.bayesian_optimizers +.. autosummary:: + :toctree: generated/ + :template: class.rst + + BaseBayesianOptimizer + SmacOptimizer + +Spaces +====== + +Converters +---------- +.. currentmodule:: mlos_core.spaces.converters.flaml +.. autosummary:: + :toctree: generated/ + :template: function.rst + + configspace_to_flaml_space + +Space Adapters +-------------- +.. currentmodule:: mlos_core.spaces.adapters +.. autosummary:: + :toctree: generated/ + + :template: class.rst + + SpaceAdapterType + SpaceAdapterFactory + + :template: function.rst + + SpaceAdapterFactory.create + +.. currentmodule:: mlos_core.spaces.adapters.adapter +.. autosummary:: + :toctree: generated/ + :template: class.rst + + BaseSpaceAdapter + +.. currentmodule:: mlos_core.spaces.adapters.identity_adapter +.. autosummary:: + :toctree: generated/ + :template: class.rst + + IdentityAdapter + +.. currentmodule:: mlos_core.spaces.adapters.llamatune +.. autosummary:: + :toctree: generated/ + :template: class.rst + + LlamaTuneAdapter + +############################# +mlos-bench API +############################# + +This is a list of major functions and classes provided by `mlos_bench`. + +.. currentmodule:: mlos_bench + +Main +==== + +:doc:`run.py ` + + The script to run the benchmarks or the optimization loop. + + Also available as `mlos_bench` command line tool. + +.. note:: + The are `json config examples `_ and `json schemas `_ on the main `source code `_ repository site. + +Benchmark Environments +====================== +.. currentmodule:: mlos_bench.environments +.. autosummary:: + :toctree: generated/ + :template: class.rst + + Status + Environment + CompositeEnv + MockEnv + +Local Environments +------------------- + +.. currentmodule:: mlos_bench.environments.local +.. autosummary:: + :toctree: generated/ + :template: class.rst + + LocalEnv + LocalFileShareEnv + +Remote Environments +------------------- + +.. currentmodule:: mlos_bench.environments.remote +.. autosummary:: + :toctree: generated/ + :template: class.rst + + RemoteEnv + OSEnv + VMEnv + HostEnv + +Tunable Parameters +================== +.. currentmodule:: mlos_bench.tunables +.. autosummary:: + :toctree: generated/ + :template: class.rst + + Tunable + TunableGroups + +Service Mix-ins +=============== +.. currentmodule:: mlos_bench.services +.. autosummary:: + :toctree: generated/ + :template: class.rst + + Service + FileShareService + +.. currentmodule:: mlos_bench.services.config_persistence +.. autosummary:: + :toctree: generated/ + :template: class.rst + + ConfigPersistenceService + +Local Services +--------------- +.. currentmodule:: mlos_bench.services.local +.. autosummary:: + :toctree: generated/ + :template: class.rst + + LocalExecService + +Remote Azure Services +--------------------- + +.. currentmodule:: mlos_bench.services.remote.azure +.. autosummary:: + :toctree: generated/ + :template: class.rst + + AzureVMService + AzureFileShareService + +Optimizer Adapters +================== +.. currentmodule:: mlos_bench.optimizers +.. autosummary:: + :toctree: generated/ + :template: class.rst + + Optimizer + MockOptimizer + MlosCoreOptimizer + +Storage +======= +Base Runtime Backends +--------------------- +.. currentmodule:: mlos_bench.storage +.. autosummary:: + :toctree: generated/ + :template: class.rst + + Storage + +.. currentmodule:: mlos_bench.storage.storage_factory +.. autosummary:: + :toctree: generated/ + :template: function.rst + + from_config + +SQL DB Storage Backend +---------------------- +.. currentmodule:: mlos_bench.storage.sql.storage +.. autosummary:: + :toctree: generated/ + :template: class.rst + + SqlStorage + +Analysis Client Access APIs +--------------------------- +.. currentmodule:: mlos_bench.storage.base_experiment_data +.. autosummary:: + :toctree: generated/ + :template: class.rst + + ExperimentData + +.. currentmodule:: mlos_bench.storage.base_trial_data +.. autosummary:: + :toctree: generated/ + :template: class.rst + + TrialData + +.. currentmodule:: mlos_bench.storage.base_tunable_config_data +.. autosummary:: + :toctree: generated/ + :template: class.rst + + TunableConfigData + +.. currentmodule:: mlos_bench.storage.base_tunable_config_trial_group_data +.. autosummary:: + :toctree: generated/ + :template: class.rst + + TunableConfigTrialGroupData + +############################# +mlos-viz API +############################# + +This is a list of major functions and classes provided by `mlos_viz`. + +.. currentmodule:: mlos_viz + +.. currentmodule:: mlos_viz +.. autosummary:: + :toctree: generated/ + :template: class.rst + + MlosVizMethod + +.. currentmodule:: mlos_viz +.. autosummary:: + :toctree: generated/ + :template: function.rst + + plot From 102d169c557e484644448fb20a9147e38c76b42c Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 18 Oct 2024 06:55:50 -0500 Subject: [PATCH 014/100] Makefile improvements --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a3bd62d2a0c..5ba0c6a7e82 100644 --- a/Makefile +++ b/Makefile @@ -659,7 +659,7 @@ clean-doc-env: COMMON_DOC_FILES := build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp doc/source/*.rst doc/source/_templates/*.rst doc/source/conf.py -SPHINX_API_RST_FILES := doc/source/index.rst +SPHINX_API_RST_FILES := doc/source/index.rst doc/source/overview.rst ifeq ($(SKIP_COVERAGE),) doc/build/html/index.html: build/pytest.${CONDA_ENV_NAME}.build-stamp @@ -670,7 +670,9 @@ endif SPHINXOPTS ?= # -v # be verbose SPHINXOPTS += -W -w $(PWD)/doc/build/sphinx-build.warn.log -j auto -doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile doc/source/conf.py doc/copy-source-tree-docs.sh $(MD_FILES) +doc/build/html/index.html: build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp +doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile doc/source/conf.py doc/source/conf.py +doc/build/html/index.html: doc/copy-source-tree-docs.sh $(MD_FILES) @rm -rf doc/build @mkdir -p doc/build @rm -f doc/build/log.txt From 303dc274d67f647c02ff70b891a6ef2077a97fc3 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 18 Oct 2024 06:56:35 -0500 Subject: [PATCH 015/100] Work towards overview improvement --- doc/source/conf.py | 2 + doc/source/index.rst | 6 + doc/source/overview.rst | 285 +-------------------- mlos_core/mlos_core/optimizers/__init__.py | 1 + 4 files changed, 13 insertions(+), 281 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index f0be7f6637b..de2de4c168e 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -63,6 +63,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + "sphinx.ext.autodoc", "autoapi.extension", "nbsphinx", "sphinx.ext.doctest", @@ -72,6 +73,7 @@ "matplotlib.sphinxext.plot_directive", "myst_parser", ] +autodoc_typehints = "both" source_suffix = { ".rst": "restructuredtext", diff --git a/doc/source/index.rst b/doc/source/index.rst index f04f4244b74..eaee2523705 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -19,6 +19,12 @@ See below for additional documentation sections. source_tree_docs/mlos_bench/index source_tree_docs/mlos_viz/index +.. toctree:: + :caption: Overview + :maxdepth: 2 + + overview + .. toctree:: :caption: API Reference :maxdepth: 2 diff --git a/doc/source/overview.rst b/doc/source/overview.rst index 800f9749c90..3bad8805c04 100644 --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -10,289 +10,12 @@ mlos-core API This is a list of major functions and classes provided by `mlos_core`. -.. currentmodule:: mlos_core +.. autoapisummary:: mlos_core Optimizers ========== -.. currentmodule:: mlos_core.optimizers -.. autosummary:: - :toctree: generated/ +.. autoapisummary:: mlos_core.optimizers.OptimizerType - :template: class.rst +.. autoapisummary:: mlos_core.optimizers.OptimizerFactory - OptimizerType - OptimizerFactory - - :template: function.rst - - OptimizerFactory.create - -.. currentmodule:: mlos_core.optimizers.optimizer -.. autosummary:: - :toctree: generated/ - :template: class.rst - - BaseOptimizer - -.. currentmodule:: mlos_core.optimizers.random_optimizer -.. autosummary:: - :toctree: generated/ - :template: class.rst - - RandomOptimizer - -.. currentmodule:: mlos_core.optimizers.flaml_optimizer -.. autosummary:: - :toctree: generated/ - :template: class.rst - - FlamlOptimizer - -.. currentmodule:: mlos_core.optimizers.bayesian_optimizers -.. autosummary:: - :toctree: generated/ - :template: class.rst - - BaseBayesianOptimizer - SmacOptimizer - -Spaces -====== - -Converters ----------- -.. currentmodule:: mlos_core.spaces.converters.flaml -.. autosummary:: - :toctree: generated/ - :template: function.rst - - configspace_to_flaml_space - -Space Adapters --------------- -.. currentmodule:: mlos_core.spaces.adapters -.. autosummary:: - :toctree: generated/ - - :template: class.rst - - SpaceAdapterType - SpaceAdapterFactory - - :template: function.rst - - SpaceAdapterFactory.create - -.. currentmodule:: mlos_core.spaces.adapters.adapter -.. autosummary:: - :toctree: generated/ - :template: class.rst - - BaseSpaceAdapter - -.. currentmodule:: mlos_core.spaces.adapters.identity_adapter -.. autosummary:: - :toctree: generated/ - :template: class.rst - - IdentityAdapter - -.. currentmodule:: mlos_core.spaces.adapters.llamatune -.. autosummary:: - :toctree: generated/ - :template: class.rst - - LlamaTuneAdapter - -############################# -mlos-bench API -############################# - -This is a list of major functions and classes provided by `mlos_bench`. - -.. currentmodule:: mlos_bench - -Main -==== - -:doc:`run.py ` - - The script to run the benchmarks or the optimization loop. - - Also available as `mlos_bench` command line tool. - -.. note:: - The are `json config examples `_ and `json schemas `_ on the main `source code `_ repository site. - -Benchmark Environments -====================== -.. currentmodule:: mlos_bench.environments -.. autosummary:: - :toctree: generated/ - :template: class.rst - - Status - Environment - CompositeEnv - MockEnv - -Local Environments -------------------- - -.. currentmodule:: mlos_bench.environments.local -.. autosummary:: - :toctree: generated/ - :template: class.rst - - LocalEnv - LocalFileShareEnv - -Remote Environments -------------------- - -.. currentmodule:: mlos_bench.environments.remote -.. autosummary:: - :toctree: generated/ - :template: class.rst - - RemoteEnv - OSEnv - VMEnv - HostEnv - -Tunable Parameters -================== -.. currentmodule:: mlos_bench.tunables -.. autosummary:: - :toctree: generated/ - :template: class.rst - - Tunable - TunableGroups - -Service Mix-ins -=============== -.. currentmodule:: mlos_bench.services -.. autosummary:: - :toctree: generated/ - :template: class.rst - - Service - FileShareService - -.. currentmodule:: mlos_bench.services.config_persistence -.. autosummary:: - :toctree: generated/ - :template: class.rst - - ConfigPersistenceService - -Local Services ---------------- -.. currentmodule:: mlos_bench.services.local -.. autosummary:: - :toctree: generated/ - :template: class.rst - - LocalExecService - -Remote Azure Services ---------------------- - -.. currentmodule:: mlos_bench.services.remote.azure -.. autosummary:: - :toctree: generated/ - :template: class.rst - - AzureVMService - AzureFileShareService - -Optimizer Adapters -================== -.. currentmodule:: mlos_bench.optimizers -.. autosummary:: - :toctree: generated/ - :template: class.rst - - Optimizer - MockOptimizer - MlosCoreOptimizer - -Storage -======= -Base Runtime Backends ---------------------- -.. currentmodule:: mlos_bench.storage -.. autosummary:: - :toctree: generated/ - :template: class.rst - - Storage - -.. currentmodule:: mlos_bench.storage.storage_factory -.. autosummary:: - :toctree: generated/ - :template: function.rst - - from_config - -SQL DB Storage Backend ----------------------- -.. currentmodule:: mlos_bench.storage.sql.storage -.. autosummary:: - :toctree: generated/ - :template: class.rst - - SqlStorage - -Analysis Client Access APIs ---------------------------- -.. currentmodule:: mlos_bench.storage.base_experiment_data -.. autosummary:: - :toctree: generated/ - :template: class.rst - - ExperimentData - -.. currentmodule:: mlos_bench.storage.base_trial_data -.. autosummary:: - :toctree: generated/ - :template: class.rst - - TrialData - -.. currentmodule:: mlos_bench.storage.base_tunable_config_data -.. autosummary:: - :toctree: generated/ - :template: class.rst - - TunableConfigData - -.. currentmodule:: mlos_bench.storage.base_tunable_config_trial_group_data -.. autosummary:: - :toctree: generated/ - :template: class.rst - - TunableConfigTrialGroupData - -############################# -mlos-viz API -############################# - -This is a list of major functions and classes provided by `mlos_viz`. - -.. currentmodule:: mlos_viz - -.. currentmodule:: mlos_viz -.. autosummary:: - :toctree: generated/ - :template: class.rst - - MlosVizMethod - -.. currentmodule:: mlos_viz -.. autosummary:: - :toctree: generated/ - :template: function.rst - - plot +.. autoapisummary:: mlos_core.optimizers.OptimizerFactory.create diff --git a/mlos_core/mlos_core/optimizers/__init__.py b/mlos_core/mlos_core/optimizers/__init__.py index e9f402c1878..7dcfebab605 100644 --- a/mlos_core/mlos_core/optimizers/__init__.py +++ b/mlos_core/mlos_core/optimizers/__init__.py @@ -16,6 +16,7 @@ from mlos_core.spaces.adapters import SpaceAdapterFactory, SpaceAdapterType __all__ = [ + "OptimizerType", "SpaceAdapterType", "OptimizerFactory", "BaseOptimizer", From 16e0b5552e7e6df886503de79959d12bae4cf285 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 18 Oct 2024 06:57:06 -0500 Subject: [PATCH 016/100] wip --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index de2de4c168e..447a20690a8 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -123,7 +123,7 @@ # "imported-members", ] autoapi_add_toctree_entry = False -# autoapi_keep_files = True # for testing +autoapi_keep_files = True # for testing # -- Options for HTML output ------------------------------------------------- From 76e5379cf5c28f6ce46bf1901b1099c42efcdb51 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 7 Nov 2024 22:15:43 +0000 Subject: [PATCH 017/100] wip: adding cross referencing links within the docstrings --- Makefile | 3 ++ mlos_core/mlos_core/__init__.py | 30 ++++++++++++++++++- mlos_core/mlos_core/optimizers/__init__.py | 26 ++++++++++++---- .../mlos_core/spaces/adapters/__init__.py | 14 ++++++--- 4 files changed, 63 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 027fe22060c..571c54f7436 100644 --- a/Makefile +++ b/Makefile @@ -671,6 +671,9 @@ SPHINXOPTS ?= # -v # be verbose SPHINXOPTS += -W -w $(PWD)/doc/build/sphinx-build.warn.log -j auto doc/build/html/index.html: build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp +doc/build/html/index.html: $(MLOS_CORE_PYTHON_FILES) +doc/build/html/index.html: $(MLOS_BENCH_PYTHON_FILES) +doc/build/html/index.html: $(MLOS_VIZ_PYTHON_FILES) doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile doc/source/conf.py doc/source/conf.py doc/build/html/index.html: doc/copy-source-tree-docs.sh $(MD_FILES) @rm -rf doc/build diff --git a/mlos_core/mlos_core/__init__.py b/mlos_core/mlos_core/__init__.py index b8a72cef92c..ce88d276963 100644 --- a/mlos_core/mlos_core/__init__.py +++ b/mlos_core/mlos_core/__init__.py @@ -2,7 +2,35 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Basic initializer module for the mlos_core package.""" +"""Basic initializer module for the mlos_core package. + +:mod:`~mlos_core` provides the main Optimizer portions of the MLOS project for use with +autotuning purposes. Although it is generally intended to be used with +:mod:`~mlos_bench`, it can be used independently as well. + +To do this it provides a small set of wrapper classes around other OSS tuning +libraries in order to provide a consistent interface so that the rest of the code +using it can easily exchange one optimizer for another (or even stack them). + +Specifically: + +- :class:`~mlos_core.optimizers.optimizer.BaseOptimizer` is the base class for all Optimizers + + Its core methods are: + + - :meth:`~mlos_core.optimizers.optimizer.BaseOptimizer.suggest` which returns a + new configuration to evaluate + - :meth:`~mlos_core.optimizers.optimizer.BaseOptimizer.register` which registers + a "score" for an evaluated configuration with the Optimizer + +- :meth:`~mlos_core.optimizers.OptimizerFactory.create` is a factory function + that creates a new :type:`~mlos_core.optimizers.ConcreteOptimizer` instance + + To do this it uses the :class:`~mlos_core.optimizers.OptimizerType` enum to + specify which underlying optimizer to use (e.g., + :class:`~mlos_core.optimizers.OptimizerType.FLAML` or + :class:`~mlos_core.optimizers.OptimizerType.SMAC`). +""" from mlos_core.version import VERSION __version__ = VERSION diff --git a/mlos_core/mlos_core/optimizers/__init__.py b/mlos_core/mlos_core/optimizers/__init__.py index 7dcfebab605..d63888e12ce 100644 --- a/mlos_core/mlos_core/optimizers/__init__.py +++ b/mlos_core/mlos_core/optimizers/__init__.py @@ -27,34 +27,50 @@ class OptimizerType(Enum): - """Enumerate supported MlosCore optimizers.""" + """Enumerate supported mlos_core optimizers.""" RANDOM = RandomOptimizer - """An instance of RandomOptimizer class will be used.""" + """ + An instance of :class:`~mlos_core.optimizers.random_optimizer.RandomOptimizer` + class will be used. + """ FLAML = FlamlOptimizer - """An instance of FlamlOptimizer class will be used.""" + """ + An instance of :class:`~mlos_core.optimizers.flaml_optimizer.FlamlOptimizer` + class will be used. + """ SMAC = SmacOptimizer - """An instance of SmacOptimizer class will be used.""" + """ + An instance of + :class:`~mlos_core.optimizers.bayesian_optimizers.smac_optimizer.SmacOptimizer` + class will be used. + """ # To make mypy happy, we need to define a type variable for each optimizer type. # https://github.com/python/mypy/issues/12952 # ConcreteOptimizer = TypeVar('ConcreteOptimizer', *[member.value for member in OptimizerType]) # To address this, we add a test for complete coverage of the enum. + ConcreteOptimizer = TypeVar( "ConcreteOptimizer", RandomOptimizer, FlamlOptimizer, SmacOptimizer, ) +"""Type variable for concrete optimizer classes.""" DEFAULT_OPTIMIZER_TYPE = OptimizerType.FLAML +"""Default optimizer type to use if none is specified.""" class OptimizerFactory: - """Simple factory class for creating BaseOptimizer-derived objects.""" + """ + Simple factory class for creating + :class:`~mlos_core.optimizers.base_optimizer.BaseOptimizer`-derived objects. + """ # pylint: disable=too-few-public-methods diff --git a/mlos_core/mlos_core/spaces/adapters/__init__.py b/mlos_core/mlos_core/spaces/adapters/__init__.py index 1645ac9cb45..b7b69b00a9e 100644 --- a/mlos_core/mlos_core/spaces/adapters/__init__.py +++ b/mlos_core/mlos_core/spaces/adapters/__init__.py @@ -15,17 +15,19 @@ __all__ = [ "IdentityAdapter", "LlamaTuneAdapter", + "SpaceAdapterFactory", + "SpaceAdapterType", ] class SpaceAdapterType(Enum): - """Enumerate supported MlosCore space adapters.""" + """Enumerate supported mlos_core space adapters.""" IDENTITY = IdentityAdapter - """A no-op adapter will be used.""" + """A no-op adapter (:class:`IdentityAdapter`) will be used.""" LLAMATUNE = LlamaTuneAdapter - """An instance of LlamaTuneAdapter class will be used.""" + """An instance of :class:`LlamaTuneAdapter` class will be used.""" # To make mypy happy, we need to define a type variable for each optimizer type. @@ -40,10 +42,14 @@ class SpaceAdapterType(Enum): IdentityAdapter, LlamaTuneAdapter, ) +"""Type variable for concrete SpaceAdapter classes.""" class SpaceAdapterFactory: - """Simple factory class for creating BaseSpaceAdapter-derived objects.""" + """ + Simple factory class for creating + :class:`~mlos_core.spaces.adapters.adapter.BaseSpaceAdapter`-derived objects. + """ # pylint: disable=too-few-public-methods From e165c15eda9e6509dfb3e9d87c234230309aa259 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 7 Nov 2024 22:20:59 +0000 Subject: [PATCH 018/100] fixups and make it more nitpicky about references --- Makefile | 4 +++- mlos_core/mlos_core/spaces/adapters/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 571c54f7436..a7f874f2b97 100644 --- a/Makefile +++ b/Makefile @@ -668,7 +668,9 @@ endif # Treat warnings as failures. SPHINXOPTS ?= # -v # be verbose -SPHINXOPTS += -W -w $(PWD)/doc/build/sphinx-build.warn.log -j auto +SPHINXOPTS += -n -W -w $(PWD)/doc/build/sphinx-build.warn.log -j auto + +sphinx-apidoc: doc/build/html/index.html doc/build/html/index.html: build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp doc/build/html/index.html: $(MLOS_CORE_PYTHON_FILES) diff --git a/mlos_core/mlos_core/spaces/adapters/__init__.py b/mlos_core/mlos_core/spaces/adapters/__init__.py index b7b69b00a9e..5999deefc0d 100644 --- a/mlos_core/mlos_core/spaces/adapters/__init__.py +++ b/mlos_core/mlos_core/spaces/adapters/__init__.py @@ -24,10 +24,10 @@ class SpaceAdapterType(Enum): """Enumerate supported mlos_core space adapters.""" IDENTITY = IdentityAdapter - """A no-op adapter (:class:`IdentityAdapter`) will be used.""" + """A no-op adapter (:class:`.IdentityAdapter`) will be used.""" LLAMATUNE = LlamaTuneAdapter - """An instance of :class:`LlamaTuneAdapter` class will be used.""" + """An instance of :class:`.LlamaTuneAdapter` class will be used.""" # To make mypy happy, we need to define a type variable for each optimizer type. From 40f1f680c3630cd09e3b2c1ab0abced3e8a0fff6 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 8 Nov 2024 00:29:51 +0000 Subject: [PATCH 019/100] more doc cross reference fixups --- doc/source/conf.py | 22 ++++++++++++++++++- mlos_core/mlos_core/optimizers/__init__.py | 1 + .../mlos_core/spaces/adapters/__init__.py | 1 + 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 447a20690a8..7ac907141d4 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -67,7 +67,7 @@ "autoapi.extension", "nbsphinx", "sphinx.ext.doctest", - # 'sphinx.ext.intersphinx', + "sphinx.ext.intersphinx", # 'sphinx.ext.linkcode', "numpydoc", "matplotlib.sphinxext.plot_directive", @@ -75,6 +75,26 @@ ] autodoc_typehints = "both" +# Add mappings to link to external documentation. +intersphinx_mapping = { # pylint: disable=consider-using-namedtuple-or-dataclass + "python": ("https://docs.python.org/3", None), + "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), + "numpy": ("https://numpy.org/doc/stable/reference/", None), +} + +# Ignore some cross references to external things we can't intersphinx with. +nitpick_ignore = [ + # FIXME: sphinx has a hard time finding typealiases instead of classes. + ("py:class", "ConcreteOptimizer"), + ("py:class", "ConcreteSpaceAdapater"), + ("py:class", "mlos_core.spaces.converters.flaml.FlamlDomain"), +] +nitpick_ignore_regex = [ + (r"py:.*", r"ConfigSpace\..*"), + (r"py:.*", r"flaml\..*"), + (r"py:.*", r"smac\..*"), +] + source_suffix = { ".rst": "restructuredtext", # '.txt': 'markdown', diff --git a/mlos_core/mlos_core/optimizers/__init__.py b/mlos_core/mlos_core/optimizers/__init__.py index d63888e12ce..073fe7e9e67 100644 --- a/mlos_core/mlos_core/optimizers/__init__.py +++ b/mlos_core/mlos_core/optimizers/__init__.py @@ -17,6 +17,7 @@ __all__ = [ "OptimizerType", + "ConcreteOptimizer", "SpaceAdapterType", "OptimizerFactory", "BaseOptimizer", diff --git a/mlos_core/mlos_core/spaces/adapters/__init__.py b/mlos_core/mlos_core/spaces/adapters/__init__.py index 5999deefc0d..51563628c97 100644 --- a/mlos_core/mlos_core/spaces/adapters/__init__.py +++ b/mlos_core/mlos_core/spaces/adapters/__init__.py @@ -13,6 +13,7 @@ from mlos_core.spaces.adapters.llamatune import LlamaTuneAdapter __all__ = [ + "ConcreteSpaceAdapter", "IdentityAdapter", "LlamaTuneAdapter", "SpaceAdapterFactory", From fa927040f0cb95c4d369787b0f675b2ed101a9f4 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 8 Nov 2024 18:01:25 +0000 Subject: [PATCH 020/100] tweaks --- Makefile | 2 ++ doc/source/conf.py | 8 ++++++-- mlos_core/mlos_core/optimizers/__init__.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a7f874f2b97..599706caf94 100644 --- a/Makefile +++ b/Makefile @@ -728,7 +728,9 @@ build/check-doc.build-stamp: doc/build/html/index.html doc/build/html/htmlcov/in test -s doc/build/html/autoapi/mlos_viz/index.html test -s doc/build/html/autoapi/mlos_viz/dabl/index.html grep -q -e '--config CONFIG' doc/build/html/api/mlos_bench/mlos_bench.run.html + # TODO: Add sphinx-build -b linkcheck target? # Check doc logs for errors (but skip over some known ones) ... + # TODO: Remove some of these ignores. @cat doc/build/log.txt \ | egrep -C1 -e WARNING -e CRITICAL -e ERROR \ | egrep -v \ diff --git a/doc/source/conf.py b/doc/source/conf.py index 7ac907141d4..9894dd37021 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -84,12 +84,16 @@ # Ignore some cross references to external things we can't intersphinx with. nitpick_ignore = [ - # FIXME: sphinx has a hard time finding typealiases instead of classes. + # FIXME: sphinx has a hard time finding typealiases and typevars instead of classes. + ("py:class", "BaseTypeVar"), ("py:class", "ConcreteOptimizer"), - ("py:class", "ConcreteSpaceAdapater"), + ("py:class", "ConcreteSpaceAdapter"), + ("py:class", "FlamlDomain"), ("py:class", "mlos_core.spaces.converters.flaml.FlamlDomain"), + ("py:class", "mlos_bench.tunables.tunable.TunableValue"), ] nitpick_ignore_regex = [ + # Ignore some external references that don't use sphinx for their docs. (r"py:.*", r"ConfigSpace\..*"), (r"py:.*", r"flaml\..*"), (r"py:.*", r"smac\..*"), diff --git a/mlos_core/mlos_core/optimizers/__init__.py b/mlos_core/mlos_core/optimizers/__init__.py index 073fe7e9e67..b73fcaa1c92 100644 --- a/mlos_core/mlos_core/optimizers/__init__.py +++ b/mlos_core/mlos_core/optimizers/__init__.py @@ -70,7 +70,7 @@ class will be used. class OptimizerFactory: """ Simple factory class for creating - :class:`~mlos_core.optimizers.base_optimizer.BaseOptimizer`-derived objects. + :class:`~mlos_core.optimizers.optimizer.BaseOptimizer`-derived objects. """ # pylint: disable=too-few-public-methods From ae97da53f053ed09a9c8168c72e4a9e1b8ee3bd6 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 17:20:10 +0000 Subject: [PATCH 021/100] adjustments for more type lookups --- doc/source/conf.py | 61 +++++++++++++++++-- mlos_bench/mlos_bench/storage/sql/common.py | 3 +- .../mlos_bench/storage/sql/experiment.py | 3 +- .../mlos_bench/storage/sql/experiment_data.py | 3 +- mlos_bench/mlos_bench/storage/sql/trial.py | 2 +- .../storage/sql/tunable_config_data.py | 2 +- .../sql/tunable_config_trial_group_data.py | 3 +- 7 files changed, 66 insertions(+), 11 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 9894dd37021..597b287e9e3 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -77,9 +77,62 @@ # Add mappings to link to external documentation. intersphinx_mapping = { # pylint: disable=consider-using-namedtuple-or-dataclass - "python": ("https://docs.python.org/3", None), - "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), - "numpy": ("https://numpy.org/doc/stable/reference/", None), + "asyncssh": ( + "https://asyncssh.readthedocs.io/en/latest/", + None, + ), + "azure-core": ( + "https://azuresdkdocs.blob.core.windows.net/$web/python/azure-core/latest/", + None, + ), + "azure-identity": ( + "https://azuresdkdocs.blob.core.windows.net/$web/python/azure-identity/latest/", + None, + ), + "ConfigSpace": ( + "https://automl.github.io/ConfigSpace/latest/", + None, + ), + "matplotlib": ( + "https://matplotlib.org/stable/", + None, + ), + "numpy": ( + "https://numpy.org/doc/stable/", + "https://numpy.org/doc/stable/objects.inv", + ), + "pandas": ( + "https://pandas.pydata.org/docs/", + None, + ), + "python": ( + "https://docs.python.org/3/", + None, + ), + "referencing": ( + "https://referencing.readthedocs.io/en/stable/", + None, + ), + "scikit-learn": ( + "https://scikit-learn.org/stable/", + None, + ), + "scipy": ( + "https://docs.scipy.org/doc/scipy/", + None, + ), + "smac": ( + "https://automl.github.io/SMAC3/main/", + None, + ), + "sqlalchemy": ( + "http://docs.sqlalchemy.org/en/stable/", + None, + ), + "typing_extensions": ( + "https://typing-extensions.readthedocs.io/en/stable/", + None, + ), } # Ignore some cross references to external things we can't intersphinx with. @@ -94,9 +147,7 @@ ] nitpick_ignore_regex = [ # Ignore some external references that don't use sphinx for their docs. - (r"py:.*", r"ConfigSpace\..*"), (r"py:.*", r"flaml\..*"), - (r"py:.*", r"smac\..*"), ] source_suffix = { diff --git a/mlos_bench/mlos_bench/storage/sql/common.py b/mlos_bench/mlos_bench/storage/sql/common.py index 918ed54ff2a..ab827f2d997 100644 --- a/mlos_bench/mlos_bench/storage/sql/common.py +++ b/mlos_bench/mlos_bench/storage/sql/common.py @@ -6,7 +6,8 @@ from typing import Dict, Optional import pandas -from sqlalchemy import Engine, Integer, and_, func, select +from sqlalchemy import Integer, and_, func, select +from sqlalchemy.engine import Engine from mlos_bench.environments.status import Status from mlos_bench.storage.base_experiment_data import ExperimentData diff --git a/mlos_bench/mlos_bench/storage/sql/experiment.py b/mlos_bench/mlos_bench/storage/sql/experiment.py index 56a3f260498..e072de40c47 100644 --- a/mlos_bench/mlos_bench/storage/sql/experiment.py +++ b/mlos_bench/mlos_bench/storage/sql/experiment.py @@ -10,7 +10,8 @@ from typing import Any, Dict, Iterator, List, Literal, Optional, Tuple from pytz import UTC -from sqlalchemy import Connection, CursorResult, Engine, Table, column, func, select +from sqlalchemy import Connection, CursorResult, Table, column, func, select +from sqlalchemy.engine import Engine from mlos_bench.environments.status import Status from mlos_bench.storage.base_storage import Storage diff --git a/mlos_bench/mlos_bench/storage/sql/experiment_data.py b/mlos_bench/mlos_bench/storage/sql/experiment_data.py index 9103322ae89..fb542cc4f35 100644 --- a/mlos_bench/mlos_bench/storage/sql/experiment_data.py +++ b/mlos_bench/mlos_bench/storage/sql/experiment_data.py @@ -7,7 +7,8 @@ from typing import Dict, Literal, Optional import pandas -from sqlalchemy import Engine, Integer, String, func +from sqlalchemy import Integer, String, func +from sqlalchemy.engine import Engine from mlos_bench.storage.base_experiment_data import ExperimentData from mlos_bench.storage.base_trial_data import TrialData diff --git a/mlos_bench/mlos_bench/storage/sql/trial.py b/mlos_bench/mlos_bench/storage/sql/trial.py index 5942912efd2..04d786b3b09 100644 --- a/mlos_bench/mlos_bench/storage/sql/trial.py +++ b/mlos_bench/mlos_bench/storage/sql/trial.py @@ -8,7 +8,7 @@ from datetime import datetime from typing import Any, Dict, List, Literal, Optional, Tuple -from sqlalchemy import Connection, Engine +from sqlalchemy.engine import Connection, Engine from sqlalchemy.exc import IntegrityError from mlos_bench.environments.status import Status diff --git a/mlos_bench/mlos_bench/storage/sql/tunable_config_data.py b/mlos_bench/mlos_bench/storage/sql/tunable_config_data.py index 40225039be5..fa861029ee8 100644 --- a/mlos_bench/mlos_bench/storage/sql/tunable_config_data.py +++ b/mlos_bench/mlos_bench/storage/sql/tunable_config_data.py @@ -5,7 +5,7 @@ """An interface to access the tunable config data stored in SQL DB.""" import pandas -from sqlalchemy import Engine +from sqlalchemy.engine import Engine from mlos_bench.storage.base_tunable_config_data import TunableConfigData from mlos_bench.storage.sql.schema import DbSchema diff --git a/mlos_bench/mlos_bench/storage/sql/tunable_config_trial_group_data.py b/mlos_bench/mlos_bench/storage/sql/tunable_config_trial_group_data.py index 0e8c022e7f0..2631c1c6eb6 100644 --- a/mlos_bench/mlos_bench/storage/sql/tunable_config_trial_group_data.py +++ b/mlos_bench/mlos_bench/storage/sql/tunable_config_trial_group_data.py @@ -7,7 +7,8 @@ from typing import TYPE_CHECKING, Dict, Optional import pandas -from sqlalchemy import Engine, Integer, func +from sqlalchemy import Integer, func +from sqlalchemy.engine import Engine from mlos_bench.storage.base_tunable_config_data import TunableConfigData from mlos_bench.storage.base_tunable_config_trial_group_data import ( From e29366b636da2d862bd43f0eb7b1c3bc27654dca Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 19:54:36 +0000 Subject: [PATCH 022/100] Tweaks --- doc/requirements.txt | 2 + doc/source/conf.py | 149 +++++++++++------- .../mlos_bench/environments/mock_env.py | 3 +- .../mlos_bench/services/config_persistence.py | 2 +- .../services/types/config_loader_type.py | 2 +- .../base_tunable_config_trial_group_data.py | 2 +- mlos_bench/mlos_bench/storage/sql/schema.py | 2 +- .../mlos_bench/storage/sql/trial_data.py | 2 +- 8 files changed, 98 insertions(+), 66 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 4505ee58bfc..4611ad88df6 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,6 +1,8 @@ setuptools-scm>=8.1.0 sphinx sphinx-autoapi +sphinx-autodoc-typehints +intersphinx_registry nbsphinx jupyter_core>=4.11.2 # nbsphix dependency - addresses CVE-2022-39286 nbconvert diff --git a/doc/source/conf.py b/doc/source/conf.py index 597b287e9e3..880391c2e89 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -23,6 +23,7 @@ from logging import warning +from intersphinx_registry import get_intersphinx_mapping import sphinx_rtd_theme # pylint: disable=unused-import @@ -64,86 +65,114 @@ # ones. extensions = [ "sphinx.ext.autodoc", + "sphinx_autodoc_typehints", "autoapi.extension", "nbsphinx", "sphinx.ext.doctest", "sphinx.ext.intersphinx", - # 'sphinx.ext.linkcode', + "sphinx.ext.linkcode", "numpydoc", "matplotlib.sphinxext.plot_directive", "myst_parser", ] autodoc_typehints = "both" +typehints_fully_qualified = True +typehints_defaults = "braces" +typehints_use_signature = True +typehints_use_signature_return = True + + +def linkcode_resolve(domain: str, info: dict): + """linkcode extension override to link to the source code on GitHub.""" + if domain != "py": + return None + if not info["module"]: + return None + if not info["module"].startswith("mlos_"): + return None + filename = info["module"].replace(".", "/") + return f"https://github.com/microsoft/MLOS/tree/main/{filename}.py" + + # Add mappings to link to external documentation. -intersphinx_mapping = { # pylint: disable=consider-using-namedtuple-or-dataclass - "asyncssh": ( - "https://asyncssh.readthedocs.io/en/latest/", - None, - ), - "azure-core": ( - "https://azuresdkdocs.blob.core.windows.net/$web/python/azure-core/latest/", - None, - ), - "azure-identity": ( - "https://azuresdkdocs.blob.core.windows.net/$web/python/azure-identity/latest/", - None, - ), - "ConfigSpace": ( - "https://automl.github.io/ConfigSpace/latest/", - None, - ), - "matplotlib": ( - "https://matplotlib.org/stable/", - None, - ), - "numpy": ( - "https://numpy.org/doc/stable/", - "https://numpy.org/doc/stable/objects.inv", - ), - "pandas": ( - "https://pandas.pydata.org/docs/", - None, - ), - "python": ( - "https://docs.python.org/3/", - None, - ), - "referencing": ( - "https://referencing.readthedocs.io/en/stable/", - None, - ), - "scikit-learn": ( - "https://scikit-learn.org/stable/", - None, - ), - "scipy": ( - "https://docs.scipy.org/doc/scipy/", - None, - ), - "smac": ( - "https://automl.github.io/SMAC3/main/", - None, - ), - "sqlalchemy": ( - "http://docs.sqlalchemy.org/en/stable/", - None, - ), - "typing_extensions": ( - "https://typing-extensions.readthedocs.io/en/stable/", - None, - ), -} +intersphinx_mapping = get_intersphinx_mapping( + packages={ + "asyncssh", + "matplotlib", + "numpy", + "pandas", + "python", + } +) + +# TODO: convert these to registry calls once the following PR is merged: +# https://github.com/Quansight-Labs/intersphinx_registry/pull/41 +intersphinx_mapping.update( + { + "azure-core": ( + "https://azuresdkdocs.blob.core.windows.net/$web/python/azure-core/latest/", + None, + ), + "azure-identity": ( + "https://azuresdkdocs.blob.core.windows.net/$web/python/azure-identity/latest/", + None, + ), + "ConfigSpace": ( + "https://automl.github.io/ConfigSpace/latest/", + None, + ), + "referencing": ( + "https://referencing.readthedocs.io/en/stable/", + None, + ), + "smac": ( + "https://automl.github.io/SMAC3/main/", + None, + ), + "typing_extensions": ( + "https://typing-extensions.readthedocs.io/en/stable/", + None, + ), + } +) + +# We recommend adding the following config value. +# Sphinx defaults to automatically resolve *unresolved* labels using all your Intersphinx mappings. +# This behavior has unintended side-effects, namely that documentations local references can +# suddenly resolve to an external location. +# See also: +# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#confval-intersphinx_disabled_reftypes +# intersphinx_disabled_reftypes = ["*"] # Ignore some cross references to external things we can't intersphinx with. +# sphinx has a hard time finding typealiases and typevars instead of classes. +# See Also: https://github.com/sphinx-doc/sphinx/issues/10974 nitpick_ignore = [ - # FIXME: sphinx has a hard time finding typealiases and typevars instead of classes. + # Internal typevars and aliases: ("py:class", "BaseTypeVar"), ("py:class", "ConcreteOptimizer"), ("py:class", "ConcreteSpaceAdapter"), + ("py:class", "DistributionName"), + ("py:class", "EnvironType"), ("py:class", "FlamlDomain"), ("py:class", "mlos_core.spaces.converters.flaml.FlamlDomain"), + ("py:class", "TunableValue"), ("py:class", "mlos_bench.tunables.tunable.TunableValue"), + ("py:class", "TunableValueType"), + ("py:class", "TunableValueTypeName"), + ("py:class", "T_co"), + # Literals in base_storage.py + ("py:class", "min"), + ("py:class", "max"), + # Literal in context handlers signatures. + ("py:class", "False"), + # External typevars and aliases: + ("py:class", "CoroReturnType"), + ("py:class", "FutureReturnType"), + ("py:class", "typing.Literal"), + ("py:class", "typing_extensions.Literal"), + ("py:class", "numpy.typing.NDArray"), ] nitpick_ignore_regex = [ # Ignore some external references that don't use sphinx for their docs. diff --git a/mlos_bench/mlos_bench/environments/mock_env.py b/mlos_bench/mlos_bench/environments/mock_env.py index 6d3309f35b5..8298f94fe17 100644 --- a/mlos_bench/mlos_bench/environments/mock_env.py +++ b/mlos_bench/mlos_bench/environments/mock_env.py @@ -14,7 +14,8 @@ from mlos_bench.environments.base_environment import Environment from mlos_bench.environments.status import Status from mlos_bench.services.base_service import Service -from mlos_bench.tunables import Tunable, TunableGroups, TunableValue +from mlos_bench.tunables.tunable import Tunable, TunableValue +from mlos_bench.tunables.tunable_groups import TunableGroups _LOG = logging.getLogger(__name__) diff --git a/mlos_bench/mlos_bench/services/config_persistence.py b/mlos_bench/mlos_bench/services/config_persistence.py index 72bfad007de..cd0f42bac40 100644 --- a/mlos_bench/mlos_bench/services/config_persistence.py +++ b/mlos_bench/mlos_bench/services/config_persistence.py @@ -25,7 +25,7 @@ import json5 # To read configs with comments and other JSON5 syntax features from jsonschema import SchemaError, ValidationError -from mlos_bench.config.schemas import ConfigSchema +from mlos_bench.config.schemas.config_schemas import ConfigSchema from mlos_bench.environments.base_environment import Environment from mlos_bench.optimizers.base_optimizer import Optimizer from mlos_bench.services.base_service import Service diff --git a/mlos_bench/mlos_bench/services/types/config_loader_type.py b/mlos_bench/mlos_bench/services/types/config_loader_type.py index 33adac67eb4..f0362e8d88d 100644 --- a/mlos_bench/mlos_bench/services/types/config_loader_type.py +++ b/mlos_bench/mlos_bench/services/types/config_loader_type.py @@ -16,7 +16,7 @@ runtime_checkable, ) -from mlos_bench.config.schemas import ConfigSchema +from mlos_bench.config.schemas.config_schemas import ConfigSchema from mlos_bench.tunables.tunable import TunableValue # Avoid's circular import issues. diff --git a/mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py b/mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py index c01c7544b39..797204199a2 100644 --- a/mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py +++ b/mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py @@ -120,5 +120,5 @@ def results_df(self) -> pandas.DataFrame: See Also -------- - ExperimentData.results + :py:attr:`mlos_bench.storage.base_experiment_data.ExperimentData.results_df` """ diff --git a/mlos_bench/mlos_bench/storage/sql/schema.py b/mlos_bench/mlos_bench/storage/sql/schema.py index 3900568b75d..af0c9fe9879 100644 --- a/mlos_bench/mlos_bench/storage/sql/schema.py +++ b/mlos_bench/mlos_bench/storage/sql/schema.py @@ -11,7 +11,6 @@ Column, DateTime, Dialect, - Engine, Float, ForeignKeyConstraint, Integer, @@ -23,6 +22,7 @@ UniqueConstraint, create_mock_engine, ) +from sqlalchemy.engine import Engine _LOG = logging.getLogger(__name__) diff --git a/mlos_bench/mlos_bench/storage/sql/trial_data.py b/mlos_bench/mlos_bench/storage/sql/trial_data.py index ac57b7b5c03..dcf2ffd3376 100644 --- a/mlos_bench/mlos_bench/storage/sql/trial_data.py +++ b/mlos_bench/mlos_bench/storage/sql/trial_data.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Optional import pandas -from sqlalchemy import Engine +from sqlalchemy.engine import Engine from mlos_bench.environments.status import Status from mlos_bench.storage.base_trial_data import TrialData From 8609e457d0b0e49469414ca288d75163a3c23fa9 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 20:23:20 +0000 Subject: [PATCH 023/100] fixups --- doc/source/conf.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 880391c2e89..21dbaaac79b 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -18,8 +18,10 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. # +import json import os import sys +from typing import Dict from logging import warning @@ -83,7 +85,22 @@ typehints_use_signature_return = True -def linkcode_resolve(domain: str, info: dict): +_base_path = os.path.abspath(os.path.join(__file__, "../../..")) +_path_cache: Dict[str, bool] = {} + + +def _check_path(path: str) -> bool: + """Check if a path exists and cache the result.""" + path = os.path.join(_base_path, path) + if path in _path_cache: + result = _path_cache[path] + else: + result = os.path.exists(path) + _path_cache[path] = result + return result + + +def linkcode_resolve(domain: str, info: Dict[str, str]): """linkcode extension override to link to the source code on GitHub.""" if domain != "py": return None @@ -91,8 +108,15 @@ def linkcode_resolve(domain: str, info: dict): return None if not info["module"].startswith("mlos_"): return None + package = info["module"].split(".")[0] filename = info["module"].replace(".", "/") - return f"https://github.com/microsoft/MLOS/tree/main/{filename}.py" + path = f"{package}/{filename}.py" + if not _check_path(path): + path = f"{package}/{filename}/__init__.py" + if not _check_path(path): + warning(f"linkcode_resolve failed to find {path}") + warning(f"linkcode_resolve info: {json.dumps(info, indent=2)}") + return f"https://github.com/microsoft/MLOS/tree/main/{path}" # Add mappings to link to external documentation. @@ -173,12 +197,16 @@ def linkcode_resolve(domain: str, info: dict): ("py:class", "typing.Literal"), ("py:class", "typing_extensions.Literal"), ("py:class", "numpy.typing.NDArray"), + # External classes that refuse to resolve: + ("py:class", "contextlib.nullcontext"), + ("py:class", "sqlalchemy.engine.Engine"), ] nitpick_ignore_regex = [ # Ignore some external references that don't use sphinx for their docs. (r"py:.*", r"flaml\..*"), ] +# Which documents to include in the build. source_suffix = { ".rst": "restructuredtext", # '.txt': 'markdown', From 22c7f1b4ee82fcfd0c1f1287d0a86bcc007014ae Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 20:59:53 +0000 Subject: [PATCH 024/100] Rewrite documentation on documentation --- doc/README.md | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/doc/README.md b/doc/README.md index eea9dd955c4..8a08459cd7b 100644 --- a/doc/README.md +++ b/doc/README.md @@ -2,14 +2,92 @@ Documentation is generated using [`sphinx`](https://www.sphinx-doc.org/). +The configuration for this is in [`doc/source/conf.py`](./source/conf.py). + +We use the [`autoapi`](https://sphinx-autoapi.readthedocs.io/en/latest/) extension to generate documentation automatically from the docstrings in our python code. + +Additionally, we also use the [`copy-source-tree-docs.sh`](./copy-source-tree-docs.sh) script to copy a few Markdown files from the root of the repository to the `doc/source` build directory automatically to include them in the documentation. + +Those are included in the [`index.rst`](./source/index.rst) file which is the main entry point for the documentation and about the only manually maintained rst file. + +## Writing Documentation + +When writing docstrings, use the [`numpydoc`](https://numpydoc.readthedocs.io/en/latest/format.html) style. + +Each top level module should include a docstring that describes the module and its purpose and usage. + +These string should be written for consumption by both users and developers. + +Other function and method docstrings that aren't typically intended for users can be written for developers. + +### Cross Referencing + +You can include links between the documentation using [cross-referencing](https://www.sphinx-doc.org/en/master/usage/domains/python.html#python-xref-roles) links in the docstring. + +For instance: + +```python +""" +My docstring that references another module :py:mod:`fully.qualified.module.name`. + +Or else, a class :py:class:`fully.qualified.module.name.ClassName`. + +Or else, a class name :py:class:`.ClassName` that is in the same module. + +Or else, a class method :py:meth:`~.ClassName.method` but without the leading class name. +""" +``` + +These links will be automatically resolved by `sphinx` and checked using the `nitpick` option to ensure we have well-formed links in the documentation. + +### Example Code + +Ideally, each main class should also inclue example code that demonstrates how to use the class. + +This code should be included in the docstring and should be runnable via [`doctest`](https://docs.python.org/3/library/doctest.html). + +For instance: + +```python +class MyClass: + """ + My class that does something. + + Examples + -------- + >>> from my_module import MyClass + >>> my_class = MyClass() + >>> my_class.do_something() + Expected output + + """ + ... +``` + +## Building the documentation + ```sh make -C .. doc ``` -## Testing with Docker +> This will also run some checks on the documentation. + +## Testing + +### Manually with Docker ```sh ./nginx-docker.sh restart ``` > Now browse to `http://localhost:8080` + +## Troubleshooting + +We use the [`intersphinx`](https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html) extension to link between external modules and the [`nitpick`](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-nitpicky) option to ensure that all references resolve correctly. + +Unfortunately, this process is not perfect and sometimes we need to provide [`nitpick_ignore`](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-nitpick_ignore)s in the [`doc/source/conf.py`](./source/conf.py) file. + +In particular, currently `TypeVar` and `TypeAliases` are not resolved correctly and we need to ignore those. + +In other cases, specifying the full path to the module in the cross-reference or the `import` can help. From 573b68f414b816b124823fbb76bc6a8b9bad2817 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 21:02:38 +0000 Subject: [PATCH 025/100] references in the doc testing --- CONTRIBUTING.md | 2 ++ MAINTAINING.md | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c32eaa836ec..163f48cb6c5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,6 +86,8 @@ We expect development to follow a typical "forking" style workflow: make doc-test ``` + > See the [documentation README](./doc/README.md) for more information on documentation and its testing. + 1. Submit changes for inclusion as a [Pull Request on Github](https://github.com/microsoft/MLOS/pulls). Some notes on organizing changes to help reviewers: diff --git a/MAINTAINING.md b/MAINTAINING.md index 54f60c016ab..9a7f6060365 100644 --- a/MAINTAINING.md +++ b/MAINTAINING.md @@ -2,6 +2,10 @@ Some notes for maintainers. +## Documentation + +See the [documentation README](./doc/README.md) for more information on writing (and testing) documentation. + ## Releasing 1. Bump the version using the [`update-version.sh`](./scripts/update-version.sh) script: From 26ef02ee030043a38045dd1efe8accf99946627a Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 21:32:54 +0000 Subject: [PATCH 026/100] Remove overview rst and add mlos_bench run usage --- Makefile | 8 ++++++-- doc/source/index.rst | 12 ++++++------ doc/source/mlos_bench.run.usage.rst | 7 +++++++ doc/source/overview.rst | 21 --------------------- 4 files changed, 19 insertions(+), 29 deletions(-) create mode 100644 doc/source/mlos_bench.run.usage.rst delete mode 100644 doc/source/overview.rst diff --git a/Makefile b/Makefile index 599706caf94..7871f242fb5 100644 --- a/Makefile +++ b/Makefile @@ -659,7 +659,7 @@ clean-doc-env: COMMON_DOC_FILES := build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp doc/source/*.rst doc/source/_templates/*.rst doc/source/conf.py -SPHINX_API_RST_FILES := doc/source/index.rst doc/source/overview.rst +SPHINX_API_RST_FILES := doc/source/index.rst doc/source/mlos_bench.run.usage.rst ifeq ($(SKIP_COVERAGE),) doc/build/html/index.html: build/pytest.${CONDA_ENV_NAME}.build-stamp @@ -690,6 +690,10 @@ doc/build/html/index.html: doc/copy-source-tree-docs.sh $(MD_FILES) # Copy some of the source tree markdown docs into place. ./doc/copy-source-tree-docs.sh + # Generate the help output from mlos_bench CLI for the docs. + mkdir -p doc/source/generated/ + conda run -n ${CONDA_ENV_NAME} mlos_bench --help > doc/source/generated/mlos_bench.run.usage.txt + # Build the rst files into html. conda run -n ${CONDA_ENV_NAME} $(MAKE) SPHINXOPTS="$(SPHINXOPTS)" -C doc/ $(MAKEFLAGS) html \ >> doc/build/log.txt 2>&1 \ @@ -727,7 +731,7 @@ build/check-doc.build-stamp: doc/build/html/index.html doc/build/html/htmlcov/in test -s doc/build/html/autoapi/mlos_bench/index.html test -s doc/build/html/autoapi/mlos_viz/index.html test -s doc/build/html/autoapi/mlos_viz/dabl/index.html - grep -q -e '--config CONFIG' doc/build/html/api/mlos_bench/mlos_bench.run.html + grep -q -e '--config CONFIG' doc/build/html//mlos_bench.run.usage.html # TODO: Add sphinx-build -b linkcheck target? # Check doc logs for errors (but skip over some known ones) ... # TODO: Remove some of these ignores. diff --git a/doc/source/index.rst b/doc/source/index.rst index eaee2523705..2d8fa1317ad 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -19,12 +19,6 @@ See below for additional documentation sections. source_tree_docs/mlos_bench/index source_tree_docs/mlos_viz/index -.. toctree:: - :caption: Overview - :maxdepth: 2 - - overview - .. toctree:: :caption: API Reference :maxdepth: 2 @@ -33,6 +27,12 @@ See below for additional documentation sections. autoapi/mlos_bench/index autoapi/mlos_viz/index +.. toctree:: + :caption: mlos_bench CLI usage + :maxdepth: 1 + + mlos_bench.run.usage + .. toctree:: :maxdepth: 1 :caption: References diff --git a/doc/source/mlos_bench.run.usage.rst b/doc/source/mlos_bench.run.usage.rst new file mode 100644 index 00000000000..b1168e39327 --- /dev/null +++ b/doc/source/mlos_bench.run.usage.rst @@ -0,0 +1,7 @@ +mlos_bench CLI usage +================================== + +Here is the current `--help` output for the `mlos_bench` CLI: + +.. literalinclude:: ./generated/mlos_bench.run.usage.txt + :language: none diff --git a/doc/source/overview.rst b/doc/source/overview.rst deleted file mode 100644 index 3bad8805c04..00000000000 --- a/doc/source/overview.rst +++ /dev/null @@ -1,21 +0,0 @@ -########################## -MLOS Package APIs Overview -########################## - -This is a list of major functions and classes provided by the MLOS packages. - -############################# -mlos-core API -############################# - -This is a list of major functions and classes provided by `mlos_core`. - -.. autoapisummary:: mlos_core - -Optimizers -========== -.. autoapisummary:: mlos_core.optimizers.OptimizerType - -.. autoapisummary:: mlos_core.optimizers.OptimizerFactory - -.. autoapisummary:: mlos_core.optimizers.OptimizerFactory.create From 668a08ee8de507cb838e9aefbf60a9fe7d17b30b Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 21:33:04 +0000 Subject: [PATCH 027/100] Remove some old error ignores --- Makefile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Makefile b/Makefile index 7871f242fb5..7a0af171d16 100644 --- a/Makefile +++ b/Makefile @@ -740,13 +740,7 @@ build/check-doc.build-stamp: doc/build/html/index.html doc/build/html/htmlcov/in | egrep -v \ -e "warnings.warn\(f'\"{wd.path}\" is shallow and may cause errors'\)" \ -e "No such file or directory: '.*.examples'.( \[docutils\]\s*)?$$" \ - -e 'Problems with "include" directive path:' \ - -e 'duplicate object description' \ - -e "document isn't included in any toctree" \ - -e "more than one target found for cross-reference" \ -e "toctree contains reference to nonexisting document 'auto_examples/index'" \ - -e "failed to import function 'create' from module '(SpaceAdapter|Optimizer)Factory'" \ - -e "No module named '(SpaceAdapter|Optimizer)Factory'" \ -e '^make.*resetting jobserver mode' \ -e 'from cryptography.hazmat.primitives.ciphers.algorithms import' \ | grep -v '^\s*$$' \ From 36fdbaab0698e4f48427863ae7d0463b4cca15c7 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 21:33:15 +0000 Subject: [PATCH 028/100] Those were redundant --- doc/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 21dbaaac79b..2c8da2883b7 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -81,8 +81,8 @@ typehints_fully_qualified = True typehints_defaults = "braces" -typehints_use_signature = True -typehints_use_signature_return = True +# typehints_use_signature = True +# typehints_use_signature_return = True _base_path = os.path.abspath(os.path.join(__file__, "../../..")) From 9304499a0533f32ecc3ea8a0bb870dade9b5cd7d Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 21:33:25 +0000 Subject: [PATCH 029/100] Start running doctests --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index cd3bdbb9916..0b7a0bb564f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,6 +38,7 @@ addopts = -vv -l --ff --nf + --doctest-modules -n auto # --dist loadgroup # --log-level=DEBUG From 56d440e12e26a2762e35598644907b08657a2c97 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 23:07:25 +0000 Subject: [PATCH 030/100] fixups --- doc/README.md | 4 ++- .../mlos_bench/optimizers/base_optimizer.py | 2 +- .../services/types/vm_provisioner_type.py | 10 +++--- mlos_core/mlos_core/optimizers/optimizer.py | 36 +++++++++---------- 4 files changed, 27 insertions(+), 25 deletions(-) diff --git a/doc/README.md b/doc/README.md index 8a08459cd7b..6af139da1ea 100644 --- a/doc/README.md +++ b/doc/README.md @@ -70,7 +70,9 @@ class MyClass: make -C .. doc ``` -> This will also run some checks on the documentation. +This will also run some checks on the documentation. + +> When running this command in a tight loop, it may be useful to run with `SKIP_COVERAGE=true` to avoid re-running the test and coverage checks each time a python file changes. ## Testing diff --git a/mlos_bench/mlos_bench/optimizers/base_optimizer.py b/mlos_bench/mlos_bench/optimizers/base_optimizer.py index d9a854b476e..1d20f0cf31b 100644 --- a/mlos_bench/mlos_bench/optimizers/base_optimizer.py +++ b/mlos_bench/mlos_bench/optimizers/base_optimizer.py @@ -206,7 +206,7 @@ def name(self) -> str: @property def targets(self) -> Dict[str, Literal["min", "max"]]: - """A dictionary of {target: direction} of optimization targets.""" + """Returns a dictionary of optimization targets and their direction.""" return { opt_target: "min" if opt_dir == 1 else "max" for (opt_target, opt_dir) in self._opt_targets.items() diff --git a/mlos_bench/mlos_bench/services/types/vm_provisioner_type.py b/mlos_bench/mlos_bench/services/types/vm_provisioner_type.py index 69d24f3fd38..d6c4f1f1c60 100644 --- a/mlos_bench/mlos_bench/services/types/vm_provisioner_type.py +++ b/mlos_bench/mlos_bench/services/types/vm_provisioner_type.py @@ -27,7 +27,7 @@ def vm_provision(self, params: dict) -> Tuple["Status", dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -63,7 +63,7 @@ def vm_start(self, params: dict) -> Tuple["Status", dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -79,7 +79,7 @@ def vm_stop(self, params: dict) -> Tuple["Status", dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -95,7 +95,7 @@ def vm_restart(self, params: dict) -> Tuple["Status", dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -111,7 +111,7 @@ def vm_deprovision(self, params: dict) -> Tuple["Status", dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ diff --git a/mlos_core/mlos_core/optimizers/optimizer.py b/mlos_core/mlos_core/optimizers/optimizer.py index d7da71ae864..1592b6d5d7e 100644 --- a/mlos_core/mlos_core/optimizers/optimizer.py +++ b/mlos_core/mlos_core/optimizers/optimizer.py @@ -84,16 +84,16 @@ def register( Parameters ---------- - configs : pd.DataFrame + configs : pandas.DataFrame Dataframe of configs / parameters. The columns are parameter names and the rows are the configs. - scores : pd.DataFrame + scores : pandas.DataFrame Scores from running the configs. The index is the same as the index of the configs. - context : pd.DataFrame + context : pandas.DataFrame Not Yet Implemented. - metadata : Optional[pd.DataFrame] + metadata : Optional[pandas.DataFrame] Metadata returned by the backend optimizer's suggest method. """ # Do some input validation. @@ -134,13 +134,13 @@ def _register( Parameters ---------- - configs : pd.DataFrame + configs : pandas.DataFrame Dataframe of configs / parameters. The columns are parameter names and the rows are the configs. - scores : pd.DataFrame + scores : pandas.DataFrame Scores from running the configs. The index is the same as the index of the configs. - context : pd.DataFrame + context : pandas.DataFrame Not Yet Implemented. """ pass # pylint: disable=unnecessary-pass # pragma: no cover @@ -157,7 +157,7 @@ def suggest( Parameters ---------- - context : pd.DataFrame + context : pandas.DataFrame Not Yet Implemented. defaults : bool Whether or not to return the default config instead of an optimizer guided one. @@ -165,10 +165,10 @@ def suggest( Returns ------- - configuration : pd.DataFrame + configuration : pandas.DataFrame Pandas dataframe with a single row. Column names are the parameter names. - metadata : Optional[pd.DataFrame] + metadata : Optional[pandas.DataFrame] The metadata associated with the given configuration used for evaluations. Backend optimizer specific. """ @@ -203,15 +203,15 @@ def _suggest( Parameters ---------- - context : pd.DataFrame + context : pandas.DataFrame Not Yet Implemented. Returns ------- - configuration : pd.DataFrame + configuration : pandas.DataFrame Pandas dataframe with a single row. Column names are the parameter names. - metadata : Optional[pd.DataFrame] + metadata : Optional[pandas.DataFrame] The metadata associated with the given configuration used for evaluations. Backend optimizer specific. """ @@ -232,12 +232,12 @@ def register_pending( Parameters ---------- - configs : pd.DataFrame + configs : pandas.DataFrame Dataframe of configs / parameters. The columns are parameter names and the rows are the configs. - context : pd.DataFrame + context : pandas.DataFrame Not Yet Implemented. - metadata : Optional[pd.DataFrame] + metadata : Optional[pandas.DataFrame] Metadata returned by the backend optimizer's suggest method. """ pass # pylint: disable=unnecessary-pass # pragma: no cover @@ -248,7 +248,7 @@ def get_observations(self) -> Tuple[pd.DataFrame, pd.DataFrame, Optional[pd.Data Returns ------- - observations : Tuple[pd.DataFrame, pd.DataFrame, Optional[pd.DataFrame]] + observations : Tuple[pandas.DataFrame, pandas.DataFrame, Optional[pandas.DataFrame]] A triplet of (config, score, context) DataFrames of observations. """ if len(self._observations) == 0: @@ -281,7 +281,7 @@ def get_best_observations( Returns ------- - observations : Tuple[pd.DataFrame, pd.DataFrame, Optional[pd.DataFrame]] + observations : Tuple[pandas.DataFrame, pandas.DataFrame, Optional[pandas.DataFrame]] A triplet of best (config, score, context) DataFrames of best observations. """ if len(self._observations) == 0: From 02478d4be2bbe4980258765344f70747f6816784 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 23:07:50 +0000 Subject: [PATCH 031/100] start converting to napolean for numpy parsing --- doc/source/conf.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 2c8da2883b7..9aaa44f7c9f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -67,23 +67,37 @@ # ones. extensions = [ "sphinx.ext.autodoc", - "sphinx_autodoc_typehints", + # "sphinx_autodoc_typehints", "autoapi.extension", "nbsphinx", "sphinx.ext.doctest", "sphinx.ext.intersphinx", "sphinx.ext.linkcode", - "numpydoc", + # "numpydoc", + "sphinx.ext.napoleon", "matplotlib.sphinxext.plot_directive", "myst_parser", ] -autodoc_typehints = "both" - -typehints_fully_qualified = True -typehints_defaults = "braces" -# typehints_use_signature = True -# typehints_use_signature_return = True - +# autodoc_typehints = "both" +# autodoc_typehints_description_target = "documented" + +# typehints_fully_qualified = True +# typehints_defaults = "braces" +## typehints_use_signature = True +## typehints_use_signature_return = True + +napoleon_numpy_docstring = True +napoleon_include_init_with_doc = False +napoleon_include_private_with_doc = False +napoleon_include_special_with_doc = False +napoleon_use_admonition_for_examples = False +napoleon_use_admonition_for_notes = False +napoleon_use_admonition_for_references = False +napoleon_use_ivar = False +napoleon_use_param = True +napoleon_use_rtype = True +napoleon_use_keyword = True +napoleon_custom_sections = None _base_path = os.path.abspath(os.path.join(__file__, "../../..")) _path_cache: Dict[str, bool] = {} From c4c504b8b05e995d0b607c8093e072ca3823148a Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 23:08:02 +0000 Subject: [PATCH 032/100] add more docstrings --- mlos_core/mlos_core/optimizers/__init__.py | 18 +++++++++++++++- .../mlos_core/optimizers/flaml_optimizer.py | 5 ++++- .../mlos_core/spaces/adapters/__init__.py | 21 ++++++++++++++++++- .../mlos_core/spaces/adapters/adapter.py | 21 ++++++++++++++----- .../mlos_core/spaces/adapters/llamatune.py | 11 +++++++++- .../mlos_core/spaces/converters/__init__.py | 8 ++++++- .../mlos_core/spaces/converters/flaml.py | 3 +++ setup.cfg | 4 ++-- 8 files changed, 79 insertions(+), 12 deletions(-) diff --git a/mlos_core/mlos_core/optimizers/__init__.py b/mlos_core/mlos_core/optimizers/__init__.py index b73fcaa1c92..04ca091f14b 100644 --- a/mlos_core/mlos_core/optimizers/__init__.py +++ b/mlos_core/mlos_core/optimizers/__init__.py @@ -2,7 +2,23 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Basic initializer module for the mlos_core optimizers.""" +"""Basic initializer module for the mlos_core optimizers. + +Optimizers are the main component of the :py:mod:`mlos_core` package. +They act as a wrapper around other OSS tuning libraries to provide a consistent API +interface to allow experimenting with different autotuning algorithms. + +The :class:`~mlos_core.optimizers.optimizer.BaseOptimizer` class is the base class +for all Optimizers and provides the core +:py:meth:`~mlos_core.optimizers.optimizer.BaseOptimizer.suggest` and +:py:meth:`~mlos_core.optimizers.optimizer.BaseOptimizer.register` methods. + +This module also provides a simple factory class for creating optimizers. + +For instance: + +TODO: Add example usage here. +""" from enum import Enum from typing import List, Optional, TypeVar diff --git a/mlos_core/mlos_core/optimizers/flaml_optimizer.py b/mlos_core/mlos_core/optimizers/flaml_optimizer.py index e5272f103ec..d3dcca11b47 100644 --- a/mlos_core/mlos_core/optimizers/flaml_optimizer.py +++ b/mlos_core/mlos_core/optimizers/flaml_optimizer.py @@ -2,7 +2,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Contains the FlamlOptimizer class.""" +"""Contains the FlamlOptimizer class + +See Also: `Flaml `_. +""" from typing import Dict, List, NamedTuple, Optional, Tuple, Union from warnings import warn diff --git a/mlos_core/mlos_core/spaces/adapters/__init__.py b/mlos_core/mlos_core/spaces/adapters/__init__.py index 51563628c97..e30b625b897 100644 --- a/mlos_core/mlos_core/spaces/adapters/__init__.py +++ b/mlos_core/mlos_core/spaces/adapters/__init__.py @@ -2,7 +2,26 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Basic initializer module for the mlos_core space adapters.""" +"""Basic initializer module for the mlos_core space adapters. + +Space adapters provide a mechanism for automatic transformation of the original +:py:class:`ConfigSpace.ConfigurationSpace` provided to the optimizer into a new +space that is more suitable for the optimizer. + +By default the :py:class:`.IdentityAdapter` is used, which does not perform any +transformation. +But, for instance, the :py:class:`.LlamaTuneAdapter` can be used to automatically +transform the space to a lower dimensional one. + +See the :py:mod:`mlos_bench.optimizers` module for more information on how to do +this with :py:mod:`mlos_bench`. + +This module provides a simple factory class for creating space adapters. + +For instance: + +TODO: Add example usage here. +""" from enum import Enum from typing import Optional, TypeVar diff --git a/mlos_core/mlos_core/spaces/adapters/adapter.py b/mlos_core/mlos_core/spaces/adapters/adapter.py index 2d48a14c317..2d03918d94a 100644 --- a/mlos_core/mlos_core/spaces/adapters/adapter.py +++ b/mlos_core/mlos_core/spaces/adapters/adapter.py @@ -2,7 +2,17 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Contains the BaseSpaceAdapter abstract class.""" +"""Contains the BaseSpaceAdapter abstract class. + +As mentioned in :py:mod:`mlos_core.spaces.adapters`, the space adapters provide a +mechanism for automatic transformation of the original +:py:class:`ConfigSpace.ConfigurationSpace` provided to the Optimizer into a new +space for the Optimizer to search over. + +It's main APIs are the :py:meth:`~.BaseSpaceAdapter.transform` and +:py:meth:`~.BaseSpaceAdapter.inverse_transform` methods, which are used to translate +configurations from one space to another. +""" from abc import ABCMeta, abstractmethod @@ -47,8 +57,9 @@ def target_parameter_space(self) -> ConfigSpace.ConfigurationSpace: def transform(self, configuration: pd.DataFrame) -> pd.DataFrame: """ Translates a configuration, which belongs to the target parameter space, to the - original parameter space. This method is called by the `suggest` method of the - `BaseOptimizer` class. + original parameter space. This method is called by the + :py:meth:`~mlos_core.optimizers.optimizer.BaseOptimizer.suggest` method of the + :py:class:`~mlos_core.optimizers.optimizer.BaseOptimizer` class. Parameters ---------- @@ -69,8 +80,8 @@ def inverse_transform(self, configurations: pd.DataFrame) -> pd.DataFrame: """ Translates a configuration, which belongs to the original parameter space, to the target parameter space. This method is called by the `register` method of - the `BaseOptimizer` class, and performs the inverse operation of - `BaseSpaceAdapter.transform` method. + the :py:class:`~mlos_core.optimizers.optimizer.BaseOptimizer` class, and + performs the inverse operation of :py:meth:`~.BaseSpaceAdapter.transform` method. Parameters ---------- diff --git a/mlos_core/mlos_core/spaces/adapters/llamatune.py b/mlos_core/mlos_core/spaces/adapters/llamatune.py index 5a39f863a56..f988f7a22e6 100644 --- a/mlos_core/mlos_core/spaces/adapters/llamatune.py +++ b/mlos_core/mlos_core/spaces/adapters/llamatune.py @@ -2,7 +2,16 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Implementation of LlamaTune space adapter.""" +"""Implementation of LlamaTune space adapter. + +LlamaTune is a technique that transforms the original parameter space into a +lower-dimensional space to try and improve the sample efficiency of the underlying +optimizer by making use of the inherent parameter sensitivity correlations in most +systems. + +See Also: `LlamaTune: Sample-Efficient DBMS Configuration Tuning +`_. +""" import os from typing import Dict, List, Optional, Union from warnings import warn diff --git a/mlos_core/mlos_core/spaces/converters/__init__.py b/mlos_core/mlos_core/spaces/converters/__init__.py index 2360bda24f8..ab7736c607e 100644 --- a/mlos_core/mlos_core/spaces/converters/__init__.py +++ b/mlos_core/mlos_core/spaces/converters/__init__.py @@ -2,4 +2,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Space converters init file.""" +"""Space converters init file. + +Space converters are helper functions that translate a +:py:class:`ConfigSpace.ConfigurationSpace` that :py:mod:`mlos_core` Optimizers take +as input to the underlying Optimizer's parameter description language (in case it +doesn't use :py:class:`ConfigSpace.ConfigurationSpace`). +""" diff --git a/mlos_core/mlos_core/spaces/converters/flaml.py b/mlos_core/mlos_core/spaces/converters/flaml.py index 71370853e4a..5b42ad55c5c 100644 --- a/mlos_core/mlos_core/spaces/converters/flaml.py +++ b/mlos_core/mlos_core/spaces/converters/flaml.py @@ -22,7 +22,10 @@ FlamlDomain: TypeAlias = flaml.tune.sample.Domain +"""Flaml domain type alias.""" + FlamlSpace: TypeAlias = Dict[str, flaml.tune.sample.Domain] +"""Flaml space type alias - a `Dict[str, FlamlDomain]`""" def configspace_to_flaml_space( diff --git a/setup.cfg b/setup.cfg index 0b7a0bb564f..0e0ffbe7c5c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,8 +38,8 @@ addopts = -vv -l --ff --nf - --doctest-modules -n auto + --doctest-modules # --dist loadgroup # --log-level=DEBUG # Moved these to Makefile (coverage is expensive and we only need it in the pipelines generally). @@ -47,7 +47,7 @@ addopts = testpaths = mlos_core mlos_bench # Ignore some upstream deprecation warnings. filterwarnings = - ignore:.*(get_hyperparam|get_dictionary).*:DeprecationWarning:smac:0 + ignore:.*(get_hyperparam|get_dictionary|get_parents_of|(list\(.*values\(\)\))).*:DeprecationWarning:smac:0 ignore:.*(Please leave at default or explicitly set .size=None).*:DeprecationWarning:smac:0 ignore:.*(Trying to register a configuration that was not previously suggested).*:UserWarning:.*llamatune.*:0 ignore:.*(DISPLAY environment variable is set).*:UserWarning:.*conftest.*:0 From 7e3c77144b97d2805baed2e9f13af2996d7afc73 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 23:09:47 +0000 Subject: [PATCH 033/100] fixups --- mlos_bench/mlos_bench/services/remote/ssh/ssh_fileshare.py | 4 ++-- mlos_core/mlos_core/tests/spaces/spaces_test.py | 2 +- mlos_viz/mlos_viz/base.py | 6 +++--- mlos_viz/mlos_viz/util.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mlos_bench/mlos_bench/services/remote/ssh/ssh_fileshare.py b/mlos_bench/mlos_bench/services/remote/ssh/ssh_fileshare.py index 137ab024d1c..657da670d97 100644 --- a/mlos_bench/mlos_bench/services/remote/ssh/ssh_fileshare.py +++ b/mlos_bench/mlos_bench/services/remote/ssh/ssh_fileshare.py @@ -50,8 +50,8 @@ async def _start_file_copy( Local path to the file/dir. remote_path : str Remote path to the file/dir. - recursive : bool, optional - _description_, by default True + recursive : bool + Whether to copy recursively. By default True. Raises ------ diff --git a/mlos_core/mlos_core/tests/spaces/spaces_test.py b/mlos_core/mlos_core/tests/spaces/spaces_test.py index b98d40d6270..a54359be906 100644 --- a/mlos_core/mlos_core/tests/spaces/spaces_test.py +++ b/mlos_core/mlos_core/tests/spaces/spaces_test.py @@ -85,7 +85,7 @@ def sample(self, config_space: OptimizerSpace, n_samples: int = 1) -> npt.NDArra ---------- config_space : CS.ConfigurationSpace Configuration space to sample from. - n_samples : int, optional + n_samples : int Number of samples to use, by default 1. """ diff --git a/mlos_viz/mlos_viz/base.py b/mlos_viz/mlos_viz/base.py index 0c6d58cd7f8..69a490ecce3 100644 --- a/mlos_viz/mlos_viz/base.py +++ b/mlos_viz/mlos_viz/base.py @@ -223,12 +223,12 @@ def limit_top_n_configs( The ExperimentData (e.g., obtained from the storage layer) to operate on. results_df : Optional[pandas.DataFrame] The results dataframe to augment, by default None to use the results_df property. - objectives : Iterable[str], optional + objectives : Iterable[str] Which result column(s) to use for sorting the configs, and in which direction ("min" or "max"). By default None to automatically select the experiment objectives. - top_n_configs : int, optional - How many configs to return, including the default, by default 20. + top_n_configs : int + How many configs to return, including the default, by default 10. method: Literal["mean", "median", "p50", "p75", "p90", "p95", "p99"] = "mean", Which statistical method to use when sorting the config groups before determining the cutoff, by default "mean". diff --git a/mlos_viz/mlos_viz/util.py b/mlos_viz/mlos_viz/util.py index cefc3080d9c..d0c7aa56526 100644 --- a/mlos_viz/mlos_viz/util.py +++ b/mlos_viz/mlos_viz/util.py @@ -22,12 +22,12 @@ def expand_results_data_args( Parameters ---------- - exp_data : Optional[ExperimentData], optional + exp_data : Optional[ExperimentData] ExperimentData to operate on. - results_df : Optional[pandas.DataFrame], optional + results_df : Optional[pandas.DataFrame] Optional results_df argument. Defaults to exp_data.results_df property. - objectives : Optional[Dict[str, Literal["min", "max"]]], optional + objectives : Optional[Dict[str, Literal["min", "max"]]] Optional objectives set to operate on. Defaults to exp_data.objectives property. From af53e68dcb623cd34b94a3297b5ecbe10420cc09 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 23:22:05 +0000 Subject: [PATCH 034/100] fixups --- .../remote/azure/azure_network_services.py | 4 ++-- .../mlos_bench/services/remote/azure/azure_saas.py | 8 ++++---- .../services/remote/azure/azure_vm_services.py | 12 ++++++------ .../services/remote/ssh/ssh_host_service.py | 6 +++--- .../mlos_bench/services/remote/ssh/ssh_service.py | 4 +++- .../mlos_bench/services/types/host_ops_type.py | 6 +++--- .../services/types/host_provisioner_type.py | 6 +++--- .../services/types/network_provisioner_type.py | 4 ++-- .../mlos_bench/services/types/os_ops_type.py | 4 ++-- .../services/types/remote_config_type.py | 2 +- .../mlos_bench/storage/base_experiment_data.py | 2 +- mlos_bench/mlos_bench/storage/base_storage.py | 14 +++++++------- mlos_bench/mlos_bench/util.py | 4 ++-- .../bayesian_optimizers/bayesian_optimizer.py | 8 ++++---- mlos_core/mlos_core/spaces/adapters/adapter.py | 8 ++++---- mlos_core/mlos_core/spaces/converters/util.py | 8 ++++---- mlos_core/mlos_core/util.py | 4 ++-- mlos_viz/mlos_viz/__init__.py | 2 +- mlos_viz/mlos_viz/base.py | 4 ++-- mlos_viz/mlos_viz/dabl.py | 2 +- 20 files changed, 57 insertions(+), 55 deletions(-) diff --git a/mlos_bench/mlos_bench/services/remote/azure/azure_network_services.py b/mlos_bench/mlos_bench/services/remote/azure/azure_network_services.py index 29552de4f04..2746caa7320 100644 --- a/mlos_bench/mlos_bench/services/remote/azure/azure_network_services.py +++ b/mlos_bench/mlos_bench/services/remote/azure/azure_network_services.py @@ -125,7 +125,7 @@ def provision_network(self, params: dict) -> Tuple[Status, dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is the input `params` plus the parameters extracted from the response JSON, or {} if the status is FAILED. Status is one of {PENDING, SUCCEEDED, FAILED} @@ -146,7 +146,7 @@ def deprovision_network(self, params: dict, ignore_errors: bool = True) -> Tuple Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ diff --git a/mlos_bench/mlos_bench/services/remote/azure/azure_saas.py b/mlos_bench/mlos_bench/services/remote/azure/azure_saas.py index 042e599f0be..fcd99991f87 100644 --- a/mlos_bench/mlos_bench/services/remote/azure/azure_saas.py +++ b/mlos_bench/mlos_bench/services/remote/azure/azure_saas.py @@ -130,7 +130,7 @@ def configure(self, config: Dict[str, Any], params: Dict[str, Any]) -> Tuple[Sta Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -202,7 +202,7 @@ def _config_one( Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -236,7 +236,7 @@ def _config_many(self, config: Dict[str, Any], params: Dict[str, Any]) -> Tuple[ Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -259,7 +259,7 @@ def _config_batch(self, config: Dict[str, Any], params: Dict[str, Any]) -> Tuple Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ diff --git a/mlos_bench/mlos_bench/services/remote/azure/azure_vm_services.py b/mlos_bench/mlos_bench/services/remote/azure/azure_vm_services.py index b62ede5fab5..856f5e99126 100644 --- a/mlos_bench/mlos_bench/services/remote/azure/azure_vm_services.py +++ b/mlos_bench/mlos_bench/services/remote/azure/azure_vm_services.py @@ -280,7 +280,7 @@ def provision_host(self, params: dict) -> Tuple[Status, dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is the input `params` plus the parameters extracted from the response JSON, or {} if the status is FAILED. Status is one of {PENDING, SUCCEEDED, FAILED} @@ -298,7 +298,7 @@ def deprovision_host(self, params: dict) -> Tuple[Status, dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -340,7 +340,7 @@ def deallocate_host(self, params: dict) -> Tuple[Status, dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -375,7 +375,7 @@ def start_host(self, params: dict) -> Tuple[Status, dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -412,7 +412,7 @@ def stop_host(self, params: dict, force: bool = False) -> Tuple[Status, dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -452,7 +452,7 @@ def restart_host(self, params: dict, force: bool = False) -> Tuple[Status, dict] Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ diff --git a/mlos_bench/mlos_bench/services/remote/ssh/ssh_host_service.py b/mlos_bench/mlos_bench/services/remote/ssh/ssh_host_service.py index 36f1f7866ba..83fc9374989 100644 --- a/mlos_bench/mlos_bench/services/remote/ssh/ssh_host_service.py +++ b/mlos_bench/mlos_bench/services/remote/ssh/ssh_host_service.py @@ -208,7 +208,7 @@ def _exec_os_op(self, cmd_opts_list: List[str], params: dict) -> Tuple[Status, d Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -249,7 +249,7 @@ def shutdown(self, params: dict, force: bool = False) -> Tuple[Status, dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -274,7 +274,7 @@ def reboot(self, params: dict, force: bool = False) -> Tuple[Status, dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. Status is one of {PENDING, SUCCEEDED, FAILED} """ diff --git a/mlos_bench/mlos_bench/services/remote/ssh/ssh_service.py b/mlos_bench/mlos_bench/services/remote/ssh/ssh_service.py index 706764a1f1a..2029de916e0 100644 --- a/mlos_bench/mlos_bench/services/remote/ssh/ssh_service.py +++ b/mlos_bench/mlos_bench/services/remote/ssh/ssh_service.py @@ -115,7 +115,9 @@ def connection_lost(self, exc: Optional[Exception]) -> None: return super().connection_lost(exc) async def connection(self) -> Optional[SSHClientConnection]: - """Waits for and returns the SSHClientConnection to be established or lost.""" + """Waits for and returns the asyncssh.connection.SSHClientConnection to be + established or lost. + """ _LOG.debug("%s: Waiting for connection to be available.", current_thread().name) await self._conn_event.wait() _LOG.debug("%s: Connection available for %s", current_thread().name, self._connection_id) diff --git a/mlos_bench/mlos_bench/services/types/host_ops_type.py b/mlos_bench/mlos_bench/services/types/host_ops_type.py index 166406714da..1ac485530a1 100644 --- a/mlos_bench/mlos_bench/services/types/host_ops_type.py +++ b/mlos_bench/mlos_bench/services/types/host_ops_type.py @@ -25,7 +25,7 @@ def start_host(self, params: dict) -> Tuple["Status", dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -43,7 +43,7 @@ def stop_host(self, params: dict, force: bool = False) -> Tuple["Status", dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -61,7 +61,7 @@ def restart_host(self, params: dict, force: bool = False) -> Tuple["Status", dic Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ diff --git a/mlos_bench/mlos_bench/services/types/host_provisioner_type.py b/mlos_bench/mlos_bench/services/types/host_provisioner_type.py index 1df0716fa13..af5b776a6c5 100644 --- a/mlos_bench/mlos_bench/services/types/host_provisioner_type.py +++ b/mlos_bench/mlos_bench/services/types/host_provisioner_type.py @@ -27,7 +27,7 @@ def provision_host(self, params: dict) -> Tuple["Status", dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -64,7 +64,7 @@ def deprovision_host(self, params: dict) -> Tuple["Status", dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -84,7 +84,7 @@ def deallocate_host(self, params: dict) -> Tuple["Status", dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ diff --git a/mlos_bench/mlos_bench/services/types/network_provisioner_type.py b/mlos_bench/mlos_bench/services/types/network_provisioner_type.py index 3525fbdee13..8332cd5a586 100644 --- a/mlos_bench/mlos_bench/services/types/network_provisioner_type.py +++ b/mlos_bench/mlos_bench/services/types/network_provisioner_type.py @@ -27,7 +27,7 @@ def provision_network(self, params: dict) -> Tuple["Status", dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -71,7 +71,7 @@ def deprovision_network( Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ diff --git a/mlos_bench/mlos_bench/services/types/os_ops_type.py b/mlos_bench/mlos_bench/services/types/os_ops_type.py index 8b727f87a6a..dfb5133e035 100644 --- a/mlos_bench/mlos_bench/services/types/os_ops_type.py +++ b/mlos_bench/mlos_bench/services/types/os_ops_type.py @@ -27,7 +27,7 @@ def shutdown(self, params: dict, force: bool = False) -> Tuple["Status", dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ @@ -45,7 +45,7 @@ def reboot(self, params: dict, force: bool = False) -> Tuple["Status", dict]: Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ diff --git a/mlos_bench/mlos_bench/services/types/remote_config_type.py b/mlos_bench/mlos_bench/services/types/remote_config_type.py index 7e8d0a6e772..5a75dac382d 100644 --- a/mlos_bench/mlos_bench/services/types/remote_config_type.py +++ b/mlos_bench/mlos_bench/services/types/remote_config_type.py @@ -27,7 +27,7 @@ def configure(self, config: Dict[str, Any], params: Dict[str, Any]) -> Tuple["St Returns ------- - result : (Status, dict={}) + result : (Status, dict) A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ diff --git a/mlos_bench/mlos_bench/storage/base_experiment_data.py b/mlos_bench/mlos_bench/storage/base_experiment_data.py index 97f946ef92b..a56e2cc1685 100644 --- a/mlos_bench/mlos_bench/storage/base_experiment_data.py +++ b/mlos_bench/mlos_bench/storage/base_experiment_data.py @@ -78,7 +78,7 @@ def objectives(self) -> Dict[str, Literal["min", "max"]]: Returns ------- - objectives : Dict[str, objective] + objectives : Dict[str, Literal["min", "max"]] A dictionary of the experiment's objective names (optimization_targets) and their directions (e.g., min or max). """ diff --git a/mlos_bench/mlos_bench/storage/base_storage.py b/mlos_bench/mlos_bench/storage/base_storage.py index 867c4e0bc02..15e08e9b789 100644 --- a/mlos_bench/mlos_bench/storage/base_storage.py +++ b/mlos_bench/mlos_bench/storage/base_storage.py @@ -258,7 +258,7 @@ def load_telemetry(self, trial_id: int) -> List[Tuple[datetime, str, Any]]: Returns ------- - metrics : List[Tuple[datetime, str, Any]] + metrics : List[Tuple[datetime.datetime, str, Any]] Telemetry data. """ @@ -298,7 +298,7 @@ def pending_trials( Parameters ---------- - timestamp : datetime + timestamp : datetime.datetime The time in UTC to check for scheduled trials. running : bool If True, include the trials that are already running. @@ -323,7 +323,7 @@ def new_trial( ---------- tunables : TunableGroups Tunable parameters to use for the trial. - ts_start : Optional[datetime] + ts_start : Optional[datetime.datetime] Timestamp of the trial start (can be in the future). config : dict Key/value pairs of additional non-tunable parameters of the trial. @@ -360,7 +360,7 @@ def _new_trial( ---------- tunables : TunableGroups Tunable parameters to use for the trial. - ts_start : Optional[datetime] + ts_start : Optional[datetime.datetime] Timestamp of the trial start (can be in the future). config : dict Key/value pairs of additional non-tunable parameters of the trial. @@ -460,7 +460,7 @@ def update( ---------- status : Status Status of the experiment run. - timestamp: datetime + timestamp: datetime.datetime Timestamp of the status and metrics. metrics : Optional[Dict[str, Any]] One or several metrics of the experiment run. @@ -499,9 +499,9 @@ def update_telemetry( ---------- status : Status Current status of the trial. - timestamp: datetime + timestamp: datetime.datetime Timestamp of the status (but not the metrics). - metrics : List[Tuple[datetime, str, Any]] + metrics : List[Tuple[datetime.datetime, str, Any]] Telemetry data. """ _LOG.info("Store telemetry: %s :: %s %d records", self, status, len(metrics)) diff --git a/mlos_bench/mlos_bench/util.py b/mlos_bench/mlos_bench/util.py index 945359bcddd..cb66c742387 100644 --- a/mlos_bench/mlos_bench/util.py +++ b/mlos_bench/mlos_bench/util.py @@ -353,7 +353,7 @@ def utcify_timestamp(timestamp: datetime, *, origin: Literal["utc", "local"]) -> Parameters ---------- - timestamp : datetime + timestamp : datetime.datetime A timestamp to convert to UTC. Note: The original datetime may or may not have tzinfo associated with it. @@ -367,7 +367,7 @@ def utcify_timestamp(timestamp: datetime, *, origin: Literal["utc", "local"]) -> Returns ------- - datetime + datetime.datetime A datetime with zoneinfo in UTC. """ if timestamp.tzinfo is not None or origin == "local": diff --git a/mlos_core/mlos_core/optimizers/bayesian_optimizers/bayesian_optimizer.py b/mlos_core/mlos_core/optimizers/bayesian_optimizers/bayesian_optimizer.py index a39a5516e83..cfdde1656dd 100644 --- a/mlos_core/mlos_core/optimizers/bayesian_optimizers/bayesian_optimizer.py +++ b/mlos_core/mlos_core/optimizers/bayesian_optimizers/bayesian_optimizer.py @@ -29,11 +29,11 @@ def surrogate_predict( Parameters ---------- - configs : pd.DataFrame + configs : pandas.DataFrame Dataframe of configs / parameters. The columns are parameter names and the rows are the configs. - context : pd.DataFrame + context : pandas.DataFrame Not Yet Implemented. """ pass # pylint: disable=unnecessary-pass # pragma: no cover @@ -51,11 +51,11 @@ def acquisition_function( Parameters ---------- - configs : pd.DataFrame + configs : pandas.DataFrame Dataframe of configs / parameters. The columns are parameter names and the rows are the configs. - context : pd.DataFrame + context : pandas.DataFrame Not Yet Implemented. """ pass # pylint: disable=unnecessary-pass # pragma: no cover diff --git a/mlos_core/mlos_core/spaces/adapters/adapter.py b/mlos_core/mlos_core/spaces/adapters/adapter.py index 2d03918d94a..d8df6b22191 100644 --- a/mlos_core/mlos_core/spaces/adapters/adapter.py +++ b/mlos_core/mlos_core/spaces/adapters/adapter.py @@ -63,13 +63,13 @@ def transform(self, configuration: pd.DataFrame) -> pd.DataFrame: Parameters ---------- - configuration : pd.DataFrame + configuration : pandas.DataFrame Pandas dataframe with a single row. Column names are the parameter names of the target parameter space. Returns ------- - configuration : pd.DataFrame + configuration : pandas.DataFrame Pandas dataframe with a single row, containing the translated configuration. Column names are the parameter names of the original parameter space. """ @@ -85,14 +85,14 @@ def inverse_transform(self, configurations: pd.DataFrame) -> pd.DataFrame: Parameters ---------- - configurations : pd.DataFrame + configurations : pandas.DataFrame Dataframe of configurations / parameters, which belong to the original parameter space. The columns are the parameter names the original parameter space and the rows are the configurations. Returns ------- - configurations : pd.DataFrame + configurations : pandas.DataFrame Dataframe of the translated configurations / parameters. The columns are the parameter names of the target parameter space and the rows are the configurations. diff --git a/mlos_core/mlos_core/spaces/converters/util.py b/mlos_core/mlos_core/spaces/converters/util.py index 4393890595e..1f190931283 100644 --- a/mlos_core/mlos_core/spaces/converters/util.py +++ b/mlos_core/mlos_core/spaces/converters/util.py @@ -20,12 +20,12 @@ def monkey_patch_hp_quantization(hp: Hyperparameter) -> Hyperparameter: Parameters ---------- - hp : Hyperparameter + hp : ConfigSpace.hyperparameters.Hyperparameter ConfigSpace hyperparameter to patch. Returns ------- - hp : Hyperparameter + hp : ConfigSpace.hyperparameters.Hyperparameter Patched hyperparameter. """ if not isinstance(hp, NumericalHyperparameter): @@ -72,12 +72,12 @@ def monkey_patch_cs_quantization(cs: ConfigurationSpace) -> ConfigurationSpace: Parameters ---------- - cs : ConfigurationSpace + cs : ConfigSpace.ConfigurationSpace ConfigSpace to patch. Returns ------- - cs : ConfigurationSpace + cs : ConfigSpace.ConfigurationSpace Patched ConfigSpace. """ for hp in cs.values(): diff --git a/mlos_core/mlos_core/util.py b/mlos_core/mlos_core/util.py index 027bfd0e35b..5a052f5f034 100644 --- a/mlos_core/mlos_core/util.py +++ b/mlos_core/mlos_core/util.py @@ -21,7 +21,7 @@ def config_to_dataframe(config: Configuration) -> pd.DataFrame: Returns ------- - pd.DataFrame + pandas.DataFrame A DataFrame with a single row, containing the config's parameters. """ return pd.DataFrame([dict(config)]) @@ -56,7 +56,7 @@ def normalize_config( Parameters ---------- - config_space : ConfigurationSpace + config_space : ConfigSpace.ConfigurationSpace The parameter space to use. config : dict The configuration to convert. diff --git a/mlos_viz/mlos_viz/__init__.py b/mlos_viz/mlos_viz/__init__.py index 1dfc795d437..fc96215bbe3 100644 --- a/mlos_viz/mlos_viz/__init__.py +++ b/mlos_viz/mlos_viz/__init__.py @@ -63,7 +63,7 @@ def plot( ---------- exp_data: ExperimentData The experiment data to plot. - results_df : Optional["pandas.DataFrame"] + results_df : Optional[pandas.DataFrame] Optional results_df to plot. If not provided, defaults to exp_data.results_df property. objectives : Optional[Dict[str, Literal["min", "max"]]] diff --git a/mlos_viz/mlos_viz/base.py b/mlos_viz/mlos_viz/base.py index 69a490ecce3..c0115ff945a 100644 --- a/mlos_viz/mlos_viz/base.py +++ b/mlos_viz/mlos_viz/base.py @@ -348,7 +348,7 @@ def plot_optimizer_trends( ---------- exp_data : ExperimentData The ExperimentData (e.g., obtained from the storage layer) to plot. - results_df : Optional["pandas.DataFrame"] + results_df : Optional[pandas.DataFrame] Optional results_df to plot. If not provided, defaults to exp_data.results_df property. objectives : Optional[Dict[str, Literal["min", "max"]]] @@ -438,7 +438,7 @@ def plot_top_n_configs( ---------- exp_data: ExperimentData The experiment data to plot. - results_df : Optional["pandas.DataFrame"] + results_df : Optional[pandas.DataFrame] Optional results_df to plot. If not provided, defaults to exp_data.results_df property. objectives : Optional[Dict[str, Literal["min", "max"]]] diff --git a/mlos_viz/mlos_viz/dabl.py b/mlos_viz/mlos_viz/dabl.py index 3f8ac640ad9..4eb81f65de7 100644 --- a/mlos_viz/mlos_viz/dabl.py +++ b/mlos_viz/mlos_viz/dabl.py @@ -26,7 +26,7 @@ def plot( ---------- exp_data : ExperimentData The ExperimentData (e.g., obtained from the storage layer) to plot. - results_df : Optional["pandas.DataFrame"] + results_df : Optional[pandas.DataFrame] Optional results_df to plot. If not provided, defaults to exp_data.results_df property. objectives : Optional[Dict[str, Literal["min", "max"]]] From f2e6aa79fa5c7221222245249eb16eef054937a0 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 11 Nov 2024 23:35:15 +0000 Subject: [PATCH 035/100] Docstring fixups --- doc/source/conf.py | 2 ++ mlos_bench/mlos_bench/dict_templater.py | 3 ++- mlos_bench/mlos_bench/environments/base_environment.py | 4 ++-- mlos_bench/mlos_bench/environments/composite_env.py | 4 ++-- mlos_bench/mlos_bench/environments/local/local_env.py | 2 +- .../mlos_bench/environments/local/local_fileshare_env.py | 2 +- mlos_bench/mlos_bench/environments/mock_env.py | 4 ++-- mlos_bench/mlos_bench/environments/remote/remote_env.py | 4 ++-- mlos_bench/mlos_bench/event_loop_context.py | 2 +- mlos_bench/mlos_bench/optimizers/base_optimizer.py | 2 +- mlos_bench/mlos_bench/optimizers/convert_configspace.py | 6 +++--- mlos_bench/mlos_bench/services/local/temp_dir_context.py | 2 +- .../services/remote/azure/azure_network_services.py | 2 +- mlos_bench/mlos_bench/services/remote/ssh/ssh_service.py | 2 +- mlos_bench/mlos_bench/services/types/authenticator_type.py | 2 +- mlos_bench/mlos_bench/services/types/local_exec_type.py | 2 +- .../mlos_bench/services/types/network_provisioner_type.py | 2 +- mlos_bench/mlos_bench/tunables/tunable.py | 4 ++-- mlos_bench/mlos_bench/tunables/tunable_groups.py | 4 ++-- mlos_core/mlos_core/util.py | 2 +- 20 files changed, 30 insertions(+), 27 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 9aaa44f7c9f..62cae3b906f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -214,6 +214,8 @@ def linkcode_resolve(domain: str, info: Dict[str, str]): # External classes that refuse to resolve: ("py:class", "contextlib.nullcontext"), ("py:class", "sqlalchemy.engine.Engine"), + ("py:exc", "jsonschema.exceptions.SchemaError"), + ("py:exc", "jsonschema.exceptions.ValidationError"), ] nitpick_ignore_regex = [ # Ignore some external references that don't use sphinx for their docs. diff --git a/mlos_bench/mlos_bench/dict_templater.py b/mlos_bench/mlos_bench/dict_templater.py index fcd75cf1b92..742f572ff07 100644 --- a/mlos_bench/mlos_bench/dict_templater.py +++ b/mlos_bench/mlos_bench/dict_templater.py @@ -51,7 +51,8 @@ def expand_vars( Raises ------ - ValueError on unsupported nested types. + ValueError + On unsupported nested types. """ self._dict = deepcopy(self._template_dict) self._dict = self._expand_vars(self._dict, extra_source_dict, use_os_env) diff --git a/mlos_bench/mlos_bench/environments/base_environment.py b/mlos_bench/mlos_bench/environments/base_environment.py index ba346b67654..0819c6b7df4 100644 --- a/mlos_bench/mlos_bench/environments/base_environment.py +++ b/mlos_bench/mlos_bench/environments/base_environment.py @@ -422,7 +422,7 @@ def run(self) -> Tuple[Status, datetime, Optional[Dict[str, TunableValue]]]: Returns ------- - (status, timestamp, output) : (Status, datetime, dict) + (status, timestamp, output) : (Status, datetime.datetime, dict) 3-tuple of (Status, timestamp, output) values, where `output` is a dict with the results or None if the status is not COMPLETED. If run script is a benchmark, then the score is usually expected to @@ -439,7 +439,7 @@ def status(self) -> Tuple[Status, datetime, List[Tuple[datetime, str, Any]]]: Returns ------- - (benchmark_status, timestamp, telemetry) : (Status, datetime, list) + (benchmark_status, timestamp, telemetry) : (Status, datetime.datetime, list) 3-tuple of (benchmark status, timestamp, telemetry) values. `timestamp` is UTC time stamp of the status; it's current time by default. `telemetry` is a list (maybe empty) of (timestamp, metric, value) triplets. diff --git a/mlos_bench/mlos_bench/environments/composite_env.py b/mlos_bench/mlos_bench/environments/composite_env.py index a7edb7bb280..d316ca0696a 100644 --- a/mlos_bench/mlos_bench/environments/composite_env.py +++ b/mlos_bench/mlos_bench/environments/composite_env.py @@ -207,7 +207,7 @@ def run(self) -> Tuple[Status, datetime, Optional[Dict[str, TunableValue]]]: Returns ------- - (status, timestamp, output) : (Status, datetime, dict) + (status, timestamp, output) : (Status, datetime.datetime, dict) 3-tuple of (Status, timestamp, output) values, where `output` is a dict with the results or None if the status is not COMPLETED. If run script is a benchmark, then the score is usually expected to @@ -238,7 +238,7 @@ def status(self) -> Tuple[Status, datetime, List[Tuple[datetime, str, Any]]]: Returns ------- - (benchmark_status, timestamp, telemetry) : (Status, datetime, list) + (benchmark_status, timestamp, telemetry) : (Status, datetime.datetime, list) 3-tuple of (benchmark status, timestamp, telemetry) values. `timestamp` is UTC time stamp of the status; it's current time by default. `telemetry` is a list (maybe empty) of (timestamp, metric, value) triplets. diff --git a/mlos_bench/mlos_bench/environments/local/local_env.py b/mlos_bench/mlos_bench/environments/local/local_env.py index 754cdd34065..93927eea9f0 100644 --- a/mlos_bench/mlos_bench/environments/local/local_env.py +++ b/mlos_bench/mlos_bench/environments/local/local_env.py @@ -167,7 +167,7 @@ def run(self) -> Tuple[Status, datetime, Optional[Dict[str, TunableValue]]]: Returns ------- - (status, timestamp, output) : (Status, datetime, dict) + (status, timestamp, output) : (Status, datetime.datetime, dict) 3-tuple of (Status, timestamp, output) values, where `output` is a dict with the results or None if the status is not COMPLETED. If run script is a benchmark, then the score is usually expected to diff --git a/mlos_bench/mlos_bench/environments/local/local_fileshare_env.py b/mlos_bench/mlos_bench/environments/local/local_fileshare_env.py index 351ed9c480a..70281b5a036 100644 --- a/mlos_bench/mlos_bench/environments/local/local_fileshare_env.py +++ b/mlos_bench/mlos_bench/environments/local/local_fileshare_env.py @@ -175,7 +175,7 @@ def run(self) -> Tuple[Status, datetime, Optional[Dict[str, TunableValue]]]: Returns ------- - (status, timestamp, output) : (Status, datetime, dict) + (status, timestamp, output) : (Status, datetime.datetime, dict) 3-tuple of (Status, timestamp, output) values, where `output` is a dict with the results or None if the status is not COMPLETED. If run script is a benchmark, then the score is usually expected to diff --git a/mlos_bench/mlos_bench/environments/mock_env.py b/mlos_bench/mlos_bench/environments/mock_env.py index 8298f94fe17..a4d61fa9e37 100644 --- a/mlos_bench/mlos_bench/environments/mock_env.py +++ b/mlos_bench/mlos_bench/environments/mock_env.py @@ -88,7 +88,7 @@ def run(self) -> Tuple[Status, datetime, Optional[Dict[str, TunableValue]]]: Returns ------- - (status, timestamp, output) : (Status, datetime, dict) + (status, timestamp, output) : (Status, datetime.datetime, dict) 3-tuple of (Status, timestamp, output) values, where `output` is a dict with the results or None if the status is not COMPLETED. The keys of the `output` dict are the names of the metrics @@ -107,7 +107,7 @@ def status(self) -> Tuple[Status, datetime, List[Tuple[datetime, str, Any]]]: Returns ------- - (benchmark_status, timestamp, telemetry) : (Status, datetime, list) + (benchmark_status, timestamp, telemetry) : (Status, datetime.datetime, list) 3-tuple of (benchmark status, timestamp, telemetry) values. `timestamp` is UTC time stamp of the status; it's current time by default. `telemetry` is a list (maybe empty) of (timestamp, metric, value) triplets. diff --git a/mlos_bench/mlos_bench/environments/remote/remote_env.py b/mlos_bench/mlos_bench/environments/remote/remote_env.py index c3535d1a6a0..0f6264e7d64 100644 --- a/mlos_bench/mlos_bench/environments/remote/remote_env.py +++ b/mlos_bench/mlos_bench/environments/remote/remote_env.py @@ -138,7 +138,7 @@ def run(self) -> Tuple[Status, datetime, Optional[Dict[str, TunableValue]]]: Returns ------- - (status, timestamp, output) : (Status, datetime, dict) + (status, timestamp, output) : (Status, datetime.datetime, dict) 3-tuple of (Status, timestamp, output) values, where `output` is a dict with the results or None if the status is not COMPLETED. If run script is a benchmark, then the score is usually expected to @@ -180,7 +180,7 @@ def _remote_exec( Returns ------- - result : (Status, datetime, dict) + result : (Status, datetime.datetime, dict) 3-tuple of Status, timestamp, and dict with the benchmark/script results. Status is one of {PENDING, SUCCEEDED, FAILED, TIMED_OUT} """ diff --git a/mlos_bench/mlos_bench/event_loop_context.py b/mlos_bench/mlos_bench/event_loop_context.py index 65285e5d66a..8144bc399d2 100644 --- a/mlos_bench/mlos_bench/event_loop_context.py +++ b/mlos_bench/mlos_bench/event_loop_context.py @@ -94,7 +94,7 @@ def run_coroutine(self, coro: Coroutine[Any, Any, CoroReturnType]) -> FutureRetu Returns ------- - Future[CoroReturnType] + concurrent.futures.Future[CoroReturnType] A future that will be completed when the coroutine completes. """ assert self._event_loop_thread_refcnt > 0 diff --git a/mlos_bench/mlos_bench/optimizers/base_optimizer.py b/mlos_bench/mlos_bench/optimizers/base_optimizer.py index 1d20f0cf31b..8abddc4660e 100644 --- a/mlos_bench/mlos_bench/optimizers/base_optimizer.py +++ b/mlos_bench/mlos_bench/optimizers/base_optimizer.py @@ -186,7 +186,7 @@ def config_space(self) -> ConfigurationSpace: Returns ------- - ConfigurationSpace + ConfigSpace.ConfigurationSpace The ConfigSpace representation of the tunable parameters. """ if self._config_space is None: diff --git a/mlos_bench/mlos_bench/optimizers/convert_configspace.py b/mlos_bench/mlos_bench/optimizers/convert_configspace.py index 755918fa99d..3545936623c 100644 --- a/mlos_bench/mlos_bench/optimizers/convert_configspace.py +++ b/mlos_bench/mlos_bench/optimizers/convert_configspace.py @@ -73,7 +73,7 @@ def _tunable_to_configspace( Returns ------- - cs : ConfigurationSpace + cs : ConfigSpace.ConfigurationSpace A ConfigurationSpace object that corresponds to the Tunable. """ # pylint: disable=too-complex @@ -206,7 +206,7 @@ def tunable_groups_to_configspace( Returns ------- - configspace : ConfigurationSpace + configspace : ConfigSpace.ConfigurationSpace A new ConfigurationSpace instance that corresponds to the input TunableGroups. """ space = ConfigurationSpace(seed=seed) @@ -234,7 +234,7 @@ def tunable_values_to_configuration(tunables: TunableGroups) -> Configuration: Returns ------- - Configuration + ConfigSpace.Configuration A ConfigSpace Configuration. """ values: Dict[str, TunableValue] = {} diff --git a/mlos_bench/mlos_bench/services/local/temp_dir_context.py b/mlos_bench/mlos_bench/services/local/temp_dir_context.py index e65a45934b7..bf730ae3452 100644 --- a/mlos_bench/mlos_bench/services/local/temp_dir_context.py +++ b/mlos_bench/mlos_bench/services/local/temp_dir_context.py @@ -77,7 +77,7 @@ def temp_dir_context( Returns ------- - temp_dir_context : TemporaryDirectory + temp_dir_context : tempfile.TemporaryDirectory Temporary directory context to use in the `with` clause. """ temp_dir = path or self._temp_dir diff --git a/mlos_bench/mlos_bench/services/remote/azure/azure_network_services.py b/mlos_bench/mlos_bench/services/remote/azure/azure_network_services.py index 2746caa7320..4f11e89aa2f 100644 --- a/mlos_bench/mlos_bench/services/remote/azure/azure_network_services.py +++ b/mlos_bench/mlos_bench/services/remote/azure/azure_network_services.py @@ -140,7 +140,7 @@ def deprovision_network(self, params: dict, ignore_errors: bool = True) -> Tuple ---------- params : dict Flat dictionary of (key, value) pairs of tunable parameters. - ignore_errors : boolean + ignore_errors : bool Whether to ignore errors (default) encountered during the operation (e.g., due to dependent resources still in use). diff --git a/mlos_bench/mlos_bench/services/remote/ssh/ssh_service.py b/mlos_bench/mlos_bench/services/remote/ssh/ssh_service.py index 2029de916e0..7e33d715ec3 100644 --- a/mlos_bench/mlos_bench/services/remote/ssh/ssh_service.py +++ b/mlos_bench/mlos_bench/services/remote/ssh/ssh_service.py @@ -178,7 +178,7 @@ async def get_client_connection( Returns ------- - Tuple[SSHClientConnection, SshClient] + Tuple[asyncssh.connection.SSHClientConnection, SshClient] A tuple of (SSHClientConnection, SshClient). """ _LOG.debug("%s: get_client_connection: %s", current_thread().name, connect_params) diff --git a/mlos_bench/mlos_bench/services/types/authenticator_type.py b/mlos_bench/mlos_bench/services/types/authenticator_type.py index b01c30d42de..4d06a7867ff 100644 --- a/mlos_bench/mlos_bench/services/types/authenticator_type.py +++ b/mlos_bench/mlos_bench/services/types/authenticator_type.py @@ -39,6 +39,6 @@ def get_credential(self) -> T_co: Returns ------- - credential : T + credential : T_co Cloud-specific credential object. """ diff --git a/mlos_bench/mlos_bench/services/types/local_exec_type.py b/mlos_bench/mlos_bench/services/types/local_exec_type.py index d0c8c357f0d..96d5042d3e9 100644 --- a/mlos_bench/mlos_bench/services/types/local_exec_type.py +++ b/mlos_bench/mlos_bench/services/types/local_exec_type.py @@ -71,6 +71,6 @@ def temp_dir_context( Returns ------- - temp_dir_context : TemporaryDirectory + temp_dir_context : tempfile.TemporaryDirectory Temporary directory context to use in the `with` clause. """ diff --git a/mlos_bench/mlos_bench/services/types/network_provisioner_type.py b/mlos_bench/mlos_bench/services/types/network_provisioner_type.py index 8332cd5a586..542028b7eaf 100644 --- a/mlos_bench/mlos_bench/services/types/network_provisioner_type.py +++ b/mlos_bench/mlos_bench/services/types/network_provisioner_type.py @@ -65,7 +65,7 @@ def deprovision_network( ---------- params : dict Flat dictionary of (key, value) pairs of tunable parameters. - ignore_errors : boolean + ignore_errors : bool Whether to ignore errors (default) encountered during the operation (e.g., due to dependent resources still in use). diff --git a/mlos_bench/mlos_bench/tunables/tunable.py b/mlos_bench/mlos_bench/tunables/tunable.py index 4d2781ad11b..120be58238d 100644 --- a/mlos_bench/mlos_bench/tunables/tunable.py +++ b/mlos_bench/mlos_bench/tunables/tunable.py @@ -406,7 +406,7 @@ def in_range(self, value: Union[int, float, str, None]) -> bool: @property def category(self) -> Optional[str]: - """Get the current value of the tunable as a number.""" + """Get the current value of the tunable as a string.""" if self.is_categorical: return nullable(str, self._current_value) else: @@ -556,7 +556,7 @@ def range(self) -> Union[Tuple[int, int], Tuple[float, float]]: Returns ------- - range : (number, number) + range : Union[Tuple[int, int], Tuple[float, float]] A 2-tuple of numbers that represents the range of the tunable. Numbers can be int or float, depending on the type of the tunable. """ diff --git a/mlos_bench/mlos_bench/tunables/tunable_groups.py b/mlos_bench/mlos_bench/tunables/tunable_groups.py index 45d8a02f9c9..78b4c720a4a 100644 --- a/mlos_bench/mlos_bench/tunables/tunable_groups.py +++ b/mlos_bench/mlos_bench/tunables/tunable_groups.py @@ -172,13 +172,13 @@ def __setitem__( self._index[name][name] = value return self._index[name][name] - def __iter__(self) -> Generator[Tuple[Tunable, CovariantTunableGroup], None, None]: + def __iter__(self) -> Generator[Tuple[Tunable, CovariantTunableGroup]]: """ An iterator over all tunables in the group. Returns ------- - [(tunable, group), ...] : iter(Tunable, CovariantTunableGroup) + [(tunable, group), ...] : Generator[Tuple[Tunable, CovariantTunableGroup]] An iterator over all tunables in all groups. Each element is a 2-tuple of an instance of the Tunable parameter and covariant group it belongs to. """ diff --git a/mlos_core/mlos_core/util.py b/mlos_core/mlos_core/util.py index 5a052f5f034..2e1c382a31a 100644 --- a/mlos_core/mlos_core/util.py +++ b/mlos_core/mlos_core/util.py @@ -63,7 +63,7 @@ def normalize_config( Returns ------- - cs_config: Configuration + cs_config: ConfigSpace.Configuration A valid ConfigSpace configuration with inactive parameters removed. """ cs_config = Configuration(config_space, values=config, allow_inactive_with_values=True) From 5eb79ac53fe030d772026a5b7060c7cd060814b1 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 18:45:05 +0000 Subject: [PATCH 036/100] Makefile improvements and remove unnecessary templates --- Makefile | 13 ++++++++----- doc/source/_templates/class.rst | 16 ---------------- doc/source/_templates/function.rst | 12 ------------ doc/source/_templates/numpydoc_docstring.py | 20 -------------------- 4 files changed, 8 insertions(+), 53 deletions(-) delete mode 100644 doc/source/_templates/class.rst delete mode 100644 doc/source/_templates/function.rst delete mode 100644 doc/source/_templates/numpydoc_docstring.py diff --git a/Makefile b/Makefile index 7a0af171d16..c96fd3d0099 100644 --- a/Makefile +++ b/Makefile @@ -672,11 +672,18 @@ SPHINXOPTS += -n -W -w $(PWD)/doc/build/sphinx-build.warn.log -j auto sphinx-apidoc: doc/build/html/index.html +doc/source/generated/mlos_bench.run.usage.txt: build/conda-env.${CONDA_ENV_NAME}.build-stamp +doc/source/generated/mlos_bench.run.usage.txt: $(MLOS_BENCH_PYTHON_FILES) + # Generate the help output from mlos_bench CLI for the docs. + mkdir -p doc/source/generated/ + conda run -n ${CONDA_ENV_NAME} mlos_bench --help > doc/source/generated/mlos_bench.run.usage.txt + doc/build/html/index.html: build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp +doc/build/html/index.html: doc/source/generated/mlos_bench.run.usage.txt doc/build/html/index.html: $(MLOS_CORE_PYTHON_FILES) doc/build/html/index.html: $(MLOS_BENCH_PYTHON_FILES) doc/build/html/index.html: $(MLOS_VIZ_PYTHON_FILES) -doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile doc/source/conf.py doc/source/conf.py +doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile doc/source/conf.py doc/build/html/index.html: doc/copy-source-tree-docs.sh $(MD_FILES) @rm -rf doc/build @mkdir -p doc/build @@ -690,10 +697,6 @@ doc/build/html/index.html: doc/copy-source-tree-docs.sh $(MD_FILES) # Copy some of the source tree markdown docs into place. ./doc/copy-source-tree-docs.sh - # Generate the help output from mlos_bench CLI for the docs. - mkdir -p doc/source/generated/ - conda run -n ${CONDA_ENV_NAME} mlos_bench --help > doc/source/generated/mlos_bench.run.usage.txt - # Build the rst files into html. conda run -n ${CONDA_ENV_NAME} $(MAKE) SPHINXOPTS="$(SPHINXOPTS)" -C doc/ $(MAKEFLAGS) html \ >> doc/build/log.txt 2>&1 \ diff --git a/doc/source/_templates/class.rst b/doc/source/_templates/class.rst deleted file mode 100644 index 3eef9746722..00000000000 --- a/doc/source/_templates/class.rst +++ /dev/null @@ -1,16 +0,0 @@ -:mod:`{{module}}`.{{objname}} -{{ underline }}============== - -.. currentmodule:: {{ module }} - -.. autoclass:: {{ objname }} - - {% block methods %} - .. automethod:: __init__ - {% endblock %} - -.. include:: {{module}}.{{objname}}.examples - -.. raw:: html - -
diff --git a/doc/source/_templates/function.rst b/doc/source/_templates/function.rst deleted file mode 100644 index 4ba355d57c8..00000000000 --- a/doc/source/_templates/function.rst +++ /dev/null @@ -1,12 +0,0 @@ -:mod:`{{module}}`.{{objname}} -{{ underline }}==================== - -.. currentmodule:: {{ module }} - -.. autofunction:: {{ objname }} - -.. include:: {{module}}.{{objname}}.examples - -.. raw:: html - -
diff --git a/doc/source/_templates/numpydoc_docstring.py b/doc/source/_templates/numpydoc_docstring.py deleted file mode 100644 index 71acc5df9f0..00000000000 --- a/doc/source/_templates/numpydoc_docstring.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# -{{index}} -{{summary}} -{{extended_summary}} -{{parameters}} -{{returns}} -{{yields}} -{{other_parameters}} -{{attributes}} -{{raises}} -{{warns}} -{{warnings}} -{{see_also}} -{{notes}} -{{references}} -{{examples}} -{{methods}} From 2ca9caf93e70144364ca41a22811aa9c60c5fefa Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 18:46:40 +0000 Subject: [PATCH 037/100] use improvements to intersphinx registry --- doc/requirements.txt | 3 +-- doc/source/conf.py | 34 +++++++--------------------------- 2 files changed, 8 insertions(+), 29 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 4611ad88df6..e104e2a760f 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,8 +1,7 @@ setuptools-scm>=8.1.0 sphinx sphinx-autoapi -sphinx-autodoc-typehints -intersphinx_registry +intersphinx_registry>=0.2410.14 # https://github.com/Quansight-Labs/intersphinx_registry/pull/41 nbsphinx jupyter_core>=4.11.2 # nbsphix dependency - addresses CVE-2022-39286 nbconvert diff --git a/doc/source/conf.py b/doc/source/conf.py index 62cae3b906f..090e67c78d5 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -137,41 +137,21 @@ def linkcode_resolve(domain: str, info: Dict[str, str]): intersphinx_mapping = get_intersphinx_mapping( packages={ "asyncssh", + "azure-core", + "azure-identity", + "configspace", "matplotlib", "numpy", "pandas", "python", + "referencing", + "smac", + "typing_extensions", } ) - -# TODO: convert these to registry calls once the following PR is merged: -# https://github.com/Quansight-Labs/intersphinx_registry/pull/41 intersphinx_mapping.update( { - "azure-core": ( - "https://azuresdkdocs.blob.core.windows.net/$web/python/azure-core/latest/", - None, - ), - "azure-identity": ( - "https://azuresdkdocs.blob.core.windows.net/$web/python/azure-identity/latest/", - None, - ), - "ConfigSpace": ( - "https://automl.github.io/ConfigSpace/latest/", - None, - ), - "referencing": ( - "https://referencing.readthedocs.io/en/stable/", - None, - ), - "smac": ( - "https://automl.github.io/SMAC3/main/", - None, - ), - "typing_extensions": ( - "https://typing-extensions.readthedocs.io/en/stable/", - None, - ), + # "ConfigSpace": ("https://automl.github.io/ConfigSpace/latest/", None), } ) From d5476e936ca74dc6e09d8c81aef1c73d4730f8dd Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 18:47:48 +0000 Subject: [PATCH 038/100] remove old modules and tweaks to theme --- doc/requirements.txt | 1 - doc/source/conf.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index e104e2a760f..fac2db1fa4c 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -6,7 +6,6 @@ nbsphinx jupyter_core>=4.11.2 # nbsphix dependency - addresses CVE-2022-39286 nbconvert mistune>=2.0.3 # Address CVE-2022-34749 -numpydoc sphinx-rtd-theme myst-parser diff --git a/doc/source/conf.py b/doc/source/conf.py index 090e67c78d5..28fbb6bac79 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -26,7 +26,6 @@ from logging import warning from intersphinx_registry import get_intersphinx_mapping -import sphinx_rtd_theme # pylint: disable=unused-import sys.path.insert(0, os.path.abspath("../../mlos_core/mlos_core")) @@ -210,7 +209,7 @@ def linkcode_resolve(domain: str, info: Dict[str, str]): } # Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] +# templates_path = ["_templates"] # Generate the plots for the gallery # plot_gallery = True @@ -259,7 +258,6 @@ def linkcode_resolve(domain: str, info: Dict[str, str]): # a list of builtin themes. # html_theme = "sphinx_rtd_theme" -# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, From 625cf2b90597fa82b4843c5011b5835fb5dca1bb Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 18:48:21 +0000 Subject: [PATCH 039/100] remove old things and cleanup confs --- doc/source/conf.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 28fbb6bac79..c170216443a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -66,24 +66,15 @@ # ones. extensions = [ "sphinx.ext.autodoc", - # "sphinx_autodoc_typehints", "autoapi.extension", "nbsphinx", - "sphinx.ext.doctest", "sphinx.ext.intersphinx", "sphinx.ext.linkcode", - # "numpydoc", "sphinx.ext.napoleon", "matplotlib.sphinxext.plot_directive", "myst_parser", ] -# autodoc_typehints = "both" -# autodoc_typehints_description_target = "documented" - -# typehints_fully_qualified = True -# typehints_defaults = "braces" -## typehints_use_signature = True -## typehints_use_signature_return = True +autodoc_typehints = "both" # signature and description napoleon_numpy_docstring = True napoleon_include_init_with_doc = False From 202dcc6127a4b89999ec29d7d27a37578b8ab860 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 18:48:33 +0000 Subject: [PATCH 040/100] napolean tweaks --- doc/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index c170216443a..c7ec0f0bb34 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -77,9 +77,9 @@ autodoc_typehints = "both" # signature and description napoleon_numpy_docstring = True -napoleon_include_init_with_doc = False +napoleon_include_init_with_doc = True napoleon_include_private_with_doc = False -napoleon_include_special_with_doc = False +napoleon_include_special_with_doc = True napoleon_use_admonition_for_examples = False napoleon_use_admonition_for_notes = False napoleon_use_admonition_for_references = False From 96b96e97b6d4aaca38c3c575289fda4f6d990e88 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 18:49:47 +0000 Subject: [PATCH 041/100] use correct source for Literal --- doc/source/conf.py | 6 +++++- mlos_bench/mlos_bench/environments/base_environment.py | 2 +- mlos_bench/mlos_bench/environments/composite_env.py | 4 +--- mlos_bench/mlos_bench/environments/local/local_env.py | 3 +-- mlos_bench/mlos_bench/optimizers/base_optimizer.py | 3 +-- mlos_bench/mlos_bench/optimizers/mlos_core_optimizer.py | 3 +-- mlos_bench/mlos_bench/schedulers/base_scheduler.py | 3 +-- mlos_bench/mlos_bench/services/base_service.py | 4 +--- mlos_bench/mlos_bench/storage/base_storage.py | 4 +--- mlos_bench/mlos_bench/tests/event_loop_context_test.py | 3 +-- 10 files changed, 14 insertions(+), 21 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index c7ec0f0bb34..83d81c5d51c 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -178,7 +178,6 @@ def linkcode_resolve(domain: str, info: Dict[str, str]): # External typevars and aliases: ("py:class", "CoroReturnType"), ("py:class", "FutureReturnType"), - ("py:class", "typing.Literal"), ("py:class", "typing_extensions.Literal"), ("py:class", "numpy.typing.NDArray"), # External classes that refuse to resolve: @@ -186,10 +185,15 @@ def linkcode_resolve(domain: str, info: Dict[str, str]): ("py:class", "sqlalchemy.engine.Engine"), ("py:exc", "jsonschema.exceptions.SchemaError"), ("py:exc", "jsonschema.exceptions.ValidationError"), + # ("py:func", "create_connection"), + # ("py:class", "SSHClient"), + # ("py:class", "SSHKey"), ] nitpick_ignore_regex = [ # Ignore some external references that don't use sphinx for their docs. (r"py:.*", r"flaml\..*"), + # Ignore some external inheritance classes that don't resolve. + # (r"py:.*", r"asyncssh\..*"), ] # Which documents to include in the build. diff --git a/mlos_bench/mlos_bench/environments/base_environment.py b/mlos_bench/mlos_bench/environments/base_environment.py index 0819c6b7df4..8c3ab6c3fa9 100644 --- a/mlos_bench/mlos_bench/environments/base_environment.py +++ b/mlos_bench/mlos_bench/environments/base_environment.py @@ -15,6 +15,7 @@ Dict, Iterable, List, + Literal, Optional, Sequence, Tuple, @@ -23,7 +24,6 @@ ) from pytz import UTC -from typing_extensions import Literal from mlos_bench.config.schemas import ConfigSchema from mlos_bench.dict_templater import DictTemplater diff --git a/mlos_bench/mlos_bench/environments/composite_env.py b/mlos_bench/mlos_bench/environments/composite_env.py index d316ca0696a..d81bd2f729c 100644 --- a/mlos_bench/mlos_bench/environments/composite_env.py +++ b/mlos_bench/mlos_bench/environments/composite_env.py @@ -7,9 +7,7 @@ import logging from datetime import datetime from types import TracebackType -from typing import Any, Dict, List, Optional, Tuple, Type - -from typing_extensions import Literal +from typing import Any, Dict, List, Literal, Optional, Tuple, Type from mlos_bench.environments.base_environment import Environment from mlos_bench.environments.status import Status diff --git a/mlos_bench/mlos_bench/environments/local/local_env.py b/mlos_bench/mlos_bench/environments/local/local_env.py index 93927eea9f0..7508b6678ff 100644 --- a/mlos_bench/mlos_bench/environments/local/local_env.py +++ b/mlos_bench/mlos_bench/environments/local/local_env.py @@ -11,10 +11,9 @@ from datetime import datetime from tempfile import TemporaryDirectory from types import TracebackType -from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Type, Union +from typing import Any, Dict, Iterable, List, Literal, Mapping, Optional, Tuple, Type, Union import pandas -from typing_extensions import Literal from mlos_bench.environments.base_environment import Environment from mlos_bench.environments.script_env import ScriptEnv diff --git a/mlos_bench/mlos_bench/optimizers/base_optimizer.py b/mlos_bench/mlos_bench/optimizers/base_optimizer.py index 8abddc4660e..14eb3eba6ce 100644 --- a/mlos_bench/mlos_bench/optimizers/base_optimizer.py +++ b/mlos_bench/mlos_bench/optimizers/base_optimizer.py @@ -9,10 +9,9 @@ import logging from abc import ABCMeta, abstractmethod from types import TracebackType -from typing import Dict, Optional, Sequence, Tuple, Type, Union +from typing import Dict, Literal, Optional, Sequence, Tuple, Type, Union from ConfigSpace import ConfigurationSpace -from typing_extensions import Literal from mlos_bench.config.schemas import ConfigSchema from mlos_bench.environments.status import Status diff --git a/mlos_bench/mlos_bench/optimizers/mlos_core_optimizer.py b/mlos_bench/mlos_bench/optimizers/mlos_core_optimizer.py index 649b070123b..f9d5685ae8f 100644 --- a/mlos_bench/mlos_bench/optimizers/mlos_core_optimizer.py +++ b/mlos_bench/mlos_bench/optimizers/mlos_core_optimizer.py @@ -7,10 +7,9 @@ import logging import os from types import TracebackType -from typing import Dict, Optional, Sequence, Tuple, Type, Union +from typing import Dict, Literal, Optional, Sequence, Tuple, Type, Union import pandas as pd -from typing_extensions import Literal from mlos_bench.environments.status import Status from mlos_bench.optimizers.base_optimizer import Optimizer diff --git a/mlos_bench/mlos_bench/schedulers/base_scheduler.py b/mlos_bench/mlos_bench/schedulers/base_scheduler.py index f38e51e7133..5d51650a59f 100644 --- a/mlos_bench/mlos_bench/schedulers/base_scheduler.py +++ b/mlos_bench/mlos_bench/schedulers/base_scheduler.py @@ -9,10 +9,9 @@ from abc import ABCMeta, abstractmethod from datetime import datetime from types import TracebackType -from typing import Any, Dict, List, Optional, Tuple, Type +from typing import Any, Dict, List, Literal, Optional, Tuple, Type from pytz import UTC -from typing_extensions import Literal from mlos_bench.config.schemas import ConfigSchema from mlos_bench.environments.base_environment import Environment diff --git a/mlos_bench/mlos_bench/services/base_service.py b/mlos_bench/mlos_bench/services/base_service.py index c5d9b78c873..8afb0b55f71 100644 --- a/mlos_bench/mlos_bench/services/base_service.py +++ b/mlos_bench/mlos_bench/services/base_service.py @@ -7,9 +7,7 @@ import json import logging from types import TracebackType -from typing import Any, Callable, Dict, List, Optional, Set, Type, Union - -from typing_extensions import Literal +from typing import Any, Callable, Dict, List, Literal, Optional, Set, Type, Union from mlos_bench.config.schemas import ConfigSchema from mlos_bench.services.types.config_loader_type import SupportsConfigLoading diff --git a/mlos_bench/mlos_bench/storage/base_storage.py b/mlos_bench/mlos_bench/storage/base_storage.py index 15e08e9b789..442e497c909 100644 --- a/mlos_bench/mlos_bench/storage/base_storage.py +++ b/mlos_bench/mlos_bench/storage/base_storage.py @@ -8,9 +8,7 @@ from abc import ABCMeta, abstractmethod from datetime import datetime from types import TracebackType -from typing import Any, Dict, Iterator, List, Optional, Tuple, Type - -from typing_extensions import Literal +from typing import Any, Dict, Iterator, List, Literal, Optional, Tuple, Type from mlos_bench.config.schemas import ConfigSchema from mlos_bench.dict_templater import DictTemplater diff --git a/mlos_bench/mlos_bench/tests/event_loop_context_test.py b/mlos_bench/mlos_bench/tests/event_loop_context_test.py index eb92c4c1326..09f94eeb91d 100644 --- a/mlos_bench/mlos_bench/tests/event_loop_context_test.py +++ b/mlos_bench/mlos_bench/tests/event_loop_context_test.py @@ -10,10 +10,9 @@ from asyncio import AbstractEventLoop from threading import Thread from types import TracebackType -from typing import Optional, Type +from typing import Literal, Optional, Type import pytest -from typing_extensions import Literal from mlos_bench.event_loop_context import EventLoopContext From 7f1d160ef07f73a3eda8efaebb99077841f1ab05 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 18:49:59 +0000 Subject: [PATCH 042/100] cleanup --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 83d81c5d51c..47c97017ffc 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -245,7 +245,7 @@ def linkcode_resolve(domain: str, info: Dict[str, str]): # "imported-members", ] autoapi_add_toctree_entry = False -autoapi_keep_files = True # for testing +# autoapi_keep_files = True # for testing # -- Options for HTML output ------------------------------------------------- From 07f32a58cc02354d54b8d74fcd8fc9b149df9d66 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 18:50:24 +0000 Subject: [PATCH 043/100] cleanup --- doc/source/index.rst | 15 ++++++++++----- doc/source/mlos_bench.run.usage.rst | 2 +- mlos_viz/README.md | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index 2d8fa1317ad..68bd53dbef9 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,27 +1,32 @@ -Welcome to the MLOS documentation! -================================== +MLOS Documentation +================== .. image:: badges/tests.svg .. image:: badges/coverage.svg :target: htmlcov/index.html -`MLOS `_ is a project to enable autotuning for systems via automated benchmarking including managing the storage and visualization of the results. +`MLOS `_ is a project to enable autotuning for systems via `automated benchmarking `_ including managing the storage and `visualization `_ of the results. See below for additional documentation sections. +Here is some documentation pulled from the markdown files in the `MLOS source tree `_: + .. toctree:: - :maxdepth: 2 :caption: Source Tree Documentation + :maxdepth: 5 source_tree_docs/index source_tree_docs/mlos_core/index source_tree_docs/mlos_bench/index source_tree_docs/mlos_viz/index +Here is some documentation pulled from the Python docstrings in the `MLOS source tree `_: + .. toctree:: :caption: API Reference - :maxdepth: 2 + :titlesonly: + :maxdepth: 5 autoapi/mlos_core/index autoapi/mlos_bench/index diff --git a/doc/source/mlos_bench.run.usage.rst b/doc/source/mlos_bench.run.usage.rst index b1168e39327..ca9accdce45 100644 --- a/doc/source/mlos_bench.run.usage.rst +++ b/doc/source/mlos_bench.run.usage.rst @@ -1,7 +1,7 @@ mlos_bench CLI usage ================================== -Here is the current `--help` output for the `mlos_bench` CLI: +Here is the current ``--help`` output for the ``mlos_bench`` CLI: .. literalinclude:: ./generated/mlos_bench.run.usage.txt :language: none diff --git a/mlos_viz/README.md b/mlos_viz/README.md index e4c5c907426..dd744442c28 100644 --- a/mlos_viz/README.md +++ b/mlos_viz/README.md @@ -1,4 +1,4 @@ -# mlos_viz +# mlos-viz The [`mlos_viz`](./) module is an aid to visualizing experiment benchmarking and optimization results generated and stored by [`mlos_bench`](../mlos_bench/). From 8bef4f2f51a468a16fdc99a9cfc72f837dcfbf47 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 19:03:46 +0000 Subject: [PATCH 044/100] improve docs --- README.md | 5 ++++- mlos_bench/mlos_bench/util.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2c1160dc938..dbc48ffb4fc 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,10 @@ Details on using a local version from git are available in [CONTRIBUTING.md](./C Working example of tuning `sqlite` with MLOS. -These can be used as starting points for new autotuning projects. +These can be used as starting points for new autotuning projects outside of the main MLOS repository if you want to keep your tuning experiment configs separate from the MLOS codebase. + +Alternatively, we accept PRs to add new examples to the main MLOS repository! +See [mlos_bench/config](./mlos_bench/mlos_bench/config/) and [CONTRIBUTING.md](./CONTRIBUTING.md) for more details. ### Publications diff --git a/mlos_bench/mlos_bench/util.py b/mlos_bench/mlos_bench/util.py index cb66c742387..0a75ec9a7d2 100644 --- a/mlos_bench/mlos_bench/util.py +++ b/mlos_bench/mlos_bench/util.py @@ -50,8 +50,16 @@ def strtobool(val: str) -> bool: """ Convert a string representation of truth to true (1) or false (0). - True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values are 'n', 'no', - 'f', 'false', 'off', and '0'. Raises ValueError if 'val' is anything else. + Parameters + ---------- + val : str + True values are 'y', 'yes', 't', 'true', 'on', and '1'; + False values are 'n', 'no', 'f', 'false', 'off', and '0'. + + Raises + ------ + ValueError + If 'val' is anything else. """ val = val.lower() if val in {"y", "yes", "t", "true", "on", "1"}: From 7cc7297c10b016c15df5403e905097f2222e876b Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 19:05:54 +0000 Subject: [PATCH 045/100] tweaks --- mlos_bench/mlos_bench/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlos_bench/mlos_bench/util.py b/mlos_bench/mlos_bench/util.py index 0a75ec9a7d2..9f2d0101445 100644 --- a/mlos_bench/mlos_bench/util.py +++ b/mlos_bench/mlos_bench/util.py @@ -72,7 +72,7 @@ def strtobool(val: str) -> bool: def preprocess_dynamic_configs(*, dest: dict, source: Optional[dict] = None) -> dict: """ - Replaces all $name values in the destination config with the corresponding value + Replaces all ``$name`` values in the destination config with the corresponding value from the source config. Parameters From aebea53399ff1f54545b7a7f118cc2cb9fc70f56 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 19:17:36 +0000 Subject: [PATCH 046/100] improve docs --- doc/copy-source-tree-docs.sh | 1 + mlos_bench/mlos_bench/util.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/copy-source-tree-docs.sh b/doc/copy-source-tree-docs.sh index 0eeac02fdc3..027a7448e70 100755 --- a/doc/copy-source-tree-docs.sh +++ b/doc/copy-source-tree-docs.sh @@ -24,6 +24,7 @@ for readme_file_path in README.md mlos_core/README.md mlos_bench/README.md mlos_ cp "$readme_file_path" "doc/source/source_tree_docs/$file_dir/index.md" # Tweak source source code links. + # FIXME: This sed expression doesn't work in MacOS. sed -i -r -e "s|\]\(([^:#)]+)(#[a-zA-Z0-9_-]+)?\)|\]\(https://github.com/microsoft/MLOS/tree/main/$file_dir/\1\2\)|g" \ "doc/source/source_tree_docs/$file_dir/index.md" # Tweak the lexers for local expansion by pygments instead of github's. diff --git a/mlos_bench/mlos_bench/util.py b/mlos_bench/mlos_bench/util.py index 9f2d0101445..dde7c2da8a9 100644 --- a/mlos_bench/mlos_bench/util.py +++ b/mlos_bench/mlos_bench/util.py @@ -39,9 +39,19 @@ from mlos_bench.services.base_service import Service from mlos_bench.storage.base_storage import Storage -# BaseTypeVar is a generic with a constraint of the three base classes. BaseTypeVar = TypeVar("BaseTypeVar", "Environment", "Optimizer", "Scheduler", "Service", "Storage") +""" +BaseTypeVar is a generic with a constraint of the main base classes (e.g., +:py:class:`~mlos_bench.environments.base_environment.Environment`, +:py:class:`~mlos_bench.optimizers.base_optimizer.Optimizer`, +:py:class:`~mlos_bench.schedulers.base_scheduler.Scheduler`, +:py:class:`~mlos_bench.services.base_service.Service`, +:py:class:`~mlos_bench.storage.base_storage.Storage`, +etc.). +""" + BaseTypes = Union["Environment", "Optimizer", "Scheduler", "Service", "Storage"] +"""Similar to :py:data:`.BaseTypeVar`, BaseTypes is a Union of the main base classes.""" # Adjusted from https://github.com/python/cpython/blob/v3.11.10/Lib/distutils/util.py#L308 From 447116ac7b5c9714ec82b818d292b81b40e01a9a Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 19:47:47 +0000 Subject: [PATCH 047/100] improvements to docs --- doc/source/conf.py | 6 ++++-- mlos_bench/mlos_bench/environments/script_env.py | 4 ++-- mlos_core/mlos_core/__init__.py | 5 +++-- mlos_core/mlos_core/optimizers/__init__.py | 5 +++-- .../bayesian_optimizers/smac_optimizer.py | 6 +++--- mlos_core/mlos_core/optimizers/flaml_optimizer.py | 4 ++-- mlos_core/mlos_core/optimizers/optimizer.py | 9 +++++++-- mlos_core/mlos_core/optimizers/random_optimizer.py | 14 +++++--------- mlos_core/mlos_core/spaces/__init__.py | 2 +- mlos_core/mlos_core/spaces/adapters/__init__.py | 7 ++++--- mlos_core/mlos_core/spaces/adapters/llamatune.py | 6 +++--- mlos_core/mlos_core/spaces/converters/flaml.py | 2 +- mlos_core/mlos_core/spaces/converters/util.py | 5 ++++- 13 files changed, 42 insertions(+), 33 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 47c97017ffc..dca10105760 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -77,7 +77,7 @@ autodoc_typehints = "both" # signature and description napoleon_numpy_docstring = True -napoleon_include_init_with_doc = True +napoleon_include_init_with_doc = False napoleon_include_private_with_doc = False napoleon_include_special_with_doc = True napoleon_use_admonition_for_examples = False @@ -244,8 +244,10 @@ def linkcode_resolve(domain: str, info: Dict[str, str]): # - mlos_bench.environments.local.local_env.LocalEnv # "imported-members", ] +autoapi_python_class_content = "both" +autoapi_member_order = "groupwise" autoapi_add_toctree_entry = False -# autoapi_keep_files = True # for testing +autoapi_keep_files = True # for testing # -- Options for HTML output ------------------------------------------------- diff --git a/mlos_bench/mlos_bench/environments/script_env.py b/mlos_bench/mlos_bench/environments/script_env.py index 7938ab65f80..06171a65fdf 100644 --- a/mlos_bench/mlos_bench/environments/script_env.py +++ b/mlos_bench/mlos_bench/environments/script_env.py @@ -37,7 +37,7 @@ def __init__( # pylint: disable=too-many-arguments Parameters ---------- - name: str + name : str Human-readable name of the environment. config : dict Free-format dictionary that contains the benchmark environment @@ -57,7 +57,7 @@ def __init__( # pylint: disable=too-many-arguments to be mixed in into the "const_args" section of the local config. tunables : TunableGroups A collection of tunable parameters for *all* environments. - service: Service + service : Service An optional service object (e.g., providing methods to deploy or reboot a VM, etc.). """ diff --git a/mlos_core/mlos_core/__init__.py b/mlos_core/mlos_core/__init__.py index ce88d276963..0fc024142f6 100644 --- a/mlos_core/mlos_core/__init__.py +++ b/mlos_core/mlos_core/__init__.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Basic initializer module for the mlos_core package. +"""mlos_core is a wrapper around other OSS tuning libraries to provide a consistent +interface for autotuning experimentation. :mod:`~mlos_core` provides the main Optimizer portions of the MLOS project for use with autotuning purposes. Although it is generally intended to be used with @@ -23,7 +24,7 @@ - :meth:`~mlos_core.optimizers.optimizer.BaseOptimizer.register` which registers a "score" for an evaluated configuration with the Optimizer -- :meth:`~mlos_core.optimizers.OptimizerFactory.create` is a factory function +- :meth:`mlos_core.optimizers.OptimizerFactory.create` is a factory function that creates a new :type:`~mlos_core.optimizers.ConcreteOptimizer` instance To do this it uses the :class:`~mlos_core.optimizers.OptimizerType` enum to diff --git a/mlos_core/mlos_core/optimizers/__init__.py b/mlos_core/mlos_core/optimizers/__init__.py index 04ca091f14b..3d58364bd85 100644 --- a/mlos_core/mlos_core/optimizers/__init__.py +++ b/mlos_core/mlos_core/optimizers/__init__.py @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Basic initializer module for the mlos_core optimizers. +"""Initializer module for the mlos_core optimizers. Optimizers are the main component of the :py:mod:`mlos_core` package. They act as a wrapper around other OSS tuning libraries to provide a consistent API @@ -13,7 +13,8 @@ :py:meth:`~mlos_core.optimizers.optimizer.BaseOptimizer.suggest` and :py:meth:`~mlos_core.optimizers.optimizer.BaseOptimizer.register` methods. -This module also provides a simple factory class for creating optimizers. +This module also provides a simple :py:class:`~.OptimizerFactory` class to +:py:meth:`~.OptimizerFactory.create` an Optimizer. For instance: diff --git a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py index fc801d7d05c..c7f0009afb9 100644 --- a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py +++ b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. # """ -Contains the wrapper class for SMAC Bayesian optimizers. +Contains the wrapper class for the :py:class:`.SmacOptimizer`. See Also: """ @@ -93,10 +93,10 @@ def __init__( Useful if you want to explicitly control the number of random configs evaluated at start. - use_default_config: bool + use_default_config : bool Whether to use the default config for the first trial after random initialization. - n_random_probability: float + n_random_probability : float Probability of choosing to evaluate a random configuration during optimization. Defaults to `0.1`. Setting this to a higher value favors exploration over exploitation. """ diff --git a/mlos_core/mlos_core/optimizers/flaml_optimizer.py b/mlos_core/mlos_core/optimizers/flaml_optimizer.py index d3dcca11b47..6a92aeec23f 100644 --- a/mlos_core/mlos_core/optimizers/flaml_optimizer.py +++ b/mlos_core/mlos_core/optimizers/flaml_optimizer.py @@ -2,9 +2,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Contains the FlamlOptimizer class +"""Contains the :py:class:`.FlamlOptimizer` class. -See Also: `Flaml `_. +See Also: `Flaml `_ """ from typing import Dict, List, NamedTuple, Optional, Tuple, Union diff --git a/mlos_core/mlos_core/optimizers/optimizer.py b/mlos_core/mlos_core/optimizers/optimizer.py index 1592b6d5d7e..476f005da45 100644 --- a/mlos_core/mlos_core/optimizers/optimizer.py +++ b/mlos_core/mlos_core/optimizers/optimizer.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Contains the BaseOptimizer abstract class.""" +"""Contains the :py:class:`.BaseOptimizer` abstract class. +""" import collections from abc import ABCMeta, abstractmethod @@ -18,7 +19,10 @@ class BaseOptimizer(metaclass=ABCMeta): - """Optimizer abstract base class defining the basic interface.""" + """Optimizer abstract base class defining the basic interface: + :py:meth:`~.BaseOptimizer.suggest`, + :py:meth:`~.BaseOptimizer.register`, + """ # pylint: disable=too-many-instance-attributes @@ -39,6 +43,7 @@ def __init__( The parameter space to optimize. optimization_targets : List[str] The names of the optimization targets to minimize. + To maximize a target, use the negative of the target when registering scores. objective_weights : Optional[List[float]] Optional list of weights of optimization targets. space_adapter : BaseSpaceAdapter diff --git a/mlos_core/mlos_core/optimizers/random_optimizer.py b/mlos_core/mlos_core/optimizers/random_optimizer.py index 661a48a373c..a086c9d8042 100644 --- a/mlos_core/mlos_core/optimizers/random_optimizer.py +++ b/mlos_core/mlos_core/optimizers/random_optimizer.py @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Contains the RandomOptimizer class.""" +"""RandomOptimizer class.""" from typing import Optional, Tuple from warnings import warn @@ -14,13 +14,9 @@ class RandomOptimizer(BaseOptimizer): """ - Optimizer class that produces random suggestions. Useful for baseline comparison - against Bayesian optimizers. + Optimizer class that produces random suggestions. - Parameters - ---------- - parameter_space : ConfigSpace.ConfigurationSpace - The parameter space to optimize. + Useful for baseline comparison against Bayesian optimizers. """ def _register( @@ -38,11 +34,11 @@ def _register( Parameters ---------- - configs : pd.DataFrame + configs : pandas.DataFrame Dataframe of configs / parameters. The columns are parameter names and the rows are the configs. - scores : pd.DataFrame + scores : pandas.DataFrame Scores from running the configs. The index is the same as the index of the configs. context : None diff --git a/mlos_core/mlos_core/spaces/__init__.py b/mlos_core/mlos_core/spaces/__init__.py index 8de6887783d..cc81a5dcc49 100644 --- a/mlos_core/mlos_core/spaces/__init__.py +++ b/mlos_core/mlos_core/spaces/__init__.py @@ -2,4 +2,4 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Space adapters and converters init file.""" +"""Space adapters and converters.""" diff --git a/mlos_core/mlos_core/spaces/adapters/__init__.py b/mlos_core/mlos_core/spaces/adapters/__init__.py index e30b625b897..1eac7577f98 100644 --- a/mlos_core/mlos_core/spaces/adapters/__init__.py +++ b/mlos_core/mlos_core/spaces/adapters/__init__.py @@ -13,10 +13,11 @@ But, for instance, the :py:class:`.LlamaTuneAdapter` can be used to automatically transform the space to a lower dimensional one. -See the :py:mod:`mlos_bench.optimizers` module for more information on how to do -this with :py:mod:`mlos_bench`. +See the :py:mod:`mlos_bench.optimizers.mlos_core_optimizer` module for more +information on how to do this with :py:mod:`mlos_bench`. -This module provides a simple factory class for creating space adapters. +This module provides a simple :py:class:`.SpaceAdapterFactory` class to +:py:meth:`~.SpaceAdapterFactory.create` space adapters. For instance: diff --git a/mlos_core/mlos_core/spaces/adapters/llamatune.py b/mlos_core/mlos_core/spaces/adapters/llamatune.py index f988f7a22e6..711747c27b6 100644 --- a/mlos_core/mlos_core/spaces/adapters/llamatune.py +++ b/mlos_core/mlos_core/spaces/adapters/llamatune.py @@ -62,11 +62,11 @@ def __init__( # pylint: disable=too-many-arguments ---------- orig_parameter_space : ConfigSpace.ConfigurationSpace The original (user-provided) parameter space to optimize. - num_low_dims: int + num_low_dims : int Number of dimensions used in the low-dimensional parameter search space. - special_param_values_dict: Optional[dict] + special_param_values_dict : Optional[dict] Dictionary of special - max_unique_values_per_param: Optional[int]: + max_unique_values_per_param : Optional[int] Number of unique values per parameter. Used to discretize the parameter space. If `None` space discretization is disabled. """ diff --git a/mlos_core/mlos_core/spaces/converters/flaml.py b/mlos_core/mlos_core/spaces/converters/flaml.py index 5b42ad55c5c..d0dc5e9b67b 100644 --- a/mlos_core/mlos_core/spaces/converters/flaml.py +++ b/mlos_core/mlos_core/spaces/converters/flaml.py @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Contains space converters for FLAML.""" +"""Contains space converters for :py:class:`~mlos_core.optimizers.flaml_optimizer`""" import sys from typing import TYPE_CHECKING, Dict diff --git a/mlos_core/mlos_core/spaces/converters/util.py b/mlos_core/mlos_core/spaces/converters/util.py index 1f190931283..8fb6bd06d1a 100644 --- a/mlos_core/mlos_core/spaces/converters/util.py +++ b/mlos_core/mlos_core/spaces/converters/util.py @@ -16,7 +16,10 @@ def monkey_patch_hp_quantization(hp: Hyperparameter) -> Hyperparameter: Monkey-patch quantization into the Hyperparameter. Temporary workaround to dropped quantization support in ConfigSpace 1.0 - See Also: + + See Also + -------- + Parameters ---------- From f034afa7b9f525a9d86f32bd9e1adb0ac38a62ef Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 19:50:59 +0000 Subject: [PATCH 048/100] fixup --- mlos_bench/mlos_bench/environments/script_env.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mlos_bench/mlos_bench/environments/script_env.py b/mlos_bench/mlos_bench/environments/script_env.py index 06171a65fdf..48db44fc35c 100644 --- a/mlos_bench/mlos_bench/environments/script_env.py +++ b/mlos_bench/mlos_bench/environments/script_env.py @@ -45,11 +45,13 @@ def __init__( # pylint: disable=too-many-arguments and the `const_args` sections. It must also have at least one of the following parameters: {`setup`, `run`, `teardown`}. Additional parameters: - * `shell_env_params` - an array of parameters to pass to the script - as shell environment variables, and - * `shell_env_params_rename` - a dictionary of {to: from} mappings - of the script parameters. If not specified, replace all - non-alphanumeric characters with underscores. + + - `shell_env_params` - an array of parameters to pass to the script + as shell environment variables, and + - `shell_env_params_rename` - a dictionary of {to: from} mappings + of the script parameters. If not specified, replace all + non-alphanumeric characters with underscores. + If neither `shell_env_params` nor `shell_env_params_rename` are specified, *no* additional shell parameters will be passed to the script. global_config : dict From 496162fae6af6c1c3a93d12d66350500c7d66572 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 20:13:25 +0000 Subject: [PATCH 049/100] improvements --- Makefile | 2 -- mlos_core/mlos_core/__init__.py | 35 ++++++++++++++++++++++----------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index c96fd3d0099..eec7baa3061 100644 --- a/Makefile +++ b/Makefile @@ -735,9 +735,7 @@ build/check-doc.build-stamp: doc/build/html/index.html doc/build/html/htmlcov/in test -s doc/build/html/autoapi/mlos_viz/index.html test -s doc/build/html/autoapi/mlos_viz/dabl/index.html grep -q -e '--config CONFIG' doc/build/html//mlos_bench.run.usage.html - # TODO: Add sphinx-build -b linkcheck target? # Check doc logs for errors (but skip over some known ones) ... - # TODO: Remove some of these ignores. @cat doc/build/log.txt \ | egrep -C1 -e WARNING -e CRITICAL -e ERROR \ | egrep -v \ diff --git a/mlos_core/mlos_core/__init__.py b/mlos_core/mlos_core/__init__.py index 0fc024142f6..d41ada2df4d 100644 --- a/mlos_core/mlos_core/__init__.py +++ b/mlos_core/mlos_core/__init__.py @@ -5,9 +5,13 @@ """mlos_core is a wrapper around other OSS tuning libraries to provide a consistent interface for autotuning experimentation. -:mod:`~mlos_core` provides the main Optimizer portions of the MLOS project for use with -autotuning purposes. Although it is generally intended to be used with -:mod:`~mlos_bench`, it can be used independently as well. +:py:mod:`mlos_core` can be installed from `pypi `_ +with ``pip install mlos-core`` from and provides the main +:py:mod:`Optimizer ` portions of the MLOS project for use with +autotuning purposes. +Although it is generally intended to be used with :py:mod:`mlos_bench` to help +automate the generation of ``(config, score)`` pairs to register with the Optimizer, +it can be used independently as well. To do this it provides a small set of wrapper classes around other OSS tuning libraries in order to provide a consistent interface so that the rest of the code @@ -15,22 +19,31 @@ Specifically: -- :class:`~mlos_core.optimizers.optimizer.BaseOptimizer` is the base class for all Optimizers +- :py:class:`~mlos_core.optimizers.optimizer.BaseOptimizer` is the base class for all Optimizers Its core methods are: - - :meth:`~mlos_core.optimizers.optimizer.BaseOptimizer.suggest` which returns a + - :py:meth:`~mlos_core.optimizers.optimizer.BaseOptimizer.suggest` which returns a new configuration to evaluate - - :meth:`~mlos_core.optimizers.optimizer.BaseOptimizer.register` which registers + - :py:meth:`~mlos_core.optimizers.optimizer.BaseOptimizer.register` which registers a "score" for an evaluated configuration with the Optimizer -- :meth:`mlos_core.optimizers.OptimizerFactory.create` is a factory function - that creates a new :type:`~mlos_core.optimizers.ConcreteOptimizer` instance +- :py:meth:`mlos_core.optimizers.OptimizerFactory.create` is a factory function + that creates a new :py:type:`~mlos_core.optimizers.ConcreteOptimizer` instance - To do this it uses the :class:`~mlos_core.optimizers.OptimizerType` enum to + To do this it uses the :py:class:`~mlos_core.optimizers.OptimizerType` enum to specify which underlying optimizer to use (e.g., - :class:`~mlos_core.optimizers.OptimizerType.FLAML` or - :class:`~mlos_core.optimizers.OptimizerType.SMAC`). + :py:class:`~mlos_core.optimizers.OptimizerType.FLAML` or + :py:class:`~mlos_core.optimizers.OptimizerType.SMAC`). + +Examples +-------- +TODO: Add example usage here. + +See Also +-------- +`mlos_core source `_ +For additional documentation and examples in `README.md` and other markdown. """ from mlos_core.version import VERSION From 6a86421848fb0fd28e337545c101e600eb852870 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 20:23:09 +0000 Subject: [PATCH 050/100] more docs --- mlos_core/mlos_core/__init__.py | 5 +++-- mlos_core/mlos_core/optimizers/__init__.py | 14 +++++++++++--- mlos_core/mlos_core/spaces/adapters/__init__.py | 14 +++++++++++--- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/mlos_core/mlos_core/__init__.py b/mlos_core/mlos_core/__init__.py index d41ada2df4d..669bf4c8db8 100644 --- a/mlos_core/mlos_core/__init__.py +++ b/mlos_core/mlos_core/__init__.py @@ -42,8 +42,9 @@ See Also -------- -`mlos_core source `_ -For additional documentation and examples in `README.md` and other markdown. +`mlos_core/README.md +`_ +for additional documentation in the source tree. """ from mlos_core.version import VERSION diff --git a/mlos_core/mlos_core/optimizers/__init__.py b/mlos_core/mlos_core/optimizers/__init__.py index 3d58364bd85..92b70699f68 100644 --- a/mlos_core/mlos_core/optimizers/__init__.py +++ b/mlos_core/mlos_core/optimizers/__init__.py @@ -16,9 +16,15 @@ This module also provides a simple :py:class:`~.OptimizerFactory` class to :py:meth:`~.OptimizerFactory.create` an Optimizer. -For instance: - +Examples +-------- TODO: Add example usage here. + +See Also +-------- +`mlos_core/optimizers/README.md +`_ +for additional documentation in the source tree. """ from enum import Enum @@ -78,7 +84,9 @@ class will be used. FlamlOptimizer, SmacOptimizer, ) -"""Type variable for concrete optimizer classes.""" +"""Type variable for concrete optimizer classes. +(e.g., :class:`~mlos_core.optimizers.bayesian_optimizers.smac_optimizer.SmacOptimizer`, etc.) +""" DEFAULT_OPTIMIZER_TYPE = OptimizerType.FLAML """Default optimizer type to use if none is specified.""" diff --git a/mlos_core/mlos_core/spaces/adapters/__init__.py b/mlos_core/mlos_core/spaces/adapters/__init__.py index 1eac7577f98..6486a5a9fde 100644 --- a/mlos_core/mlos_core/spaces/adapters/__init__.py +++ b/mlos_core/mlos_core/spaces/adapters/__init__.py @@ -19,9 +19,15 @@ This module provides a simple :py:class:`.SpaceAdapterFactory` class to :py:meth:`~.SpaceAdapterFactory.create` space adapters. -For instance: - +Examples +-------- TODO: Add example usage here. + +See Also +-------- +`mlos_core/spaces/adapters/README.md +`_ +for additional documentation in the source tree. """ from enum import Enum @@ -63,7 +69,9 @@ class SpaceAdapterType(Enum): IdentityAdapter, LlamaTuneAdapter, ) -"""Type variable for concrete SpaceAdapter classes.""" +"""Type variable for concrete SpaceAdapter classes +(e.g., :class:`~mlos_core.spaces.adapters.identity_adapter.IdentityAdapter`, etc.) +""" class SpaceAdapterFactory: From c46cdd4fd923df7619575b8c1566a25ddb45c364 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 20:45:59 +0000 Subject: [PATCH 051/100] tweaks --- mlos_core/mlos_core/__init__.py | 2 +- mlos_core/mlos_core/optimizers/__init__.py | 2 +- mlos_core/mlos_core/optimizers/flaml_optimizer.py | 4 +++- mlos_core/mlos_core/spaces/adapters/__init__.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mlos_core/mlos_core/__init__.py b/mlos_core/mlos_core/__init__.py index 669bf4c8db8..14f6eae4a69 100644 --- a/mlos_core/mlos_core/__init__.py +++ b/mlos_core/mlos_core/__init__.py @@ -44,7 +44,7 @@ -------- `mlos_core/README.md `_ -for additional documentation in the source tree. +for additional documentation and examples in the source tree. """ from mlos_core.version import VERSION diff --git a/mlos_core/mlos_core/optimizers/__init__.py b/mlos_core/mlos_core/optimizers/__init__.py index 92b70699f68..c5f34714ab6 100644 --- a/mlos_core/mlos_core/optimizers/__init__.py +++ b/mlos_core/mlos_core/optimizers/__init__.py @@ -24,7 +24,7 @@ -------- `mlos_core/optimizers/README.md `_ -for additional documentation in the source tree. +for additional documentation and examples in the source tree. """ from enum import Enum diff --git a/mlos_core/mlos_core/optimizers/flaml_optimizer.py b/mlos_core/mlos_core/optimizers/flaml_optimizer.py index 6a92aeec23f..8fafd6f0b0c 100644 --- a/mlos_core/mlos_core/optimizers/flaml_optimizer.py +++ b/mlos_core/mlos_core/optimizers/flaml_optimizer.py @@ -4,7 +4,9 @@ # """Contains the :py:class:`.FlamlOptimizer` class. -See Also: `Flaml `_ +See Also +-------- +`Flaml `_ """ from typing import Dict, List, NamedTuple, Optional, Tuple, Union diff --git a/mlos_core/mlos_core/spaces/adapters/__init__.py b/mlos_core/mlos_core/spaces/adapters/__init__.py index 6486a5a9fde..8e67b0dda38 100644 --- a/mlos_core/mlos_core/spaces/adapters/__init__.py +++ b/mlos_core/mlos_core/spaces/adapters/__init__.py @@ -27,7 +27,7 @@ -------- `mlos_core/spaces/adapters/README.md `_ -for additional documentation in the source tree. +for additional documentation and examples in the source tree. """ from enum import Enum From 0c3e7cea424e41c72cdd3a937b5d882c70cc6a87 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 21:38:19 +0000 Subject: [PATCH 052/100] moar docs! --- mlos_core/mlos_core/__init__.py | 62 ++++++++++++++++++- .../bayesian_optimizers/smac_optimizer.py | 25 +++++++- .../mlos_core/optimizers/flaml_optimizer.py | 2 +- mlos_core/mlos_core/optimizers/optimizer.py | 5 ++ 4 files changed, 89 insertions(+), 5 deletions(-) diff --git a/mlos_core/mlos_core/__init__.py b/mlos_core/mlos_core/__init__.py index 14f6eae4a69..55cd336e1ad 100644 --- a/mlos_core/mlos_core/__init__.py +++ b/mlos_core/mlos_core/__init__.py @@ -28,6 +28,9 @@ - :py:meth:`~mlos_core.optimizers.optimizer.BaseOptimizer.register` which registers a "score" for an evaluated configuration with the Optimizer + Each operates on Pandas :py:class:`DataFrames ` as the lingua + franca for data science. + - :py:meth:`mlos_core.optimizers.OptimizerFactory.create` is a factory function that creates a new :py:type:`~mlos_core.optimizers.ConcreteOptimizer` instance @@ -38,7 +41,58 @@ Examples -------- -TODO: Add example usage here. +>>> # Import the necessary classes. +>>> import pandas +>>> from ConfigSpace import ConfigurationSpace, UniformIntegerHyperparameter +>>> from mlos_core.optimizers import OptimizerFactory, OptimizerType +>>> from mlos_core.spaces.adapters import SpaceAdapterFactory, SpaceAdapterType +>>> # Create a simple ConfigurationSpace with a single integer hyperparameter. +>>> cs = ConfigurationSpace(seed=1234) +>>> _ = cs.add(UniformIntegerHyperparameter("x", lower=0, upper=10)) +>>> # Create a new optimizer instance using the SMAC optimizer. +>>> opt_args = {"seed": 1234, "max_trials": 100} +>>> space_adpaters_kwargs = {} # no additional args for this example +>>> opt = OptimizerFactory.create( +... parameter_space=cs, +... optimization_targets=["y"], +... optimizer_type=OptimizerType.SMAC, +... optimizer_kwargs=opt_args, +... space_adapter_type=SpaceAdapterType.IDENTITY, # or LLAMATUNE +... space_adapter_kwargs=space_adpaters_kwargs, +... ) +>>> # Get a new configuration suggestion. +>>> (config_df, _metadata_df) = opt.suggest() +>>> # Examine the suggested configuration. +>>> assert len(config_df) == 1 +>>> config_df.iloc[0] +x 3 +Name: 0, dtype: int64 +>>> # Register the configuration and its corresponding target value. +>>> score = 42 # a made up score +>>> scores_df = pandas.DataFrame({"y": [score]}) +>>> opt.register(configs=config_df, scores=scores_df) +>>> # Get a new configuration suggestion. +>>> (config_df, _metadata_df) = opt.suggest() +>>> config_df.iloc[0] +x 10 +Name: 0, dtype: int64 +>>> score = 7 # a better made up score +>>> # optimizers minimize, so lower is better +>>> # use negative score to maximize +>>> # convert to a DataFrame again +>>> scores_df = pandas.DataFrame({"y": [score]}) +>>> opt.register(configs=config_df, scores=scores_df) +>>> # Get the best observations. +>>> (configs_df, scores_df, _contexts_df) = opt.get_best_observations() +>>> # default is only return one +>>> assert len(configs_df) == 1 +>>> assert len(scores_df) == 1 +>>> configs_df.iloc[0] +x 10 +Name: 1, dtype: int64 +>>> scores_df.iloc[0] +y 7 +Name: 1, dtype: int64 See Also -------- @@ -49,3 +103,9 @@ from mlos_core.version import VERSION __version__ = VERSION + + +if __name__ == "__main__": + import doctest + + doctest.testmod() diff --git a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py index c7f0009afb9..e245c055243 100644 --- a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py +++ b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py @@ -5,7 +5,9 @@ """ Contains the wrapper class for the :py:class:`.SmacOptimizer`. -See Also: +See Also +-------- +`SMAC3 Documentation `_ """ from logging import warning @@ -85,7 +87,8 @@ def __init__( Default depends on max_trials and number of parameters and max_ratio. Note: it can sometimes be useful to set this to 1 when pre-warming the optimizer from historical data. - See Also: mlos_bench.optimizer.bulk_register + See Also: + :py:meth:`mlos_bench.optimizers.base_optimizer.Optimizer.bulk_register` max_ratio : Optional[int] Maximum ratio of max_trials to be random configs to be evaluated @@ -193,6 +196,7 @@ def __init__( if max_ratio is not None: assert isinstance(max_ratio, float) and 0.0 <= max_ratio <= 1.0 initial_design_args["max_ratio"] = max_ratio + self._max_ratio = max_ratio # Use the default InitialDesign from SMAC. # (currently SBOL instead of LatinHypercube due to better uniformity @@ -232,6 +236,18 @@ def __del__(self) -> None: # Best-effort attempt to clean up, in case the user forgets to call .cleanup() self.cleanup() + @property + def max_ratio(self) -> Optional[float]: + """ + Gets the `max_ratio` parameter used in py:meth:`constructor <.__init__>` of + this SmacOptimizer. + + Returns + ------- + float + """ + return self._max_ratio + @property def n_random_init(self) -> int: """ @@ -240,7 +256,10 @@ def n_random_init(self) -> int: Note: This may not be equal to the value passed to the initializer, due to logic present in the SMAC. - See Also: max_ratio + + See Also + -------- + :py:attr:`.max_ratio` Returns ------- diff --git a/mlos_core/mlos_core/optimizers/flaml_optimizer.py b/mlos_core/mlos_core/optimizers/flaml_optimizer.py index 8fafd6f0b0c..52b42451f1d 100644 --- a/mlos_core/mlos_core/optimizers/flaml_optimizer.py +++ b/mlos_core/mlos_core/optimizers/flaml_optimizer.py @@ -6,7 +6,7 @@ See Also -------- -`Flaml `_ +`Flaml Documentation `_ """ from typing import Dict, List, NamedTuple, Optional, Tuple, Union diff --git a/mlos_core/mlos_core/optimizers/optimizer.py b/mlos_core/mlos_core/optimizers/optimizer.py index 476f005da45..23f76168fe7 100644 --- a/mlos_core/mlos_core/optimizers/optimizer.py +++ b/mlos_core/mlos_core/optimizers/optimizer.py @@ -50,9 +50,14 @@ def __init__( The space adapter class to employ for parameter space transformations. """ self.parameter_space: ConfigSpace.ConfigurationSpace = parameter_space + """The parameter space to optimize.""" + self.optimizer_parameter_space: ConfigSpace.ConfigurationSpace = ( parameter_space if space_adapter is None else space_adapter.target_parameter_space ) + """The parameter space used by the optimizer (in case a + :py:mod:`SpaceAdapter ` is used). + """ if space_adapter is not None and space_adapter.orig_parameter_space != parameter_space: raise ValueError("Given parameter space differs from the one given to space adapter") From 7721b9ce6312222cb0c8d5e04d5f3e14ae09e5c0 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 21:49:15 +0000 Subject: [PATCH 053/100] unnecessary --- mlos_core/mlos_core/__init__.py | 11 ++++++----- setup.cfg | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mlos_core/mlos_core/__init__.py b/mlos_core/mlos_core/__init__.py index 55cd336e1ad..a2b98aa4f8e 100644 --- a/mlos_core/mlos_core/__init__.py +++ b/mlos_core/mlos_core/__init__.py @@ -67,7 +67,7 @@ >>> config_df.iloc[0] x 3 Name: 0, dtype: int64 ->>> # Register the configuration and its corresponding target value. +>>> # Register the configuration and its corresponding target value >>> score = 42 # a made up score >>> scores_df = pandas.DataFrame({"y": [score]}) >>> opt.register(configs=config_df, scores=scores_df) @@ -77,14 +77,15 @@ x 10 Name: 0, dtype: int64 >>> score = 7 # a better made up score ->>> # optimizers minimize, so lower is better ->>> # use negative score to maximize ->>> # convert to a DataFrame again +>>> # Optimizers minimize by convention, so a lower score is better +>>> # You can use a negative score to maximize values instead +>>> # +>>> # Convert it to a DataFrame again >>> scores_df = pandas.DataFrame({"y": [score]}) >>> opt.register(configs=config_df, scores=scores_df) >>> # Get the best observations. >>> (configs_df, scores_df, _contexts_df) = opt.get_best_observations() ->>> # default is only return one +>>> # The default is to only return one >>> assert len(configs_df) == 1 >>> assert len(scores_df) == 1 >>> configs_df.iloc[0] diff --git a/setup.cfg b/setup.cfg index 0e0ffbe7c5c..f8b0c945f2d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,7 +44,7 @@ addopts = # --log-level=DEBUG # Moved these to Makefile (coverage is expensive and we only need it in the pipelines generally). #--cov=mlos_core --cov-report=xml -testpaths = mlos_core mlos_bench +testpaths = mlos_core mlos_bench mlos_viz # Ignore some upstream deprecation warnings. filterwarnings = ignore:.*(get_hyperparam|get_dictionary|get_parents_of|(list\(.*values\(\)\))).*:DeprecationWarning:smac:0 From ba4fe091867b4457674826a12f035f868d67c409 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 22:16:14 +0000 Subject: [PATCH 054/100] more docs --- doc/source/conf.py | 7 +------ mlos_viz/mlos_viz/__init__.py | 13 +++++++++---- mlos_viz/mlos_viz/base.py | 18 ++++++++++-------- mlos_viz/mlos_viz/dabl.py | 12 ++++++++++-- mlos_viz/mlos_viz/util.py | 4 ++-- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index dca10105760..5ed5595ffbb 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -141,7 +141,7 @@ def linkcode_resolve(domain: str, info: Dict[str, str]): ) intersphinx_mapping.update( { - # "ConfigSpace": ("https://automl.github.io/ConfigSpace/latest/", None), + "dabl": ("https://dabl.github.io/stable/", None), } ) @@ -185,15 +185,10 @@ def linkcode_resolve(domain: str, info: Dict[str, str]): ("py:class", "sqlalchemy.engine.Engine"), ("py:exc", "jsonschema.exceptions.SchemaError"), ("py:exc", "jsonschema.exceptions.ValidationError"), - # ("py:func", "create_connection"), - # ("py:class", "SSHClient"), - # ("py:class", "SSHKey"), ] nitpick_ignore_regex = [ # Ignore some external references that don't use sphinx for their docs. (r"py:.*", r"flaml\..*"), - # Ignore some external inheritance classes that don't resolve. - # (r"py:.*", r"asyncssh\..*"), ] # Which documents to include in the build. diff --git a/mlos_viz/mlos_viz/__init__.py b/mlos_viz/mlos_viz/__init__.py index fc96215bbe3..783285d939f 100644 --- a/mlos_viz/mlos_viz/__init__.py +++ b/mlos_viz/mlos_viz/__init__.py @@ -3,7 +3,12 @@ # Licensed under the MIT License. # """mlos_viz is a framework to help visualizing, explain, and gain insights from results -from the mlos_bench framework for benchmarking and optimization automation. +from the :py:mod:`mlos_bench` framework for benchmarking and optimization automation. + +Its main entrypoint is the :py:func:`plot` function, which can be used to +automatically visualize :py:class:`~.ExperimentData` from :py:mod:`mlos_bench` using +other libraries for automatic data correlation and visualization like +:external:py:func:`dabl `. """ from enum import Enum @@ -64,11 +69,11 @@ def plot( exp_data: ExperimentData The experiment data to plot. results_df : Optional[pandas.DataFrame] - Optional results_df to plot. - If not provided, defaults to exp_data.results_df property. + Optional `results_df` to plot. + If not provided, defaults to :py:attr:`.ExperimentData.results_df` property. objectives : Optional[Dict[str, Literal["min", "max"]]] Optional objectives to plot. - If not provided, defaults to exp_data.objectives property. + If not provided, defaults to :py:attr:`.ExperimentData.objectives` property. plotter_method: MlosVizMethod The method to use for visualizing the experiment results. filter_warnings: bool diff --git a/mlos_viz/mlos_viz/base.py b/mlos_viz/mlos_viz/base.py index c0115ff945a..ceffe5b7d53 100644 --- a/mlos_viz/mlos_viz/base.py +++ b/mlos_viz/mlos_viz/base.py @@ -222,11 +222,12 @@ def limit_top_n_configs( exp_data : Optional[ExperimentData] The ExperimentData (e.g., obtained from the storage layer) to operate on. results_df : Optional[pandas.DataFrame] - The results dataframe to augment, by default None to use the results_df property. + The results dataframe to augment, by default None to use + :py:attr:`.ExperimentData.results_df` property. objectives : Iterable[str] Which result column(s) to use for sorting the configs, and in which direction ("min" or "max"). - By default None to automatically select the experiment objectives. + By default None to automatically select the :py:attr:`.ExperimentData.objectives`. top_n_configs : int How many configs to return, including the default, by default 10. method: Literal["mean", "median", "p50", "p75", "p90", "p95", "p99"] = "mean", @@ -350,10 +351,10 @@ def plot_optimizer_trends( The ExperimentData (e.g., obtained from the storage layer) to plot. results_df : Optional[pandas.DataFrame] Optional results_df to plot. - If not provided, defaults to exp_data.results_df property. + If not provided, defaults to :py:attr:`.ExperimentData.results_df` property. objectives : Optional[Dict[str, Literal["min", "max"]]] Optional objectives to plot. - If not provided, defaults to exp_data.objectives property. + If not provided, defaults to :py:attr:`.ExperimentData.objectives` property. """ (results_df, obj_cols) = expand_results_data_args(exp_data, results_df, objectives) (results_df, groupby_columns, groupby_column) = _add_groupby_desc_column(results_df) @@ -430,7 +431,8 @@ def plot_top_n_configs( ) -> None: # pylint: disable=too-many-locals """ - Plots the top-N configs along with the default config for the given ExperimentData. + Plots the top-N configs along with the default config for the given + :py:class:`.ExperimentData`. Intended to be used from a Jupyter notebook. @@ -440,14 +442,14 @@ def plot_top_n_configs( The experiment data to plot. results_df : Optional[pandas.DataFrame] Optional results_df to plot. - If not provided, defaults to exp_data.results_df property. + If not provided, defaults to :py:attr:`.ExperimentData.results_df` property. objectives : Optional[Dict[str, Literal["min", "max"]]] Optional objectives to plot. - If not provided, defaults to exp_data.objectives property. + If not provided, defaults to :py:attr:`.ExperimentData.objectives` property. with_scatter_plot : bool Whether to also add scatter plot to the output figure. kwargs : dict - Remaining keyword arguments are passed along to the limit_top_n_configs function. + Remaining keyword arguments are passed along to the :py:func:`limit_top_n_configs` function. """ (results_df, _obj_cols) = expand_results_data_args(exp_data, results_df, objectives) top_n_config_args = _get_kwarg_defaults(limit_top_n_configs, **kwargs) diff --git a/mlos_viz/mlos_viz/dabl.py b/mlos_viz/mlos_viz/dabl.py index 4eb81f65de7..9cea97f2765 100644 --- a/mlos_viz/mlos_viz/dabl.py +++ b/mlos_viz/mlos_viz/dabl.py @@ -2,7 +2,14 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Small wrapper functions for dabl plotting functions via mlos_bench data.""" +""" +Small wrapper functions for plotting :py:mod:`mlos_bench` data via +:external:py:func:`dabl.plot`. + +See Also +-------- +`dabl `_ for more information on the dabl library. +""" import warnings from typing import Dict, Literal, Optional @@ -20,7 +27,8 @@ def plot( objectives: Optional[Dict[str, Literal["min", "max"]]] = None, ) -> None: """ - Plots the Experiment results data using dabl. + Plots the :py:class:`~mlos_bench.storage.base_storage.Storage.Experiment` + results data using :external:py:func:`dabl.plot`. Parameters ---------- diff --git a/mlos_viz/mlos_viz/util.py b/mlos_viz/mlos_viz/util.py index d0c7aa56526..2e537021125 100644 --- a/mlos_viz/mlos_viz/util.py +++ b/mlos_viz/mlos_viz/util.py @@ -26,10 +26,10 @@ def expand_results_data_args( ExperimentData to operate on. results_df : Optional[pandas.DataFrame] Optional results_df argument. - Defaults to exp_data.results_df property. + If not provided, defaults to :py:attr:`.ExperimentData.results_df` property. objectives : Optional[Dict[str, Literal["min", "max"]]] Optional objectives set to operate on. - Defaults to exp_data.objectives property. + If not provided, defaults to :py:attr:`.ExperimentData.objectives` property. Returns ------- From 50075a88f9d7b29701e050e8041786cab2577da2 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 22:17:42 +0000 Subject: [PATCH 055/100] reformat docstrings --- .../environments/local/local_env.py | 13 ++++++++++++- mlos_bench/mlos_bench/util.py | 3 +-- mlos_core/mlos_core/__init__.py | 3 ++- mlos_core/mlos_core/optimizers/__init__.py | 19 +++++++++---------- .../bayesian_optimizers/smac_optimizer.py | 4 ++-- .../mlos_core/optimizers/flaml_optimizer.py | 3 ++- mlos_core/mlos_core/optimizers/optimizer.py | 7 +++---- .../mlos_core/spaces/adapters/__init__.py | 10 +++++----- .../mlos_core/spaces/adapters/adapter.py | 6 ++++-- .../mlos_core/spaces/adapters/llamatune.py | 3 ++- .../mlos_core/spaces/converters/__init__.py | 3 ++- mlos_viz/mlos_viz/__init__.py | 3 ++- mlos_viz/mlos_viz/dabl.py | 4 ++-- 13 files changed, 48 insertions(+), 33 deletions(-) diff --git a/mlos_bench/mlos_bench/environments/local/local_env.py b/mlos_bench/mlos_bench/environments/local/local_env.py index 7508b6678ff..4b4909acb27 100644 --- a/mlos_bench/mlos_bench/environments/local/local_env.py +++ b/mlos_bench/mlos_bench/environments/local/local_env.py @@ -11,7 +11,18 @@ from datetime import datetime from tempfile import TemporaryDirectory from types import TracebackType -from typing import Any, Dict, Iterable, List, Literal, Mapping, Optional, Tuple, Type, Union +from typing import ( + Any, + Dict, + Iterable, + List, + Literal, + Mapping, + Optional, + Tuple, + Type, + Union, +) import pandas diff --git a/mlos_bench/mlos_bench/util.py b/mlos_bench/mlos_bench/util.py index dde7c2da8a9..d6dc0a6c4a2 100644 --- a/mlos_bench/mlos_bench/util.py +++ b/mlos_bench/mlos_bench/util.py @@ -40,8 +40,7 @@ from mlos_bench.storage.base_storage import Storage BaseTypeVar = TypeVar("BaseTypeVar", "Environment", "Optimizer", "Scheduler", "Service", "Storage") -""" -BaseTypeVar is a generic with a constraint of the main base classes (e.g., +"""BaseTypeVar is a generic with a constraint of the main base classes (e.g., :py:class:`~mlos_bench.environments.base_environment.Environment`, :py:class:`~mlos_bench.optimizers.base_optimizer.Optimizer`, :py:class:`~mlos_bench.schedulers.base_scheduler.Scheduler`, diff --git a/mlos_core/mlos_core/__init__.py b/mlos_core/mlos_core/__init__.py index a2b98aa4f8e..353bfb290b3 100644 --- a/mlos_core/mlos_core/__init__.py +++ b/mlos_core/mlos_core/__init__.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""mlos_core is a wrapper around other OSS tuning libraries to provide a consistent +""" +mlos_core is a wrapper around other OSS tuning libraries to provide a consistent interface for autotuning experimentation. :py:mod:`mlos_core` can be installed from `pypi `_ diff --git a/mlos_core/mlos_core/optimizers/__init__.py b/mlos_core/mlos_core/optimizers/__init__.py index c5f34714ab6..e8e91e903b6 100644 --- a/mlos_core/mlos_core/optimizers/__init__.py +++ b/mlos_core/mlos_core/optimizers/__init__.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Initializer module for the mlos_core optimizers. +""" +Initializer module for the mlos_core optimizers. Optimizers are the main component of the :py:mod:`mlos_core` package. They act as a wrapper around other OSS tuning libraries to provide a consistent API @@ -54,20 +55,17 @@ class OptimizerType(Enum): """Enumerate supported mlos_core optimizers.""" RANDOM = RandomOptimizer - """ - An instance of :class:`~mlos_core.optimizers.random_optimizer.RandomOptimizer` + """An instance of :class:`~mlos_core.optimizers.random_optimizer.RandomOptimizer` class will be used. """ FLAML = FlamlOptimizer - """ - An instance of :class:`~mlos_core.optimizers.flaml_optimizer.FlamlOptimizer` + """An instance of :class:`~mlos_core.optimizers.flaml_optimizer.FlamlOptimizer` class will be used. """ SMAC = SmacOptimizer - """ - An instance of + """An instance of :class:`~mlos_core.optimizers.bayesian_optimizers.smac_optimizer.SmacOptimizer` class will be used. """ @@ -84,7 +82,9 @@ class will be used. FlamlOptimizer, SmacOptimizer, ) -"""Type variable for concrete optimizer classes. +""" +Type variable for concrete optimizer classes. + (e.g., :class:`~mlos_core.optimizers.bayesian_optimizers.smac_optimizer.SmacOptimizer`, etc.) """ @@ -93,8 +93,7 @@ class will be used. class OptimizerFactory: - """ - Simple factory class for creating + """Simple factory class for creating :class:`~mlos_core.optimizers.optimizer.BaseOptimizer`-derived objects. """ diff --git a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py index e245c055243..2af22b996a2 100644 --- a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py +++ b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py @@ -239,8 +239,8 @@ def __del__(self) -> None: @property def max_ratio(self) -> Optional[float]: """ - Gets the `max_ratio` parameter used in py:meth:`constructor <.__init__>` of - this SmacOptimizer. + Gets the `max_ratio` parameter used in py:meth:`constructor <.__init__>` of this + SmacOptimizer. Returns ------- diff --git a/mlos_core/mlos_core/optimizers/flaml_optimizer.py b/mlos_core/mlos_core/optimizers/flaml_optimizer.py index 52b42451f1d..bf7fe3ecc9f 100644 --- a/mlos_core/mlos_core/optimizers/flaml_optimizer.py +++ b/mlos_core/mlos_core/optimizers/flaml_optimizer.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Contains the :py:class:`.FlamlOptimizer` class. +""" +Contains the :py:class:`.FlamlOptimizer` class. See Also -------- diff --git a/mlos_core/mlos_core/optimizers/optimizer.py b/mlos_core/mlos_core/optimizers/optimizer.py index 23f76168fe7..64a693d6c18 100644 --- a/mlos_core/mlos_core/optimizers/optimizer.py +++ b/mlos_core/mlos_core/optimizers/optimizer.py @@ -2,8 +2,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Contains the :py:class:`.BaseOptimizer` abstract class. -""" +"""Contains the :py:class:`.BaseOptimizer` abstract class.""" import collections from abc import ABCMeta, abstractmethod @@ -55,8 +54,8 @@ def __init__( self.optimizer_parameter_space: ConfigSpace.ConfigurationSpace = ( parameter_space if space_adapter is None else space_adapter.target_parameter_space ) - """The parameter space used by the optimizer (in case a - :py:mod:`SpaceAdapter ` is used). + """The parameter space used by the optimizer (in case a :py:mod:`SpaceAdapter + ` is used). """ if space_adapter is not None and space_adapter.orig_parameter_space != parameter_space: diff --git a/mlos_core/mlos_core/spaces/adapters/__init__.py b/mlos_core/mlos_core/spaces/adapters/__init__.py index 8e67b0dda38..642daf240a2 100644 --- a/mlos_core/mlos_core/spaces/adapters/__init__.py +++ b/mlos_core/mlos_core/spaces/adapters/__init__.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Basic initializer module for the mlos_core space adapters. +""" +Basic initializer module for the mlos_core space adapters. Space adapters provide a mechanism for automatic transformation of the original :py:class:`ConfigSpace.ConfigurationSpace` provided to the optimizer into a new @@ -69,14 +70,13 @@ class SpaceAdapterType(Enum): IdentityAdapter, LlamaTuneAdapter, ) -"""Type variable for concrete SpaceAdapter classes -(e.g., :class:`~mlos_core.spaces.adapters.identity_adapter.IdentityAdapter`, etc.) +"""Type variable for concrete SpaceAdapter classes (e.g., +:class:`~mlos_core.spaces.adapters.identity_adapter.IdentityAdapter`, etc.) """ class SpaceAdapterFactory: - """ - Simple factory class for creating + """Simple factory class for creating :class:`~mlos_core.spaces.adapters.adapter.BaseSpaceAdapter`-derived objects. """ diff --git a/mlos_core/mlos_core/spaces/adapters/adapter.py b/mlos_core/mlos_core/spaces/adapters/adapter.py index d8df6b22191..b4b53b73a08 100644 --- a/mlos_core/mlos_core/spaces/adapters/adapter.py +++ b/mlos_core/mlos_core/spaces/adapters/adapter.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Contains the BaseSpaceAdapter abstract class. +""" +Contains the BaseSpaceAdapter abstract class. As mentioned in :py:mod:`mlos_core.spaces.adapters`, the space adapters provide a mechanism for automatic transformation of the original @@ -81,7 +82,8 @@ def inverse_transform(self, configurations: pd.DataFrame) -> pd.DataFrame: Translates a configuration, which belongs to the original parameter space, to the target parameter space. This method is called by the `register` method of the :py:class:`~mlos_core.optimizers.optimizer.BaseOptimizer` class, and - performs the inverse operation of :py:meth:`~.BaseSpaceAdapter.transform` method. + performs the inverse operation of :py:meth:`~.BaseSpaceAdapter.transform` + method. Parameters ---------- diff --git a/mlos_core/mlos_core/spaces/adapters/llamatune.py b/mlos_core/mlos_core/spaces/adapters/llamatune.py index 711747c27b6..625dd886d08 100644 --- a/mlos_core/mlos_core/spaces/adapters/llamatune.py +++ b/mlos_core/mlos_core/spaces/adapters/llamatune.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Implementation of LlamaTune space adapter. +""" +Implementation of LlamaTune space adapter. LlamaTune is a technique that transforms the original parameter space into a lower-dimensional space to try and improve the sample efficiency of the underlying diff --git a/mlos_core/mlos_core/spaces/converters/__init__.py b/mlos_core/mlos_core/spaces/converters/__init__.py index ab7736c607e..3abebbfbe54 100644 --- a/mlos_core/mlos_core/spaces/converters/__init__.py +++ b/mlos_core/mlos_core/spaces/converters/__init__.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Space converters init file. +""" +Space converters init file. Space converters are helper functions that translate a :py:class:`ConfigSpace.ConfigurationSpace` that :py:mod:`mlos_core` Optimizers take diff --git a/mlos_viz/mlos_viz/__init__.py b/mlos_viz/mlos_viz/__init__.py index 783285d939f..8450e32b12b 100644 --- a/mlos_viz/mlos_viz/__init__.py +++ b/mlos_viz/mlos_viz/__init__.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""mlos_viz is a framework to help visualizing, explain, and gain insights from results +""" +mlos_viz is a framework to help visualizing, explain, and gain insights from results from the :py:mod:`mlos_bench` framework for benchmarking and optimization automation. Its main entrypoint is the :py:func:`plot` function, which can be used to diff --git a/mlos_viz/mlos_viz/dabl.py b/mlos_viz/mlos_viz/dabl.py index 9cea97f2765..c2fd0af8c35 100644 --- a/mlos_viz/mlos_viz/dabl.py +++ b/mlos_viz/mlos_viz/dabl.py @@ -27,8 +27,8 @@ def plot( objectives: Optional[Dict[str, Literal["min", "max"]]] = None, ) -> None: """ - Plots the :py:class:`~mlos_bench.storage.base_storage.Storage.Experiment` - results data using :external:py:func:`dabl.plot`. + Plots the :py:class:`~mlos_bench.storage.base_storage.Storage.Experiment` results + data using :external:py:func:`dabl.plot`. Parameters ---------- From 2b14658a2801f928dbadc08636f3b68d432c3c3d Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 22:20:24 +0000 Subject: [PATCH 056/100] formatting --- .../mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py | 3 +-- mlos_viz/mlos_viz/base.py | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py index 2af22b996a2..42d3280aeee 100644 --- a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py +++ b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py @@ -86,8 +86,7 @@ def __init__( Number of points evaluated at start to bootstrap the optimizer. Default depends on max_trials and number of parameters and max_ratio. Note: it can sometimes be useful to set this to 1 when pre-warming the - optimizer from historical data. - See Also: + optimizer from historical data. See Also: :py:meth:`mlos_bench.optimizers.base_optimizer.Optimizer.bulk_register` max_ratio : Optional[int] diff --git a/mlos_viz/mlos_viz/base.py b/mlos_viz/mlos_viz/base.py index ceffe5b7d53..e7a52f1bd8c 100644 --- a/mlos_viz/mlos_viz/base.py +++ b/mlos_viz/mlos_viz/base.py @@ -449,7 +449,8 @@ def plot_top_n_configs( with_scatter_plot : bool Whether to also add scatter plot to the output figure. kwargs : dict - Remaining keyword arguments are passed along to the :py:func:`limit_top_n_configs` function. + Remaining keyword arguments are passed along to the + :py:func:`limit_top_n_configs` function. """ (results_df, _obj_cols) = expand_results_data_args(exp_data, results_df, objectives) top_n_config_args = _get_kwarg_defaults(limit_top_n_configs, **kwargs) From 57cf01e6975d58bcac6f6f59c7a89c7f04c6454c Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 22:26:33 +0000 Subject: [PATCH 057/100] formatting --- mlos_bench/mlos_bench/util.py | 3 +-- mlos_core/mlos_core/optimizers/optimizer.py | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mlos_bench/mlos_bench/util.py b/mlos_bench/mlos_bench/util.py index d6dc0a6c4a2..e7cd5a89862 100644 --- a/mlos_bench/mlos_bench/util.py +++ b/mlos_bench/mlos_bench/util.py @@ -45,8 +45,7 @@ :py:class:`~mlos_bench.optimizers.base_optimizer.Optimizer`, :py:class:`~mlos_bench.schedulers.base_scheduler.Scheduler`, :py:class:`~mlos_bench.services.base_service.Service`, -:py:class:`~mlos_bench.storage.base_storage.Storage`, -etc.). +:py:class:`~mlos_bench.storage.base_storage.Storage`, etc.). """ BaseTypes = Union["Environment", "Optimizer", "Scheduler", "Service", "Storage"] diff --git a/mlos_core/mlos_core/optimizers/optimizer.py b/mlos_core/mlos_core/optimizers/optimizer.py index 64a693d6c18..84f0f8fab61 100644 --- a/mlos_core/mlos_core/optimizers/optimizer.py +++ b/mlos_core/mlos_core/optimizers/optimizer.py @@ -54,8 +54,10 @@ def __init__( self.optimizer_parameter_space: ConfigSpace.ConfigurationSpace = ( parameter_space if space_adapter is None else space_adapter.target_parameter_space ) - """The parameter space used by the optimizer (in case a :py:mod:`SpaceAdapter - ` is used). + """ + The parameter space actually used by the optimizer. + + (in case a :py:mod:`SpaceAdapter ` is used) """ if space_adapter is not None and space_adapter.orig_parameter_space != parameter_space: From e64418013b5a2cafadceae771270fbebbf07d62a Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 22:27:10 +0000 Subject: [PATCH 058/100] comments --- doc/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 5ed5595ffbb..db266c18f49 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -241,8 +241,8 @@ def linkcode_resolve(domain: str, info: Dict[str, str]): ] autoapi_python_class_content = "both" autoapi_member_order = "groupwise" -autoapi_add_toctree_entry = False -autoapi_keep_files = True # for testing +autoapi_add_toctree_entry = False # handled manually +# autoapi_keep_files = True # for testing # -- Options for HTML output ------------------------------------------------- From 838e2f1574978a62e4ae9cfd878162923dfd1ef0 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 23:02:46 +0000 Subject: [PATCH 059/100] more docs --- mlos_bench/mlos_bench/__init__.py | 65 ++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index d0214754411..72d6fb22401 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -2,8 +2,69 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""mlos_bench is a framework to help automate benchmarking and and OS/application -parameter autotuning. +"""mlos_bench is a framework to help automate benchmarking and OS/application +parameter autotuning and the data management of the results. + +It can be installed from `pypi `_ via +``pip install mlos-bench`` and executed using the ``mlos_bench`` +`command <../../mlos_bench.run.usage.html>`_ using a collection of `json` +`configs `_. + +It is intended to be used with :py:mod:`mlos_core` via +:py:class:`~mlos_bench.optimizers.mlos_core_optimizer.MlosCoreOptimizer` to help +navigate complex parameter spaces more effeciently, though other +:py:mod:`~mlos_bench.optimizers` are also available to +help customize the search process easily by simply swapping out the +:py:class:`~mlos_bench.optimizers.base_optimizer.Optimizer` class in the associated +json configs. +For instance, +:py:class:`~mlos_bench.optimizers.grid_search_optimizer.GridSearchOptimizer` can be +used to perform a grid search over the parameter space instead. + +The other core classes in this package are: + +- :py:mod:`~mlos_bench.environments` which provide abstractions for representing an + execution environment. + + These are generally the target of the optimization process and are used to + evaluate the performance of a given configuration, though can also be used to + simply run a single benchmark. + They can be used, for instance, to :py:mod:`provision VMs + `, run benchmarks or execute any other + arbitrary code on a :py:mod:`remote machine `, + and many other things. + +- :py:mod:`~mlos_bench.environments` are often associated with + :py:mod:`~mlos_bench.tunables` which provide a language for specifying the set of + configuration parameters that can be optimized or searched over with the + :py:mod:`~mlos_bench.optimizers`. + +- :py:mod:`~mlos_bench.storage` which provides abstractions for storing and + retrieving data from the experiments. + + For instance, nearly any :py:mod:`SQL ` backend that + `sqlalchemy `_ supports can be used. + +See below for more information on the classes in this package. + +Notes +----- +Note that while the docstrings in this package are generated from the source code +and hence sometimes more focused on the implementation details, most user +interactions with the package will be through the +`json configs `_. +Even so it may be useful to look at the source code to understand how those are +interpretted. + +Examples +-------- +TODO: Add examples + +See Also +-------- +`mlos_bench/README.md +`_ +for additional documentation and examples in the source tree. """ from mlos_bench.version import VERSION From 62a00a58b9b1631d7345383fb1744620aee059f5 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 23:16:34 +0000 Subject: [PATCH 060/100] more docs --- mlos_bench/mlos_bench/__init__.py | 17 ++++++++++++ mlos_bench/mlos_bench/config/__init__.py | 2 +- .../config/schemas/config_schemas.py | 26 +++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index 72d6fb22401..f81585c74a2 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -39,6 +39,23 @@ configuration parameters that can be optimized or searched over with the :py:mod:`~mlos_bench.optimizers`. +- :py:mod:`~mlos_bench.services` provide the necessary abstractions to run interact + with the :py:mod:`~mlos_bench.environments` in different settings. + + For instance, the + :py:class:`~mlos_bench.services.remote.azure.azure_vm_services.AzureVMService` + can be used to run commands on Azure VMs for a remote + :py:mod:`~mlos_bench.environments.remote.vm_env.VMEnv`. + + Alternatively, one could swap out that service for + :py:class:`~mlos_bench.services.remote.ssh.ssh_host_services.SshHostService` in + order to target a different VM without having to change the + :py:class:`~mlos_bench.environments.base_environment.Environment` configuration at + all. + + This is particularly useful when running the same benchmark on different + ecosystems and makes the configs more modular and composable. + - :py:mod:`~mlos_bench.storage` which provides abstractions for storing and retrieving data from the experiments. diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index b78386118c6..306d58f02f2 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -2,4 +2,4 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""mlos_bench.config.""" +"""Basic initializer for the mlos_bench config subtree.""" diff --git a/mlos_bench/mlos_bench/config/schemas/config_schemas.py b/mlos_bench/mlos_bench/config/schemas/config_schemas.py index b7ce402b5d5..5f145f6bdad 100644 --- a/mlos_bench/mlos_bench/config/schemas/config_schemas.py +++ b/mlos_bench/mlos_bench/config/schemas/config_schemas.py @@ -22,12 +22,19 @@ # The path to find all config schemas. CONFIG_SCHEMA_DIR = path_join(path.dirname(__file__), abs_path=True) +"""The local directory where all config schemas are stored.""" # Allow skipping schema validation for tight dev cycle changes. # It is used in `ConfigSchema.validate()` method below. # NOTE: this may cause pytest to fail if it's expecting exceptions # to be raised for invalid configs. _VALIDATION_ENV_FLAG = "MLOS_BENCH_SKIP_SCHEMA_VALIDATION" +"""The special environment flag to set to skip schema validation. + +Useful for local development when you're making a lot of changes to the config or +adding new classes that aren't in the main repo yet. +""" + _SKIP_VALIDATION = environ.get(_VALIDATION_ENV_FLAG, "false").lower() in { "true", "y", @@ -105,22 +112,41 @@ def registry(self) -> Registry: SCHEMA_STORE = SchemaStore() +"""Static SchemaStore instance used for storing and retrieving schemas for config validation.""" class ConfigSchema(Enum): """An enum to help describe schema types and help validate configs against them.""" CLI = path_join(CONFIG_SCHEMA_DIR, "cli/cli-schema.json") + """Schema for command line arguments.""" + GLOBALS = path_join(CONFIG_SCHEMA_DIR, "cli/globals-schema.json") + """Schema for global variables.""" + ENVIRONMENT = path_join(CONFIG_SCHEMA_DIR, "environments/environment-schema.json") + """Schema for :py:mod:`~mlos_bench.environments` configurations.""" + OPTIMIZER = path_join(CONFIG_SCHEMA_DIR, "optimizers/optimizer-schema.json") + """Schema for :py:mod:`~mlos_bench.optimizers` configurations.""" + SCHEDULER = path_join(CONFIG_SCHEMA_DIR, "schedulers/scheduler-schema.json") + """Schema for :py:mod:`~mlos_bench.schedulers` configurations.""" + SERVICE = path_join(CONFIG_SCHEMA_DIR, "services/service-schema.json") + """Schema for :py:mod:`~mlos_bench.services` configurations.""" + STORAGE = path_join(CONFIG_SCHEMA_DIR, "storage/storage-schema.json") + """Schema for :py:mod:`~mlos_bench.storage` configurations.""" + TUNABLE_PARAMS = path_join(CONFIG_SCHEMA_DIR, "tunables/tunable-params-schema.json") + """Schema for :py:mod:`~mlos_bench.tunables.tunable_groups` configurations.""" + TUNABLE_VALUES = path_join(CONFIG_SCHEMA_DIR, "tunables/tunable-values-schema.json") + """Schema for :py:mod:`~mlos_bench.tunables.tunable_groups` values configurations.""" UNIFIED = path_join(CONFIG_SCHEMA_DIR, "mlos-bench-config-schema.json") + """Global combined schema file (e.g., for ``*.mlos.json`` files).""" @property def schema(self) -> dict: From 9f10f6274d4f6e6048bb482a237ed4adb78fcdda Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 23:25:08 +0000 Subject: [PATCH 061/100] more docs --- mlos_bench/mlos_bench/__init__.py | 6 ++++-- mlos_bench/mlos_bench/config/schemas/config_schemas.py | 6 ++++++ mlos_bench/mlos_bench/dict_templater.py | 7 ++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index f81585c74a2..1163223b5cb 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -48,10 +48,12 @@ :py:mod:`~mlos_bench.environments.remote.vm_env.VMEnv`. Alternatively, one could swap out that service for - :py:class:`~mlos_bench.services.remote.ssh.ssh_host_services.SshHostService` in + :py:class:`~mlos_bench.services.remote.ssh.ssh_host_service.SshHostService` in order to target a different VM without having to change the :py:class:`~mlos_bench.environments.base_environment.Environment` configuration at - all. + all since they both implement the same + :py:class:`~mlos_bench.services.types.remote_exec_type.SupportsRemoteExec` + :py:mod:`Services type` interfaces. This is particularly useful when running the same benchmark on different ecosystems and makes the configs more modular and composable. diff --git a/mlos_bench/mlos_bench/config/schemas/config_schemas.py b/mlos_bench/mlos_bench/config/schemas/config_schemas.py index 5f145f6bdad..699e76c5fe0 100644 --- a/mlos_bench/mlos_bench/config/schemas/config_schemas.py +++ b/mlos_bench/mlos_bench/config/schemas/config_schemas.py @@ -4,6 +4,12 @@ # """A simple class for describing where to find different config schemas and validating configs against them. + +See Also +-------- +`mlos_bench/config/README.md +`_ +for additional documentation and examples in the source tree. """ import json # schema files are pure json - no comments diff --git a/mlos_bench/mlos_bench/dict_templater.py b/mlos_bench/mlos_bench/dict_templater.py index 742f572ff07..e0c3b7778ea 100644 --- a/mlos_bench/mlos_bench/dict_templater.py +++ b/mlos_bench/mlos_bench/dict_templater.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Simple class to help with nested dictionary $var templating.""" +"""Simple class to help with nested dictionary ``$var`` templating in configuration +file expansions.""" from copy import deepcopy from string import Template @@ -12,7 +13,7 @@ class DictTemplater: # pylint: disable=too-few-public-methods - """Simple class to help with nested dictionary $var templating.""" + """Simple class to help with nested dictionary ``$var`` templating.""" def __init__(self, source_dict: Dict[str, Any]): """ @@ -65,7 +66,7 @@ def _expand_vars( extra_source_dict: Optional[Dict[str, Any]], use_os_env: bool, ) -> Any: - """Recursively expand $var strings in the currently operating dictionary.""" + """Recursively expand ``$var`` strings in the currently operating dictionary.""" if isinstance(value, str): # First try to expand all $vars internally. value = Template(value).safe_substitute(self._dict) From 89ef7316fa40b1e72fa7bee0eed3d115723627f1 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 23:31:57 +0000 Subject: [PATCH 062/100] formatting --- doc/source/mlos_bench.run.usage.rst | 2 +- mlos_bench/mlos_bench/__init__.py | 5 +++-- .../config/schemas/config_schemas.py | 18 ++++++++++++------ mlos_bench/mlos_bench/dict_templater.py | 5 +++-- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/doc/source/mlos_bench.run.usage.rst b/doc/source/mlos_bench.run.usage.rst index ca9accdce45..b589a55e712 100644 --- a/doc/source/mlos_bench.run.usage.rst +++ b/doc/source/mlos_bench.run.usage.rst @@ -1,5 +1,5 @@ mlos_bench CLI usage -================================== +==================== Here is the current ``--help`` output for the ``mlos_bench`` CLI: diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index 1163223b5cb..85c1bf683b9 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -2,8 +2,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""mlos_bench is a framework to help automate benchmarking and OS/application -parameter autotuning and the data management of the results. +""" +mlos_bench is a framework to help automate benchmarking and OS/application parameter +autotuning and the data management of the results. It can be installed from `pypi `_ via ``pip install mlos-bench`` and executed using the ``mlos_bench`` diff --git a/mlos_bench/mlos_bench/config/schemas/config_schemas.py b/mlos_bench/mlos_bench/config/schemas/config_schemas.py index 699e76c5fe0..4f297871139 100644 --- a/mlos_bench/mlos_bench/config/schemas/config_schemas.py +++ b/mlos_bench/mlos_bench/config/schemas/config_schemas.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""A simple class for describing where to find different config schemas and validating +""" +A simple class for describing where to find different config schemas and validating configs against them. See Also @@ -35,10 +36,11 @@ # NOTE: this may cause pytest to fail if it's expecting exceptions # to be raised for invalid configs. _VALIDATION_ENV_FLAG = "MLOS_BENCH_SKIP_SCHEMA_VALIDATION" -"""The special environment flag to set to skip schema validation. +""" +The special environment flag to set to skip schema validation. -Useful for local development when you're making a lot of changes to the config or -adding new classes that aren't in the main repo yet. +Useful for local development when you're making a lot of changes to the config or adding +new classes that aren't in the main repo yet. """ _SKIP_VALIDATION = environ.get(_VALIDATION_ENV_FLAG, "false").lower() in { @@ -118,7 +120,9 @@ def registry(self) -> Registry: SCHEMA_STORE = SchemaStore() -"""Static SchemaStore instance used for storing and retrieving schemas for config validation.""" +"""Static SchemaStore instance used for storing and retrieving schemas for config +validation. +""" class ConfigSchema(Enum): @@ -149,7 +153,9 @@ class ConfigSchema(Enum): """Schema for :py:mod:`~mlos_bench.tunables.tunable_groups` configurations.""" TUNABLE_VALUES = path_join(CONFIG_SCHEMA_DIR, "tunables/tunable-values-schema.json") - """Schema for :py:mod:`~mlos_bench.tunables.tunable_groups` values configurations.""" + """Schema for :py:mod:`~mlos_bench.tunables.tunable_groups` values + configurations. + """ UNIFIED = path_join(CONFIG_SCHEMA_DIR, "mlos-bench-config-schema.json") """Global combined schema file (e.g., for ``*.mlos.json`` files).""" diff --git a/mlos_bench/mlos_bench/dict_templater.py b/mlos_bench/mlos_bench/dict_templater.py index e0c3b7778ea..dd1d1f78afd 100644 --- a/mlos_bench/mlos_bench/dict_templater.py +++ b/mlos_bench/mlos_bench/dict_templater.py @@ -2,8 +2,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Simple class to help with nested dictionary ``$var`` templating in configuration -file expansions.""" +"""Simple class to help with nested dictionary ``$var`` templating in configuration file +expansions. +""" from copy import deepcopy from string import Template From 09f1850dd22ba75f53f03b973d34ab4cb37d6c6a Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 12 Nov 2024 23:46:33 +0000 Subject: [PATCH 063/100] don't run anything via pytest parsing --- .../boot/scripts/local/create_new_grub_cfg.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/mlos_bench/mlos_bench/config/environments/os/linux/boot/scripts/local/create_new_grub_cfg.py b/mlos_bench/mlos_bench/config/environments/os/linux/boot/scripts/local/create_new_grub_cfg.py index 9b75f040080..c0cc5b3ea84 100755 --- a/mlos_bench/mlos_bench/config/environments/os/linux/boot/scripts/local/create_new_grub_cfg.py +++ b/mlos_bench/mlos_bench/config/environments/os/linux/boot/scripts/local/create_new_grub_cfg.py @@ -14,10 +14,16 @@ JSON_CONFIG_FILE = "config-boot-time.json" NEW_CFG = "zz-mlos-boot-params.cfg" -with open(JSON_CONFIG_FILE, "r", encoding="UTF-8") as fh_json, open( - NEW_CFG, "w", encoding="UTF-8" -) as fh_config: - for key, val in json.load(fh_json).items(): - fh_config.write( - 'GRUB_CMDLINE_LINUX_DEFAULT="$' f'{{GRUB_CMDLINE_LINUX_DEFAULT}} {key}={val}"\n' - ) + +def _write_config() -> None: + with open(JSON_CONFIG_FILE, "r", encoding="UTF-8") as fh_json, open( + NEW_CFG, "w", encoding="UTF-8" + ) as fh_config: + for key, val in json.load(fh_json).items(): + fh_config.write( + 'GRUB_CMDLINE_LINUX_DEFAULT="$' f'{{GRUB_CMDLINE_LINUX_DEFAULT}} {key}={val}"\n' + ) + + +if __name__ == "__main__": + _write_config() From 0625a187cc193bbd029d526913cf39eca5ae434d Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 13 Nov 2024 00:48:32 +0000 Subject: [PATCH 064/100] ignore generated files --- .vscode/settings.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 982eb42c8dd..7d7a270a9a0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,10 @@ // vim: set ft=jsonc: { "makefile.extensionOutputFolder": "./.vscode", + "files.exclude": { + "**/__pycache__/": true, + "doc/source/autoapi/": true + }, // Note: this only works in WSL/Linux currently. "python.defaultInterpreterPath": "${env:HOME}/.conda/envs/mlos/bin/python", // For Windows it should be this instead: From 5ada076732896c12294137287ce48d710cdab138 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 13 Nov 2024 00:48:46 +0000 Subject: [PATCH 065/100] add a somehow missing test case --- .../test-cases/good/partial/grid_search_opt_minimal.jsonc | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 mlos_bench/mlos_bench/tests/config/schemas/optimizers/test-cases/good/partial/grid_search_opt_minimal.jsonc diff --git a/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test-cases/good/partial/grid_search_opt_minimal.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test-cases/good/partial/grid_search_opt_minimal.jsonc new file mode 100644 index 00000000000..0311708350f --- /dev/null +++ b/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test-cases/good/partial/grid_search_opt_minimal.jsonc @@ -0,0 +1,4 @@ +{ + "class": "mlos_bench.optimizers.GridSearchOptimizer" + // no config required +} From b3c2536a21c6ab75bc04563fa832565123afefac Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 13 Nov 2024 02:09:52 +0000 Subject: [PATCH 066/100] tweak --- .../test-cases/good/partial/grid_search_opt_minimal.jsonc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test-cases/good/partial/grid_search_opt_minimal.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test-cases/good/partial/grid_search_opt_minimal.jsonc index 0311708350f..297e3a62c8e 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test-cases/good/partial/grid_search_opt_minimal.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test-cases/good/partial/grid_search_opt_minimal.jsonc @@ -1,4 +1,4 @@ { - "class": "mlos_bench.optimizers.GridSearchOptimizer" + "class": "mlos_bench.optimizers.grid_search_optimizer.GridSearchOptimizer" // no config required } From 80887192777acdd4b76acc13e605826a1afc10fd Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 13 Nov 2024 17:23:29 +0000 Subject: [PATCH 067/100] fixup for keeping files based on github actions --- Makefile | 2 +- doc/source/conf.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index eec7baa3061..d465b2b70ea 100644 --- a/Makefile +++ b/Makefile @@ -668,7 +668,7 @@ endif # Treat warnings as failures. SPHINXOPTS ?= # -v # be verbose -SPHINXOPTS += -n -W -w $(PWD)/doc/build/sphinx-build.warn.log -j auto +SPHINXOPTS += -n -W -w $(CURDIR)/doc/build/sphinx-build.warn.log -j auto sphinx-apidoc: doc/build/html/index.html diff --git a/doc/source/conf.py b/doc/source/conf.py index db266c18f49..b9043e9214f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -123,6 +123,13 @@ def linkcode_resolve(domain: str, info: Dict[str, str]): return f"https://github.com/microsoft/MLOS/tree/main/{path}" +def is_on_github_actions(): + """Check if the documentation is being built on GitHub Actions.""" + if "CI" not in os.environ or not os.environ["CI"] or "GITHUB_RUN_ID" not in os.environ: + return False + return True + + # Add mappings to link to external documentation. intersphinx_mapping = get_intersphinx_mapping( packages={ @@ -242,7 +249,7 @@ def linkcode_resolve(domain: str, info: Dict[str, str]): autoapi_python_class_content = "both" autoapi_member_order = "groupwise" autoapi_add_toctree_entry = False # handled manually -# autoapi_keep_files = True # for testing +autoapi_keep_files = not is_on_github_actions() # for local testing # -- Options for HTML output ------------------------------------------------- From b40ba84a3f04a5ed2159cd3f49978f269af8632e Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 13 Nov 2024 18:26:28 +0000 Subject: [PATCH 068/100] revert pylint tweak --- mlos_bench/mlos_bench/tunables/tunable_groups.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlos_bench/mlos_bench/tunables/tunable_groups.py b/mlos_bench/mlos_bench/tunables/tunable_groups.py index 78b4c720a4a..99772acfd05 100644 --- a/mlos_bench/mlos_bench/tunables/tunable_groups.py +++ b/mlos_bench/mlos_bench/tunables/tunable_groups.py @@ -172,13 +172,13 @@ def __setitem__( self._index[name][name] = value return self._index[name][name] - def __iter__(self) -> Generator[Tuple[Tunable, CovariantTunableGroup]]: + def __iter__(self) -> Generator[Tuple[Tunable, CovariantTunableGroup], None, None]: """ An iterator over all tunables in the group. Returns ------- - [(tunable, group), ...] : Generator[Tuple[Tunable, CovariantTunableGroup]] + [(tunable, group), ...] : Generator[Tuple[Tunable, CovariantTunableGroup], None, None] An iterator over all tunables in all groups. Each element is a 2-tuple of an instance of the Tunable parameter and covariant group it belongs to. """ From 3dcd0beab50adc5c15345ac3c87417980749d5df Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 13 Nov 2024 18:52:12 +0000 Subject: [PATCH 069/100] ignore some generated files to let vscode enumerate things quicker --- .vscode/settings.json | 12 ++++++++++-- pyproject.toml | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 7d7a270a9a0..5c75f4e35c6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,8 +2,16 @@ { "makefile.extensionOutputFolder": "./.vscode", "files.exclude": { - "**/__pycache__/": true, - "doc/source/autoapi/": true + ".git/": true, + ".mypy_cache/": true, + ".pytest_cache/": true, + "**/__pycache__/": true, + "**/node_modules/": true, + "**/*.egg-info": true, + "doc/source/autoapi/": true, + "doc/build/doctrees/": true, + "doc/build/html/": true, + "htmlcov/": true, }, // Note: this only works in WSL/Linux currently. "python.defaultInterpreterPath": "${env:HOME}/.conda/envs/mlos/bin/python", diff --git a/pyproject.toml b/pyproject.toml index 5464aa60b65..1ece657210e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,3 +68,17 @@ disable = [ [tool.pylint.string] check-quote-consistency = true check-str-concat-over-line-jumps = true + +[tool.pyright] +exclude = [ + ".git", + ".mypy_cache", + ".pytest_cache", + "**/node_modules", + "**/__pycache__", + "**/*.egg-info", + "doc/source/autoapi", + "doc/build/html", + "doc/build/doctrees", + "htmlcov", +] From 424f1fc73e2918ce459a38b2aea9e4081b23be3b Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 13 Nov 2024 18:54:01 +0000 Subject: [PATCH 070/100] always upload the artifact (for testing) --- .github/workflows/devcontainer.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index c0d41a3d72c..ae009107508 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -198,7 +198,6 @@ jobs: rm -f doc/build/html/htmlcov/.gitignore - uses: actions/upload-artifact@v4 - if: github.ref == 'refs/heads/main' with: name: docs path: doc/build/html From 9810161965220938aac04ab67a87b560df827be9 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 13 Nov 2024 19:36:07 +0000 Subject: [PATCH 071/100] Some mlos_bench examples --- mlos_bench/mlos_bench/__init__.py | 36 ++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index 85c1bf683b9..3f5314d2f0a 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -78,13 +78,47 @@ Examples -------- -TODO: Add examples + +Here is an example that shows how to run a simple benchmark using the command line. + +The entry point for these configs can be found `here +`_. + +> Note: we show the command wrapped in python here for testing purposes. + +>>> from subprocess import run +>>> # Alternatively replace test-cli-local-env-bench.jsonc with +>>> # test-cli-local-env-opt.jsonc for one that does an optimization loop. +>>> cmd = r'''mlos_bench \\ +... --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc \\ +... --globals experiment_test_local.jsonc \\ +... --tunable_values tunable-values/tunable-values-local.jsonc''' +>>> print(f"Here's the shell command you'd actually run:\\n# {cmd}") +Here's the shell command you'd actually run: +# mlos_bench \\ + --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc \\ + --globals experiment_test_local.jsonc \\ + --tunable_values tunable-values/tunable-values-local.jsonc +>>> # Now we run the command and check the output. +>>> result = run(cmd, shell=True, capture_output=True, text=True, check=True) +>>> assert result.returncode == 0 +>>> lines = result.stderr.splitlines() +>>> first_line = lines[0] +>>> last_line = lines[-1] +>>> expected = "INFO Launch: mlos_bench" +>>> assert first_line.endswith(expected) +>>> expected = "INFO Final score: {'score': 123.4, 'total_time': 123.4, 'throughput': 1234567.0}" +>>> assert last_line.endswith(expected) See Also -------- `mlos_bench/README.md `_ for additional documentation and examples in the source tree. + +There is also a working example of using ``mlos_bench`` in a separate repo for the +configs in the `sqlite-autotuning +`_ repo. """ from mlos_bench.version import VERSION From c9e43a9366abdc8c90ca025e4d82907ea90c17c9 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 13 Nov 2024 21:40:45 +0000 Subject: [PATCH 072/100] more docs --- doc/source/mlos_bench.run.usage.rst | 2 +- mlos_bench/mlos_bench/__init__.py | 5 +- .../config/schemas/config_schemas.py | 7 ++- .../mlos_bench/environments/__init__.py | 47 ++++++++++++++++++- mlos_bench/mlos_bench/event_loop_context.py | 11 +++-- mlos_bench/mlos_bench/launcher.py | 4 +- mlos_bench/mlos_bench/os_environ.py | 13 +++-- mlos_bench/mlos_bench/run.py | 6 +-- mlos_bench/mlos_bench/storage/__init__.py | 2 +- mlos_bench/mlos_bench/storage/sql/__init__.py | 2 +- 10 files changed, 78 insertions(+), 21 deletions(-) diff --git a/doc/source/mlos_bench.run.usage.rst b/doc/source/mlos_bench.run.usage.rst index b589a55e712..782da2924e8 100644 --- a/doc/source/mlos_bench.run.usage.rst +++ b/doc/source/mlos_bench.run.usage.rst @@ -1,7 +1,7 @@ mlos_bench CLI usage ==================== -Here is the current ``--help`` output for the ``mlos_bench`` CLI: +Here is the current ``--help`` output for the ``mlos_bench`` :py:mod:`CLI script `: .. literalinclude:: ./generated/mlos_bench.run.usage.txt :language: none diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index 3f5314d2f0a..ac8e8b4532a 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -30,7 +30,7 @@ These are generally the target of the optimization process and are used to evaluate the performance of a given configuration, though can also be used to simply run a single benchmark. - They can be used, for instance, to :py:mod:`provision VMs + They can be used, for instance, to provision a :py:mod:`VM `, run benchmarks or execute any other arbitrary code on a :py:mod:`remote machine `, and many other things. @@ -84,9 +84,8 @@ The entry point for these configs can be found `here `_. -> Note: we show the command wrapped in python here for testing purposes. - >>> from subprocess import run +>>> # Note: we show the command wrapped in python here for testing purposes. >>> # Alternatively replace test-cli-local-env-bench.jsonc with >>> # test-cli-local-env-opt.jsonc for one that does an optimization loop. >>> cmd = r'''mlos_bench \\ diff --git a/mlos_bench/mlos_bench/config/schemas/config_schemas.py b/mlos_bench/mlos_bench/config/schemas/config_schemas.py index 4f297871139..dd3f8c6b081 100644 --- a/mlos_bench/mlos_bench/config/schemas/config_schemas.py +++ b/mlos_bench/mlos_bench/config/schemas/config_schemas.py @@ -158,7 +158,12 @@ class ConfigSchema(Enum): """ UNIFIED = path_join(CONFIG_SCHEMA_DIR, "mlos-bench-config-schema.json") - """Global combined schema file (e.g., for ``*.mlos.json`` files).""" + """Global combined schema file (e.g., for ``*.mlos.json`` files). + + See Also + -------- + + """ @property def schema(self) -> dict: diff --git a/mlos_bench/mlos_bench/environments/__init__.py b/mlos_bench/mlos_bench/environments/__init__.py index ff649af50ef..569147b7c01 100644 --- a/mlos_bench/mlos_bench/environments/__init__.py +++ b/mlos_bench/mlos_bench/environments/__init__.py @@ -2,7 +2,52 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Tunable Environments for mlos_bench.""" +"""Tunable Environments for mlos_bench. + +Environments are classes that represent an execution setting (environment) for +running a benchmark or tuning process. + +An Environment goes through a series of phases (e.g., +:py:meth:`~.Environment.setup`, :py:meth:`~.Environment.run`, +:py:meth:`~.Environment.teardown`) that can be used to prepare a VM, workload, etc.; +run a benchmark, script, etc.; and clean up afterwards. + +They can be stacked together with the :py:class:`.CompositeEnv` class to +represent complex setups (e.g., an appication running on a VM). + +Each environment can use +:py:class:`~mlos_bench.tunables.tunable_groups.TunableGroups` to specify the set of +configuration parameters that can be optimized or searched. +At each iteration of the optimization process, the optimizer will generate a set of +values for the `Tunables` that the environment can use to configure itself. + +Environments can also reference :py:mod:`~mlos_bench.services` that provide the +necessary support to enact the actions that environment needs for each of its +phases depending upon where its being deployed (e.g., local machine, remote machine, +cloud provider VM, etc.) + +See below for the set of Environments currently available in this package. +Note that additional ones can also be created by extending the base +:py:class:`.Environment` class. + +Examples +-------- +While this documentation is generated from the source code and is intended to be a +useful reference on the internal details, most users will be more interested in +generating json configs to be used with the ``mlos_bench`` command line tool. + +TODO: Add examples here. + +See Also +-------- +`mlos_bench/environments/README.md +`_ +for additional documentation in the source tree. + +`mlos_bench/config/environments/README.md +`_ +for additional config examples in the source tree. +""" from mlos_bench.environments.base_environment import Environment from mlos_bench.environments.composite_env import CompositeEnv diff --git a/mlos_bench/mlos_bench/event_loop_context.py b/mlos_bench/mlos_bench/event_loop_context.py index 8144bc399d2..39896030872 100644 --- a/mlos_bench/mlos_bench/event_loop_context.py +++ b/mlos_bench/mlos_bench/event_loop_context.py @@ -19,8 +19,11 @@ from typing_extensions import TypeAlias CoroReturnType = TypeVar("CoroReturnType") # pylint: disable=invalid-name +"""Type variable for the return type of an :external:py:mod:`asyncio` coroutine.""" + if sys.version_info >= (3, 9): FutureReturnType: TypeAlias = Future[CoroReturnType] + """Type variable for the return type of a :py:class:`~concurrent.futures.Future`.""" else: FutureReturnType: TypeAlias = Future @@ -29,15 +32,15 @@ class EventLoopContext: """ - EventLoopContext encapsulates a background thread for asyncio event loop processing - as an aid for context managers. + EventLoopContext encapsulates a background thread for :external:py:mod:`asyncio` + event loop processing as an aid for context managers. There is generally only expected to be one of these, either as a base class instance if it's specific to that functionality or for the full mlos_bench process to support parallel trial runners, for instance. - It's enter() and exit() routines are expected to be called from the caller's context - manager routines (e.g., __enter__ and __exit__). + It's :py:meth:`.enter` and :py:meth:`.exit` routines are expected to be called + from the caller's context manager routines (e.g., __enter__ and __exit__). """ def __init__(self) -> None: diff --git a/mlos_bench/mlos_bench/launcher.py b/mlos_bench/mlos_bench/launcher.py index 339a11963de..b970b98456e 100644 --- a/mlos_bench/mlos_bench/launcher.py +++ b/mlos_bench/mlos_bench/launcher.py @@ -6,8 +6,8 @@ A helper class to load the configuration files, parse the command line parameters, and instantiate the main components of mlos_bench system. -It is used in `mlos_bench.run` module to run the benchmark/optimizer from the -command line. +It is used in the :py:mod:`mlos_bench.run` module to run the benchmark/optimizer +from the command line. """ import argparse diff --git a/mlos_bench/mlos_bench/os_environ.py b/mlos_bench/mlos_bench/os_environ.py index f750f120389..7a870f94ec5 100644 --- a/mlos_bench/mlos_bench/os_environ.py +++ b/mlos_bench/mlos_bench/os_environ.py @@ -4,13 +4,16 @@ # """ Simple platform agnostic abstraction for the OS environment variables. Meant as a -replacement for os.environ vs nt.environ. +replacement for :external:py:data:`os.environ` vs ``nt.environ``. Example ------- -from mlos_bench.os_env import environ -environ['FOO'] = 'bar' -environ.get('PWD') +>>> # Import the environ object. +>>> from mlos_bench.os_env import environ +>>> # Set an environment variable. +>>> environ["FOO"] = "bar" +>>> # Get an environment variable. +>>> pwd = environ.get("PWD") """ import os @@ -33,7 +36,9 @@ import nt # type: ignore[import-not-found] # pylint: disable=import-error # (3.8) environ: EnvironType = nt.environ + """A platform agnostic abstraction for the OS environment variables.""" else: environ: EnvironType = os.environ + """A platform agnostic abstraction for the OS environment variables.""" __all__ = ["environ"] diff --git a/mlos_bench/mlos_bench/run.py b/mlos_bench/mlos_bench/run.py index a554f1a803d..1c993ea0247 100755 --- a/mlos_bench/mlos_bench/run.py +++ b/mlos_bench/mlos_bench/run.py @@ -4,11 +4,11 @@ # Licensed under the MIT License. # """ -OS Autotune main optimization loop. +mlos_bench main optimization loop and benchmark runner CLI. -Note: this script is also available as a CLI tool via pip under the name "mlos_bench". +Note: this script is also available as a CLI tool via ``pip`` under the name ``mlos_bench``. -See `--help` output for details. +See the current ``--help`` `output for details `_. """ import logging diff --git a/mlos_bench/mlos_bench/storage/__init__.py b/mlos_bench/mlos_bench/storage/__init__.py index 64e70c20f76..2f75f7253f4 100644 --- a/mlos_bench/mlos_bench/storage/__init__.py +++ b/mlos_bench/mlos_bench/storage/__init__.py @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Interfaces to the storage backends for OS Autotune.""" +"""Interfaces to the storage backends for mlos_bench.""" from mlos_bench.storage.base_storage import Storage from mlos_bench.storage.storage_factory import from_config diff --git a/mlos_bench/mlos_bench/storage/sql/__init__.py b/mlos_bench/mlos_bench/storage/sql/__init__.py index 9d749ed35d1..eb8867032ad 100644 --- a/mlos_bench/mlos_bench/storage/sql/__init__.py +++ b/mlos_bench/mlos_bench/storage/sql/__init__.py @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Interfaces to the SQL-based storage backends for OS Autotune.""" +"""Interfaces to the SQL-based storage backends for mlos_bench.""" from mlos_bench.storage.sql.storage import SqlStorage __all__ = [ From 44d38747a0211c47c2629dc016aa6dcc9976c449 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 13 Nov 2024 21:40:57 +0000 Subject: [PATCH 073/100] hack to resolve internal typevars --- doc/source/conf.py | 85 +++++++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 28 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index b9043e9214f..a099fd451df 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -21,12 +21,15 @@ import json import os import sys -from typing import Dict +from typing import Dict, Union, Tuple from logging import warning +from docutils.nodes import Element from intersphinx_registry import get_intersphinx_mapping - +from sphinx.application import Sphinx as SphinxApp +from sphinx.environment import BuildEnvironment +from sphinx.addnodes import pending_xref sys.path.insert(0, os.path.abspath("../../mlos_core/mlos_core")) sys.path.insert(1, os.path.abspath("../../mlos_bench/mlos_bench")) @@ -152,40 +155,66 @@ def is_on_github_actions(): } ) -# We recommend adding the following config value. -# Sphinx defaults to automatically resolve *unresolved* labels using all your Intersphinx mappings. -# This behavior has unintended side-effects, namely that documentations local references can -# suddenly resolve to an external location. -# See also: -# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#confval-intersphinx_disabled_reftypes -# intersphinx_disabled_reftypes = ["*"] +# Hack to resolve type aliases as attributes instead of classes. +# See Also: https://github.com/sphinx-doc/sphinx/issues/10785 + +# Type alias resolution map +# (original, refname) -> new +CUSTOM_REF_TYPE_MAP: Dict[Tuple[str, str], str] = { + # Internal typevars and aliases: + ("BaseTypeVar", "class"): "data", + ("ConcreteOptimizer", "class"): "data", + ("ConcreteSpaceAdapter", "class"): "data", + ("DistributionName", "class"): "data", + ("FlamlDomain", "class"): "data", + ("mlos_core.spaces.converters.flaml.FlamlDomain", "class"): "data", + ("TunableValue", "class"): "data", + ("mlos_bench.tunables.tunable.TunableValue", "class"): "data", + ("TunableValueType", "class"): "data", + ("TunableValueTypeName", "class"): "data", + ("T_co", "class"): "data", + ("CoroReturnType", "class"): "data", + ("FutureReturnType", "class"): "data", +} + + +def resolve_type_aliases( + app: SphinxApp, + env: BuildEnvironment, + node: pending_xref, + contnode: Element, +) -> Union[Element, None]: + """Resolve :class: references to our type aliases as :attr: instead.""" + if node["refdomain"] != "py": + return None + (orig_type, reftarget) = (node["reftype"], node["reftarget"]) + new_type = CUSTOM_REF_TYPE_MAP.get((reftarget, orig_type)) + if new_type: + # warning(f"Resolved {orig_type} {reftarget} to {new_type}") + return app.env.get_domain("py").resolve_xref( + env, + node["refdoc"], + app.builder, + new_type, + reftarget, + node, + contnode, + ) + return None + + +def setup(app: SphinxApp) -> None: + """Connect the missing-reference event to resolve type aliases.""" + app.connect("missing-reference", resolve_type_aliases) + # Ignore some cross references to external things we can't intersphinx with. # sphinx has a hard time finding typealiases and typevars instead of classes. # See Also: https://github.com/sphinx-doc/sphinx/issues/10974 nitpick_ignore = [ # Internal typevars and aliases: - ("py:class", "BaseTypeVar"), - ("py:class", "ConcreteOptimizer"), - ("py:class", "ConcreteSpaceAdapter"), - ("py:class", "DistributionName"), ("py:class", "EnvironType"), - ("py:class", "FlamlDomain"), - ("py:class", "mlos_core.spaces.converters.flaml.FlamlDomain"), - ("py:class", "TunableValue"), - ("py:class", "mlos_bench.tunables.tunable.TunableValue"), - ("py:class", "TunableValueType"), - ("py:class", "TunableValueTypeName"), - ("py:class", "T_co"), - # Literals in base_storage.py - ("py:class", "min"), - ("py:class", "max"), - # Literal in context handlers signatures. - ("py:class", "False"), # External typevars and aliases: - ("py:class", "CoroReturnType"), - ("py:class", "FutureReturnType"), - ("py:class", "typing_extensions.Literal"), ("py:class", "numpy.typing.NDArray"), # External classes that refuse to resolve: ("py:class", "contextlib.nullcontext"), From 7acd53a9e286851473e1c760fc8b989e01faf6a9 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 13 Nov 2024 22:01:53 +0000 Subject: [PATCH 074/100] fixup --- mlos_bench/mlos_bench/os_environ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlos_bench/mlos_bench/os_environ.py b/mlos_bench/mlos_bench/os_environ.py index 7a870f94ec5..c83ce05b343 100644 --- a/mlos_bench/mlos_bench/os_environ.py +++ b/mlos_bench/mlos_bench/os_environ.py @@ -9,7 +9,7 @@ Example ------- >>> # Import the environ object. ->>> from mlos_bench.os_env import environ +>>> from mlos_bench.os_environ import environ >>> # Set an environment variable. >>> environ["FOO"] = "bar" >>> # Get an environment variable. From 8aa72a80e4f6a4cde51f232852e4cd58fe992d0b Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 13 Nov 2024 23:21:57 +0000 Subject: [PATCH 075/100] comments --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 1ece657210e..b5014df8a87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ disable = [ check-quote-consistency = true check-str-concat-over-line-jumps = true +# Tell the vscode python extension to ignore some autogenerated files. [tool.pyright] exclude = [ ".git", From be9b1385a208244e44812e0a0c17c322021d3f8f Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 14 Nov 2024 18:22:36 +0000 Subject: [PATCH 076/100] more rules on how to schema interpret some things --- .vscode/settings.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 5c75f4e35c6..759b34a412a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -38,6 +38,8 @@ "mlos_bench/mlos_bench/tests/config/environments/**/*.json", "mlos_bench/mlos_bench/config/environments/**/*.jsonc", "mlos_bench/mlos_bench/config/environments/**/*.json", + "!mlos_bench/mlos_bench/tests/config/environments/**/*-tunables.jsonc", + "!mlos_bench/mlos_bench/tests/config/environments/**/*-tunables.json", "!mlos_bench/mlos_bench/config/environments/**/*-tunables.jsonc", "!mlos_bench/mlos_bench/config/environments/**/*-tunables.json" ], From 920c3333411e605d5ed1b33dd5bddf8eee47d46a Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 14 Nov 2024 18:22:50 +0000 Subject: [PATCH 077/100] docs on docs --- doc/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/README.md b/doc/README.md index 6af139da1ea..4ed884f8cc7 100644 --- a/doc/README.md +++ b/doc/README.md @@ -14,6 +14,8 @@ Those are included in the [`index.rst`](./source/index.rst) file which is the ma When writing docstrings, use the [`numpydoc`](https://numpydoc.readthedocs.io/en/latest/format.html) style. +Where necessary embedded [reStructuredText (rst)](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html) markup can be used to help format the documentation. + Each top level module should include a docstring that describes the module and its purpose and usage. These string should be written for consumption by both users and developers. @@ -64,10 +66,13 @@ class MyClass: ... ``` +This code will be automatically checked with `pytest` using the `--doctest-modules` option specified in [`setup.cfg`](../setup.cfg). + ## Building the documentation ```sh -make -C .. doc +# From the root of the repository +make SKIP_COVERAGE=true doc ``` This will also run some checks on the documentation. @@ -90,6 +95,6 @@ We use the [`intersphinx`](https://www.sphinx-doc.org/en/master/usage/extensions Unfortunately, this process is not perfect and sometimes we need to provide [`nitpick_ignore`](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-nitpick_ignore)s in the [`doc/source/conf.py`](./source/conf.py) file. -In particular, currently `TypeVar` and `TypeAliases` are not resolved correctly and we need to ignore those. +In particular, currently external `TypeVar` and `TypeAliases` are not resolved correctly and we need to ignore those. In other cases, specifying the full path to the module in the cross-reference or the `import` can help. From 943cb274042556403b8373881c5f6ec6f0441504 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 14 Nov 2024 18:23:26 +0000 Subject: [PATCH 078/100] more docs --- .../mlos_bench/config/schemas/__init__.py | 8 +- .../config/schemas/config_schemas.py | 94 +++++++++++++++---- .../mlos_bench/environments/__init__.py | 48 ++++++++-- mlos_bench/mlos_bench/run.py | 3 + 4 files changed, 124 insertions(+), 29 deletions(-) diff --git a/mlos_bench/mlos_bench/config/schemas/__init__.py b/mlos_bench/mlos_bench/config/schemas/__init__.py index d4987add630..5ebc80fac40 100644 --- a/mlos_bench/mlos_bench/config/schemas/__init__.py +++ b/mlos_bench/mlos_bench/config/schemas/__init__.py @@ -2,7 +2,13 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""A module for managing config schemas and their validation.""" +"""A module for managing config schemas and their validation. + +See Also +-------- +:py:mod:`~mlos_bench.config.schemas.config_schemas`: The module handling the actual + schema definitions and validation. +""" from mlos_bench.config.schemas.config_schemas import CONFIG_SCHEMA_DIR, ConfigSchema diff --git a/mlos_bench/mlos_bench/config/schemas/config_schemas.py b/mlos_bench/mlos_bench/config/schemas/config_schemas.py index dd3f8c6b081..530a6a13557 100644 --- a/mlos_bench/mlos_bench/config/schemas/config_schemas.py +++ b/mlos_bench/mlos_bench/config/schemas/config_schemas.py @@ -3,14 +3,22 @@ # Licensed under the MIT License. # """ -A simple class for describing where to find different config schemas and validating -configs against them. +A simple class for describing where to find different `json config schemas +`_ and validating configs against them. + +Used by the :py:class:`~mlos_bench.launcher.Launcher` and +:py:class:`~mlos_bench.services.config_persistence.ConfigPersistenceService` to +validate configs on load. See Also -------- +`mlos_bench/config/schemas/README.md +`_ +for additional documentation in the source tree. + `mlos_bench/config/README.md `_ -for additional documentation and examples in the source tree. +for additional config examples in the source tree. """ import json # schema files are pure json - no comments @@ -35,15 +43,15 @@ # It is used in `ConfigSchema.validate()` method below. # NOTE: this may cause pytest to fail if it's expecting exceptions # to be raised for invalid configs. -_VALIDATION_ENV_FLAG = "MLOS_BENCH_SKIP_SCHEMA_VALIDATION" +VALIDATION_ENV_FLAG = "MLOS_BENCH_SKIP_SCHEMA_VALIDATION" """ -The special environment flag to set to skip schema validation. +The special environment flag to set to skip schema validation when "true". Useful for local development when you're making a lot of changes to the config or adding new classes that aren't in the main repo yet. """ -_SKIP_VALIDATION = environ.get(_VALIDATION_ENV_FLAG, "false").lower() in { +_SKIP_VALIDATION = environ.get(VALIDATION_ENV_FLAG, "false").lower() in { "true", "y", "yes", @@ -120,8 +128,8 @@ def registry(self) -> Registry: SCHEMA_STORE = SchemaStore() -"""Static SchemaStore instance used for storing and retrieving schemas for config -validation. +"""Static :py:class:`.SchemaStore` instance used for storing and retrieving schemas +for config validation. """ @@ -129,36 +137,80 @@ class ConfigSchema(Enum): """An enum to help describe schema types and help validate configs against them.""" CLI = path_join(CONFIG_SCHEMA_DIR, "cli/cli-schema.json") - """Schema for command line arguments.""" + """ + Json config `schema + `__ + for :py:mod:`mlos_bench ` CLI configuration. + + Note: The :py:class:`mlos_bench.launcher.Launcher` class is responsible for + processing the CLI args. + """ GLOBALS = path_join(CONFIG_SCHEMA_DIR, "cli/globals-schema.json") - """Schema for global variables.""" + """ + Json config `schema + `__ + for :py:mod:`global variables `. + """ ENVIRONMENT = path_join(CONFIG_SCHEMA_DIR, "environments/environment-schema.json") - """Schema for :py:mod:`~mlos_bench.environments` configurations.""" + """ + Json config `schema + `__ + for :py:mod:`~mlos_bench.environments`. + """ OPTIMIZER = path_join(CONFIG_SCHEMA_DIR, "optimizers/optimizer-schema.json") - """Schema for :py:mod:`~mlos_bench.optimizers` configurations.""" + """ + Json config `schema + `__ + for :py:mod:`~mlos_bench.optimizers`. + """ SCHEDULER = path_join(CONFIG_SCHEMA_DIR, "schedulers/scheduler-schema.json") - """Schema for :py:mod:`~mlos_bench.schedulers` configurations.""" + """ + Json config `schema + `__ + for :py:mod:`~mlos_bench.schedulers`. + """ SERVICE = path_join(CONFIG_SCHEMA_DIR, "services/service-schema.json") - """Schema for :py:mod:`~mlos_bench.services` configurations.""" + """ + Json config `schema + `__ + for :py:mod:`~mlos_bench.services`. + """ STORAGE = path_join(CONFIG_SCHEMA_DIR, "storage/storage-schema.json") - """Schema for :py:mod:`~mlos_bench.storage` configurations.""" + """ + Json config `schema + `__ + for :py:mod:`~mlos_bench.storage` instances. + """ TUNABLE_PARAMS = path_join(CONFIG_SCHEMA_DIR, "tunables/tunable-params-schema.json") - """Schema for :py:mod:`~mlos_bench.tunables.tunable_groups` configurations.""" + """ + Json config `schema + `__ + for :py:mod:`~mlos_bench.tunables` instances. + """ TUNABLE_VALUES = path_join(CONFIG_SCHEMA_DIR, "tunables/tunable-values-schema.json") - """Schema for :py:mod:`~mlos_bench.tunables.tunable_groups` values - configurations. + """ + Json config `schema + `__ + for values of :py:mod:`~mlos_bench.tunables.tunable_groups.TunableGroups` instances. + + These can be used to specify the values of the tunables for a given experiment + using the :py:class:`~mlos_bench.optimizers.one_shot_optimizer.OneShotOptimizer` + for instance. """ UNIFIED = path_join(CONFIG_SCHEMA_DIR, "mlos-bench-config-schema.json") - """Global combined schema file (e.g., for ``*.mlos.json`` files). + """ + Combined global json `schema + `__ + use to validate any ``mlos_bench`` config file (e.g., ``*.mlos.json`` files). See Also -------- @@ -184,10 +236,12 @@ def validate(self, config: dict) -> None: Raises ------ jsonschema.exceptions.ValidationError + On validation failure. jsonschema.exceptions.SchemaError + On schema loading error. """ if _SKIP_VALIDATION: - _LOG.warning("%s is set - skip schema validation", _VALIDATION_ENV_FLAG) + _LOG.warning("%s is set - skip schema validation", VALIDATION_ENV_FLAG) else: jsonschema.Draft202012Validator( schema=self.schema, diff --git a/mlos_bench/mlos_bench/environments/__init__.py b/mlos_bench/mlos_bench/environments/__init__.py index 569147b7c01..02b2ab60f2d 100644 --- a/mlos_bench/mlos_bench/environments/__init__.py +++ b/mlos_bench/mlos_bench/environments/__init__.py @@ -9,26 +9,54 @@ An Environment goes through a series of phases (e.g., :py:meth:`~.Environment.setup`, :py:meth:`~.Environment.run`, -:py:meth:`~.Environment.teardown`) that can be used to prepare a VM, workload, etc.; -run a benchmark, script, etc.; and clean up afterwards. +:py:meth:`~.Environment.teardown`, etc.) that can be used to prepare a VM, workload, +etc.; run a benchmark, script, etc.; and clean up afterwards. -They can be stacked together with the :py:class:`.CompositeEnv` class to +Environments can be stacked together with the :py:class:`.CompositeEnv` class to represent complex setups (e.g., an appication running on a VM). +See below for the set of Environments currently available in this package. + +Note that additional ones can also be created by extending the base +:py:class:`.Environment` class and referencing them in the :py:mod:`json configs +` using the ``class`` key. + +Environment Tunables +++++++++++++++++++++ Each environment can use :py:class:`~mlos_bench.tunables.tunable_groups.TunableGroups` to specify the set of configuration parameters that can be optimized or searched. At each iteration of the optimization process, the optimizer will generate a set of -values for the `Tunables` that the environment can use to configure itself. +values for the :py:class:`Tunables ` that the +environment can use to configure itself. + +At a python level, this happens by passing a +:py:meth:`~mlos_bench.tunables.tunable_groups.TunableGroups` object to the +``tunable_groups`` parameter of the :py:class:`~.Environment` constructor. + +In the typical json user level configs, this is specified in the +``include_tunables`` section of the Environment config to include the +``TunableGroups`` definitions from other json files when the +``mlos_bench.loader.Loader`` processes the initial set of config files. + +The ``tunable_params`` setting in the ``config`` section of the Environment config +can also be used to limit *which* of the ``TunableGroups`` should be used for the +Environment. + +Since :py:mod:`json configs ` also support ``$variable`` +substitution in the values using the `globals` mechanism, this setting can used to +dynamically change the set of active TunableGroups for a given Experiment using only +`globals`, allowing for configs to be more modular and composable. + +Environment Services +++++++++++++++++++++ Environments can also reference :py:mod:`~mlos_bench.services` that provide the -necessary support to enact the actions that environment needs for each of its +necessary support to perform the actions that environment needs for each of its phases depending upon where its being deployed (e.g., local machine, remote machine, cloud provider VM, etc.) -See below for the set of Environments currently available in this package. -Note that additional ones can also be created by extending the base -:py:class:`.Environment` class. +TODO: Add more details here. Examples -------- @@ -36,6 +64,10 @@ useful reference on the internal details, most users will be more interested in generating json configs to be used with the ``mlos_bench`` command line tool. +For a working example please see the `test_local_env_bench.jsonc +`_ +file or other examples in the source tree linked below. + TODO: Add examples here. See Also diff --git a/mlos_bench/mlos_bench/run.py b/mlos_bench/mlos_bench/run.py index 1c993ea0247..0eef45e1042 100755 --- a/mlos_bench/mlos_bench/run.py +++ b/mlos_bench/mlos_bench/run.py @@ -9,6 +9,9 @@ Note: this script is also available as a CLI tool via ``pip`` under the name ``mlos_bench``. See the current ``--help`` `output for details `_. + +Note: The :py:class:`mlos_bench.launcher.Launcher` class is responsible for +processing the CLI args. """ import logging From df08c2ee84d54536b0e6138e4e26645fe5b6c1f8 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 14 Nov 2024 19:09:48 +0000 Subject: [PATCH 079/100] some more docs about environments --- .../mlos_bench/environments/__init__.py | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/mlos_bench/mlos_bench/environments/__init__.py b/mlos_bench/mlos_bench/environments/__init__.py index 02b2ab60f2d..58d7677deee 100644 --- a/mlos_bench/mlos_bench/environments/__init__.py +++ b/mlos_bench/mlos_bench/environments/__init__.py @@ -4,16 +4,26 @@ # """Tunable Environments for mlos_bench. -Environments are classes that represent an execution setting (environment) for +Environments are classes that represent an execution setting (i.e., environment) for running a benchmark or tuning process. -An Environment goes through a series of phases (e.g., +For instance, a :py:class:`~.LocalEnv` represents a local execution environment, a +:py:class:`~.RemoteEnv` represents a remote execution environment, a +:py:class:`~mlos_bench.environments.remote.vm_env.VMEnv` represents a virtual +machine, etc. + +An Environment goes through a series of *phases* (e.g., :py:meth:`~.Environment.setup`, :py:meth:`~.Environment.run`, :py:meth:`~.Environment.teardown`, etc.) that can be used to prepare a VM, workload, etc.; run a benchmark, script, etc.; and clean up afterwards. +Often, what these phases do (e.g., what commands to execute) will depend on the +specific Environment and the configs that Environment was loaded with. +This lets Environments be very flexible in what they can accomplish. Environments can be stacked together with the :py:class:`.CompositeEnv` class to -represent complex setups (e.g., an appication running on a VM). +represent complex setups (e.g., an appication running on a remote VM with a +benchmark running from a local machine). + See below for the set of Environments currently available in this package. Note that additional ones can also be created by extending the base @@ -32,7 +42,13 @@ At a python level, this happens by passing a :py:meth:`~mlos_bench.tunables.tunable_groups.TunableGroups` object to the -``tunable_groups`` parameter of the :py:class:`~.Environment` constructor. +``tunable_groups`` parameter of the :py:class:`~.Environment` constructor, but that +is typically handled by the +:py:meth:`~mlos_bench.services.config_persistence.ConfigPersistence.load_environment` +method of the +:py:meth:`~mlos_bench.services.config_persistence.ConfigPersistenceService` invoked +by the ``mlos_bench`` command line tool's :py:class:`mlos_bench.loader.Loader` +class. In the typical json user level configs, this is specified in the ``include_tunables`` section of the Environment config to include the @@ -56,7 +72,11 @@ phases depending upon where its being deployed (e.g., local machine, remote machine, cloud provider VM, etc.) -TODO: Add more details here. +Although this can be done in the Environment config directly with the +``include_services`` key, it is often more useful to do it in the global or +:py:mod:`cli config ` to allow for the same Environment to be +used in different settings (e.g., local machine, SSH accessible machine, Azure VM, +etc.) without having to change the Environment config. Examples -------- @@ -64,11 +84,13 @@ useful reference on the internal details, most users will be more interested in generating json configs to be used with the ``mlos_bench`` command line tool. -For a working example please see the `test_local_env_bench.jsonc +For a simple working user oriented example please see the `test_local_env_bench.jsonc `_ file or other examples in the source tree linked below. -TODO: Add examples here. +For more developer oriented examples please see the `mlos_bench/tests/environments +`_ +directory in the source tree. See Also -------- From fc92372d6d9a2a9a38208d22046a694f7ac7089d Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 14 Nov 2024 20:02:37 +0000 Subject: [PATCH 080/100] reformat --- doc/source/mlos_bench.run.usage.rst | 3 + mlos_bench/mlos_bench/__init__.py | 16 ++-- mlos_bench/mlos_bench/config/__init__.py | 94 ++++++++++++++++++- .../mlos_bench/config/schemas/__init__.py | 4 +- .../config/schemas/config_schemas.py | 24 ++--- .../mlos_bench/environments/__init__.py | 26 ++--- mlos_bench/mlos_bench/run.py | 6 +- mlos_core/mlos_core/__init__.py | 6 +- mlos_core/mlos_core/optimizers/__init__.py | 6 +- .../bayesian_optimizers/smac_optimizer.py | 7 +- .../mlos_core/optimizers/flaml_optimizer.py | 7 +- .../mlos_core/spaces/adapters/__init__.py | 6 +- mlos_core/mlos_core/spaces/converters/util.py | 6 +- mlos_viz/mlos_viz/dabl.py | 6 +- 14 files changed, 159 insertions(+), 58 deletions(-) diff --git a/doc/source/mlos_bench.run.usage.rst b/doc/source/mlos_bench.run.usage.rst index 782da2924e8..513d2274db5 100644 --- a/doc/source/mlos_bench.run.usage.rst +++ b/doc/source/mlos_bench.run.usage.rst @@ -3,5 +3,8 @@ mlos_bench CLI usage Here is the current ``--help`` output for the ``mlos_bench`` :py:mod:`CLI script `: +See the :py:mod:`mlos_bench.config` module documentation for more information on +configuration files. + .. literalinclude:: ./generated/mlos_bench.run.usage.txt :language: none diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index ac8e8b4532a..6e8564b639c 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -109,15 +109,15 @@ >>> expected = "INFO Final score: {'score': 123.4, 'total_time': 123.4, 'throughput': 1234567.0}" >>> assert last_line.endswith(expected) -See Also --------- -`mlos_bench/README.md -`_ -for additional documentation and examples in the source tree. +Notes +----- +- `mlos_bench/README.md + `_ + for additional documentation and examples in the source tree. -There is also a working example of using ``mlos_bench`` in a separate repo for the -configs in the `sqlite-autotuning -`_ repo. +- There is also a working example of using ``mlos_bench`` in a separate repo for the + configs in the `sqlite-autotuning + `_ repo. """ from mlos_bench.version import VERSION diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index 306d58f02f2..3a613c188bc 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -2,4 +2,96 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Basic initializer for the mlos_bench config subtree.""" +""" +A module for managing json config schemas and their validation for various +components of MLOS. + +Overview +++++++++ + +MLOS is a framework for doing benchmarking and autotuning for systems written in +python. As such, all of the code classes documented here take python objects in +their construction. + +However, most users of MLOS will interact with the system via the ``mlos_bench`` CLI +and its json config files. This module attempts to document some of those high level +interactions. + +General JSON Config Structure ++++++++++++++++++++++++++++++ + +We use `json5 `_ to parse the json files, since it +allows for inline C style comments (e.g., ``//``, ``/* */``). + +By convention files use the ``*.mlos.json`` or ``*.mlos.jsonc`` extension to indicate +that they are an ``mlos_bench`` config file. + +This allows tools that support `JSON Schema Store +`_ (e.g., `VSCode +`_) to provide helpful autocomplete and validation +of the json configs while editing. + +CLI Configs +^^^^^^^^^^^ + +Tunable Configs +^^^^^^^^^^^^^^^ + +Tunable + +Class Configs +^^^^^^^^^^^^^ + +Class style configs include most anything else and roughly take this form: + +.. code block:: json + { + // some mlos class name to load + "class": "mlos_bench.type.ClassName", + "config": { + // class specific config + "key": "value" + } + } + +Globals and Variable Substitution ++++++++++++++++++++++++++++++++++ +TODO: Document globals and variable substitution. + +Well Known Variables +++++++++++++++++++++ + +Here is a list of well known variables that are used in the config files: + +- ``$experiment_id``: A unique identifier for the experiment. + Typically provided in globals. + +Config Processing ++++++++++++++++++ + +Config files are processed by the :py:class:`~mlos_bench.launcher.Launcher` and +:py:class:`~mlos_bench.services.config_persistence.ConfigPersistenceService` classes +at startup time by the ``mlos_bench`` CLI. + +The typical entrypoint is a CLI config which references other configs, especially +the base Environment config, Services, Optimizer, and Storage. + +See `mlos_bench CLI usage `_ for more details on +those arguments. + +Schema Definitions +++++++++++++++++++ + +For further details on the schema definitions and validation, see the +:py:class:`~mlos_bench.config.schemas.config_schemas.ConfigSchema` class +documentation, which also contains links to the actual schema definitions in the +source tree (see below). + +Notes +----- +See `mlos_bench/config/README.md +`_ and +`mlos_bench/tests/config/README.md +`_ +for additional documentation and examples in the source tree. +""" diff --git a/mlos_bench/mlos_bench/config/schemas/__init__.py b/mlos_bench/mlos_bench/config/schemas/__init__.py index 5ebc80fac40..135e6e84759 100644 --- a/mlos_bench/mlos_bench/config/schemas/__init__.py +++ b/mlos_bench/mlos_bench/config/schemas/__init__.py @@ -6,8 +6,8 @@ See Also -------- -:py:mod:`~mlos_bench.config.schemas.config_schemas`: The module handling the actual - schema definitions and validation. +mlos_bench.config.schemas.config_schemas : The module handling the actual schema + definitions and validation. """ from mlos_bench.config.schemas.config_schemas import CONFIG_SCHEMA_DIR, ConfigSchema diff --git a/mlos_bench/mlos_bench/config/schemas/config_schemas.py b/mlos_bench/mlos_bench/config/schemas/config_schemas.py index 530a6a13557..2415325d036 100644 --- a/mlos_bench/mlos_bench/config/schemas/config_schemas.py +++ b/mlos_bench/mlos_bench/config/schemas/config_schemas.py @@ -10,15 +10,15 @@ :py:class:`~mlos_bench.services.config_persistence.ConfigPersistenceService` to validate configs on load. -See Also --------- -`mlos_bench/config/schemas/README.md -`_ -for additional documentation in the source tree. - -`mlos_bench/config/README.md -`_ -for additional config examples in the source tree. +Notes +----- +- See `mlos_bench/config/schemas/README.md + `_ + for additional documentation in the source tree. + +- See `mlos_bench/config/README.md + `_ + for additional config examples in the source tree. """ import json # schema files are pure json - no comments @@ -142,8 +142,10 @@ class ConfigSchema(Enum): `__ for :py:mod:`mlos_bench ` CLI configuration. - Note: The :py:class:`mlos_bench.launcher.Launcher` class is responsible for - processing the CLI args. + See Also + -------- + mlos_bench.config : documentation on the configuration system. + mlos_bench.launcher.Launcher : class is responsible for processing the CLI args. """ GLOBALS = path_join(CONFIG_SCHEMA_DIR, "cli/globals-schema.json") diff --git a/mlos_bench/mlos_bench/environments/__init__.py b/mlos_bench/mlos_bench/environments/__init__.py index 58d7677deee..1ae7122cab5 100644 --- a/mlos_bench/mlos_bench/environments/__init__.py +++ b/mlos_bench/mlos_bench/environments/__init__.py @@ -44,16 +44,17 @@ :py:meth:`~mlos_bench.tunables.tunable_groups.TunableGroups` object to the ``tunable_groups`` parameter of the :py:class:`~.Environment` constructor, but that is typically handled by the -:py:meth:`~mlos_bench.services.config_persistence.ConfigPersistence.load_environment` +:py:meth:`~mlos_bench.services.config_persistence.ConfigPersistenceService.load_environment` method of the :py:meth:`~mlos_bench.services.config_persistence.ConfigPersistenceService` invoked -by the ``mlos_bench`` command line tool's :py:class:`mlos_bench.loader.Loader` +by the ``mlos_bench`` command line tool's :py:class:`mlos_bench.launcher.Launcher` class. In the typical json user level configs, this is specified in the ``include_tunables`` section of the Environment config to include the -``TunableGroups`` definitions from other json files when the -``mlos_bench.loader.Loader`` processes the initial set of config files. +:py:class:`~mlos_bench.tunables.tunable_groups.TunableGroups` definitions from other +json files when the :py:class:`~mlos_bench.launcher.Launcher` processes the initial +set of config files. The ``tunable_params`` setting in the ``config`` section of the Environment config can also be used to limit *which* of the ``TunableGroups`` should be used for the @@ -92,15 +93,14 @@ `_ directory in the source tree. -See Also --------- -`mlos_bench/environments/README.md -`_ -for additional documentation in the source tree. - -`mlos_bench/config/environments/README.md -`_ -for additional config examples in the source tree. +Notes +----- +- See `mlos_bench/environments/README.md + `_ + for additional documentation in the source tree. +- See `mlos_bench/config/environments/README.md + `_ + for additional config examples in the source tree. """ from mlos_bench.environments.base_environment import Environment diff --git a/mlos_bench/mlos_bench/run.py b/mlos_bench/mlos_bench/run.py index 0eef45e1042..cc3cf60b8f6 100755 --- a/mlos_bench/mlos_bench/run.py +++ b/mlos_bench/mlos_bench/run.py @@ -10,8 +10,10 @@ See the current ``--help`` `output for details `_. -Note: The :py:class:`mlos_bench.launcher.Launcher` class is responsible for -processing the CLI args. +See Also +-------- +mlos_bench.config : documentation on the configuration system. +mlos_bench.launcher.Launcher : class is responsible for processing the CLI args. """ import logging diff --git a/mlos_core/mlos_core/__init__.py b/mlos_core/mlos_core/__init__.py index 353bfb290b3..13b1bf2af82 100644 --- a/mlos_core/mlos_core/__init__.py +++ b/mlos_core/mlos_core/__init__.py @@ -96,9 +96,9 @@ y 7 Name: 1, dtype: int64 -See Also --------- -`mlos_core/README.md +Notes +----- +See `mlos_core/README.md `_ for additional documentation and examples in the source tree. """ diff --git a/mlos_core/mlos_core/optimizers/__init__.py b/mlos_core/mlos_core/optimizers/__init__.py index e8e91e903b6..3b00fa00cee 100644 --- a/mlos_core/mlos_core/optimizers/__init__.py +++ b/mlos_core/mlos_core/optimizers/__init__.py @@ -21,9 +21,9 @@ -------- TODO: Add example usage here. -See Also --------- -`mlos_core/optimizers/README.md +Notes +----- +See `mlos_core/optimizers/README.md `_ for additional documentation and examples in the source tree. """ diff --git a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py index 42d3280aeee..b41016b013a 100644 --- a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py +++ b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py @@ -5,9 +5,10 @@ """ Contains the wrapper class for the :py:class:`.SmacOptimizer`. -See Also --------- -`SMAC3 Documentation `_ +Notes +----- +See the `SMAC3 Documentation `_ for +more details. """ from logging import warning diff --git a/mlos_core/mlos_core/optimizers/flaml_optimizer.py b/mlos_core/mlos_core/optimizers/flaml_optimizer.py index bf7fe3ecc9f..6c0b3a25400 100644 --- a/mlos_core/mlos_core/optimizers/flaml_optimizer.py +++ b/mlos_core/mlos_core/optimizers/flaml_optimizer.py @@ -5,9 +5,10 @@ """ Contains the :py:class:`.FlamlOptimizer` class. -See Also --------- -`Flaml Documentation `_ +Notes +----- +See the `Flaml Documentation `_ for more +details. """ from typing import Dict, List, NamedTuple, Optional, Tuple, Union diff --git a/mlos_core/mlos_core/spaces/adapters/__init__.py b/mlos_core/mlos_core/spaces/adapters/__init__.py index 642daf240a2..608993af500 100644 --- a/mlos_core/mlos_core/spaces/adapters/__init__.py +++ b/mlos_core/mlos_core/spaces/adapters/__init__.py @@ -24,9 +24,9 @@ -------- TODO: Add example usage here. -See Also --------- -`mlos_core/spaces/adapters/README.md +Notes +----- +See `mlos_core/spaces/adapters/README.md `_ for additional documentation and examples in the source tree. """ diff --git a/mlos_core/mlos_core/spaces/converters/util.py b/mlos_core/mlos_core/spaces/converters/util.py index 8fb6bd06d1a..de0edb7cd1b 100644 --- a/mlos_core/mlos_core/spaces/converters/util.py +++ b/mlos_core/mlos_core/spaces/converters/util.py @@ -17,9 +17,9 @@ def monkey_patch_hp_quantization(hp: Hyperparameter) -> Hyperparameter: Temporary workaround to dropped quantization support in ConfigSpace 1.0 - See Also - -------- - + Notes + ----- + See . Parameters ---------- diff --git a/mlos_viz/mlos_viz/dabl.py b/mlos_viz/mlos_viz/dabl.py index c2fd0af8c35..918390fdbca 100644 --- a/mlos_viz/mlos_viz/dabl.py +++ b/mlos_viz/mlos_viz/dabl.py @@ -6,9 +6,9 @@ Small wrapper functions for plotting :py:mod:`mlos_bench` data via :external:py:func:`dabl.plot`. -See Also --------- -`dabl `_ for more information on the dabl library. +Notes +----- +See `dabl `_ for more information on the dabl library. """ import warnings from typing import Dict, Literal, Optional From 673b64f6cd24f17b77db74e9eece30e316600dc4 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 14 Nov 2024 20:06:07 +0000 Subject: [PATCH 081/100] docformatting --- mlos_bench/mlos_bench/config/__init__.py | 4 ++-- mlos_bench/mlos_bench/config/schemas/__init__.py | 3 ++- mlos_bench/mlos_bench/config/schemas/config_schemas.py | 10 ++++++---- mlos_bench/mlos_bench/environments/__init__.py | 3 ++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index 3a613c188bc..d3238c970f6 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -3,8 +3,8 @@ # Licensed under the MIT License. # """ -A module for managing json config schemas and their validation for various -components of MLOS. +A module for managing json config schemas and their validation for various components of +MLOS. Overview ++++++++ diff --git a/mlos_bench/mlos_bench/config/schemas/__init__.py b/mlos_bench/mlos_bench/config/schemas/__init__.py index 135e6e84759..9f34906a76d 100644 --- a/mlos_bench/mlos_bench/config/schemas/__init__.py +++ b/mlos_bench/mlos_bench/config/schemas/__init__.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""A module for managing config schemas and their validation. +""" +A module for managing config schemas and their validation. See Also -------- diff --git a/mlos_bench/mlos_bench/config/schemas/config_schemas.py b/mlos_bench/mlos_bench/config/schemas/config_schemas.py index 2415325d036..402f96a8b9f 100644 --- a/mlos_bench/mlos_bench/config/schemas/config_schemas.py +++ b/mlos_bench/mlos_bench/config/schemas/config_schemas.py @@ -37,7 +37,9 @@ # The path to find all config schemas. CONFIG_SCHEMA_DIR = path_join(path.dirname(__file__), abs_path=True) -"""The local directory where all config schemas are stored.""" +"""The local directory where all config schemas shipped as a part of the +:py:mod:`mlos_bench` module are stored. +""" # Allow skipping schema validation for tight dev cycle changes. # It is used in `ConfigSchema.validate()` method below. @@ -128,8 +130,8 @@ def registry(self) -> Registry: SCHEMA_STORE = SchemaStore() -"""Static :py:class:`.SchemaStore` instance used for storing and retrieving schemas -for config validation. +"""Static :py:class:`.SchemaStore` instance used for storing and retrieving schemas for +config validation. """ @@ -212,7 +214,7 @@ class ConfigSchema(Enum): """ Combined global json `schema `__ - use to validate any ``mlos_bench`` config file (e.g., ``*.mlos.json`` files). + use to validate any ``mlos_bench`` config file (e.g., ``*.mlos.jsonc`` files). See Also -------- diff --git a/mlos_bench/mlos_bench/environments/__init__.py b/mlos_bench/mlos_bench/environments/__init__.py index 1ae7122cab5..ba82b26d946 100644 --- a/mlos_bench/mlos_bench/environments/__init__.py +++ b/mlos_bench/mlos_bench/environments/__init__.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Tunable Environments for mlos_bench. +""" +Tunable Environments for mlos_bench. Environments are classes that represent an execution setting (i.e., environment) for running a benchmark or tuning process. From 5b6f28bfcfda117ac630cdd68642a6830c055786 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 14 Nov 2024 20:36:04 +0000 Subject: [PATCH 082/100] add some documentation todos --- mlos_bench/mlos_bench/config/__init__.py | 105 ++++++++++++++---- .../mlos_bench/environments/__init__.py | 5 + .../environments/local/local_env.py | 5 +- .../environments/remote/remote_env.py | 2 + .../mlos_bench/environments/script_env.py | 6 +- 5 files changed, 102 insertions(+), 21 deletions(-) diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index d3238c970f6..609fa6ce239 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -3,8 +3,8 @@ # Licensed under the MIT License. # """ -A module for managing json config schemas and their validation for various components of -MLOS. +A module for managing json config schemas and their validation for various +components of MLOS. Overview ++++++++ @@ -21,10 +21,11 @@ +++++++++++++++++++++++++++++ We use `json5 `_ to parse the json files, since it -allows for inline C style comments (e.g., ``//``, ``/* */``). +allows for inline C style comments (e.g., ``//``, ``/* */``), trailing commas, etc., +so it is slightly more user friendly than strict json. -By convention files use the ``*.mlos.json`` or ``*.mlos.jsonc`` extension to indicate -that they are an ``mlos_bench`` config file. +By convention files use the ``*.mlos.json`` or ``*.mlos.jsonc`` extension to +indicate that they are an ``mlos_bench`` config file. This allows tools that support `JSON Schema Store `_ (e.g., `VSCode @@ -34,6 +35,84 @@ CLI Configs ^^^^^^^^^^^ +:py:attr:`~.mlos_bench.config.schema.config_schemas.ConfigSchema.CLI` style configs +are typically used to start the ``mlos_bench`` CLI using the ``--config`` argument +and a restricted key-value dict form where each key corresponds to a CLI argument. + +For instance: + +.. code-block:: json + + { + "experiment": "path/to/base/experiment-config.mlos.json", + "services": [ + "path/to/some/service-config.mlos.json", + ], + "globals": "path/to/basic-globals-config.mlos.json", + } + +Typically CLI configs will reference some other configs, especially the base +Environment and Services configs, but some ``globals`` may be left to be specified +on the command line. + +For instance: + +.. code-block:: shell + + mlos_bench --config path/to/cli-config.mlos.json --globals experiment-config.mlos.json + +This allows some of the ``globals`` to be specified on the CLI to alter the behavior +of a set of Experiments without having to adjust many of the other config files +themselves. + +See below for examples. + +Notes +----- +- See `mlos_bench CLI usage `_ for more details on the + CLI arguments. +- See `mlos_bench/config/cli + `_ + and `mlos_bench/tests/config/cli + `_ + for some examples of CLI configs. + +Globals and Variable Substitution ++++++++++++++++++++++++++++++++++ + +:py:attr:`Globals ` +are basically just key-value variables that can be used in other configs using +``$variable`` substituion. + +For instance: + +.. code-block:: json + + { + "experiment_id": "my_experiment", + "some_var": "some_value", + // environment variable expansion also works here + "current_dir": "$PWD", + "some_expanded_var": "$some_var: $experiment_id", + "location": "eastus", + } + +There are additional details about variable propogation in the +:py:mod:`mlos_bench.environments` module. + +Well Known Variables +^^^^^^^^^^^^^^^^^^^^ + +Here is a list of some well known variables that are provided or required by the +system and may be used in the config files: + +- ``$experiment_id``: A unique identifier for the experiment. + Typically provided in globals. +- ``$trial_id``: A unique identifier for the trial currently being executed. + This can be useful in the configs for :py:mod:`mlos_bench.environments` for + instance (e.g., when writing scripts). +- TODO: Document more variables here. + Tunable Configs ^^^^^^^^^^^^^^^ @@ -54,18 +133,6 @@ } } -Globals and Variable Substitution -+++++++++++++++++++++++++++++++++ -TODO: Document globals and variable substitution. - -Well Known Variables -++++++++++++++++++++ - -Here is a list of well known variables that are used in the config files: - -- ``$experiment_id``: A unique identifier for the experiment. - Typically provided in globals. - Config Processing +++++++++++++++++ @@ -76,8 +143,8 @@ The typical entrypoint is a CLI config which references other configs, especially the base Environment config, Services, Optimizer, and Storage. -See `mlos_bench CLI usage `_ for more details on -those arguments. +See `mlos_bench CLI usage `_ for more details on those +arguments. Schema Definitions ++++++++++++++++++ diff --git a/mlos_bench/mlos_bench/environments/__init__.py b/mlos_bench/mlos_bench/environments/__init__.py index ba82b26d946..9de6a2f1d32 100644 --- a/mlos_bench/mlos_bench/environments/__init__.py +++ b/mlos_bench/mlos_bench/environments/__init__.py @@ -80,6 +80,11 @@ used in different settings (e.g., local machine, SSH accessible machine, Azure VM, etc.) without having to change the Environment config. +Variable Propogation +++++++++++++++++++++ +TODO: Document how variable propogation works in the script environments using +required_args, const_args, etc. + Examples -------- While this documentation is generated from the source code and is intended to be a diff --git a/mlos_bench/mlos_bench/environments/local/local_env.py b/mlos_bench/mlos_bench/environments/local/local_env.py index 4b4909acb27..a18d7318b2e 100644 --- a/mlos_bench/mlos_bench/environments/local/local_env.py +++ b/mlos_bench/mlos_bench/environments/local/local_env.py @@ -2,7 +2,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Scheduler-side benchmark environment to run scripts locally.""" +"""Scheduler-side benchmark environment to run scripts locally. + +TODO: Reference the script_env.py file for the base class. +""" import json import logging diff --git a/mlos_bench/mlos_bench/environments/remote/remote_env.py b/mlos_bench/mlos_bench/environments/remote/remote_env.py index 0f6264e7d64..0b1ed314663 100644 --- a/mlos_bench/mlos_bench/environments/remote/remote_env.py +++ b/mlos_bench/mlos_bench/environments/remote/remote_env.py @@ -6,6 +6,8 @@ Remotely executed benchmark/script environment. e.g. Application Environment + +TODO: Documentat how variable propogation works in the remote environments. """ import logging diff --git a/mlos_bench/mlos_bench/environments/script_env.py b/mlos_bench/mlos_bench/environments/script_env.py index 48db44fc35c..ea1d16cf34d 100644 --- a/mlos_bench/mlos_bench/environments/script_env.py +++ b/mlos_bench/mlos_bench/environments/script_env.py @@ -2,7 +2,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Base scriptable benchmark environment.""" +"""Base scriptable benchmark environment. + +TODO: Document how variable propogation works in the script environments using +shell_env_params, required_args, const_args, etc. +""" import abc import logging From e28e4e2f6f38af834931f7c3d4820bd948afb775 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 14 Nov 2024 20:36:52 +0000 Subject: [PATCH 083/100] doc formatter --- mlos_bench/mlos_bench/config/__init__.py | 4 ++-- mlos_bench/mlos_bench/environments/local/local_env.py | 3 ++- mlos_bench/mlos_bench/environments/script_env.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index 609fa6ce239..3ae5adda4d7 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -3,8 +3,8 @@ # Licensed under the MIT License. # """ -A module for managing json config schemas and their validation for various -components of MLOS. +A module for managing json config schemas and their validation for various components of +MLOS. Overview ++++++++ diff --git a/mlos_bench/mlos_bench/environments/local/local_env.py b/mlos_bench/mlos_bench/environments/local/local_env.py index a18d7318b2e..344dd593b34 100644 --- a/mlos_bench/mlos_bench/environments/local/local_env.py +++ b/mlos_bench/mlos_bench/environments/local/local_env.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Scheduler-side benchmark environment to run scripts locally. +""" +Scheduler-side benchmark environment to run scripts locally. TODO: Reference the script_env.py file for the base class. """ diff --git a/mlos_bench/mlos_bench/environments/script_env.py b/mlos_bench/mlos_bench/environments/script_env.py index ea1d16cf34d..0dc3f493a80 100644 --- a/mlos_bench/mlos_bench/environments/script_env.py +++ b/mlos_bench/mlos_bench/environments/script_env.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Base scriptable benchmark environment. +""" +Base scriptable benchmark environment. TODO: Document how variable propogation works in the script environments using shell_env_params, required_args, const_args, etc. From 1156858f9219e3e3aaa6a29649815bf1a74d05f2 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 14 Nov 2024 21:39:20 +0000 Subject: [PATCH 084/100] more docs --- mlos_bench/mlos_bench/__init__.py | 7 +- mlos_bench/mlos_bench/config/__init__.py | 119 ++++++++++++++++-- .../mlos_bench/environments/__init__.py | 10 +- .../mlos_bench/environments/script_env.py | 5 +- 4 files changed, 121 insertions(+), 20 deletions(-) diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index 6e8564b639c..af9c2aca864 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -35,10 +35,9 @@ arbitrary code on a :py:mod:`remote machine `, and many other things. -- :py:mod:`~mlos_bench.environments` are often associated with - :py:mod:`~mlos_bench.tunables` which provide a language for specifying the set of - configuration parameters that can be optimized or searched over with the - :py:mod:`~mlos_bench.optimizers`. +- Environments are often associated with :py:mod:`~mlos_bench.tunables` which + provide a language for specifying the set of configuration parameters that can be + optimized or searched over with the :py:mod:`~mlos_bench.optimizers`. - :py:mod:`~mlos_bench.services` provide the necessary abstractions to run interact with the :py:mod:`~mlos_bench.environments` in different settings. diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index 3ae5adda4d7..b912b76484c 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -3,19 +3,22 @@ # Licensed under the MIT License. # """ -A module for managing json config schemas and their validation for various components of -MLOS. +A module for and documentation about the structure and mangement of json configs, +their schemas and validation for various components of MLOS. + +.. contents:: Table of Contents + :depth: 3 Overview ++++++++ -MLOS is a framework for doing benchmarking and autotuning for systems written in -python. As such, all of the code classes documented here take python objects in -their construction. +MLOS is a framework for doing benchmarking and autotuning for systems. +The bulk of the code to do that is written in python. As such, all of the code +classes documented here take python objects in their construction. However, most users of MLOS will interact with the system via the ``mlos_bench`` CLI -and its json config files. This module attempts to document some of those high level -interactions. +and its json config files and their own scripts for MLOS to invoke. This module +attempts to document some of those high level interactions. General JSON Config Structure +++++++++++++++++++++++++++++ @@ -35,7 +38,7 @@ CLI Configs ^^^^^^^^^^^ -:py:attr:`~.mlos_bench.config.schema.config_schemas.ConfigSchema.CLI` style configs +:py:attr:`~.mlos_bench.config.schemas.config_schemas.ConfigSchema.CLI` style configs are typically used to start the ``mlos_bench`` CLI using the ``--config`` argument and a restricted key-value dict form where each key corresponds to a CLI argument. @@ -43,6 +46,7 @@ .. code-block:: json + // cli-config.mlos.json { "experiment": "path/to/base/experiment-config.mlos.json", "services": [ @@ -51,6 +55,12 @@ "globals": "path/to/basic-globals-config.mlos.json", } + // basic-globals-config.mlos.json + { + "location": "westus", + "vm_size": "Standard_D2s_v5", + } + Typically CLI configs will reference some other configs, especially the base Environment and Services configs, but some ``globals`` may be left to be specified on the command line. @@ -61,6 +71,14 @@ mlos_bench --config path/to/cli-config.mlos.json --globals experiment-config.mlos.json +where ``experiment-config.mlos.json`` might look something like this: + +.. code-block:: json + { + "experiment_id": "my_experiment", + "some_var": "some_value", + } + This allows some of the ``globals`` to be specified on the CLI to alter the behavior of a set of Experiments without having to adjust many of the other config files themselves. @@ -82,12 +100,14 @@ :py:attr:`Globals ` are basically just key-value variables that can be used in other configs using -``$variable`` substituion. +``$variable`` substituion via the +:py:meth:`~mlos_bench.dict_templater.DictTemplater.expand_vars` method. For instance: .. code-block:: json + // globals-config.mlos.json { "experiment_id": "my_experiment", "some_var": "some_value", @@ -116,23 +136,96 @@ Tunable Configs ^^^^^^^^^^^^^^^ -Tunable +There are two forms of tunable configs: + +- "TunableParams" style configs + + Which are used to define the set of + :py:mod:`~mlos_bench.tunables.tunable_groups.TunableGroups` (i.e., tunable + parameters). + + .. code-block:: json + + // env-tunables.json + { + // a group of tunables that are tuned together + "covariant_group_name": [ + { + "name": "tunable_name", + "type": "int", + "range": [0, 100], + "default": 50, + }, + // more tunables + ], + // another group of tunables + // both can be enabled at the same time + "another_group_name": [ + { + "name": "another_tunable_name", + "type": "categorical", + "values": ["red", "yellow", "green"], + "default": "green" + }, + // more tunables + ], + } + + Since TunableParams are associated with Environments, they are typically kept + in the same directory as that environment and named something like + ``env-tunables.json``. + +- "TunableValues" style configs which are used to specify the values for an + instantiation of a set of tunables params. + + These are essentially just a dict of the tunable names and their values. + For instance: + + .. code-block:: json + + { + "tunable_name": 25, + "another_tunable_name": "red", + } + + These can be used with the + :py:class:`~mlos_bench.optimizers.one_shot_optimizer.OneShotOptimizer` + :py:class:`~mlos_bench.optimizers.manual_optimizer.ManualOptimizer` to run a + benchmark with a particular config or set of configs. Class Configs ^^^^^^^^^^^^^ Class style configs include most anything else and roughly take this form: -.. code block:: json +.. code-block:: json + + // class configs (environments, services, etc.) { // some mlos class name to load "class": "mlos_bench.type.ClassName", "config": { // class specific config - "key": "value" + "key": "value", + "key2": "$some_var", // variable substitution is allowed here too } } +Where ``type`` is one of the core classes in the system: + +- :py:mod:`~mlos_bench.environments` +- :py:mod:`~mlos_bench.optimizers` +- :py:mod:`~mlos_bench.services` +- :py:mod:`~mlos_bench.schedulers` +- :py:mod:`~mlos_bench.storage` + +Each of which have their own submodules and classes that dictate the allowed and +expected structure of the ``config`` section. + +In certain cases (e.g., script command execution) the variable substitution rules +take on slightly different behavior +See various documentation in :py:mod:`mlos_bench.environments` for more details. + Config Processing +++++++++++++++++ @@ -155,7 +248,7 @@ source tree (see below). Notes ------ ++++++ See `mlos_bench/config/README.md `_ and `mlos_bench/tests/config/README.md diff --git a/mlos_bench/mlos_bench/environments/__init__.py b/mlos_bench/mlos_bench/environments/__init__.py index 9de6a2f1d32..31c728fde16 100644 --- a/mlos_bench/mlos_bench/environments/__init__.py +++ b/mlos_bench/mlos_bench/environments/__init__.py @@ -5,6 +5,12 @@ """ Tunable Environments for mlos_bench. +.. contents:: Table of Contents + :depth: 3 + +Overview +++++++++ + Environments are classes that represent an execution setting (i.e., environment) for running a benchmark or tuning process. @@ -86,7 +92,7 @@ required_args, const_args, etc. Examples --------- +++++++++ While this documentation is generated from the source code and is intended to be a useful reference on the internal details, most users will be more interested in generating json configs to be used with the ``mlos_bench`` command line tool. @@ -100,7 +106,7 @@ directory in the source tree. Notes ------ ++++++ - See `mlos_bench/environments/README.md `_ for additional documentation in the source tree. diff --git a/mlos_bench/mlos_bench/environments/script_env.py b/mlos_bench/mlos_bench/environments/script_env.py index 0dc3f493a80..2a0e0e6e5ba 100644 --- a/mlos_bench/mlos_bench/environments/script_env.py +++ b/mlos_bench/mlos_bench/environments/script_env.py @@ -24,7 +24,10 @@ class ScriptEnv(Environment, metaclass=abc.ABCMeta): - """Base Environment that runs scripts for setup/run/teardown.""" + """ + Base Environment that runs scripts for :py:meth:`.setup`, :py:meth:`.run`, + :py:meth:`.teardown`, etc. phases. + """ _RE_INVALID = re.compile(r"[^a-zA-Z0-9_]") From 26da3ed313ecfc34d559412c5637fe6ca16ab7bd Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 14 Nov 2024 21:47:42 +0000 Subject: [PATCH 085/100] more docs --- mlos_bench/mlos_bench/config/__init__.py | 13 +++++++++---- mlos_bench/mlos_bench/environments/script_env.py | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index b912b76484c..e1b4c6dcdfb 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -55,6 +55,8 @@ "globals": "path/to/basic-globals-config.mlos.json", } +.. code-block:: json + // basic-globals-config.mlos.json { "location": "westus", @@ -74,6 +76,8 @@ where ``experiment-config.mlos.json`` might look something like this: .. code-block:: json + + // experiment-config.mlos.json (also a set of globals) { "experiment_id": "my_experiment", "some_var": "some_value", @@ -146,7 +150,7 @@ .. code-block:: json - // env-tunables.json + // some-env-tunables.json { // a group of tunables that are tuned together "covariant_group_name": [ @@ -171,9 +175,9 @@ ], } - Since TunableParams are associated with Environments, they are typically kept - in the same directory as that environment and named something like - ``env-tunables.json``. + Since TunableParams are associated with Environments, they are typically kept + in the same directory as that environment and named something like + ``env-tunables.json``. - "TunableValues" style configs which are used to specify the values for an instantiation of a set of tunables params. @@ -183,6 +187,7 @@ .. code-block:: json + // tunable-values.mlos.json { "tunable_name": 25, "another_tunable_name": "red", diff --git a/mlos_bench/mlos_bench/environments/script_env.py b/mlos_bench/mlos_bench/environments/script_env.py index 2a0e0e6e5ba..0b135cea1ac 100644 --- a/mlos_bench/mlos_bench/environments/script_env.py +++ b/mlos_bench/mlos_bench/environments/script_env.py @@ -25,8 +25,8 @@ class ScriptEnv(Environment, metaclass=abc.ABCMeta): """ - Base Environment that runs scripts for :py:meth:`.setup`, :py:meth:`.run`, - :py:meth:`.teardown`, etc. phases. + Base Environment that runs scripts for :py:meth:`.Environment.setup`, + :py:meth:`.Environment.run`, :py:meth:`.Environment.teardown`, etc. phases. """ _RE_INVALID = re.compile(r"[^a-zA-Z0-9_]") From a5d56764a10d576814258c4f1c29111cc7dc14ac Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 14 Nov 2024 21:55:19 +0000 Subject: [PATCH 086/100] more docs --- mlos_bench/mlos_bench/optimizers/__init__.py | 5 ++++- mlos_bench/mlos_bench/optimizers/manual_optimizer.py | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mlos_bench/mlos_bench/optimizers/__init__.py b/mlos_bench/mlos_bench/optimizers/__init__.py index 106b0fc496b..4ab1b6c3469 100644 --- a/mlos_bench/mlos_bench/optimizers/__init__.py +++ b/mlos_bench/mlos_bench/optimizers/__init__.py @@ -2,7 +2,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Interfaces and wrapper classes for optimizers to be used in Autotune.""" +""" +Interfaces and wrapper classes for optimizers to be used in mlos_bench for +autotuning or benchmarking. +""" from mlos_bench.optimizers.base_optimizer import Optimizer from mlos_bench.optimizers.manual_optimizer import ManualOptimizer diff --git a/mlos_bench/mlos_bench/optimizers/manual_optimizer.py b/mlos_bench/mlos_bench/optimizers/manual_optimizer.py index d9f48a4e193..c3a8069779f 100644 --- a/mlos_bench/mlos_bench/optimizers/manual_optimizer.py +++ b/mlos_bench/mlos_bench/optimizers/manual_optimizer.py @@ -2,7 +2,15 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Optimizer for mlos_bench that proposes an explicit sequence of configurations.""" +""" +Manual config suggestor (Optimizer) for mlos_bench that proposes an explicit +sequence of configurations. + +This is useful for testing and validation, as it allows you to run a sequence of +configurations in a cyclic fashion. + +TODO: Add an example configuration. +""" import logging from typing import Dict, List, Optional From ef43373fc42fed14fca2b90f88ccc0f743a6aaa7 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 14 Nov 2024 17:51:21 -0600 Subject: [PATCH 087/100] Apply suggestions from code review Co-authored-by: Sergiy Matusevych --- doc/source/conf.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index a099fd451df..5a3771714f9 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -99,9 +99,8 @@ def _check_path(path: str) -> bool: """Check if a path exists and cache the result.""" path = os.path.join(_base_path, path) - if path in _path_cache: - result = _path_cache[path] - else: + result = _path_cache.get(path) + if result is None: result = os.path.exists(path) _path_cache[path] = result return result @@ -128,9 +127,7 @@ def linkcode_resolve(domain: str, info: Dict[str, str]): def is_on_github_actions(): """Check if the documentation is being built on GitHub Actions.""" - if "CI" not in os.environ or not os.environ["CI"] or "GITHUB_RUN_ID" not in os.environ: - return False - return True + return os.environ.get("CI") and os.environ.get("GITHUB_RUN_ID") # Add mappings to link to external documentation. @@ -183,7 +180,7 @@ def resolve_type_aliases( env: BuildEnvironment, node: pending_xref, contnode: Element, -) -> Union[Element, None]: +) -> Optional[Element]: """Resolve :class: references to our type aliases as :attr: instead.""" if node["refdomain"] != "py": return None From 4664abbb9a72a4d63c123d269c8008fc40f7cf50 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 14 Nov 2024 23:51:52 +0000 Subject: [PATCH 088/100] more docs --- mlos_bench/mlos_bench/__init__.py | 41 +++++++++---------- mlos_bench/mlos_bench/storage/__init__.py | 34 ++++++++++++++- mlos_bench/mlos_bench/storage/sql/__init__.py | 3 +- .../mlos_bench/storage/storage_factory.py | 10 ++++- 4 files changed, 63 insertions(+), 25 deletions(-) diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index af9c2aca864..1520b6c0ccf 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -6,19 +6,18 @@ mlos_bench is a framework to help automate benchmarking and OS/application parameter autotuning and the data management of the results. -It can be installed from `pypi `_ via -``pip install mlos-bench`` and executed using the ``mlos_bench`` -`command <../../mlos_bench.run.usage.html>`_ using a collection of `json` -`configs `_. +It can be installed from `pypi `_ via ``pip +install mlos-bench`` and executed using the ``mlos_bench`` `command +<../../mlos_bench.run.usage.html>`_ using a collection of `json` `configs +`_. It is intended to be used with :py:mod:`mlos_core` via :py:class:`~mlos_bench.optimizers.mlos_core_optimizer.MlosCoreOptimizer` to help navigate complex parameter spaces more effeciently, though other -:py:mod:`~mlos_bench.optimizers` are also available to -help customize the search process easily by simply swapping out the +:py:mod:`~mlos_bench.optimizers` are also available to help customize the search +process easily by simply swapping out the :py:class:`~mlos_bench.optimizers.base_optimizer.Optimizer` class in the associated -json configs. -For instance, +json configs. For instance, :py:class:`~mlos_bench.optimizers.grid_search_optimizer.GridSearchOptimizer` can be used to perform a grid search over the parameter space instead. @@ -29,11 +28,10 @@ These are generally the target of the optimization process and are used to evaluate the performance of a given configuration, though can also be used to - simply run a single benchmark. - They can be used, for instance, to provision a :py:mod:`VM - `, run benchmarks or execute any other - arbitrary code on a :py:mod:`remote machine `, - and many other things. + simply run a single benchmark. They can be used, for instance, to provision a + :py:mod:`VM `, run benchmarks or execute + any other arbitrary code on a :py:mod:`remote machine + `, and many other things. - Environments are often associated with :py:mod:`~mlos_bench.tunables` which provide a language for specifying the set of configuration parameters that can be @@ -43,8 +41,8 @@ with the :py:mod:`~mlos_bench.environments` in different settings. For instance, the - :py:class:`~mlos_bench.services.remote.azure.azure_vm_services.AzureVMService` - can be used to run commands on Azure VMs for a remote + :py:class:`~mlos_bench.services.remote.azure.azure_vm_services.AzureVMService` can + be used to run commands on Azure VMs for a remote :py:mod:`~mlos_bench.environments.remote.vm_env.VMEnv`. Alternatively, one could swap out that service for @@ -70,9 +68,9 @@ ----- Note that while the docstrings in this package are generated from the source code and hence sometimes more focused on the implementation details, most user -interactions with the package will be through the -`json configs `_. -Even so it may be useful to look at the source code to understand how those are +interactions with the package will be through the `json configs +`_. Even +so it may be useful to look at the source code to understand how those are interpretted. Examples @@ -110,12 +108,11 @@ Notes ----- -- `mlos_bench/README.md - `_ +- `mlos_bench/README.md `_ for additional documentation and examples in the source tree. -- There is also a working example of using ``mlos_bench`` in a separate repo for the - configs in the `sqlite-autotuning +- There is also a working example of using ``mlos_bench`` in a *separate config + repo* (the more expected case for most users) in the `sqlite-autotuning `_ repo. """ from mlos_bench.version import VERSION diff --git a/mlos_bench/mlos_bench/storage/__init__.py b/mlos_bench/mlos_bench/storage/__init__.py index 2f75f7253f4..b4d3b65d5bd 100644 --- a/mlos_bench/mlos_bench/storage/__init__.py +++ b/mlos_bench/mlos_bench/storage/__init__.py @@ -2,7 +2,39 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Interfaces to the storage backends for mlos_bench.""" +"""Interfaces to the storage backends for mlos_bench. + +Storage backends (for instance :py:mod:`~mlos_bench.storage.sql`) are used to store +and retrieve the results of experiments and implement a persistent queue for +:py:mod:`~mlos_bench.schedulers`. + +The :py:class:`~mlos_bench.storage.base_storage.Storage` class is the main interface +and provides the ability to + +- Create or reload a new :py:class:`~.Storage.Experiment` with one or more + associated :py:class:`~.Storage.Trial` instances which are used by the + :py:mod:`~mlos_bench.schedulers` during ``mlos_bench`` run time to execute + `Trials`. +- Retrieve the :py:class:`~mlos_bench.storage.base_trial_data.TrialData` results + with a :py:class:`~mlos_bench.storage.base_experiment_data.ExperimentData` + instance. These can be especially useful with :py:mod:`mlos_viz` for interactive + exploration in a Jupyter Notebook interface, for instance. + +The :py:func:`.from_config` :py:mod:`.storage_factory` function can be used to get a +:py:class:`.Storage` instance from a +:py:attr:`~mlos_bench.config.schemas.config_schemas.ConfigSchema.STORAGE` type json +config. + +Example +------- +TODO: Add example usage. + +Notes +----- +- See `sqlite-autotuning notebooks + `_ + for additional examples. +""" from mlos_bench.storage.base_storage import Storage from mlos_bench.storage.storage_factory import from_config diff --git a/mlos_bench/mlos_bench/storage/sql/__init__.py b/mlos_bench/mlos_bench/storage/sql/__init__.py index eb8867032ad..ecfe2a3345d 100644 --- a/mlos_bench/mlos_bench/storage/sql/__init__.py +++ b/mlos_bench/mlos_bench/storage/sql/__init__.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Interfaces to the SQL-based storage backends for mlos_bench.""" +"""Interfaces to the SQL-based storage backends for mlos_bench. +""" from mlos_bench.storage.sql.storage import SqlStorage __all__ = [ diff --git a/mlos_bench/mlos_bench/storage/storage_factory.py b/mlos_bench/mlos_bench/storage/storage_factory.py index ea0201717d4..8980323a781 100644 --- a/mlos_bench/mlos_bench/storage/storage_factory.py +++ b/mlos_bench/mlos_bench/storage/storage_factory.py @@ -2,7 +2,15 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Factory method to create a new Storage instance from configs.""" +""" +Factory method to create a new :py:class:`.Storage` instance from a +:py:attr:`~mlos_bench.config.schemas.config_schemas.ConfigSchema.STORAGE` type json +config. + +See Also +-------- +mlos_bench.storage : For example usage. +""" from typing import Any, Dict, List, Optional From 3afe9956a0d85a9b60b605be83825b8544faf141 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 15 Nov 2024 00:01:24 +0000 Subject: [PATCH 089/100] formatting --- mlos_bench/mlos_bench/config/__init__.py | 4 ++-- mlos_bench/mlos_bench/environments/script_env.py | 3 ++- mlos_bench/mlos_bench/optimizers/__init__.py | 5 ++--- mlos_bench/mlos_bench/optimizers/manual_optimizer.py | 4 ++-- mlos_bench/mlos_bench/storage/__init__.py | 3 ++- mlos_bench/mlos_bench/storage/sql/__init__.py | 3 +-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index e1b4c6dcdfb..3620d1b405b 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -3,8 +3,8 @@ # Licensed under the MIT License. # """ -A module for and documentation about the structure and mangement of json configs, -their schemas and validation for various components of MLOS. +A module for and documentation about the structure and mangement of json configs, their +schemas and validation for various components of MLOS. .. contents:: Table of Contents :depth: 3 diff --git a/mlos_bench/mlos_bench/environments/script_env.py b/mlos_bench/mlos_bench/environments/script_env.py index 0b135cea1ac..49eb9b5bb6f 100644 --- a/mlos_bench/mlos_bench/environments/script_env.py +++ b/mlos_bench/mlos_bench/environments/script_env.py @@ -26,7 +26,8 @@ class ScriptEnv(Environment, metaclass=abc.ABCMeta): """ Base Environment that runs scripts for :py:meth:`.Environment.setup`, - :py:meth:`.Environment.run`, :py:meth:`.Environment.teardown`, etc. phases. + :py:meth:`.Environment.run`, :py:meth:`.Environment.teardown`, etc. + phases. """ _RE_INVALID = re.compile(r"[^a-zA-Z0-9_]") diff --git a/mlos_bench/mlos_bench/optimizers/__init__.py b/mlos_bench/mlos_bench/optimizers/__init__.py index 4ab1b6c3469..99890cb2a93 100644 --- a/mlos_bench/mlos_bench/optimizers/__init__.py +++ b/mlos_bench/mlos_bench/optimizers/__init__.py @@ -2,9 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -""" -Interfaces and wrapper classes for optimizers to be used in mlos_bench for -autotuning or benchmarking. +"""Interfaces and wrapper classes for optimizers to be used in mlos_bench for autotuning +or benchmarking. """ from mlos_bench.optimizers.base_optimizer import Optimizer diff --git a/mlos_bench/mlos_bench/optimizers/manual_optimizer.py b/mlos_bench/mlos_bench/optimizers/manual_optimizer.py index c3a8069779f..e9c3ecde192 100644 --- a/mlos_bench/mlos_bench/optimizers/manual_optimizer.py +++ b/mlos_bench/mlos_bench/optimizers/manual_optimizer.py @@ -3,8 +3,8 @@ # Licensed under the MIT License. # """ -Manual config suggestor (Optimizer) for mlos_bench that proposes an explicit -sequence of configurations. +Manual config suggestor (Optimizer) for mlos_bench that proposes an explicit sequence of +configurations. This is useful for testing and validation, as it allows you to run a sequence of configurations in a cyclic fashion. diff --git a/mlos_bench/mlos_bench/storage/__init__.py b/mlos_bench/mlos_bench/storage/__init__.py index b4d3b65d5bd..2315622d2ed 100644 --- a/mlos_bench/mlos_bench/storage/__init__.py +++ b/mlos_bench/mlos_bench/storage/__init__.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Interfaces to the storage backends for mlos_bench. +""" +Interfaces to the storage backends for mlos_bench. Storage backends (for instance :py:mod:`~mlos_bench.storage.sql`) are used to store and retrieve the results of experiments and implement a persistent queue for diff --git a/mlos_bench/mlos_bench/storage/sql/__init__.py b/mlos_bench/mlos_bench/storage/sql/__init__.py index ecfe2a3345d..eb8867032ad 100644 --- a/mlos_bench/mlos_bench/storage/sql/__init__.py +++ b/mlos_bench/mlos_bench/storage/sql/__init__.py @@ -2,8 +2,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Interfaces to the SQL-based storage backends for mlos_bench. -""" +"""Interfaces to the SQL-based storage backends for mlos_bench.""" from mlos_bench.storage.sql.storage import SqlStorage __all__ = [ From 9725ad7a498dbfa1be8dab59815936548feda9ed Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 15 Nov 2024 00:14:12 +0000 Subject: [PATCH 090/100] rework --- mlos_bench/mlos_bench/environments/script_env.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mlos_bench/mlos_bench/environments/script_env.py b/mlos_bench/mlos_bench/environments/script_env.py index 49eb9b5bb6f..9dc6d661433 100644 --- a/mlos_bench/mlos_bench/environments/script_env.py +++ b/mlos_bench/mlos_bench/environments/script_env.py @@ -24,10 +24,9 @@ class ScriptEnv(Environment, metaclass=abc.ABCMeta): - """ - Base Environment that runs scripts for :py:meth:`.Environment.setup`, - :py:meth:`.Environment.run`, :py:meth:`.Environment.teardown`, etc. - phases. + """Base Environment that runs scripts for the different phases (e.g., + :py:meth:`.Environment.setup`, :py:meth:`.Environment.run`, + :py:meth:`.Environment.teardown`, etc.) """ _RE_INVALID = re.compile(r"[^a-zA-Z0-9_]") From b24cb2143ffb7e2400c45fae58f91c7fc029688b Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 15 Nov 2024 00:16:26 +0000 Subject: [PATCH 091/100] docstyle --- mlos_bench/mlos_bench/__init__.py | 1 - mlos_bench/mlos_bench/config/__init__.py | 2 +- mlos_bench/mlos_bench/environments/__init__.py | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index 1520b6c0ccf..0f623d40bcb 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -75,7 +75,6 @@ Examples -------- - Here is an example that shows how to run a simple benchmark using the command line. The entry point for these configs can be found `here diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index 3620d1b405b..c1b34553ccc 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -253,7 +253,7 @@ source tree (see below). Notes -+++++ +----- See `mlos_bench/config/README.md `_ and `mlos_bench/tests/config/README.md diff --git a/mlos_bench/mlos_bench/environments/__init__.py b/mlos_bench/mlos_bench/environments/__init__.py index 31c728fde16..2ddb7c60b7c 100644 --- a/mlos_bench/mlos_bench/environments/__init__.py +++ b/mlos_bench/mlos_bench/environments/__init__.py @@ -92,7 +92,7 @@ required_args, const_args, etc. Examples -++++++++ +-------- While this documentation is generated from the source code and is intended to be a useful reference on the internal details, most users will be more interested in generating json configs to be used with the ``mlos_bench`` command line tool. @@ -106,7 +106,7 @@ directory in the source tree. Notes -+++++ +----- - See `mlos_bench/environments/README.md `_ for additional documentation in the source tree. From 743af9cd9f2f3f1f097f405676eb06de62d9a753 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 15 Nov 2024 00:18:04 +0000 Subject: [PATCH 092/100] raw format string --- mlos_bench/mlos_bench/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index 0f623d40bcb..2cc3cfa3d6a 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -""" +r""" mlos_bench is a framework to help automate benchmarking and OS/application parameter autotuning and the data management of the results. @@ -84,15 +84,15 @@ >>> # Note: we show the command wrapped in python here for testing purposes. >>> # Alternatively replace test-cli-local-env-bench.jsonc with >>> # test-cli-local-env-opt.jsonc for one that does an optimization loop. ->>> cmd = r'''mlos_bench \\ -... --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc \\ -... --globals experiment_test_local.jsonc \\ +>>> cmd = r'''mlos_bench \ +... --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc \ +... --globals experiment_test_local.jsonc \ ... --tunable_values tunable-values/tunable-values-local.jsonc''' ->>> print(f"Here's the shell command you'd actually run:\\n# {cmd}") +>>> print(f"Here's the shell command you'd actually run:\n# {cmd}") Here's the shell command you'd actually run: -# mlos_bench \\ - --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc \\ - --globals experiment_test_local.jsonc \\ +# mlos_bench \ + --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc \ + --globals experiment_test_local.jsonc \ --tunable_values tunable-values/tunable-values-local.jsonc >>> # Now we run the command and check the output. >>> result = run(cmd, shell=True, capture_output=True, text=True, check=True) From 496594722cb56b8b73f30229ae272629c24af4a9 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 15 Nov 2024 16:50:41 +0000 Subject: [PATCH 093/100] use a single long line for platform compatibility (windows doesn't like backslashes for command line continuation) --- mlos_bench/mlos_bench/__init__.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index 2cc3cfa3d6a..9b68daa8715 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -84,16 +84,13 @@ >>> # Note: we show the command wrapped in python here for testing purposes. >>> # Alternatively replace test-cli-local-env-bench.jsonc with >>> # test-cli-local-env-opt.jsonc for one that does an optimization loop. ->>> cmd = r'''mlos_bench \ -... --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc \ -... --globals experiment_test_local.jsonc \ -... --tunable_values tunable-values/tunable-values-local.jsonc''' +>>> cmd = "mlos_bench \ +... --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc \ +... --globals experiment_test_local.jsonc \ +... --tunable_values tunable-values/tunable-values-local.jsonc" >>> print(f"Here's the shell command you'd actually run:\n# {cmd}") Here's the shell command you'd actually run: -# mlos_bench \ - --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc \ - --globals experiment_test_local.jsonc \ - --tunable_values tunable-values/tunable-values-local.jsonc +# mlos_bench --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc --globals experiment_test_local.jsonc --tunable_values tunable-values/tunable-values-local.jsonc >>> # Now we run the command and check the output. >>> result = run(cmd, shell=True, capture_output=True, text=True, check=True) >>> assert result.returncode == 0 @@ -113,7 +110,7 @@ - There is also a working example of using ``mlos_bench`` in a *separate config repo* (the more expected case for most users) in the `sqlite-autotuning `_ repo. -""" +""" # pylint: disable=line-too-long # noqa: E501 from mlos_bench.version import VERSION __version__ = VERSION From 2646ccddf34ae662a0d8c890b76eb12cda073677 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 15 Nov 2024 16:50:52 +0000 Subject: [PATCH 094/100] note the extension --- mlos_bench/mlos_bench/config/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index c1b34553ccc..f19cf95b32c 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -32,8 +32,9 @@ This allows tools that support `JSON Schema Store `_ (e.g., `VSCode -`_) to provide helpful autocomplete and validation -of the json configs while editing. +`_ with an `extension +`_) to +provide helpful autocomplete and validation of the json configs while editing. CLI Configs ^^^^^^^^^^^ From 18ff25cb93f395cde24db275f016e98b3084d982 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 15 Nov 2024 17:57:46 +0000 Subject: [PATCH 095/100] Adding more docs about storage --- mlos_bench/mlos_bench/__init__.py | 16 ++++++++++++ mlos_bench/mlos_bench/storage/__init__.py | 18 ++++++++++--- .../storage/base_experiment_data.py | 9 ++++++- .../mlos_bench/storage/base_trial_data.py | 6 ++++- .../storage/base_tunable_config_data.py | 6 ++++- .../base_tunable_config_trial_group_data.py | 6 ++++- mlos_bench/mlos_bench/storage/sql/__init__.py | 25 ++++++++++++++++++- .../mlos_bench/storage/sql/experiment.py | 4 ++- .../mlos_bench/storage/sql/experiment_data.py | 4 ++- mlos_bench/mlos_bench/storage/sql/schema.py | 11 +++++++- mlos_bench/mlos_bench/storage/sql/storage.py | 3 ++- mlos_bench/mlos_bench/storage/sql/trial.py | 5 +++- .../mlos_bench/storage/sql/trial_data.py | 4 ++- .../storage/sql/tunable_config_data.py | 4 ++- .../sql/tunable_config_trial_group_data.py | 4 ++- 15 files changed, 109 insertions(+), 16 deletions(-) diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index 9b68daa8715..1f54f78c440 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -62,6 +62,19 @@ For instance, nearly any :py:mod:`SQL ` backend that `sqlalchemy `_ supports can be used. +The data management and automation portions of experiment data is a key component of +MLOS as it provides a unified way to manage experiment data across different +Environments, enabling more reusable visualization and analysis by mapping benchmark +metrics into common semantic types (e.g., via `OpenTelemetry +`_). + +Without this most experiments are effectively siloed and require custom, and more +critically, non-reusable scripts to setup and later parse results and are hence +harder to scale to many users. + +With these features as a part of the MLOS ecosystem, benchmarking can become a +*service* that any developer, admin, research, etc. can use and adapt. + See below for more information on the classes in this package. Notes @@ -107,6 +120,9 @@ - `mlos_bench/README.md `_ for additional documentation and examples in the source tree. +- `mlos_bench/DEVNOTES.md `_ + for additional developer notes in the source tree. + - There is also a working example of using ``mlos_bench`` in a *separate config repo* (the more expected case for most users) in the `sqlite-autotuning `_ repo. diff --git a/mlos_bench/mlos_bench/storage/__init__.py b/mlos_bench/mlos_bench/storage/__init__.py index 2315622d2ed..f5d62d0d6ce 100644 --- a/mlos_bench/mlos_bench/storage/__init__.py +++ b/mlos_bench/mlos_bench/storage/__init__.py @@ -16,10 +16,22 @@ associated :py:class:`~.Storage.Trial` instances which are used by the :py:mod:`~mlos_bench.schedulers` during ``mlos_bench`` run time to execute `Trials`. + + In MLOS terms, an *Experiment* is a group of *Trials* that share the same scripts + and target system. + + A *Trial* is a single run of the target system with a specific *Configuration* + (e.g., set of tunable parameter values). + (Note: other systems may call this a *sample*) + - Retrieve the :py:class:`~mlos_bench.storage.base_trial_data.TrialData` results - with a :py:class:`~mlos_bench.storage.base_experiment_data.ExperimentData` - instance. These can be especially useful with :py:mod:`mlos_viz` for interactive - exploration in a Jupyter Notebook interface, for instance. + with the :py:attr:`~mlos_bench.storage.base_experiment_data.ExperimentData.trials` + property on a :py:class:`~mlos_bench.storage.base_experiment_data.ExperimentData` + instance via the :py:class:`~.Storage` instance's + :py:attr:`~mlos_bench.storage.base_storage.Storage.experiments` property. + + These can be especially useful with :py:mod:`mlos_viz` for interactive exploration + in a Jupyter Notebook interface, for instance. The :py:func:`.from_config` :py:mod:`.storage_factory` function can be used to get a :py:class:`.Storage` instance from a diff --git a/mlos_bench/mlos_bench/storage/base_experiment_data.py b/mlos_bench/mlos_bench/storage/base_experiment_data.py index a56e2cc1685..a11c12a5ac0 100644 --- a/mlos_bench/mlos_bench/storage/base_experiment_data.py +++ b/mlos_bench/mlos_bench/storage/base_experiment_data.py @@ -2,7 +2,14 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Base interface for accessing the stored benchmark experiment data.""" +"""Base interface for accessing the stored benchmark experiment data. + +An experiment is a collection of trials that are run with a given set of scripts and +target system. + +Each trial is associated with a configuration (e.g., set of tunable parameters), but +multiple trials may use the same config (e.g., for repeat run variability analysis). +""" from abc import ABCMeta, abstractmethod from typing import TYPE_CHECKING, Dict, Literal, Optional, Tuple diff --git a/mlos_bench/mlos_bench/storage/base_trial_data.py b/mlos_bench/mlos_bench/storage/base_trial_data.py index 4782aa92b36..86df6c43cda 100644 --- a/mlos_bench/mlos_bench/storage/base_trial_data.py +++ b/mlos_bench/mlos_bench/storage/base_trial_data.py @@ -2,7 +2,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Base interface for accessing the stored benchmark trial data.""" +"""Base interface for accessing the stored benchmark trial data. + +A single trial is a single run of an experiment with a given configuration (e.g., +set of tunable parameters). +""" from abc import ABCMeta, abstractmethod from datetime import datetime from typing import TYPE_CHECKING, Any, Dict, Optional diff --git a/mlos_bench/mlos_bench/storage/base_tunable_config_data.py b/mlos_bench/mlos_bench/storage/base_tunable_config_data.py index 62751deb8e9..43ffdbabb46 100644 --- a/mlos_bench/mlos_bench/storage/base_tunable_config_data.py +++ b/mlos_bench/mlos_bench/storage/base_tunable_config_data.py @@ -2,7 +2,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Base interface for accessing the stored benchmark (tunable) config data.""" +"""Base interface for accessing the stored benchmark (tunable) config data. + +Note: a configuration in this context is the set of tunable parameter values and can +be used by one or more trials. +""" from abc import ABCMeta, abstractmethod from typing import Any, Dict, Optional diff --git a/mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py b/mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py index 797204199a2..88687a01d42 100644 --- a/mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py +++ b/mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py @@ -2,7 +2,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Base interface for accessing the stored benchmark config trial group data.""" +"""Base interface for accessing the stored benchmark config trial group data. + +Since a single config may be used by multiple trials, we can group them together for +easier analysis. +""" from abc import ABCMeta, abstractmethod from typing import TYPE_CHECKING, Any, Dict, Optional diff --git a/mlos_bench/mlos_bench/storage/sql/__init__.py b/mlos_bench/mlos_bench/storage/sql/__init__.py index eb8867032ad..84c357e7d62 100644 --- a/mlos_bench/mlos_bench/storage/sql/__init__.py +++ b/mlos_bench/mlos_bench/storage/sql/__init__.py @@ -2,7 +2,30 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Interfaces to the SQL-based storage backends for mlos_bench.""" +"""Interfaces to the SQL-based storage backends for mlos_bench using `SQLAlchemy +`_. + +In general any SQL system supported by SQLAlchemy can be used, but the default is a +local SQLite instance. + +Although the schema is defined (and printable) by the +:py:mod:`mlos_bench.storage.sql.schema` module so direct queries are possible, users +are expected to interact with the data using the +:py:class:`~mlos_bench.storage.sql.experiment_data.ExperimentSqlData` and +:py:class:`~mlos_bench.storage.sql.trial_data.TrialSqlData` interfaces, which can be +obtained from the initial :py:class:`.SqlStorage` instance obtained by +:py:func:`mlos_bench.storage.storage_factory.from_config`. + +Examples +-------- +TODO: Add example usage. + +Notes +----- +See the `mlos_bench/config/storage +`_ +tree for some configuration examples. +""" from mlos_bench.storage.sql.storage import SqlStorage __all__ = [ diff --git a/mlos_bench/mlos_bench/storage/sql/experiment.py b/mlos_bench/mlos_bench/storage/sql/experiment.py index e072de40c47..e128f64b223 100644 --- a/mlos_bench/mlos_bench/storage/sql/experiment.py +++ b/mlos_bench/mlos_bench/storage/sql/experiment.py @@ -2,7 +2,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Saving and restoring the benchmark data using SQLAlchemy.""" +""":py:class:`.Storage.Experiment` interface implementation for saving and restoring +the benchmark experiment data using `SQLAlchemy `_ backend. +""" import hashlib import logging diff --git a/mlos_bench/mlos_bench/storage/sql/experiment_data.py b/mlos_bench/mlos_bench/storage/sql/experiment_data.py index fb542cc4f35..f24ed82268b 100644 --- a/mlos_bench/mlos_bench/storage/sql/experiment_data.py +++ b/mlos_bench/mlos_bench/storage/sql/experiment_data.py @@ -2,7 +2,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""An interface to access the experiment benchmark data stored in SQL DB.""" +"""An interface to access the benchmark experiment data stored in SQL DB using the +:py:class:`.ExperimentData` interface. +""" import logging from typing import Dict, Literal, Optional diff --git a/mlos_bench/mlos_bench/storage/sql/schema.py b/mlos_bench/mlos_bench/storage/sql/schema.py index af0c9fe9879..0da09e774bd 100644 --- a/mlos_bench/mlos_bench/storage/sql/schema.py +++ b/mlos_bench/mlos_bench/storage/sql/schema.py @@ -2,7 +2,16 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""DB schema definition.""" +"""DB schema definition for the +:py:class:`~mlos_bench.storage.sql.storage.SqlStorage` backend. + +Notes +----- +The SQL statements are generated by SQLAlchemy, but can be obtained using +``repr`` or ``str`` (e.g., via ``print()``) on this object. +The ``mlos_bench`` CLI will do this automatically if the logging level is set to +``DEBUG``. +""" import logging from typing import Any, List diff --git a/mlos_bench/mlos_bench/storage/sql/storage.py b/mlos_bench/mlos_bench/storage/sql/storage.py index 3a272ff19cb..e60b32282e2 100644 --- a/mlos_bench/mlos_bench/storage/sql/storage.py +++ b/mlos_bench/mlos_bench/storage/sql/storage.py @@ -21,7 +21,8 @@ class SqlStorage(Storage): - """An implementation of the Storage interface using SQLAlchemy backend.""" + """An implementation of the :py:class:`~.Storage` interface using SQLAlchemy + backend.""" def __init__( self, diff --git a/mlos_bench/mlos_bench/storage/sql/trial.py b/mlos_bench/mlos_bench/storage/sql/trial.py index 04d786b3b09..0951e702647 100644 --- a/mlos_bench/mlos_bench/storage/sql/trial.py +++ b/mlos_bench/mlos_bench/storage/sql/trial.py @@ -2,7 +2,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Saving and updating benchmark data using SQLAlchemy backend.""" +""":py:class:`.Storage.Trial` interface implementation for saving and restoring +the benchmark trial data using `SQLAlchemy `_ backend. +""" + import logging from datetime import datetime diff --git a/mlos_bench/mlos_bench/storage/sql/trial_data.py b/mlos_bench/mlos_bench/storage/sql/trial_data.py index dcf2ffd3376..60027f24194 100644 --- a/mlos_bench/mlos_bench/storage/sql/trial_data.py +++ b/mlos_bench/mlos_bench/storage/sql/trial_data.py @@ -2,7 +2,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""An interface to access the benchmark trial data stored in SQL DB.""" +"""An interface to access the benchmark trial data stored in SQL DB using the +:py:class:`.TrialData` interface. +""" from datetime import datetime from typing import TYPE_CHECKING, Optional diff --git a/mlos_bench/mlos_bench/storage/sql/tunable_config_data.py b/mlos_bench/mlos_bench/storage/sql/tunable_config_data.py index fa861029ee8..97fb5d3d0b6 100644 --- a/mlos_bench/mlos_bench/storage/sql/tunable_config_data.py +++ b/mlos_bench/mlos_bench/storage/sql/tunable_config_data.py @@ -2,7 +2,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""An interface to access the tunable config data stored in SQL DB.""" +"""An interface to access the tunable config data stored in a SQL DB using the +:py:class:`.TunableConfigData` interface. +""" import pandas from sqlalchemy.engine import Engine diff --git a/mlos_bench/mlos_bench/storage/sql/tunable_config_trial_group_data.py b/mlos_bench/mlos_bench/storage/sql/tunable_config_trial_group_data.py index 2631c1c6eb6..5348f507e24 100644 --- a/mlos_bench/mlos_bench/storage/sql/tunable_config_trial_group_data.py +++ b/mlos_bench/mlos_bench/storage/sql/tunable_config_trial_group_data.py @@ -2,7 +2,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""An interface to access the tunable config trial group data stored in SQL DB.""" +"""An interface to access the tunable config trial group data stored in a SQL DB +using the :py:class:`.TunableConfigTrialGroupData` interface. +""" from typing import TYPE_CHECKING, Dict, Optional From 15499c3fac35dd7a783db406333079e4f2e3a426 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 15 Nov 2024 17:58:50 +0000 Subject: [PATCH 096/100] formatting --- mlos_bench/mlos_bench/storage/base_experiment_data.py | 3 ++- mlos_bench/mlos_bench/storage/base_trial_data.py | 7 ++++--- mlos_bench/mlos_bench/storage/base_tunable_config_data.py | 3 ++- .../storage/base_tunable_config_trial_group_data.py | 3 ++- mlos_bench/mlos_bench/storage/sql/schema.py | 5 +++-- mlos_bench/mlos_bench/storage/sql/storage.py | 3 ++- .../storage/sql/tunable_config_trial_group_data.py | 4 ++-- 7 files changed, 17 insertions(+), 11 deletions(-) diff --git a/mlos_bench/mlos_bench/storage/base_experiment_data.py b/mlos_bench/mlos_bench/storage/base_experiment_data.py index a11c12a5ac0..d214c98630b 100644 --- a/mlos_bench/mlos_bench/storage/base_experiment_data.py +++ b/mlos_bench/mlos_bench/storage/base_experiment_data.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Base interface for accessing the stored benchmark experiment data. +""" +Base interface for accessing the stored benchmark experiment data. An experiment is a collection of trials that are run with a given set of scripts and target system. diff --git a/mlos_bench/mlos_bench/storage/base_trial_data.py b/mlos_bench/mlos_bench/storage/base_trial_data.py index 86df6c43cda..b4b5936a290 100644 --- a/mlos_bench/mlos_bench/storage/base_trial_data.py +++ b/mlos_bench/mlos_bench/storage/base_trial_data.py @@ -2,10 +2,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Base interface for accessing the stored benchmark trial data. +""" +Base interface for accessing the stored benchmark trial data. -A single trial is a single run of an experiment with a given configuration (e.g., -set of tunable parameters). +A single trial is a single run of an experiment with a given configuration (e.g., set of +tunable parameters). """ from abc import ABCMeta, abstractmethod from datetime import datetime diff --git a/mlos_bench/mlos_bench/storage/base_tunable_config_data.py b/mlos_bench/mlos_bench/storage/base_tunable_config_data.py index 43ffdbabb46..c925fa7b0ec 100644 --- a/mlos_bench/mlos_bench/storage/base_tunable_config_data.py +++ b/mlos_bench/mlos_bench/storage/base_tunable_config_data.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Base interface for accessing the stored benchmark (tunable) config data. +""" +Base interface for accessing the stored benchmark (tunable) config data. Note: a configuration in this context is the set of tunable parameter values and can be used by one or more trials. diff --git a/mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py b/mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py index 88687a01d42..d17fc74a9db 100644 --- a/mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py +++ b/mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Base interface for accessing the stored benchmark config trial group data. +""" +Base interface for accessing the stored benchmark config trial group data. Since a single config may be used by multiple trials, we can group them together for easier analysis. diff --git a/mlos_bench/mlos_bench/storage/sql/schema.py b/mlos_bench/mlos_bench/storage/sql/schema.py index 0da09e774bd..e6e36ea2d14 100644 --- a/mlos_bench/mlos_bench/storage/sql/schema.py +++ b/mlos_bench/mlos_bench/storage/sql/schema.py @@ -2,8 +2,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""DB schema definition for the -:py:class:`~mlos_bench.storage.sql.storage.SqlStorage` backend. +""" +DB schema definition for the :py:class:`~mlos_bench.storage.sql.storage.SqlStorage` +backend. Notes ----- diff --git a/mlos_bench/mlos_bench/storage/sql/storage.py b/mlos_bench/mlos_bench/storage/sql/storage.py index e60b32282e2..495b38e6546 100644 --- a/mlos_bench/mlos_bench/storage/sql/storage.py +++ b/mlos_bench/mlos_bench/storage/sql/storage.py @@ -22,7 +22,8 @@ class SqlStorage(Storage): """An implementation of the :py:class:`~.Storage` interface using SQLAlchemy - backend.""" + backend. + """ def __init__( self, diff --git a/mlos_bench/mlos_bench/storage/sql/tunable_config_trial_group_data.py b/mlos_bench/mlos_bench/storage/sql/tunable_config_trial_group_data.py index 5348f507e24..6fac71be15f 100644 --- a/mlos_bench/mlos_bench/storage/sql/tunable_config_trial_group_data.py +++ b/mlos_bench/mlos_bench/storage/sql/tunable_config_trial_group_data.py @@ -2,8 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""An interface to access the tunable config trial group data stored in a SQL DB -using the :py:class:`.TunableConfigTrialGroupData` interface. +"""An interface to access the tunable config trial group data stored in a SQL DB using +the :py:class:`.TunableConfigTrialGroupData` interface. """ from typing import TYPE_CHECKING, Dict, Optional From e0c29558570c2c746f761be1bae4fecde608fa78 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 15 Nov 2024 18:06:12 +0000 Subject: [PATCH 097/100] todo comments --- mlos_bench/mlos_bench/optimizers/__init__.py | 2 ++ mlos_bench/mlos_bench/services/__init__.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mlos_bench/mlos_bench/optimizers/__init__.py b/mlos_bench/mlos_bench/optimizers/__init__.py index 99890cb2a93..12ee5e4265d 100644 --- a/mlos_bench/mlos_bench/optimizers/__init__.py +++ b/mlos_bench/mlos_bench/optimizers/__init__.py @@ -4,6 +4,8 @@ # """Interfaces and wrapper classes for optimizers to be used in mlos_bench for autotuning or benchmarking. + +TODO: Improve documentation here. """ from mlos_bench.optimizers.base_optimizer import Optimizer diff --git a/mlos_bench/mlos_bench/services/__init__.py b/mlos_bench/mlos_bench/services/__init__.py index b768afb09c8..234317c37c6 100644 --- a/mlos_bench/mlos_bench/services/__init__.py +++ b/mlos_bench/mlos_bench/services/__init__.py @@ -2,7 +2,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Services for implementing Environments for mlos_bench.""" +"""Services for implementing Environments for mlos_bench. + +TODO: Improve documentation here. +""" from mlos_bench.services.base_fileshare import FileShareService from mlos_bench.services.base_service import Service From 3f89042e05d7777931d394312716d5445c98ee30 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 15 Nov 2024 18:22:51 +0000 Subject: [PATCH 098/100] more docstrings and formatting --- mlos_bench/mlos_bench/optimizers/__init__.py | 5 +++-- mlos_bench/mlos_bench/services/__init__.py | 3 ++- .../mlos_bench/storage/base_experiment_data.py | 4 ++++ mlos_bench/mlos_bench/storage/base_storage.py | 15 ++++++++++++++- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/mlos_bench/mlos_bench/optimizers/__init__.py b/mlos_bench/mlos_bench/optimizers/__init__.py index 12ee5e4265d..7f2d6310c9a 100644 --- a/mlos_bench/mlos_bench/optimizers/__init__.py +++ b/mlos_bench/mlos_bench/optimizers/__init__.py @@ -2,8 +2,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Interfaces and wrapper classes for optimizers to be used in mlos_bench for autotuning -or benchmarking. +""" +Interfaces and wrapper classes for optimizers to be used in mlos_bench for autotuning or +benchmarking. TODO: Improve documentation here. """ diff --git a/mlos_bench/mlos_bench/services/__init__.py b/mlos_bench/mlos_bench/services/__init__.py index 234317c37c6..65ffc8e8d80 100644 --- a/mlos_bench/mlos_bench/services/__init__.py +++ b/mlos_bench/mlos_bench/services/__init__.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Services for implementing Environments for mlos_bench. +""" +Services for implementing Environments for mlos_bench. TODO: Improve documentation here. """ diff --git a/mlos_bench/mlos_bench/storage/base_experiment_data.py b/mlos_bench/mlos_bench/storage/base_experiment_data.py index d214c98630b..4ec935f68fd 100644 --- a/mlos_bench/mlos_bench/storage/base_experiment_data.py +++ b/mlos_bench/mlos_bench/storage/base_experiment_data.py @@ -10,6 +10,10 @@ Each trial is associated with a configuration (e.g., set of tunable parameters), but multiple trials may use the same config (e.g., for repeat run variability analysis). + +See Also +-------- +ExperimentData.trials : Retrieves a dictionary of the experiment's trials' data. """ from abc import ABCMeta, abstractmethod diff --git a/mlos_bench/mlos_bench/storage/base_storage.py b/mlos_bench/mlos_bench/storage/base_storage.py index 442e497c909..50e5025ad64 100644 --- a/mlos_bench/mlos_bench/storage/base_storage.py +++ b/mlos_bench/mlos_bench/storage/base_storage.py @@ -2,7 +2,20 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Base interface for saving and restoring the benchmark data.""" +""" +Base interface for saving and restoring the benchmark data. + +See Also +-------- +mlos_bench.storage.base_storage.Storage.experiments : + Retrieves a dictionary of the Experiments' data. +mlos_bench.storage.base_experiment_data.ExperimentData.results_df : + Retrieves a pandas DataFrame of the Experiment's trials' results data. +mlos_bench.storage.base_experiment_data.ExperimentData.trials : + Retrieves a dictionary of the Experiment's trials' data. +mlos_bench.storage.base_experiment_data.ExperimentData.configs : + Retrieves a dictionary of the Experiment's sampled configs data. +""" import logging from abc import ABCMeta, abstractmethod From 634a4713f455e6143d55d9e89af1316634faf491 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 15 Nov 2024 18:23:29 +0000 Subject: [PATCH 099/100] fixup --- mlos_bench/mlos_bench/storage/base_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlos_bench/mlos_bench/storage/base_storage.py b/mlos_bench/mlos_bench/storage/base_storage.py index 50e5025ad64..65a02d2c894 100644 --- a/mlos_bench/mlos_bench/storage/base_storage.py +++ b/mlos_bench/mlos_bench/storage/base_storage.py @@ -13,7 +13,7 @@ Retrieves a pandas DataFrame of the Experiment's trials' results data. mlos_bench.storage.base_experiment_data.ExperimentData.trials : Retrieves a dictionary of the Experiment's trials' data. -mlos_bench.storage.base_experiment_data.ExperimentData.configs : +mlos_bench.storage.base_experiment_data.ExperimentData.tunable_configs : Retrieves a dictionary of the Experiment's sampled configs data. """ From 5782c5a8a950e0f13582e48bf2a1d56c5a4c49ef Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 15 Nov 2024 18:30:29 +0000 Subject: [PATCH 100/100] Add some high level references for user access of data --- mlos_bench/mlos_bench/storage/__init__.py | 6 ++++++ .../mlos_bench/storage/base_experiment_data.py | 12 +++++++++++- mlos_bench/mlos_bench/storage/base_storage.py | 5 +++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/mlos_bench/mlos_bench/storage/__init__.py b/mlos_bench/mlos_bench/storage/__init__.py index f5d62d0d6ce..840a6d87fce 100644 --- a/mlos_bench/mlos_bench/storage/__init__.py +++ b/mlos_bench/mlos_bench/storage/__init__.py @@ -42,6 +42,12 @@ ------- TODO: Add example usage. +See Also +-------- +mlos_bench.storage.base_storage : Base interface for backends. +mlos_bench.storage.base_experiment_data : Base interface for ExperimentData. +mlos_bench.storage.base_trial_data : Base interface for TrialData. + Notes ----- - See `sqlite-autotuning notebooks diff --git a/mlos_bench/mlos_bench/storage/base_experiment_data.py b/mlos_bench/mlos_bench/storage/base_experiment_data.py index 4ec935f68fd..098c1bca26c 100644 --- a/mlos_bench/mlos_bench/storage/base_experiment_data.py +++ b/mlos_bench/mlos_bench/storage/base_experiment_data.py @@ -13,7 +13,17 @@ See Also -------- -ExperimentData.trials : Retrieves a dictionary of the experiment's trials' data. +ExperimentData.results_df : + Retrieves a pandas DataFrame of the Experiment's trials' results data. +ExperimentData.trials : + Retrieves a dictionary of the Experiment's trials' data. +ExperimentData.tunable_configs : + Retrieves a dictionary of the Experiment's sampled configs data. +ExperimentData.tunable_config_trial_groups : + Retrieves a dictionary of the Experiment's trials' data, grouped by shared + tunable config. +mlos_bench.storage.base_trial_data.TrialData : + Base interface for accessing the stored benchmark trial data. """ from abc import ABCMeta, abstractmethod diff --git a/mlos_bench/mlos_bench/storage/base_storage.py b/mlos_bench/mlos_bench/storage/base_storage.py index 65a02d2c894..a6b3a1aa90a 100644 --- a/mlos_bench/mlos_bench/storage/base_storage.py +++ b/mlos_bench/mlos_bench/storage/base_storage.py @@ -15,6 +15,11 @@ Retrieves a dictionary of the Experiment's trials' data. mlos_bench.storage.base_experiment_data.ExperimentData.tunable_configs : Retrieves a dictionary of the Experiment's sampled configs data. +mlos_bench.storage.base_experiment_data.ExperimentData.tunable_config_trial_groups : + Retrieves a dictionary of the Experiment's trials' data, grouped by shared + tunable config. +mlos_bench.storage.base_trial_data.TrialData : + Base interface for accessing the stored benchmark trial data. """ import logging