Merge branch 'v2_final' into patch_v2_final

scripts/yahpo.py

@@ -1,6 +1,7 @@
-from yahpo_gym import benchmark_set
 import argparse
 
+from yahpo_gym import benchmark_set
+
 
 class ParseKwargs(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):

yahpo_gym/docs/source/conf.py

@@ -9,35 +9,33 @@
 import os
 import sys
 
+import yahpo_gym
+
 # import pathlib
 # from yahpo_gym.local_config import LocalConfiguration
 # if not os.path.exists(pathlib.Path("~/.config/yahpo_gym").expanduser().absolute()):
 #     LocalConfiguration().init_config("~/.config/yahpo_gym")
 
-import yahpo_gym
-
-
 
 # -- Path setup --------------------------------------------------------------
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-sys.path.insert(0, os.path.abspath('../yahpo_gym'))
-
+sys.path.insert(0, os.path.abspath("../yahpo_gym"))
 
 
 # -- Project information -----------------------------------------------------
 
-project = 'yahpo_gym'
-copyright = '2021, Florian Pfisterer, Lennart Schneider'
-author = 'Florian Pfisterer, Lennart Schneider'
+project = "yahpo_gym"
+copyright = "2021, Florian Pfisterer, Lennart Schneider"
+author = "Florian Pfisterer, Lennart Schneider"
 
 # The short X.Y version
-version = '0.1'
+version = "0.1"
 # The full version, including alpha/beta/rc tags
-release = '0.1.0'
+release = "0.1.0"
 
 
 # -- General configuration ---------------------------------------------------
@@ -50,25 +48,25 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.intersphinx',
-    'sphinx.ext.coverage',
-    'sphinx.ext.mathjax',
-    'sphinx.ext.ifconfig',
-    'sphinx.ext.githubpages',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.coverage",
+    "sphinx.ext.mathjax",
+    "sphinx.ext.ifconfig",
+    "sphinx.ext.githubpages",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -83,15 +81,15 @@
 exclude_patterns = []
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -102,7 +100,7 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
@@ -118,7 +116,7 @@
 # -- Options for HTMLHelp output ---------------------------------------------
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'yahpo_gymdoc'
+htmlhelp_basename = "yahpo_gymdoc"
 
 
 # -- Options for LaTeX output ------------------------------------------------
@@ -127,15 +125,12 @@
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',
-
     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
@@ -145,19 +140,21 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'yahpo_gym.tex', 'yahpo\\_gym Documentation',
-     'Florian Pfisterer, Lennart Schneider', 'manual'),
+    (
+        master_doc,
+        "yahpo_gym.tex",
+        "yahpo\\_gym Documentation",
+        "Florian Pfisterer, Lennart Schneider",
+        "manual",
+    ),
 ]
 
 
 # -- Options for manual page output ------------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'yahpo_gym', 'yahpo_gym Documentation',
-     [author], 1)
-]
+man_pages = [(master_doc, "yahpo_gym", "yahpo_gym Documentation", [author], 1)]
 
 
 # -- Options for Texinfo output ----------------------------------------------
@@ -166,9 +163,15 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'yahpo_gym', 'yahpo_gym Documentation',
-     author, 'yahpo_gym', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        "yahpo_gym",
+        "yahpo_gym Documentation",
+        author,
+        "yahpo_gym",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 
@@ -187,15 +190,14 @@
 # epub_uid = ''
 
 # A list of files that should not be packed into the epub file.
-epub_exclude_files = ['search.html']
+epub_exclude_files = ["search.html"]
 
 
 # -- Extension configuration -------------------------------------------------
 
 # -- Options for intersphinx extension ---------------------------------------
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/': None}
-
-autoclass_content = 'both'
+intersphinx_mapping = {"https://docs.python.org/": None}
 
+autoclass_content = "both"

yahpo_gym/docs/source/frequently_asked.rst

@@ -55,7 +55,7 @@ Using F1 scores for rbv2_*
 
 `F1` scores in the `rbv2_*` scenarios are only available for binary classification datasets. 
 On multiclass datasets, the corresponding `F1` score is imputed with `0` and returned by the surrogate model.
-The information on which `id` corresponds to a multiclass dataset can be obtained from the entry `is_multicrit` in `BenchmarkSet.config.config`.
+The information on which `id` corresponds to a multiclass dataset can be obtained from the entry `is_multiclass` in `BenchmarkSet.config.config`.
 
 Memory Estimation for rbv2_*
 =======================

yahpo_gym/setup.py

@@ -1,7 +1,8 @@
 import sys
 from codecs import open
 from os import path
-from setuptools import setup, find_packages
+
+from setuptools import find_packages, setup
 
 here = path.abspath(path.dirname(__file__))
 

yahpo_gym/tests/test_benchmark_set.py

@@ -1,8 +1,8 @@
 import random
 import time
-import pytest
 
 import ConfigSpace
+import pytest
 
 import yahpo_gym
 from yahpo_gym.benchmark_set import BenchmarkSet

yahpo_gym/tests/test_config.py

@@ -1,6 +1,7 @@
-import pytest
 import os
 
+import pytest
+
 
 def test_config():
     import yahpo_gym.configuration as cfg

yahpo_gym/tests/test_get_suites.py

@@ -1,4 +1,5 @@
 import pytest
+
 from yahpo_gym.get_suite import get_suite
 
 

yahpo_gym/tests/test_objective_timed.py

@@ -1,6 +1,7 @@
-import pytest
 import time
+
 import ConfigSpace
+import pytest
 
 import yahpo_gym
 from yahpo_gym.benchmark_set import BenchmarkSet

yahpo_gym/yahpo_gym/benchmark_set.py

@@ -25,6 +25,7 @@ def __init__(
         multithread: bool = True,
         check: bool = True,
         noisy: bool = False,
+        data_path=None,
     ):
         """
         Interface for a benchmark scenario.
@@ -50,10 +51,14 @@ def __init__(
             Should input to objective_function be checked for validity? Initialized to `True`, but can be disabled for speedups.
         noisy: bool
             Use stochastic surrogate models? Initialized to `False`.
+        data_path: str
+            Optional path to the data directory. If not provided, the default data path as indicated by the local config is used.
         """
 
         assert scenario is not None, "Please provide a valid scenario."
         self.config = cfg(scenario)
+        if data_path is not None:
+            self.config.config["basedir"] = data_path  # override the default path
         self.encoding = self._get_encoding()
         self.config_space = self._get_config_space()
         self.active_session = active_session
@@ -90,6 +95,8 @@ def objective_function(
         configuration: Dict
             A valid dict or list of dicts containing hyperparameters to be evaluated.
             Attention: `configuration` is not checked for internal validity for speed purposes.
+        seed: int
+            Seed for the random number generator. Initialized to `None`.
         logging: bool
             Should the evaluation be logged in the `archive`? Initialized to `False`.
         multithread: bool
@@ -162,6 +169,8 @@ def objective_function_timed(
         configuration: Dict
             A valid dict or list of dicts containing hyperparameters to be evaluated.
             Attention: `configuration` is not checked for internal validity for speed purposes.
+        seed: int
+            Seed for the random number generator. Initialized to `None`.
         logging: bool
             Should the evaluation be logged in the `archive`? Initialized to `False`.
         multithread: bool
@@ -348,7 +357,7 @@ def target_stats(self):
             * statistic :: min or max
             * value :: value of minimum/maximum
             * scenario :: the scenario
-            " instance :: the instance
+            * instance :: the instance
         If no instance is set, all instances for a given scenario are returned.
         """
         df = pd.read_csv(
@@ -462,7 +471,7 @@ def _eval_random(self):
     def _infer_quant(self):
         offsets = []
         runtimes = []
-        for i in range(15):
+        for _ in range(15):
             start_time = time.time()
             results = self._eval_random()
             runtimes += [results[self.config.runtime_name]]
@@ -486,7 +495,6 @@ def _get_model_path(self):
     from yahpo_gym import benchmark_set
 
     bench = benchmark_set.BenchmarkSet("iaml_super")
-    bench.instances
     bench.set_instance("1067")
     config = {
         "task_id": "1067",

yahpo_gym/yahpo_gym/benchmarks/__init__.py

@@ -1 +1 @@
-from yahpo_gym.benchmarks import lcbench, nb301, rbv2, iaml, fair  # noqa: F401
+from yahpo_gym.benchmarks import fair, iaml, lcbench, nb301, rbv2  # noqa: F401

yahpo_gym/yahpo_gym/benchmarks/rbv2.py

@@ -28,7 +28,7 @@
     "instance_names": "task_id",
     "runtime_name": "timetrain",
     # "memory_name": "memory",
-    "is_multicrit": {
+    "is_multiclass": {
         "3": False,
         "11": True,
         "12": True,

yahpo_gym/yahpo_gym/configuration.py

@@ -24,7 +24,8 @@ def __init__(self, config_dict: Dict):
     def get_path(self, key: str):
         return f"{self.config_path}/{self.config[key]}"
 
-    def _get_default_dict(self):
+    @staticmethod
+    def _get_default_dict():
         return {
             "basedir": local_config.data_path,
             "config_id": "",
@@ -164,7 +165,7 @@ def __str__(self):
         return out
 
 
-def cfg(key: str|None=None, **kwargs):
+def cfg(key: str | None = None, **kwargs):
     """
     Shorthand access to 'ConfigDict'.
 

yahpo_gym/yahpo_gym/get_suite.py

@@ -1,4 +1,5 @@
 import pandas as pd
+
 from yahpo_gym.local_config import local_config
 
 

yahpo_gym/yahpo_gym/local_config.py

@@ -1,5 +1,6 @@
-from pathlib import Path
 import os
+from pathlib import Path
+
 import yaml
 
 
@@ -31,13 +32,12 @@ def init_config(self, data_path: str = ""):
         Initialize a new local configuration.
 
         This writes a local configuration file to the specified 'settings_path'.
-        The
         It is currently used to globally store the following information
         'data_path': A path to the metadata required for inference.
 
         Parameters
         ----------
-        settings_path: str
+        data_path : str
             Path to the directory where surrogate models and metadata are saved.
         """
         os.makedirs(os.path.dirname(self.settings_path), exist_ok=True)
@@ -62,7 +62,6 @@ def set_data_path(self, data_path: str | Path):
             yaml.dump(config, fh)
 
     def _load_config(self):
-        config = {"data_path": ""}
         try:
             with self.settings_path.open("r") as fh:
                 config = yaml.load(fh, Loader=yaml.FullLoader)