Thomas Holvoet: Interrupt optimization through IncorporateRunResultCa…

…llback (#765)

development/_modules/smac/callbacks.html

@@ -284,7 +284,7 @@
            <div itemprop="articleBody">
 
   <h1>Source code for smac.callbacks</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">TYPE_CHECKING</span>
+<span></span><span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">TYPE_CHECKING</span><span class="p">,</span> <span class="n">Optional</span>
 
 <span class="k">if</span> <span class="n">TYPE_CHECKING</span><span class="p">:</span>
     <span class="kn">from</span> <span class="nn">smac.optimizer.smbo</span> <span class="kn">import</span> <span class="n">SMBO</span>
@@ -317,14 +317,18 @@ <h1>Source code for smac.callbacks</h1><div class="highlight"><pre>
 
 <div class="viewcode-block" id="IncorporateRunResultCallback"><a class="viewcode-back" href="../../apidoc/smac.callbacks.html#smac.callbacks.IncorporateRunResultCallback">[docs]</a><span class="k">class</span> <span class="nc">IncorporateRunResultCallback</span><span class="p">:</span>
 
-    <span class="sd">&quot;&quot;&quot;Callback to react on a new run result. Called after the finished run is added to the runhistory.&quot;&quot;&quot;</span>
+    <span class="sd">&quot;&quot;&quot;Callback to react on a new run result.</span>
+
+<span class="sd">    Called after the finished run is added to the runhistory.</span>
+<span class="sd">    Optionally return `False` to (gracefully) stop the optimization.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span>
             <span class="bp">self</span><span class="p">,</span> <span class="n">smbo</span><span class="p">:</span> <span class="s1">&#39;SMBO&#39;</span><span class="p">,</span>
             <span class="n">run_info</span><span class="p">:</span> <span class="n">RunInfo</span><span class="p">,</span>
             <span class="n">result</span><span class="p">:</span> <span class="n">RunValue</span><span class="p">,</span>
             <span class="n">time_left</span><span class="p">:</span> <span class="nb">float</span><span class="p">,</span>
-    <span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]:</span>
         <span class="k">pass</span></div>
 </pre></div>
 

development/_modules/smac/optimizer/smbo.html

@@ -587,7 +587,7 @@ <h1>Source code for smac.optimizer.smbo</h1><div class="highlight"><pre>
                 <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">stats</span><span class="o">.</span><span class="n">is_budget_exhausted</span><span class="p">():</span>
                     <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Exhausted configuration budget&quot;</span><span class="p">)</span>
                 <span class="k">else</span><span class="p">:</span>
-                    <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Shutting down because a configuration returned status STOP&quot;</span><span class="p">)</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Shutting down because a configuration or callback returned status STOP&quot;</span><span class="p">)</span>
 
                 <span class="c1"># The budget can be exhausted  for 2 reasons: number of ta runs or</span>
                 <span class="c1"># time. If the number of ta runs is reached, but there is still budget,</span>
@@ -775,7 +775,12 @@ <h1>Source code for smac.optimizer.smbo</h1><div class="highlight"><pre>
         <span class="p">)</span>
 
         <span class="k">for</span> <span class="n">callback</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_callbacks</span><span class="p">[</span><span class="s1">&#39;_incorporate_run_results&#39;</span><span class="p">]:</span>
-            <span class="n">callback</span><span class="p">(</span><span class="n">smbo</span><span class="o">=</span><span class="bp">self</span><span class="p">,</span> <span class="n">run_info</span><span class="o">=</span><span class="n">run_info</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">,</span> <span class="n">time_left</span><span class="o">=</span><span class="n">time_left</span><span class="p">)</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="n">callback</span><span class="p">(</span><span class="n">smbo</span><span class="o">=</span><span class="bp">self</span><span class="p">,</span> <span class="n">run_info</span><span class="o">=</span><span class="n">run_info</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">,</span> <span class="n">time_left</span><span class="o">=</span><span class="n">time_left</span><span class="p">)</span>
+            <span class="c1"># If a callback returns False, the optimization loop should be interrupted</span>
+            <span class="c1"># the other callbacks are still being called</span>
+            <span class="k">if</span> <span class="n">response</span> <span class="ow">is</span> <span class="kc">False</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;An IncorporateRunResultCallback returned False, requesting abort.&quot;</span><span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_stop</span> <span class="o">=</span> <span class="kc">True</span>
 
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">scenario</span><span class="o">.</span><span class="n">save_results_instantly</span><span class="p">:</span>  <span class="c1"># type: ignore[attr-defined] # noqa F821</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>

development/_sources/callbacks.rst.txt

@@ -0,0 +1,19 @@
+Callbacks
+---------
+
+Callbacks allow customizing the behavior of SMAC to ones needs. Currently, the list of implemented callbacks is
+very limited, but they can easily be added.
+
+If you want to create a new callback, please check `smac.callbacks` and create a new pull request.
+
+
+IncorporateRunResultCallback 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Callback to react on a new run result.
+
+Called after the finished run is added to the runhistory.
+Optionally return `False` to (gracefully) stop the optimization.
+
+
+

development/_sources/examples/SMAC4BB/SMAC4BB_rosenbrock_example.rst.txt

@@ -95,7 +95,7 @@ Using the black-box optimization interface of SMAC
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 0 minutes  23.626 seconds)
+   **Total running time of the script:** ( 0 minutes  21.098 seconds)
 
 
 .. _sphx_glr_download_examples_SMAC4BB_SMAC4BB_rosenbrock_example.py:

development/_sources/examples/SMAC4BB/sg_execution_times.rst.txt

@@ -5,8 +5,8 @@
 
 Computation times
 =================
-**00:23.626** total execution time for **examples_SMAC4BB** files:
+**00:21.098** total execution time for **examples_SMAC4BB** files:
 
 +----------------------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_examples_SMAC4BB_SMAC4BB_rosenbrock_example.py` (``SMAC4BB_rosenbrock_example.py``) | 00:23.626 | 0.0 MB |
+| :ref:`sphx_glr_examples_SMAC4BB_SMAC4BB_rosenbrock_example.py` (``SMAC4BB_rosenbrock_example.py``) | 00:21.098 | 0.0 MB |
 +----------------------------------------------------------------------------------------------------+-----------+--------+

development/_sources/examples/SMAC4HPO/SMAC4HPO_acq_rosenbrock_example.rst.txt

@@ -109,7 +109,7 @@ Changing the acquisition function
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 0 minutes  3.859 seconds)
+   **Total running time of the script:** ( 0 minutes  3.484 seconds)
 
 
 .. _sphx_glr_download_examples_SMAC4HPO_SMAC4HPO_acq_rosenbrock_example.py:

development/_sources/examples/SMAC4HPO/SMAC4HPO_gradientboosting_example.rst.txt

@@ -118,7 +118,7 @@ We optimize a GradientBoosting on an artificially created binary classification
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 2 minutes  9.768 seconds)
+   **Total running time of the script:** ( 2 minutes  13.068 seconds)
 
 
 .. _sphx_glr_download_examples_SMAC4HPO_SMAC4HPO_gradientboosting_example.py:

development/_sources/examples/SMAC4HPO/SMAC4HPO_rosenbrock_example.rst.txt

@@ -96,7 +96,7 @@ Using SMAC tuned for HPO for black-box optimization
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 0 minutes  3.915 seconds)
+   **Total running time of the script:** ( 0 minutes  3.910 seconds)
 
 
 .. _sphx_glr_download_examples_SMAC4HPO_SMAC4HPO_rosenbrock_example.py:

development/_sources/examples/SMAC4HPO/SMAC4HPO_svm_example.rst.txt

@@ -163,7 +163,7 @@ Note: SMAC-documentation uses linenumbers to generate docs from this file.
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 0 minutes  27.469 seconds)
+   **Total running time of the script:** ( 0 minutes  26.483 seconds)
 
 
 .. _sphx_glr_download_examples_SMAC4HPO_SMAC4HPO_svm_example.py:

development/_sources/examples/SMAC4HPO/sg_execution_times.rst.txt

@@ -5,14 +5,14 @@
 
 Computation times
 =================
-**02:45.011** total execution time for **examples_SMAC4HPO** files:
+**02:46.945** total execution time for **examples_SMAC4HPO** files:
 
 +-------------------------------------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_examples_SMAC4HPO_SMAC4HPO_gradientboosting_example.py` (``SMAC4HPO_gradientboosting_example.py``) | 02:09.768 | 0.0 MB |
+| :ref:`sphx_glr_examples_SMAC4HPO_SMAC4HPO_gradientboosting_example.py` (``SMAC4HPO_gradientboosting_example.py``) | 02:13.068 | 0.0 MB |
 +-------------------------------------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_examples_SMAC4HPO_SMAC4HPO_svm_example.py` (``SMAC4HPO_svm_example.py``)                           | 00:27.469 | 0.0 MB |
+| :ref:`sphx_glr_examples_SMAC4HPO_SMAC4HPO_svm_example.py` (``SMAC4HPO_svm_example.py``)                           | 00:26.483 | 0.0 MB |
 +-------------------------------------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_examples_SMAC4HPO_SMAC4HPO_rosenbrock_example.py` (``SMAC4HPO_rosenbrock_example.py``)             | 00:03.915 | 0.0 MB |
+| :ref:`sphx_glr_examples_SMAC4HPO_SMAC4HPO_rosenbrock_example.py` (``SMAC4HPO_rosenbrock_example.py``)             | 00:03.910 | 0.0 MB |
 +-------------------------------------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_examples_SMAC4HPO_SMAC4HPO_acq_rosenbrock_example.py` (``SMAC4HPO_acq_rosenbrock_example.py``)     | 00:03.859 | 0.0 MB |
+| :ref:`sphx_glr_examples_SMAC4HPO_SMAC4HPO_acq_rosenbrock_example.py` (``SMAC4HPO_acq_rosenbrock_example.py``)     | 00:03.484 | 0.0 MB |
 +-------------------------------------------------------------------------------------------------------------------+-----------+--------+

development/_sources/examples/SMAC4MF/SMAC4MF_mlp_example.rst.txt

@@ -187,7 +187,7 @@ optimize the average accuracy on a 5-fold cross validation.
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 1 minutes  58.904 seconds)
+   **Total running time of the script:** ( 1 minutes  53.036 seconds)
 
 
 .. _sphx_glr_download_examples_SMAC4MF_SMAC4MF_mlp_example.py:

development/_sources/examples/SMAC4MF/SMAC4MF_sgd_example.rst.txt

@@ -117,6 +117,8 @@ digit-2 vs digit-3.
       warnings.warn("{} is intended to be used "
     /home/runner/work/SMAC3/SMAC3/doc/../smac/intensification/parallel_scheduling.py:152: UserWarning: Hyperband is intended to be used with more than 1 worker but num_workers=1
       warnings.warn("{} is intended to be used "
+    /home/runner/work/SMAC3/SMAC3/doc/../smac/intensification/parallel_scheduling.py:152: UserWarning: Hyperband is intended to be used with more than 1 worker but num_workers=1
+      warnings.warn("{} is intended to be used "
     Optimized Value: 0.0038
 
 
@@ -272,7 +274,7 @@ digit-2 vs digit-3.
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 1 minutes  48.303 seconds)
+   **Total running time of the script:** ( 1 minutes  47.518 seconds)
 
 
 .. _sphx_glr_download_examples_SMAC4MF_SMAC4MF_sgd_example.py:

development/_sources/examples/SMAC4MF/sg_execution_times.rst.txt

@@ -5,10 +5,10 @@
 
 Computation times
 =================
-**03:47.207** total execution time for **examples_SMAC4MF** files:
+**03:40.553** total execution time for **examples_SMAC4MF** files:
 
 +--------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_examples_SMAC4MF_SMAC4MF_mlp_example.py` (``SMAC4MF_mlp_example.py``) | 01:58.904 | 0.0 MB |
+| :ref:`sphx_glr_examples_SMAC4MF_SMAC4MF_mlp_example.py` (``SMAC4MF_mlp_example.py``) | 01:53.036 | 0.0 MB |
 +--------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_examples_SMAC4MF_SMAC4MF_sgd_example.py` (``SMAC4MF_sgd_example.py``) | 01:48.303 | 0.0 MB |
+| :ref:`sphx_glr_examples_SMAC4MF_SMAC4MF_sgd_example.py` (``SMAC4MF_sgd_example.py``) | 01:47.518 | 0.0 MB |
 +--------------------------------------------------------------------------------------+-----------+--------+

development/_sources/examples/function_minimization/fmin_rosenbrock_example.rst.txt

@@ -20,7 +20,7 @@ scipy-style fmin interface
 
  .. code-block:: none
 
-    INFO:smac.utils.io.cmd_reader.CMDReader:Output to smac3-output_2021-08-26_07:17:21_075792
+    INFO:smac.utils.io.cmd_reader.CMDReader:Output to smac3-output_2021-08-26_08:03:43_223636
     INFO:smac.facade.smac_hpo_facade.SMAC4HPO:Optimizing a deterministic scenario for quality without a tuner timeout - will make SMAC deterministic and only evaluate one configuration per iteration!
     INFO:smac.initial_design.sobol_design.SobolDesign:Running initial design for 2 configurations
     INFO:smac.facade.smac_hpo_facade.SMAC4HPO:<class 'smac.facade.smac_hpo_facade.SMAC4HPO'>
@@ -38,8 +38,8 @@ scipy-style fmin interface
     INFO:smac.stats.stats.Stats:#Submitted target algorithm runs: 10 / 10.0
     INFO:smac.stats.stats.Stats:#Finished target algorithm runs: 10 / 10.0
     INFO:smac.stats.stats.Stats:#Configurations: 10
-    INFO:smac.stats.stats.Stats:Used wallclock time: 3.46 / inf sec 
-    INFO:smac.stats.stats.Stats:Used target algorithm runtime: 0.07 / inf sec
+    INFO:smac.stats.stats.Stats:Used wallclock time: 3.25 / inf sec 
+    INFO:smac.stats.stats.Stats:Used target algorithm runtime: 0.06 / inf sec
     INFO:smac.stats.stats.Stats:##########################################################
     INFO:smac.facade.smac_hpo_facade.SMAC4HPO:Final Incumbent: Configuration:
       x1, Value: -1.345753380449037
@@ -97,7 +97,7 @@ scipy-style fmin interface
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 0 minutes  3.467 seconds)
+   **Total running time of the script:** ( 0 minutes  3.264 seconds)
 
 
 .. _sphx_glr_download_examples_function_minimization_fmin_rosenbrock_example.py:

development/_sources/examples/function_minimization/sg_execution_times.rst.txt

@@ -5,8 +5,8 @@
 
 Computation times
 =================
-**00:03.467** total execution time for **examples_function_minimization** files:
+**00:03.264** total execution time for **examples_function_minimization** files:
 
 +------------------------------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_examples_function_minimization_fmin_rosenbrock_example.py` (``fmin_rosenbrock_example.py``) | 00:03.467 | 0.0 MB |
+| :ref:`sphx_glr_examples_function_minimization_fmin_rosenbrock_example.py` (``fmin_rosenbrock_example.py``) | 00:03.264 | 0.0 MB |
 +------------------------------------------------------------------------------------------------------------+-----------+--------+

development/_sources/examples/hyperband/hyperband_mlp_example.rst.txt

@@ -51,6 +51,8 @@ optimize the average accuracy on a 5-fold cross validation.
       warnings.warn("{} is intended to be used "
     /home/runner/work/SMAC3/SMAC3/doc/../smac/intensification/parallel_scheduling.py:152: UserWarning: Hyperband is intended to be used with more than 1 worker but num_workers=1
       warnings.warn("{} is intended to be used "
+    /home/runner/work/SMAC3/SMAC3/doc/../smac/intensification/parallel_scheduling.py:152: UserWarning: Hyperband is intended to be used with more than 1 worker but num_workers=1
+      warnings.warn("{} is intended to be used "
     Optimized Value: 0.0184
 
 
@@ -188,7 +190,7 @@ optimize the average accuracy on a 5-fold cross validation.
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 1 minutes  57.703 seconds)
+   **Total running time of the script:** ( 2 minutes  0.629 seconds)
 
 
 .. _sphx_glr_download_examples_hyperband_hyperband_mlp_example.py:

development/_sources/examples/hyperband/sg_execution_times.rst.txt

@@ -5,8 +5,8 @@
 
 Computation times
 =================
-**01:57.703** total execution time for **examples_hyperband** files:
+**02:00.629** total execution time for **examples_hyperband** files:
 
 +--------------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_examples_hyperband_hyperband_mlp_example.py` (``hyperband_mlp_example.py``) | 01:57.703 | 0.0 MB |
+| :ref:`sphx_glr_examples_hyperband_hyperband_mlp_example.py` (``hyperband_mlp_example.py``) | 02:00.629 | 0.0 MB |
 +--------------------------------------------------------------------------------------------+-----------+--------+

development/_sources/examples/parallel/fmin_rosenbrock_parallel_example.rst.txt

@@ -25,7 +25,7 @@ launch parallel configurations via n_workers
 
     /opt/hostedtoolcache/Python/3.8.11/x64/lib/python3.8/site-packages/scipy/stats/_qmc.py:1078: UserWarning: The balance properties of Sobol' points require n to be a power of 2.
       warnings.warn("The balance properties of Sobol' points require"
-    Best x: [-0.83874415  0.93576137]; with cost: 8.775898
+    Best x: [-0.7883424   0.83818718]; with cost: 7.894206
 
 
 
@@ -83,7 +83,7 @@ launch parallel configurations via n_workers
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 0 minutes  30.217 seconds)
+   **Total running time of the script:** ( 0 minutes  30.818 seconds)
 
 
 .. _sphx_glr_download_examples_parallel_fmin_rosenbrock_parallel_example.py:

development/_sources/examples/parallel/parallel_sh_mlp_example.rst.txt

@@ -129,7 +129,7 @@ advantage of the idle resources.
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 1 minutes  51.582 seconds)
+   **Total running time of the script:** ( 1 minutes  48.841 seconds)
 
 
 .. _sphx_glr_download_examples_parallel_parallel_sh_mlp_example.py:

development/_sources/examples/parallel/sg_execution_times.rst.txt

@@ -5,10 +5,10 @@
 
 Computation times
 =================
-**02:21.800** total execution time for **examples_parallel** files:
+**02:19.659** total execution time for **examples_parallel** files:
 
 +-----------------------------------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_examples_parallel_parallel_sh_mlp_example.py` (``parallel_sh_mlp_example.py``)                   | 01:51.582 | 0.0 MB |
+| :ref:`sphx_glr_examples_parallel_parallel_sh_mlp_example.py` (``parallel_sh_mlp_example.py``)                   | 01:48.841 | 0.0 MB |
 +-----------------------------------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_examples_parallel_fmin_rosenbrock_parallel_example.py` (``fmin_rosenbrock_parallel_example.py``) | 00:30.217 | 0.0 MB |
+| :ref:`sphx_glr_examples_parallel_fmin_rosenbrock_parallel_example.py` (``fmin_rosenbrock_parallel_example.py``) | 00:30.818 | 0.0 MB |
 +-----------------------------------------------------------------------------------------------------------------+-----------+--------+