From 6627a056e299b2d6221ae179d1b62c20fe4fec24 Mon Sep 17 00:00:00 2001 From: chico Date: Thu, 4 Feb 2021 21:57:39 +0100 Subject: [PATCH 1/5] Almost all files for evaluation --- .pre-commit-config.yaml | 12 + autosklearn/evaluation/__init__.py | 62 ++++-- autosklearn/evaluation/abstract_evaluator.py | 223 +++++++++++++------ autosklearn/evaluation/util.py | 11 +- 4 files changed, 223 insertions(+), 85 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af23e08eff..a8dc4d6dcd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,18 @@ repos: args: [--show-error-codes] name: mypy auto-sklearn-util files: autosklearn/util + - id: mypy + args: [--show-error-codes] + name: mypy auto-sklearn-evaluation-tae + files: autosklearn/evaluation/__init__.py + - id: mypy + args: [--show-error-codes] + name: mypy auto-sklearn-evaluation-abstract + files: autosklearn/evaluation/abstract_evaluator.py + - id: mypy + args: [--show-error-codes] + name: mypy auto-sklearn-evaluation-util + files: autosklearn/evaluation/util.py - repo: https://gitlab.com/pycqa/flake8 rev: 3.8.3 hooks: diff --git a/autosklearn/evaluation/__init__.py b/autosklearn/evaluation/__init__.py index ff28d2d096..19d21e5631 100644 --- a/autosklearn/evaluation/__init__.py +++ b/autosklearn/evaluation/__init__.py @@ -7,12 +7,13 @@ from queue import Empty import time import traceback -from typing import Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast from ConfigSpace import Configuration import numpy as np import pynisher from smac.runhistory.runhistory import RunInfo, RunValue +from smac.stats.stats import Stats from smac.tae import StatusType, TAEAbortException from smac.tae.execute_func import AbstractTAFunc @@ -23,11 +24,16 @@ import autosklearn.evaluation.train_evaluator import autosklearn.evaluation.test_evaluator import autosklearn.evaluation.util -from autosklearn.util.logging_ import get_named_client_logger +from autosklearn.util.backend import Backend +from autosklearn.util.logging_ import PickableLoggerAdapter, get_named_client_logger from autosklearn.util.parallel import preload_modules -def fit_predict_try_except_decorator(ta, queue, cost_for_crash, **kwargs): +def fit_predict_try_except_decorator( + ta: Callable, + queue: multiprocessing.Queue, + cost_for_crash: float, + **kwargs: Any) -> None: try: return ta(queue=queue, **kwargs) @@ -66,7 +72,7 @@ def fit_predict_try_except_decorator(ta, queue, cost_for_crash, **kwargs): queue.close() -def get_cost_of_crash(metric): +def get_cost_of_crash(metric: Scorer) -> int: # The metric must always be defined to extract optimum/worst if not isinstance(metric, Scorer): @@ -85,7 +91,8 @@ def get_cost_of_crash(metric): return worst_possible_result -def _encode_exit_status(exit_status): +def _encode_exit_status(exit_status: multiprocessing.connection.Connection + ) -> Union[multiprocessing.connection.Connection, str]: try: json.dumps(exit_status) return exit_status @@ -97,13 +104,31 @@ def _encode_exit_status(exit_status): # easier debugging of potential crashes class ExecuteTaFuncWithQueue(AbstractTAFunc): - def __init__(self, backend, autosklearn_seed, resampling_strategy, metric, - cost_for_crash, abort_on_first_run_crash, port, pynisher_context, - initial_num_run=1, stats=None, - run_obj='quality', par_factor=1, scoring_functions=None, - output_y_hat_optimization=True, include=None, exclude=None, - memory_limit=None, disable_file_output=False, init_params=None, - budget_type=None, ta=False, **resampling_strategy_args): + def __init__( + self, + backend: Backend, + autosklearn_seed: int, + resampling_strategy: Union[str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit], + metric: Scorer, + cost_for_crash: float, + abort_on_first_run_crash: bool, + port: int, + pynisher_context: str, + initial_num_run: int = 1, + stats: Optional[Stats] = None, + run_obj: str = 'quality', + par_factor: int = 1, + scoring_functions: Optional[List[Scorer]] = None, + output_y_hat_optimization: bool = True, + include: Optional[List[str]] = None, + exclude: Optional[List[str]] = None, + memory_limit: Optional[int] = None, + disable_file_output: bool = False, + init_params: Optional[Dict[str, Any]] = None, + budget_type: Optional[str] = None, + ta: Optional[Callable] = None, + **resampling_strategy_args: Any, + ): if resampling_strategy == 'holdout': eval_function = autosklearn.evaluation.train_evaluator.eval_holdout @@ -180,7 +205,7 @@ def __init__(self, backend, autosklearn_seed, resampling_strategy, metric, self.port = port self.pynisher_context = pynisher_context if self.port is None: - self.logger = logging.getLogger("TAE") + self.logger: Union[logging.Logger, PickableLoggerAdapter] = logging.getLogger("TAE") else: self.logger = get_named_client_logger( name="TAE", @@ -272,7 +297,7 @@ def run( init_params.update(self.init_params) if self.port is None: - logger = logging.getLogger("pynisher") + logger: Union[logging.Logger, PickableLoggerAdapter] = logging.getLogger("pynisher") else: logger = get_named_client_logger( name="pynisher", @@ -324,7 +349,12 @@ def run( 'traceback': exception_traceback, 'error': error_message } - return StatusType.CRASHED, self.cost_for_crash, 0.0, additional_info + return StatusType.CRASHED, self.worst_possible_result, 0.0, cast( + # Mypy requires casting to indicate that this Dict can have other + # types (specially for Dict, List, Tuple) + Dict[str, Union[int, float, str, Dict, List, Tuple]], + additional_info + ) if obj.exit_status in (pynisher.TimeoutException, pynisher.MemorylimitException): # Even if the pynisher thinks that a timeout or memout occured, @@ -359,7 +389,7 @@ def run( elif obj.exit_status is pynisher.MemorylimitException: status = StatusType.MEMOUT additional_run_info = { - 'error': 'Memout (used more than %d MB).' % self.memory_limit + "error': 'Memout (used more than {} MB).".format(self.memory_limit) } else: raise ValueError(obj.exit_status) diff --git a/autosklearn/evaluation/abstract_evaluator.py b/autosklearn/evaluation/abstract_evaluator.py index fb3cb49eb0..a14023dc71 100644 --- a/autosklearn/evaluation/abstract_evaluator.py +++ b/autosklearn/evaluation/abstract_evaluator.py @@ -1,12 +1,17 @@ -import typing import logging +import multiprocessing import time import warnings +from typing import Any, Dict, List, Optional, TextIO, Tuple, Type, Union import numpy as np + +from sklearn.base import BaseEstimator from sklearn.dummy import DummyClassifier, DummyRegressor from sklearn.ensemble import VotingClassifier, VotingRegressor +from smac.tae import StatusType + import autosklearn.pipeline.classification import autosklearn.pipeline.regression from autosklearn.constants import ( @@ -19,8 +24,9 @@ from autosklearn.pipeline.implementations.util import ( convert_multioutput_multiclass_to_multilabel ) -from autosklearn.metrics import calculate_loss, Scorer -from autosklearn.util.logging_ import get_named_client_logger +from autosklearn.metrics import Scorer, calculate_score +from autosklearn.util.backend import Backend +from autosklearn.util.logging_ import PicklableClientLogger, get_named_client_logger from ConfigSpace import Configuration @@ -31,79 +37,125 @@ class MyDummyClassifier(DummyClassifier): - def __init__(self, configuration, random_state, init_params=None): - self.configuration = configuration - if configuration == 1: + def __init__( + self, + config: Configuration, + random_state: np.random.RandomState, + init_params: Optional[Dict[str, Any]] = None, + dataset_properties: Dict[str, Any] = {}, + include: Optional[List[str]] = None, + exclude: Optional[List[str]] = None, + ): + self.config = config + if config == 1: super(MyDummyClassifier, self).__init__(strategy="uniform") else: super(MyDummyClassifier, self).__init__(strategy="most_frequent") self.random_state = random_state self.init_params = init_params - def pre_transform(self, X, y, fit_params=None): # pylint: disable=R0201 + def pre_transform( + self, + X: np.ndarray, + y: np.ndarray, + fit_params: Optional[Dict[str, Any]] = None + ) -> Tuple[np.ndarray, Dict[str, Any]]: # pylint: disable=R0201 if fit_params is None: fit_params = {} return X, fit_params - def fit(self, X, y, sample_weight=None): + def fit(self, X: np.ndarray, y: np.ndarray, + sample_weight: Optional[Union[np.ndarray, List]] = None + ) -> DummyClassifier: return super(MyDummyClassifier, self).fit(np.ones((X.shape[0], 1)), y, sample_weight=sample_weight) - def fit_estimator(self, X, y, fit_params=None): + def fit_estimator(self, X: np.ndarray, y: np.ndarray, + fit_params: Optional[Dict[str, Any]] = None) -> DummyClassifier: return self.fit(X, y) - def predict_proba(self, X, batch_size=1000): + def predict_proba(self, X: np.ndarray, batch_size: int = 1000 + ) -> np.ndarray: new_X = np.ones((X.shape[0], 1)) probas = super(MyDummyClassifier, self).predict_proba(new_X) probas = convert_multioutput_multiclass_to_multilabel(probas).astype( np.float32) return probas - def estimator_supports_iterative_fit(self): # pylint: disable=R0201 + def estimator_supports_iterative_fit(self) -> bool: # pylint: disable=R0201 return False - def get_additional_run_info(self): # pylint: disable=R0201 + def get_additional_run_info(self) -> Optional[ + Dict[str, Union[str, int, float, Dict, List, Tuple]]]: # pylint: disable=R0201 return None class MyDummyRegressor(DummyRegressor): - def __init__(self, configuration, random_state, init_params=None): - self.configuration = configuration - if configuration == 1: + def __init__( + self, + config: Configuration, + random_state: np.random.RandomState, + init_params: Optional[Dict[str, Any]] = None, + dataset_properties: Dict[str, Any] = {}, + include: Optional[List[str]] = None, + exclude: Optional[List[str]] = None, + ): + self.config = config + if config == 1: super(MyDummyRegressor, self).__init__(strategy='mean') else: super(MyDummyRegressor, self).__init__(strategy='median') self.random_state = random_state self.init_params = init_params - def pre_transform(self, X, y, fit_params=None): + def pre_transform( + self, + X: np.ndarray, + y: np.ndarray, + fit_params: Optional[Dict[str, Any]] = None + ) -> Tuple[np.ndarray, Dict[str, Any]]: # pylint: disable=R0201 if fit_params is None: fit_params = {} return X, fit_params - def fit(self, X, y, sample_weight=None): + def fit(self, X: np.ndarray, y: np.ndarray, + sample_weight: Optional[Union[np.ndarray, List]] = None + ) -> DummyRegressor: return super(MyDummyRegressor, self).fit(np.ones((X.shape[0], 1)), y, sample_weight=sample_weight) - def fit_estimator(self, X, y, fit_params=None): + def fit_estimator(self, X: np.ndarray, y: np.ndarray, + fit_params: Optional[Dict[str, Any]] = None) -> DummyRegressor: return self.fit(X, y) - def predict(self, X, batch_size=1000): + def predict(self, X: np.ndarray, batch_size: int = 1000) -> np.ndarray: new_X = np.ones((X.shape[0], 1)) return super(MyDummyRegressor, self).predict(new_X).astype(np.float32) - def estimator_supports_iterative_fit(self): # pylint: disable=R0201 + def estimator_supports_iterative_fit(self) -> bool: # pylint: disable=R0201 return False - def get_additional_run_info(self): # pylint: disable=R0201 + def get_additional_run_info(self) -> Optional[ + Dict[str, Union[str, int, float, Dict, List, Tuple]]]: # pylint: disable=R0201 return None -def _fit_and_suppress_warnings(logger, model, X, y): - def send_warnings_to_log(message, category, filename, lineno, - file=None, line=None): - logger.debug('%s:%s: %s:%s', - filename, lineno, category.__name__, message) +def _fit_and_suppress_warnings( + logger: Union[logging.Logger, PicklableClientLogger], + model: BaseEstimator, + X: np.ndarray, + y: np.ndarray +) -> BaseEstimator: + def send_warnings_to_log( + message: Union[Warning, str], + category: Type[Warning], + filename: str, + lineno: int, + file: Optional[TextIO] = None, + line: Optional[str] = None, + ) -> None: + logger.debug('%s:%s: %s:%s' % + (filename, lineno, str(category), message)) return with warnings.catch_warnings(): @@ -114,19 +166,24 @@ def send_warnings_to_log(message, category, filename, lineno, class AbstractEvaluator(object): - def __init__(self, backend, queue, metric, - port: typing.Optional[int], - configuration=None, - scoring_functions=None, - seed=1, - output_y_hat_optimization=True, - num_run=None, - include=None, - exclude=None, - disable_file_output=False, - init_params=None, - budget=None, - budget_type=None): + def __init__( + self, + backend: Backend, + queue: multiprocessing.Queue, + metric: Scorer, + port: Optional[int], + configuration: Optional[Union[int, Configuration]] = None, + scoring_functions: Optional[List[Scorer]] = None, + seed: int = 1, + output_y_hat_optimization: bool = True, + num_run: Optional[int] = None, + include: Optional[List[str]] = None, + exclude: Optional[List[str]] = None, + disable_file_output: bool = False, + init_params: Optional[Dict[str, Any]] = None, + budget: Optional[float] = None, + budget_type: Optional[str] = None, + ): self.starttime = time.time() @@ -209,9 +266,9 @@ def __init__(self, backend, queue, metric, self.budget = budget self.budget_type = budget_type - def _get_model(self): + def _get_model(self) -> BaseEstimator: if not isinstance(self.configuration, Configuration): - model = self.model_class(configuration=self.configuration, + model = self.model_class(config=self.configuration, random_state=self.seed, init_params=self._init_params) else: @@ -264,8 +321,20 @@ def _loss(self, y_true: np.ndarray, y_hat: np.ndarray, y_true, y_hat, self.task_type, self.metric, scoring_functions=scoring_functions) - def finish_up(self, loss, train_loss, opt_pred, valid_pred, test_pred, - additional_run_info, file_output, final_call, status): + + def finish_up( + self, + loss: Union[Dict[str, float], float], + train_loss: Optional[float], + opt_pred: np.ndarray, + valid_pred: np.ndarray, + test_pred: np.ndarray, + additional_run_info: Optional[Dict[str, Union[str, int, float, Dict, List, Tuple]]], + file_output: bool, + final_call: bool, + status: StatusType, + ) -> Tuple[float, Union[float, Dict[str, float]], int, + Dict[str, Union[str, int, float, Dict, List, Tuple]]]: """This function does everything necessary after the fitting is done: * predicting @@ -277,19 +346,19 @@ def finish_up(self, loss, train_loss, opt_pred, valid_pred, test_pred, self.duration = time.time() - self.starttime if file_output: - loss_, additional_run_info_ = self.file_output( + file_out_loss, additional_run_info_ = self.file_output( opt_pred, valid_pred, test_pred, ) else: - loss_ = None + file_out_loss = None additional_run_info_ = {} validation_loss, test_loss = self.calculate_auxiliary_losses( valid_pred, test_pred, ) - if loss_ is not None: - return self.duration, loss_, self.seed, additional_run_info_ + if file_out_loss is not None: + return self.duration, file_out_loss, self.seed, additional_run_info_ if isinstance(loss, dict): loss_ = loss @@ -318,15 +387,17 @@ def finish_up(self, loss, train_loss, opt_pred, valid_pred, test_pred, rval_dict['final_queue_element'] = True self.queue.put(rval_dict) + return self.duration, loss_, self.seed, additional_run_info_ def calculate_auxiliary_losses( self, - Y_valid_pred, - Y_test_pred - ): + Y_valid_pred: np.ndarray, + Y_test_pred: np.ndarray, + ) -> Tuple[Optional[float], Optional[float]]: if Y_valid_pred is not None: if self.y_valid is not None: - validation_loss = self._loss(self.y_valid, Y_valid_pred) + validation_loss: Optional[Union[float, Dict[str, float]]] = self._loss( + self.y_valid, Y_valid_pred) if isinstance(validation_loss, dict): validation_loss = validation_loss[self.metric.name] else: @@ -336,7 +407,8 @@ def calculate_auxiliary_losses( if Y_test_pred is not None: if self.y_test is not None: - test_loss = self._loss(self.y_test, Y_test_pred) + test_loss: Optional[Union[float, Dict[str, float]]] = self._loss( + self.y_test, Y_test_pred) if isinstance(test_loss, dict): test_loss = test_loss[self.metric.name] else: @@ -347,11 +419,11 @@ def calculate_auxiliary_losses( return validation_loss, test_loss def file_output( - self, - Y_optimization_pred, - Y_valid_pred, - Y_test_pred - ): + self, + Y_optimization_pred: np.ndarray, + Y_valid_pred: np.ndarray, + Y_test_pred: np.ndarray, + ) -> Tuple[Optional[float], Dict[str, Union[str, int, float, List, Dict, Tuple]]]: # Abort if self.Y_optimization is None # self.Y_optimization can be None if we use partial-cv, then, # obviously no output should be saved. @@ -434,25 +506,43 @@ def file_output( return None, {} - def _predict_proba(self, X, model, task_type, Y_train): - def send_warnings_to_log(message, category, filename, lineno, - file=None, line=None): + def _predict_proba(self, X: np.ndarray, model: BaseEstimator, + task_type: int, Y_train: Optional[np.ndarray] = None, + ) -> np.ndarray: + def send_warnings_to_log( + message: Union[Warning, str], + category: Type[Warning], + filename: str, + lineno: int, + file: Optional[TextIO] = None, + line: Optional[str] = None, + ) -> None: self.logger.debug('%s:%s: %s:%s' % - (filename, lineno, category.__name__, message)) + (filename, lineno, str(category), message)) return with warnings.catch_warnings(): warnings.showwarning = send_warnings_to_log Y_pred = model.predict_proba(X, batch_size=1000) + if Y_train is None: + raise ValueError("Y_train is required for classification problems") + Y_pred = self._ensure_prediction_array_sizes(Y_pred, Y_train) return Y_pred - def _predict_regression(self, X, model, task_type, Y_train=None): - def send_warnings_to_log(message, category, filename, lineno, - file=None, line=None): + def _predict_regression(self, X: np.ndarray, model: BaseEstimator, + task_type: int, Y_train: Optional[np.ndarray] = None) -> np.ndarray: + def send_warnings_to_log( + message: Union[Warning, str], + category: Type[Warning], + filename: str, + lineno: int, + file: Optional[TextIO] = None, + line: Optional[str] = None, + ) -> None: self.logger.debug('%s:%s: %s:%s' % - (filename, lineno, category.__name__, message)) + (filename, lineno, str(category), message)) return with warnings.catch_warnings(): @@ -464,7 +554,8 @@ def send_warnings_to_log(message, category, filename, lineno, return Y_pred - def _ensure_prediction_array_sizes(self, prediction, Y_train): + def _ensure_prediction_array_sizes(self, prediction: np.ndarray, Y_train: np.ndarray + ) -> np.ndarray: num_classes = self.datamanager.info['label_num'] if self.task_type == MULTICLASS_CLASSIFICATION and \ diff --git a/autosklearn/evaluation/util.py b/autosklearn/evaluation/util.py index 6df8e8439d..7c9b3078e5 100644 --- a/autosklearn/evaluation/util.py +++ b/autosklearn/evaluation/util.py @@ -1,3 +1,6 @@ +from typing import Any, Dict, List, Optional, Tuple, Union + +import multiprocessing import queue @@ -6,7 +9,8 @@ ] -def read_queue(queue_): +def read_queue(queue_: multiprocessing.Queue + ) -> List[Union[str, int, float, List, Dict, Tuple]]: stack = [] while True: try: @@ -32,7 +36,7 @@ def read_queue(queue_): return stack -def empty_queue(queue_): +def empty_queue(queue_: multiprocessing.Queue) -> None: while True: try: queue_.get(block=False) @@ -42,7 +46,8 @@ def empty_queue(queue_): queue_.close() -def extract_learning_curve(stack, key=None): +def extract_learning_curve(stack: List[Dict[str, Any]], + key: Optional[str] = None) -> List[Union[float, int]]: learning_curve = [] for entry in stack: if key: From eba4c0a7c5c598103b3a2414f6881f3aa69203ba Mon Sep 17 00:00:00 2001 From: chico Date: Fri, 12 Feb 2021 12:41:18 +0100 Subject: [PATCH 2/5] Feedback from PR --- .pre-commit-config.yaml | 12 +- autosklearn/evaluation/__init__.py | 27 +- autosklearn/evaluation/abstract_evaluator.py | 43 +- autosklearn/evaluation/test_evaluator.py | 59 ++- autosklearn/evaluation/train_evaluator.py | 476 +++++++++++-------- autosklearn/evaluation/util.py | 4 +- test/test_evaluation/test_train_evaluator.py | 4 +- 7 files changed, 357 insertions(+), 268 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a8dc4d6dcd..77e6c5c512 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,16 +20,8 @@ repos: files: autosklearn/util - id: mypy args: [--show-error-codes] - name: mypy auto-sklearn-evaluation-tae - files: autosklearn/evaluation/__init__.py - - id: mypy - args: [--show-error-codes] - name: mypy auto-sklearn-evaluation-abstract - files: autosklearn/evaluation/abstract_evaluator.py - - id: mypy - args: [--show-error-codes] - name: mypy auto-sklearn-evaluation-util - files: autosklearn/evaluation/util.py + name: mypy auto-sklearn-evaluation + files: autosklearn/evaluation - repo: https://gitlab.com/pycqa/flake8 rev: 3.8.3 hooks: diff --git a/autosklearn/evaluation/__init__.py b/autosklearn/evaluation/__init__.py index 19d21e5631..6fc0a780ce 100644 --- a/autosklearn/evaluation/__init__.py +++ b/autosklearn/evaluation/__init__.py @@ -7,7 +7,7 @@ from queue import Empty import time import traceback -from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast +from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union, cast from ConfigSpace import Configuration import numpy as np @@ -72,7 +72,7 @@ def fit_predict_try_except_decorator( queue.close() -def get_cost_of_crash(metric: Scorer) -> int: +def get_cost_of_crash(metric: Scorer) -> float: # The metric must always be defined to extract optimum/worst if not isinstance(metric, Scorer): @@ -91,9 +91,11 @@ def get_cost_of_crash(metric: Scorer) -> int: return worst_possible_result -def _encode_exit_status(exit_status: multiprocessing.connection.Connection - ) -> Union[multiprocessing.connection.Connection, str]: +def _encode_exit_status(exit_status: Union[str, int, Type[BaseException]] + ) -> Union[str, int]: try: + # If it can be dumped, then it is int + exit_status = cast(int, exit_status) json.dumps(exit_status) return exit_status except (TypeError, OverflowError): @@ -286,6 +288,10 @@ def run( instance_specific: Optional[str] = None, ) -> Tuple[StatusType, float, float, Dict[str, Union[int, float, str, Dict, List, Tuple]]]: + # Additional information of each of the tae executions + # Defined upfront for mypy + additional_run_info: Dict[str, Union[int, float, str, Dict, List, Tuple]] = {} + context = multiprocessing.get_context(self.pynisher_context) preload_modules(context) queue = context.Queue() @@ -345,16 +351,11 @@ def run( except Exception as e: exception_traceback = traceback.format_exc() error_message = repr(e) - additional_info = { + additional_run_info.update({ 'traceback': exception_traceback, 'error': error_message - } - return StatusType.CRASHED, self.worst_possible_result, 0.0, cast( - # Mypy requires casting to indicate that this Dict can have other - # types (specially for Dict, List, Tuple) - Dict[str, Union[int, float, str, Dict, List, Tuple]], - additional_info - ) + }) + return StatusType.CRASHED, self.worst_possible_result, 0.0, additional_run_info if obj.exit_status in (pynisher.TimeoutException, pynisher.MemorylimitException): # Even if the pynisher thinks that a timeout or memout occured, @@ -389,7 +390,7 @@ def run( elif obj.exit_status is pynisher.MemorylimitException: status = StatusType.MEMOUT additional_run_info = { - "error': 'Memout (used more than {} MB).".format(self.memory_limit) + "error": "Memout (used more than {} MB).".format(self.memory_limit) } else: raise ValueError(obj.exit_status) diff --git a/autosklearn/evaluation/abstract_evaluator.py b/autosklearn/evaluation/abstract_evaluator.py index a14023dc71..8d9e00c4e6 100644 --- a/autosklearn/evaluation/abstract_evaluator.py +++ b/autosklearn/evaluation/abstract_evaluator.py @@ -2,7 +2,7 @@ import multiprocessing import time import warnings -from typing import Any, Dict, List, Optional, TextIO, Tuple, Type, Union +from typing import Any, Dict, List, Optional, TextIO, Tuple, Type, Union, cast import numpy as np @@ -179,7 +179,7 @@ def __init__( num_run: Optional[int] = None, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, - disable_file_output: bool = False, + disable_file_output: Union[bool, List[str]] = False, init_params: Optional[Dict[str, Any]] = None, budget: Optional[float] = None, budget_type: Optional[str] = None, @@ -209,7 +209,7 @@ def __init__( self.scoring_functions = scoring_functions if isinstance(disable_file_output, (bool, list)): - self.disable_file_output = disable_file_output + self.disable_file_output: Union[bool, List[str]] = disable_file_output else: raise ValueError('disable_file_output should be either a bool or a list') @@ -260,12 +260,15 @@ def __init__( port=self.port, ) - self.Y_optimization = None + self.Y_optimization: Optional[Union[List, np.ndarray]] = None self.Y_actual_train = None self.budget = budget self.budget_type = budget_type + # Please mypy to prevent not defined attr + self.model = self._get_model() + def _get_model(self) -> BaseEstimator: if not isinstance(self.configuration, Configuration): model = self.model_class(config=self.configuration, @@ -325,7 +328,7 @@ def _loss(self, y_true: np.ndarray, y_hat: np.ndarray, def finish_up( self, loss: Union[Dict[str, float], float], - train_loss: Optional[float], + train_loss: Optional[Union[float, Dict[str, float]]], opt_pred: np.ndarray, valid_pred: np.ndarray, test_pred: np.ndarray, @@ -431,14 +434,14 @@ def file_output( return None, {} # Abort in case of shape misalignment - if self.Y_optimization.shape[0] != Y_optimization_pred.shape[0]: + if np.shape(self.Y_optimization)[0] != Y_optimization_pred.shape[0]: return ( 1.0, { 'error': "Targets %s and prediction %s don't have " "the same length. Probably training didn't " - "finish" % (self.Y_optimization.shape, Y_optimization_pred.shape) + "finish" % (np.shape(self.Y_optimization), Y_optimization_pred.shape) }, ) @@ -469,23 +472,25 @@ def file_output( if self.disable_file_output is False: self.disable_file_output = [] + # Here onwards, the self.disable_file_output can be treated as a list + self.disable_file_output = cast(List, self.disable_file_output) + # This file can be written independently of the others down bellow if ('y_optimization' not in self.disable_file_output): if self.output_y_hat_optimization: self.backend.save_targets_ensemble(self.Y_optimization) - if hasattr(self, 'models') and len(self.models) > 0 and self.models[0] is not None: - if ('models' not in self.disable_file_output): - - if self.task_type in CLASSIFICATION_TASKS: - models = VotingClassifier(estimators=None, voting='soft', ) - else: - models = VotingRegressor(estimators=None) - models.estimators_ = self.models - else: - models = None - else: - models = None + models: Optional[BaseEstimator] = None + if hasattr(self, 'models'): + if len(self.models) > 0 and self.models[0] is not None: # type: ignore[attr-defined] + if ('models' not in self.disable_file_output): + + if self.task_type in CLASSIFICATION_TASKS: + models = VotingClassifier(estimators=None, voting='soft', ) + else: + models = VotingRegressor(estimators=None) + # Mypy cannot understand hasattr yet + models.estimators_ = self.models # type: ignore[attr-defined] self.backend.save_numrun_to_dir( seed=self.seed, diff --git a/autosklearn/evaluation/test_evaluator.py b/autosklearn/evaluation/test_evaluator.py index b4c685b4f8..4e19073e90 100644 --- a/autosklearn/evaluation/test_evaluator.py +++ b/autosklearn/evaluation/test_evaluator.py @@ -1,4 +1,11 @@ # -*- encoding: utf-8 -*- +import multiprocessing +from typing import Any, Dict, List, Optional, Tuple, Union + +from ConfigSpace import Configuration + +import numpy as np + from smac.tae import StatusType from autosklearn.evaluation.abstract_evaluator import ( @@ -6,6 +13,7 @@ _fit_and_suppress_warnings, ) from autosklearn.metrics import calculate_loss +from autosklearn.util.backend import Backend __all__ = [ @@ -16,15 +24,20 @@ class TestEvaluator(AbstractEvaluator): - def __init__(self, backend, queue, metric, - port, - configuration=None, - scoring_functions=None, - seed=1, - include=None, - exclude=None, - disable_file_output=False, - init_params=None): + def __init__( + self, + backend: Backend, + queue: multiprocessing.Queue, + metric: Scorer, + port: Optional[int], + configuration: Optional[Union[int, Configuration]] = None, + scoring_functions: Optional[List[Scorer]] = None, + seed: int = 1, + include: Optional[List[str]] = None, + exclude: Optional[List[str]] = None, + disable_file_output: bool = False, + init_params: Optional[Dict[str, Any]] = None, + ): super(TestEvaluator, self).__init__( backend=backend, queue=queue, @@ -50,7 +63,7 @@ def __init__(self, backend, queue, metric, self.model = self._get_model() - def fit_predict_and_loss(self): + def fit_predict_and_loss(self) -> None: _fit_and_suppress_warnings(self.logger, self.model, self.X_train, self.Y_train) loss, Y_pred, _, _ = self.predict_and_loss() self.finish_up( @@ -65,7 +78,9 @@ def fit_predict_and_loss(self): status=StatusType.SUCCESS, ) - def predict_and_loss(self, train=False): + def predict_and_loss( + self, train: bool = False + ) -> Tuple[Union[Dict[str, float], float], np.array, Any, Any]: if train: Y_pred = self.predict_function(self.X_train, self.model, @@ -91,10 +106,24 @@ def predict_and_loss(self, train=False): # create closure for evaluating an algorithm # Has a stupid name so pytest doesn't regard it as a test -def eval_t(queue, config, backend, metric, seed, num_run, instance, - scoring_functions, output_y_hat_optimization, include, - exclude, disable_file_output, port, init_params=None, budget_type=None, - budget=None): +def eval_t( + queue: multiprocessing.Queue, + config: Union[int, Configuration], + backend: Backend, + metric: Scorer, + seed: int, + num_run: int, + instance: Dict[str, Any], + scoring_functions: Optional[List[Scorer]], + output_y_hat_optimization: bool, + include: Optional[List[str]], + exclude: Optional[List[str]], + disable_file_output: bool, + port: Optional[int], + init_params: Optional[Dict[str, Any]] = None, + budget: Optional[float] = None, + budget_type: Optional[str] = None, +) -> None: evaluator = TestEvaluator(configuration=config, backend=backend, metric=metric, seed=seed, port=port, diff --git a/autosklearn/evaluation/train_evaluator.py b/autosklearn/evaluation/train_evaluator.py index 8657ca4062..2ddaf7a9e4 100644 --- a/autosklearn/evaluation/train_evaluator.py +++ b/autosklearn/evaluation/train_evaluator.py @@ -1,8 +1,16 @@ +import logging +import multiprocessing +from typing import Any, Dict, List, Optional, Tuple, Union, cast + import copy import json +from ConfigSpace import Configuration + import numpy as np from smac.tae import TAEAbortException, StatusType + +from sklearn.base import BaseEstimator from sklearn.model_selection import ShuffleSplit, StratifiedShuffleSplit, KFold, \ StratifiedKFold, train_test_split, BaseCrossValidator, PredefinedSplit from sklearn.model_selection._split import _RepeatedSplits, BaseShuffleSplit @@ -11,50 +19,56 @@ AbstractEvaluator, _fit_and_suppress_warnings, ) +from autosklearn.data.abstract_data_manager import AbstractDataManager from autosklearn.constants import ( CLASSIFICATION_TASKS, MULTILABEL_CLASSIFICATION, REGRESSION_TASKS, MULTIOUTPUT_REGRESSION ) +from autosklearn.pipeline.components.base import IterativeComponent +from autosklearn.metrics import Scorer +from autosklearn.util.backend import Backend +from autosklearn.util.logging_ import PicklableClientLogger __all__ = ['TrainEvaluator', 'eval_holdout', 'eval_iterative_holdout', 'eval_cv', 'eval_partial_cv', 'eval_partial_cv_iterative'] -__baseCrossValidator_defaults__ = {'GroupKFold': {'n_splits': 3}, - 'KFold': {'n_splits': 3, - 'shuffle': False, - 'random_state': None}, - 'LeaveOneGroupOut': {}, - 'LeavePGroupsOut': {'n_groups': 2}, - 'LeaveOneOut': {}, - 'LeavePOut': {'p': 2}, - 'PredefinedSplit': {}, - 'RepeatedKFold': {'n_splits': 5, - 'n_repeats': 10, - 'random_state': None}, - 'RepeatedStratifiedKFold': {'n_splits': 5, - 'n_repeats': 10, - 'random_state': None}, - 'StratifiedKFold': {'n_splits': 3, - 'shuffle': False, - 'random_state': None}, - 'TimeSeriesSplit': {'n_splits': 3, - 'max_train_size': None}, - 'GroupShuffleSplit': {'n_splits': 5, - 'test_size': None, - 'random_state': None}, - 'StratifiedShuffleSplit': {'n_splits': 10, - 'test_size': None, - 'random_state': None}, - 'ShuffleSplit': {'n_splits': 10, - 'test_size': None, - 'random_state': None} - } - - -def _get_y_array(y, task_type): +baseCrossValidator_defaults: Dict[str, Dict[str, Optional[Union[int, float, str]]]] = { + 'GroupKFold': {'n_splits': 3}, + 'KFold': {'n_splits': 3, + 'shuffle': False, + 'random_state': None}, + 'LeaveOneGroupOut': {}, + 'LeavePGroupsOut': {'n_groups': 2}, + 'LeaveOneOut': {}, + 'LeavePOut': {'p': 2}, + 'PredefinedSplit': {}, + 'RepeatedKFold': {'n_splits': 5, + 'n_repeats': 10, + 'random_state': None}, + 'RepeatedStratifiedKFold': {'n_splits': 5, + 'n_repeats': 10, + 'random_state': None}, + 'StratifiedKFold': {'n_splits': 3, + 'shuffle': False, + 'random_state': None}, + 'TimeSeriesSplit': {'n_splits': 3, + 'max_train_size': None}, + 'GroupShuffleSplit': {'n_splits': 5, + 'test_size': None, + 'random_state': None}, + 'StratifiedShuffleSplit': {'n_splits': 10, + 'test_size': None, + 'random_state': None}, + 'ShuffleSplit': {'n_splits': 10, + 'test_size': None, + 'random_state': None} + } + + +def _get_y_array(y: np.ndarray, task_type: int) -> np.ndarray: if task_type in CLASSIFICATION_TASKS and task_type != \ MULTILABEL_CLASSIFICATION: return y.ravel() @@ -62,7 +76,12 @@ def _get_y_array(y, task_type): return y -def subsample_indices(train_indices, subsample, task_type, Y_train): +def subsample_indices( + train_indices: List[int], + subsample: Optional[float], + task_type: int, + Y_train: np.ndarray +) -> List[int]: if not isinstance(subsample, float): raise ValueError( @@ -98,8 +117,16 @@ def subsample_indices(train_indices, subsample, task_type, Y_train): return train_indices -def _fit_with_budget(X_train, Y_train, budget, budget_type, logger, model, train_indices, - task_type): +def _fit_with_budget( + X_train: np.ndarray, + Y_train: np.ndarray, + budget: float, + budget_type: Optional[str], + logger: Union[logging.Logger, PicklableClientLogger], + model: BaseEstimator, + train_indices: List[int], + task_type: int, +) -> None: if ( budget_type == 'iterations' or budget_type == 'mixed' and model.estimator_supports_iterative_fit() @@ -141,22 +168,29 @@ def _fit_with_budget(X_train, Y_train, budget, budget_type, logger, model, train class TrainEvaluator(AbstractEvaluator): - def __init__(self, backend, queue, metric, - port, - configuration=None, - scoring_functions=None, - seed=1, - output_y_hat_optimization=True, - resampling_strategy=None, - resampling_strategy_args=None, - num_run=None, - budget=None, - budget_type=None, - keep_models=False, - include=None, - exclude=None, - disable_file_output=False, - init_params=None,): + def __init__( + self, + backend: Backend, + queue: multiprocessing.Queue, + metric: Scorer, + port: Optional[int], + configuration: Optional[Union[int, Configuration]] = None, + scoring_functions: Optional[List[Scorer]] = None, + seed: int = 1, + output_y_hat_optimization: bool = True, + resampling_strategy: Optional[Union[str, BaseCrossValidator, + _RepeatedSplits, BaseShuffleSplit]] = None, + resampling_strategy_args: Optional[Dict[str, Optional[Union[float, int, str]]]] = None, + num_run: Optional[int] = None, + budget: Optional[float] = None, + budget_type: Optional[str] = None, + keep_models: bool = False, + include: Optional[List[str]] = None, + exclude: Optional[List[str]] = None, + disable_file_output: bool = False, + init_params: Optional[Dict[str, Any]] = None, + ): + super().__init__( backend=backend, queue=queue, @@ -186,11 +220,11 @@ def __init__(self, backend, queue, metric, ) self.X_train = self.datamanager.data['X_train'] self.Y_train = self.datamanager.data['Y_train'] - self.Y_optimization = None + self.Y_optimization: Optional[Union[List, np.ndarray]] = None self.Y_targets = [None] * self.num_cv_folds self.Y_train_targets = np.ones(self.Y_train.shape) * np.NaN self.models = [None] * self.num_cv_folds - self.indices = [None] * self.num_cv_folds + self.indices: List[Optional[Tuple[List[int], List[int]]]] = [None] * self.num_cv_folds # Necessary for full CV. Makes full CV not write predictions if only # a subset of folds is evaluated but time is up. Complicated, because @@ -199,10 +233,14 @@ def __init__(self, backend, queue, metric, self.partial = True self.keep_models = keep_models - def fit_predict_and_loss(self, iterative=False): + def fit_predict_and_loss(self, iterative: bool = False) -> None: """Fit, predict and compute the loss for cross-validation and holdout (both iterative and non-iterative)""" + # Define beforehand for mypy + additional_run_info: Optional[ + Dict[str, Union[str, int, float, Dict, List, Tuple]]] = None + if iterative: if self.num_cv_folds == 1: @@ -232,17 +270,23 @@ def fit_predict_and_loss(self, iterative=False): Y_optimization_pred = [None] * self.num_cv_folds Y_valid_pred = [None] * self.num_cv_folds Y_test_pred = [None] * self.num_cv_folds - additional_run_info = None train_splits = [None] * self.num_cv_folds self.models = [self._get_model() for i in range(self.num_cv_folds)] iterations = [1] * self.num_cv_folds total_n_iterations = [0] * self.num_cv_folds - model_max_iter = [model.get_max_iter() for model in self.models] + # model.estimator_supports_iterative_fit -> true + # After the if above, we now estimator support iterative fit + model_max_iter = [cast(IterativeComponent, model).get_max_iter() + for model in self.models] + + if self.budget_type in ['iterations', 'mixed'] and self.budget is None: + raise ValueError(f"When budget type is {self.budget_type} the budget " + "can not be None") - if self.budget_type in ['iterations', 'mixed'] and self.budget > 0: + if self.budget_type in ['iterations', 'mixed'] and cast(float, self.budget) > 0: max_n_iter_budget = int( - np.ceil(self.budget / 100 * model_max_iter[0])) + np.ceil(cast(float, self.budget) / 100 * model_max_iter[0])) max_iter = min(model_max_iter[0], max_n_iter_budget) else: max_iter = model_max_iter[0] @@ -250,7 +294,7 @@ def fit_predict_and_loss(self, iterative=False): models_current_iters = [0] * self.num_cv_folds Xt_array = [None] * self.num_cv_folds - fit_params_array = [{}] * self.num_cv_folds + fit_params_array = [{}] * self.num_cv_folds # type: List[Dict[str, Any]] y = _get_y_array(self.Y_train, self.task_type) @@ -432,7 +476,6 @@ def fit_predict_and_loss(self, iterative=False): Y_optimization_pred = [None] * self.num_cv_folds Y_valid_pred = [None] * self.num_cv_folds Y_test_pred = [None] * self.num_cv_folds - additional_run_info = None train_splits = [None] * self.num_cv_folds y = _get_y_array(self.Y_train, self.task_type) @@ -560,20 +603,16 @@ def fit_predict_and_loss(self, iterative=False): for i in range(self.num_cv_folds) if Y_valid_pred[i] is not None]) # Average the predictions of several models - if len(Y_valid_pred.shape) == 3: + if len(np.shape(Y_valid_pred)) == 3: Y_valid_pred = np.nanmean(Y_valid_pred, axis=0) - else: - Y_valid_pred = None if self.X_test is not None: Y_test_pred = np.array([Y_test_pred[i] for i in range(self.num_cv_folds) if Y_test_pred[i] is not None]) # Average the predictions of several models - if len(Y_test_pred.shape) == 3: + if len(np.shape(Y_test_pred)) == 3: Y_test_pred = np.nanmean(Y_test_pred, axis=0) - else: - Y_test_pred = None self.Y_optimization = Y_targets self.Y_actual_train = Y_train_targets @@ -591,7 +630,8 @@ def fit_predict_and_loss(self, iterative=False): and self.model.estimator_supports_iterative_fit() ): budget_factor = self.model.get_max_iter() - n_iter = int(np.ceil(self.budget / 100 * budget_factor)) + # We check for budget being None in initialization + n_iter = int(np.ceil(cast(float, self.budget) / 100 * budget_factor)) model_current_iter = self.model.get_current_iter() if model_current_iter < n_iter: status = StatusType.DONOTADVANCE @@ -612,15 +652,15 @@ def fit_predict_and_loss(self, iterative=False): loss=opt_loss, train_loss=train_loss, opt_pred=Y_optimization_pred, - valid_pred=Y_valid_pred, - test_pred=Y_test_pred, + valid_pred=Y_valid_pred if self.X_valid is not None else None, + test_pred=Y_test_pred if self.X_test is not None else None, additional_run_info=additional_run_info, file_output=True, final_call=True, status=status, ) - def partial_fit_predict_and_loss(self, fold, iterative=False): + def partial_fit_predict_and_loss(self, fold: int, iterative: bool = False) -> None: """Fit, predict and compute the loss for eval_partial_cv (both iterative and normal)""" if fold > self.num_cv_folds: @@ -683,8 +723,9 @@ def partial_fit_predict_and_loss(self, fold, iterative=False): status=status ) - def _partial_fit_and_predict_iterative(self, fold, train_indices, test_indices, - add_model_to_self): + def _partial_fit_and_predict_iterative(self, fold: int, train_indices: List[int], + test_indices: List[int], + add_model_to_self: bool) -> None: model = self._get_model() self.indices[fold] = ((train_indices, test_indices)) @@ -704,7 +745,7 @@ def _partial_fit_and_predict_iterative(self, fold, train_indices, test_indices, total_n_iteration = 0 model_max_iter = model.get_max_iter() - if self.budget > 0: + if self.budget is not None and self.budget > 0: max_n_iter_budget = int(np.ceil(self.budget / 100 * model_max_iter)) max_iter = min(model_max_iter, max_n_iter_budget) else: @@ -795,8 +836,13 @@ def _partial_fit_and_predict_iterative(self, fold, train_indices, test_indices, ) return - def _partial_fit_and_predict_standard(self, fold, train_indices, test_indices, - add_model_to_self=False): + def _partial_fit_and_predict_standard( + self, + fold: int, train_indices: List[int], + test_indices: List[int], + add_model_to_self: bool = False + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, + Dict[str, Union[str, int, float, Dict, List, Tuple]]]: model = self._get_model() self.indices[fold] = ((train_indices, test_indices)) @@ -813,7 +859,7 @@ def _partial_fit_and_predict_standard(self, fold, train_indices, test_indices, else: self.models[fold] = model - train_indices, test_indices = self.indices[fold] + train_indices, test_indices = ((train_indices, test_indices)) self.Y_targets[fold] = self.Y_train[test_indices] self.Y_train_targets[train_indices] = self.Y_train[train_indices] @@ -831,8 +877,17 @@ def _partial_fit_and_predict_standard(self, fold, train_indices, test_indices, additional_run_info, ) - def _partial_fit_and_predict_budget(self, fold, train_indices, test_indices, - add_model_to_self=False): + def _partial_fit_and_predict_budget( + self, + fold: int, train_indices: List[int], + test_indices: List[int], + add_model_to_self: bool = False, + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, + Dict[str, Union[str, int, float, Dict, List, Tuple]]]: + + # This function is only called in the event budget is not None + # Add this statement for mypy + assert self.budget is not None model = self._get_model() self.indices[fold] = ((train_indices, test_indices)) @@ -870,7 +925,9 @@ def _partial_fit_and_predict_budget(self, fold, train_indices, test_indices, additional_run_info, ) - def _predict(self, model, test_indices, train_indices): + def _predict(self, model: BaseEstimator, test_indices: List[int], + train_indices: List[int]) -> Tuple[np.ndarray, np.ndarray, + np.ndarray, np.ndarray]: train_pred = self.predict_function(self.X_train[train_indices], model, self.task_type, self.Y_train[train_indices]) @@ -897,21 +954,22 @@ def _predict(self, model, test_indices, train_indices): return train_pred, opt_pred, valid_pred, test_pred - def get_splitter(self, D): + def get_splitter(self, D: AbstractDataManager) -> Union[BaseCrossValidator, _RepeatedSplits, + BaseShuffleSplit]: if self.resampling_strategy_args is None: self.resampling_strategy_args = {} - if not isinstance(self.resampling_strategy, str): + if self.resampling_strategy is not None and not isinstance(self.resampling_strategy, str): if issubclass(self.resampling_strategy, BaseCrossValidator) or \ issubclass(self.resampling_strategy, _RepeatedSplits) or \ issubclass(self.resampling_strategy, BaseShuffleSplit): class_name = self.resampling_strategy.__name__ - if class_name not in __baseCrossValidator_defaults__: + if class_name not in baseCrossValidator_defaults: raise ValueError('Unknown CrossValidator.') - ref_arg_dict = __baseCrossValidator_defaults__[class_name] + ref_arg_dict = baseCrossValidator_defaults[class_name] y = D.data['Y_train'] if (D.info['task'] in CLASSIFICATION_TASKS and @@ -932,7 +990,7 @@ def get_splitter(self, D): raise ValueError('Must provide parameter groups ' 'for chosen CrossValidator.') try: - if self.resampling_strategy_args['groups'].shape[0] != y.shape[0]: + if np.shape(self.resampling_strategy_args['groups'])[0] != y.shape[0]: raise ValueError('Groups must be array-like ' 'with shape (n_samples,).') except Exception: @@ -940,7 +998,7 @@ def get_splitter(self, D): 'with shape (n_samples,).') else: if 'groups' in self.resampling_strategy_args: - if self.resampling_strategy_args['groups'].shape[0] != y.shape[0]: + if np.shape(self.resampling_strategy_args['groups'])[0] != y.shape[0]: raise ValueError('Groups must be array-like' ' with shape (n_samples,).') @@ -958,6 +1016,7 @@ def get_splitter(self, D): init_dict.pop('groups', None) if 'folds' in init_dict: init_dict['n_splits'] = init_dict.pop('folds', None) + assert self.resampling_strategy is not None cv = copy.deepcopy(self.resampling_strategy)(**init_dict) if 'groups' not in self.resampling_strategy_args: @@ -969,8 +1028,9 @@ def get_splitter(self, D): shuffle = self.resampling_strategy_args.get('shuffle', True) train_size = 0.67 if self.resampling_strategy_args: - train_size = self.resampling_strategy_args.get('train_size', - train_size) + train_size_from_user = self.resampling_strategy_args.get('train_size') + if train_size_from_user is not None: + train_size = float(train_size_from_user) test_size = float("%.4f" % (1 - train_size)) if D.info['task'] in CLASSIFICATION_TASKS and D.info['task'] != MULTILABEL_CLASSIFICATION: @@ -1037,26 +1097,26 @@ def get_splitter(self, D): # create closure for evaluating an algorithm def eval_holdout( - queue, - config, - backend, - resampling_strategy, - resampling_strategy_args, - metric, - seed, - num_run, - instance, - scoring_functions, - output_y_hat_optimization, - include, - exclude, - disable_file_output, - port, - init_params=None, - iterative=False, - budget=100.0, - budget_type=None, -): + queue: multiprocessing.Queue, + config: Union[int, Configuration], + backend: Backend, + resampling_strategy: Union[str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit], + resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]], + metric: Scorer, + seed: int, + num_run: int, + instance: Union[str, bytes], + scoring_functions: Optional[List[Scorer]], + output_y_hat_optimization: bool, + include: Optional[List[str]], + exclude: Optional[List[str]], + disable_file_output: bool, + port: Optional[int], + init_params: Optional[Dict[str, Any]] = None, + budget: Optional[float] = 100.0, + budget_type: Optional[str] = None, + iterative: bool = False, +) -> None: evaluator = TrainEvaluator( backend=backend, port=port, @@ -1080,25 +1140,25 @@ def eval_holdout( def eval_iterative_holdout( - queue, - config, - backend, - resampling_strategy, - resampling_strategy_args, - metric, - seed, - num_run, - instance, - scoring_functions, - output_y_hat_optimization, - include, - exclude, - disable_file_output, - port, - init_params=None, - budget=100.0, - budget_type=None, -): + queue: multiprocessing.Queue, + config: Union[int, Configuration], + backend: Backend, + resampling_strategy: Union[str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit], + resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]], + metric: Scorer, + seed: int, + num_run: int, + instance: Union[str, bytes], + scoring_functions: Optional[List[Scorer]], + output_y_hat_optimization: bool, + include: Optional[List[str]], + exclude: Optional[List[str]], + disable_file_output: bool, + port: Optional[int], + init_params: Optional[Dict[str, Any]] = None, + budget: Optional[float] = 100.0, + budget_type: Optional[str] = None, +) -> None: return eval_holdout( queue=queue, port=port, @@ -1123,30 +1183,30 @@ def eval_iterative_holdout( def eval_partial_cv( - queue, - config, - backend, - resampling_strategy, - resampling_strategy_args, - metric, - seed, - num_run, - instance, - scoring_functions, - output_y_hat_optimization, - include, - exclude, - disable_file_output, - port, - init_params=None, - iterative=False, - budget=None, - budget_type=None, -): + queue: multiprocessing.Queue, + config: Union[int, Configuration], + backend: Backend, + resampling_strategy: Union[str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit], + resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]], + metric: Scorer, + seed: int, + num_run: int, + instance: Union[str, bytes], + scoring_functions: Optional[List[Scorer]], + output_y_hat_optimization: bool, + include: Optional[List[str]], + exclude: Optional[List[str]], + disable_file_output: bool, + port: Optional[int], + init_params: Optional[Dict[str, Any]] = None, + budget: Optional[float] = None, + budget_type: Optional[str] = None, + iterative: bool = False, +) -> None: if budget_type is not None: raise NotImplementedError() - instance = json.loads(instance) if instance is not None else {} - fold = instance['fold'] + instance_dict: Dict[str, int] = json.loads(instance) if instance is not None else {} + fold = instance_dict['fold'] evaluator = TrainEvaluator( backend=backend, @@ -1172,25 +1232,25 @@ def eval_partial_cv( def eval_partial_cv_iterative( - queue, - config, - backend, - resampling_strategy, - resampling_strategy_args, - metric, - seed, - num_run, - instance, - scoring_functions, - output_y_hat_optimization, - include, - exclude, - disable_file_output, - port, - init_params=None, - budget=None, - budget_type=None, -): + queue: multiprocessing.Queue, + config: Union[int, Configuration], + backend: Backend, + resampling_strategy: Union[str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit], + resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]], + metric: Scorer, + seed: int, + num_run: int, + instance: Union[str, bytes], + scoring_functions: Optional[List[Scorer]], + output_y_hat_optimization: bool, + include: Optional[List[str]], + exclude: Optional[List[str]], + disable_file_output: bool, + port: Optional[int], + init_params: Optional[Dict[str, Any]] = None, + budget: Optional[float] = None, + budget_type: Optional[str] = None, +) -> None: if budget_type is not None: raise NotImplementedError() return eval_partial_cv( @@ -1216,26 +1276,26 @@ def eval_partial_cv_iterative( # create closure for evaluating an algorithm def eval_cv( - queue, - config, - backend, - resampling_strategy, - resampling_strategy_args, - metric, - seed, - num_run, - instance, - scoring_functions, - output_y_hat_optimization, - include, - exclude, - disable_file_output, - port, - init_params=None, - budget=None, - budget_type=None, - iterative=False, -): + queue: multiprocessing.Queue, + config: Union[int, Configuration], + backend: Backend, + resampling_strategy: Union[str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit], + resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]], + metric: Scorer, + seed: int, + num_run: int, + instance: Union[str, bytes], + scoring_functions: Optional[List[Scorer]], + output_y_hat_optimization: bool, + include: Optional[List[str]], + exclude: Optional[List[str]], + disable_file_output: bool, + port: Optional[int], + init_params: Optional[Dict[str, Any]] = None, + budget: Optional[float] = None, + budget_type: Optional[str] = None, + iterative: bool = False, +) -> None: evaluator = TrainEvaluator( backend=backend, port=port, @@ -1260,26 +1320,26 @@ def eval_cv( def eval_iterative_cv( - queue, - config, - backend, - resampling_strategy, - resampling_strategy_args, - metric, - seed, - num_run, - instance, - scoring_functions, - output_y_hat_optimization, - include, - exclude, - disable_file_output, - port, - init_params=None, - budget=None, - budget_type=None, - iterative=True, -): + queue: multiprocessing.Queue, + config: Union[int, Configuration], + backend: Backend, + resampling_strategy: Union[str, BaseCrossValidator, _RepeatedSplits, BaseShuffleSplit], + resampling_strategy_args: Dict[str, Optional[Union[float, int, str]]], + metric: Scorer, + seed: int, + num_run: int, + instance: Union[str, bytes], + scoring_functions: Optional[List[Scorer]], + output_y_hat_optimization: bool, + include: Optional[List[str]], + exclude: Optional[List[str]], + disable_file_output: bool, + port: Optional[int], + init_params: Optional[Dict[str, Any]] = None, + budget: Optional[float] = None, + budget_type: Optional[str] = None, + iterative: bool = True, +) -> None: eval_cv( backend=backend, queue=queue, diff --git a/autosklearn/evaluation/util.py b/autosklearn/evaluation/util.py index 7c9b3078e5..e7483cbd26 100644 --- a/autosklearn/evaluation/util.py +++ b/autosklearn/evaluation/util.py @@ -10,7 +10,7 @@ def read_queue(queue_: multiprocessing.Queue - ) -> List[Union[str, int, float, List, Dict, Tuple]]: + ) -> List[Dict[str, Union[str, bool, int, float, List, Dict, Tuple]]]: stack = [] while True: try: @@ -47,7 +47,7 @@ def empty_queue(queue_: multiprocessing.Queue) -> None: def extract_learning_curve(stack: List[Dict[str, Any]], - key: Optional[str] = None) -> List[Union[float, int]]: + key: Optional[str] = None) -> List[float]: learning_curve = [] for entry in stack: if key: diff --git a/test/test_evaluation/test_train_evaluator.py b/test/test_evaluation/test_train_evaluator.py index 97acceb780..d923be5eba 100644 --- a/test/test_evaluation/test_train_evaluator.py +++ b/test/test_evaluation/test_train_evaluator.py @@ -545,7 +545,8 @@ def side_effect(self, *args, **kwargs): self.assertEqual(pipeline_mock.predict_proba.call_count, 36) @unittest.mock.patch.object(TrainEvaluator, '_loss') - def test_file_output(self, loss_mock): + @unittest.mock.patch.object(TrainEvaluator, '_get_model') + def test_file_output(self, loss_mock, model_mock): D = get_regression_datamanager() D.name = 'test' @@ -553,6 +554,7 @@ def test_file_output(self, loss_mock): configuration = unittest.mock.Mock(spec=Configuration) queue_ = multiprocessing.Queue() loss_mock.return_value = None + model_mock.return_value = None evaluator = TrainEvaluator(self.backend_mock, queue=queue_, port=self.port, From a916f3dc906960e4741d84f476bfc87234def652 Mon Sep 17 00:00:00 2001 From: chico Date: Tue, 16 Feb 2021 13:41:48 +0100 Subject: [PATCH 3/5] Feedback from comments --- autosklearn/evaluation/__init__.py | 3 ++- autosklearn/evaluation/abstract_evaluator.py | 12 +++++++----- autosklearn/evaluation/train_evaluator.py | 7 +++---- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/autosklearn/evaluation/__init__.py b/autosklearn/evaluation/__init__.py index 6fc0a780ce..6df39da521 100644 --- a/autosklearn/evaluation/__init__.py +++ b/autosklearn/evaluation/__init__.py @@ -24,6 +24,7 @@ import autosklearn.evaluation.train_evaluator import autosklearn.evaluation.test_evaluator import autosklearn.evaluation.util +from autosklearn.evaluation.train_evaluator import TYPE_ADDITIONAL_INFO from autosklearn.util.backend import Backend from autosklearn.util.logging_ import PickableLoggerAdapter, get_named_client_logger from autosklearn.util.parallel import preload_modules @@ -290,7 +291,7 @@ def run( # Additional information of each of the tae executions # Defined upfront for mypy - additional_run_info: Dict[str, Union[int, float, str, Dict, List, Tuple]] = {} + additional_run_info: TYPE_ADDITIONAL_INFO = {} context = multiprocessing.get_context(self.pynisher_context) preload_modules(context) diff --git a/autosklearn/evaluation/abstract_evaluator.py b/autosklearn/evaluation/abstract_evaluator.py index 8d9e00c4e6..dd666eb936 100644 --- a/autosklearn/evaluation/abstract_evaluator.py +++ b/autosklearn/evaluation/abstract_evaluator.py @@ -36,6 +36,10 @@ ] +# General TYPE definitions for numpy +TYPE_ADDITIONAL_INFO = Dict[str, Union[int, float, str, Dict, List, Tuple]] + + class MyDummyClassifier(DummyClassifier): def __init__( self, @@ -85,8 +89,7 @@ def predict_proba(self, X: np.ndarray, batch_size: int = 1000 def estimator_supports_iterative_fit(self) -> bool: # pylint: disable=R0201 return False - def get_additional_run_info(self) -> Optional[ - Dict[str, Union[str, int, float, Dict, List, Tuple]]]: # pylint: disable=R0201 + def get_additional_run_info(self) -> Optional[TYPE_ADDITIONAL_INFO]: # pylint: disable=R0201 return None @@ -135,8 +138,7 @@ def predict(self, X: np.ndarray, batch_size: int = 1000) -> np.ndarray: def estimator_supports_iterative_fit(self) -> bool: # pylint: disable=R0201 return False - def get_additional_run_info(self) -> Optional[ - Dict[str, Union[str, int, float, Dict, List, Tuple]]]: # pylint: disable=R0201 + def get_additional_run_info(self) -> Optional[TYPE_ADDITIONAL_INFO]: # pylint: disable=R0201 return None @@ -332,7 +334,7 @@ def finish_up( opt_pred: np.ndarray, valid_pred: np.ndarray, test_pred: np.ndarray, - additional_run_info: Optional[Dict[str, Union[str, int, float, Dict, List, Tuple]]], + additional_run_info: Optional[TYPE_ADDITIONAL_INFO], file_output: bool, final_call: bool, status: StatusType, diff --git a/autosklearn/evaluation/train_evaluator.py b/autosklearn/evaluation/train_evaluator.py index 2ddaf7a9e4..274f95e3d3 100644 --- a/autosklearn/evaluation/train_evaluator.py +++ b/autosklearn/evaluation/train_evaluator.py @@ -17,6 +17,7 @@ from autosklearn.evaluation.abstract_evaluator import ( AbstractEvaluator, + TYPE_ADDITIONAL_INFO, _fit_and_suppress_warnings, ) from autosklearn.data.abstract_data_manager import AbstractDataManager @@ -238,8 +239,7 @@ def fit_predict_and_loss(self, iterative: bool = False) -> None: holdout (both iterative and non-iterative)""" # Define beforehand for mypy - additional_run_info: Optional[ - Dict[str, Union[str, int, float, Dict, List, Tuple]]] = None + additional_run_info: Optional[TYPE_ADDITIONAL_INFO] = None if iterative: if self.num_cv_folds == 1: @@ -276,7 +276,7 @@ def fit_predict_and_loss(self, iterative: bool = False) -> None: iterations = [1] * self.num_cv_folds total_n_iterations = [0] * self.num_cv_folds # model.estimator_supports_iterative_fit -> true - # After the if above, we now estimator support iterative fit + # After the if above, we know estimator support iterative fit model_max_iter = [cast(IterativeComponent, model).get_max_iter() for model in self.models] @@ -859,7 +859,6 @@ def _partial_fit_and_predict_standard( else: self.models[fold] = model - train_indices, test_indices = ((train_indices, test_indices)) self.Y_targets[fold] = self.Y_train[test_indices] self.Y_train_targets[train_indices] = self.Y_train[train_indices] From 5ac67cbf9bef2fc070e1fcf9db47200b75d3ec21 Mon Sep 17 00:00:00 2001 From: chico Date: Tue, 16 Feb 2021 14:18:40 +0100 Subject: [PATCH 4/5] Solving rebase artifacts --- autosklearn/evaluation/abstract_evaluator.py | 7 +++---- autosklearn/evaluation/test_evaluator.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/autosklearn/evaluation/abstract_evaluator.py b/autosklearn/evaluation/abstract_evaluator.py index dd666eb936..8178688e58 100644 --- a/autosklearn/evaluation/abstract_evaluator.py +++ b/autosklearn/evaluation/abstract_evaluator.py @@ -24,7 +24,7 @@ from autosklearn.pipeline.implementations.util import ( convert_multioutput_multiclass_to_multilabel ) -from autosklearn.metrics import Scorer, calculate_score +from autosklearn.metrics import calculate_loss, Scorer from autosklearn.util.backend import Backend from autosklearn.util.logging_ import PicklableClientLogger, get_named_client_logger @@ -299,8 +299,8 @@ def _get_model(self) -> BaseEstimator: return model def _loss(self, y_true: np.ndarray, y_hat: np.ndarray, - scoring_functions: typing.Optional[typing.List[Scorer]] = None - ) -> typing.Union[float, typing.Dict[str, float]]: + scoring_functions: Optional[List[Scorer]] = None + ) -> Union[float, Dict[str, float]]: """Auto-sklearn follows a minimization goal. The calculate_loss internally translate a score function to a minimization problem. @@ -326,7 +326,6 @@ def _loss(self, y_true: np.ndarray, y_hat: np.ndarray, y_true, y_hat, self.task_type, self.metric, scoring_functions=scoring_functions) - def finish_up( self, loss: Union[Dict[str, float], float], diff --git a/autosklearn/evaluation/test_evaluator.py b/autosklearn/evaluation/test_evaluator.py index 4e19073e90..94199e47f9 100644 --- a/autosklearn/evaluation/test_evaluator.py +++ b/autosklearn/evaluation/test_evaluator.py @@ -12,7 +12,7 @@ AbstractEvaluator, _fit_and_suppress_warnings, ) -from autosklearn.metrics import calculate_loss +from autosklearn.metrics import calculate_loss, Scorer from autosklearn.util.backend import Backend From e5db241d12dccdab719c188f7fc4412ef7ff1630 Mon Sep 17 00:00:00 2001 From: chico Date: Tue, 16 Feb 2021 19:30:45 +0100 Subject: [PATCH 5/5] Revert bytes --- autosklearn/evaluation/train_evaluator.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/autosklearn/evaluation/train_evaluator.py b/autosklearn/evaluation/train_evaluator.py index 274f95e3d3..f1fd027f37 100644 --- a/autosklearn/evaluation/train_evaluator.py +++ b/autosklearn/evaluation/train_evaluator.py @@ -1104,7 +1104,7 @@ def eval_holdout( metric: Scorer, seed: int, num_run: int, - instance: Union[str, bytes], + instance: str, scoring_functions: Optional[List[Scorer]], output_y_hat_optimization: bool, include: Optional[List[str]], @@ -1147,7 +1147,7 @@ def eval_iterative_holdout( metric: Scorer, seed: int, num_run: int, - instance: Union[str, bytes], + instance: str, scoring_functions: Optional[List[Scorer]], output_y_hat_optimization: bool, include: Optional[List[str]], @@ -1190,7 +1190,7 @@ def eval_partial_cv( metric: Scorer, seed: int, num_run: int, - instance: Union[str, bytes], + instance: str, scoring_functions: Optional[List[Scorer]], output_y_hat_optimization: bool, include: Optional[List[str]], @@ -1239,7 +1239,7 @@ def eval_partial_cv_iterative( metric: Scorer, seed: int, num_run: int, - instance: Union[str, bytes], + instance: str, scoring_functions: Optional[List[Scorer]], output_y_hat_optimization: bool, include: Optional[List[str]], @@ -1283,7 +1283,7 @@ def eval_cv( metric: Scorer, seed: int, num_run: int, - instance: Union[str, bytes], + instance: str, scoring_functions: Optional[List[Scorer]], output_y_hat_optimization: bool, include: Optional[List[str]], @@ -1327,7 +1327,7 @@ def eval_iterative_cv( metric: Scorer, seed: int, num_run: int, - instance: Union[str, bytes], + instance: str, scoring_functions: Optional[List[Scorer]], output_y_hat_optimization: bool, include: Optional[List[str]],