diff --git a/examples/plugins/example_launcher_plugin/hydra_plugins/example_launcher_plugin/example_launcher.py b/examples/plugins/example_launcher_plugin/hydra_plugins/example_launcher_plugin/example_launcher.py index 3da7e27e30f..71bbf401672 100644 --- a/examples/plugins/example_launcher_plugin/hydra_plugins/example_launcher_plugin/example_launcher.py +++ b/examples/plugins/example_launcher_plugin/hydra_plugins/example_launcher_plugin/example_launcher.py @@ -6,7 +6,6 @@ from typing import Optional, Sequence from hydra.core.utils import HydraContext -from hydra.core.config_loader import ConfigLoader from hydra.core.config_store import ConfigStore from hydra.core.singleton import Singleton from hydra.core.utils import ( @@ -49,8 +48,8 @@ class LauncherConfig: class ExampleLauncher(Launcher): def __init__(self, foo: str, bar: str) -> None: self.config: Optional[DictConfig] = None - self.config_loader: Optional[ConfigLoader] = None self.task_function: Optional[TaskFunction] = None + self.hydra_context: Optional[HydraContext] = None # foo and var are coming from the the plugin's configuration self.foo = foo @@ -64,7 +63,7 @@ def setup( config: DictConfig, ) -> None: self.config = config - self.config_loader = hydra_context.config_loader + self.hydra_context = hydra_context self.task_function = task_function def launch( @@ -77,7 +76,7 @@ def launch( """ setup_globals() assert self.config is not None - assert self.config_loader is not None + assert self.hydra_context is not None assert self.task_function is not None configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) @@ -93,7 +92,7 @@ def launch( idx = initial_job_idx + idx lst = " ".join(filter_overrides(overrides)) log.info(f"\t#{idx} : {lst}") - sweep_config = self.config_loader.load_sweep_config( + sweep_config = self.hydra_context.config_loader.load_sweep_config( self.config, list(overrides) ) with open_dict(sweep_config): @@ -115,8 +114,9 @@ def launch( Singleton.set_state(state) ret = run_job( - config=sweep_config, + hydra_context=self.hydra_context, task_function=self.task_function, + config=sweep_config, job_dir_key="hydra.sweep.dir", job_subdir_key="hydra.sweep.subdir", ) diff --git a/examples/plugins/example_sweeper_plugin/hydra_plugins/example_sweeper_plugin/example_sweeper.py b/examples/plugins/example_sweeper_plugin/hydra_plugins/example_sweeper_plugin/example_sweeper.py index 02bcfd9c871..11db7df71ee 100644 --- a/examples/plugins/example_sweeper_plugin/hydra_plugins/example_sweeper_plugin/example_sweeper.py +++ b/examples/plugins/example_sweeper_plugin/hydra_plugins/example_sweeper_plugin/example_sweeper.py @@ -7,7 +7,6 @@ from typing import Any, Iterable, List, Optional, Sequence from hydra.core.utils import HydraContext -from hydra.core.config_loader import ConfigLoader from hydra.core.config_store import ConfigStore from hydra.core.override_parser.overrides_parser import OverridesParser from hydra.core.plugins import Plugins @@ -46,6 +45,7 @@ def __init__(self, max_batch_size: int, foo: str, bar: str): self.max_batch_size = max_batch_size self.config: Optional[DictConfig] = None self.launcher: Optional[Launcher] = None + self.hydra_context: Optional[HydraContext] = None self.job_results = None self.foo = foo self.bar = bar @@ -58,10 +58,10 @@ def setup( config: DictConfig, ) -> None: self.config = config - self.config_loader = hydra_context.config_loader self.launcher = Plugins.instance().instantiate_launcher( hydra_context=hydra_context, task_function=task_function, config=config ) + self.hydra_context = hydra_context def sweep(self, arguments: List[str]) -> Any: assert self.config is not None diff --git a/hydra/_internal/core_plugins/basic_launcher.py b/hydra/_internal/core_plugins/basic_launcher.py index 4ad619537fc..9d1ec1f8dca 100644 --- a/hydra/_internal/core_plugins/basic_launcher.py +++ b/hydra/_internal/core_plugins/basic_launcher.py @@ -6,11 +6,9 @@ from omegaconf import DictConfig, open_dict -from hydra.core.utils import HydraContext -from hydra.core.callbacks import Callbacks -from hydra.core.config_loader import ConfigLoader from hydra.core.config_store import ConfigStore from hydra.core.utils import ( + HydraContext, JobReturn, configure_log, filter_overrides, @@ -37,7 +35,6 @@ class BasicLauncher(Launcher): def __init__(self) -> None: super().__init__() self.config: Optional[DictConfig] = None - self.config_loader: Optional[ConfigLoader] = None self.task_function: Optional[TaskFunction] = None self.hydra_context: Optional[HydraContext] = None @@ -50,16 +47,15 @@ def setup( ) -> None: self.config = config self.hydra_context = hydra_context - self.config_loader = hydra_context.config_loader self.task_function = task_function def launch( self, job_overrides: Sequence[Sequence[str]], initial_job_idx: int ) -> Sequence[JobReturn]: setup_globals() + assert self.hydra_context is not None assert self.config is not None assert self.task_function is not None - assert self.config_loader is not None configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) sweep_dir = self.config.hydra.sweep.dir @@ -70,7 +66,7 @@ def launch( idx = initial_job_idx + idx lst = " ".join(filter_overrides(overrides)) log.info(f"\t#{idx} : {lst}") - sweep_config = self.config_loader.load_sweep_config( + sweep_config = self.hydra_context.config_loader.load_sweep_config( self.config, list(overrides) ) with open_dict(sweep_config): diff --git a/hydra/_internal/core_plugins/basic_sweeper.py b/hydra/_internal/core_plugins/basic_sweeper.py index 923cd2b89c7..0c6c080f22d 100644 --- a/hydra/_internal/core_plugins/basic_sweeper.py +++ b/hydra/_internal/core_plugins/basic_sweeper.py @@ -25,12 +25,10 @@ from omegaconf import DictConfig, OmegaConf -from hydra.core.utils import HydraContext -from hydra.core.config_loader import ConfigLoader from hydra.core.config_store import ConfigStore from hydra.core.override_parser.overrides_parser import OverridesParser from hydra.core.override_parser.types import Override -from hydra.core.utils import JobReturn +from hydra.core.utils import HydraContext, JobReturn from hydra.errors import HydraException from hydra.plugins.launcher import Launcher from hydra.plugins.sweeper import Sweeper @@ -65,7 +63,7 @@ def __init__(self, max_batch_size: Optional[int]) -> None: self.batch_index = 0 self.max_batch_size = max_batch_size - self.config_loader: Optional[ConfigLoader] = None + self.hydra_context: Optional[HydraContext] = None self.config: Optional[DictConfig] = None self.launcher: Optional[Launcher] = None @@ -78,11 +76,13 @@ def setup( ) -> None: from hydra.core.plugins import Plugins - self.config_loader = hydra_context.config_loader + self.hydra_context = hydra_context self.config = config self.launcher = Plugins.instance().instantiate_launcher( - config=config, config_loader=self.config_loader, task_function=task_function + hydra_context=hydra_context, + task_function=task_function, + config=config, ) @staticmethod @@ -130,8 +130,9 @@ def split_arguments( def sweep(self, arguments: List[str]) -> Any: assert self.config is not None assert self.launcher is not None + assert self.hydra_context is not None - parser = OverridesParser.create(config_loader=self.config_loader) + parser = OverridesParser.create(config_loader=self.hydra_context.config_loader) overrides = parser.parse_overrides(arguments) self.overrides = self.split_arguments(overrides, self.max_batch_size) diff --git a/hydra/_internal/hydra.py b/hydra/_internal/hydra.py index 5b7e3edb8da..8cd707764d6 100644 --- a/hydra/_internal/hydra.py +++ b/hydra/_internal/hydra.py @@ -5,22 +5,22 @@ import sys from argparse import ArgumentParser from collections import defaultdict -from dataclasses import dataclass from typing import Any, Callable, DefaultDict, List, Optional, Sequence, Type, Union from omegaconf import Container, DictConfig, OmegaConf, flag_override, open_dict -from hydra._internal.utils import get_column_widths, run_and_report, Callbacks +from hydra._internal.utils import Callbacks, get_column_widths, run_and_report from hydra.core.config_loader import ConfigLoader from hydra.core.config_search_path import ConfigSearchPath from hydra.core.plugins import Plugins from hydra.core.utils import ( + HydraContext, JobReturn, JobRuntime, configure_log, run_job, setup_globals, - simple_stdout_log_config, HydraContext, + simple_stdout_log_config, ) from hydra.plugins.completion_plugin import CompletionPlugin from hydra.plugins.config_source import ConfigSource @@ -95,7 +95,9 @@ def run( callbacks.on_run_start(config=cfg, config_name=config_name) ret = run_job( - hydra_context=HydraContext(callbacks=callbacks), + hydra_context=HydraContext( + config_loader=self.config_loader, callbacks=callbacks + ), task_function=task_function, config=cfg, job_dir_key="hydra.run.dir", diff --git a/hydra/_internal/utils.py b/hydra/_internal/utils.py index 4c68c364e76..f962f91bdd0 100644 --- a/hydra/_internal/utils.py +++ b/hydra/_internal/utils.py @@ -648,4 +648,4 @@ def on_job_end( job_return=job_return, reverse=True, **kwargs, - ) \ No newline at end of file + ) diff --git a/hydra/core/plugins.py b/hydra/core/plugins.py index d6ef04dc5aa..c01f7232451 100644 --- a/hydra/core/plugins.py +++ b/hydra/core/plugins.py @@ -11,10 +11,9 @@ from omegaconf import DictConfig -from hydra.core.utils import HydraContext from hydra._internal.sources_registry import SourcesRegistry -from hydra.core.config_loader import ConfigLoader from hydra.core.singleton import Singleton +from hydra.core.utils import HydraContext from hydra.plugins.completion_plugin import CompletionPlugin from hydra.plugins.config_source import ConfigSource from hydra.plugins.launcher import Launcher diff --git a/hydra/core/utils.py b/hydra/core/utils.py index e5f32b15465..1b2a3c79ebd 100644 --- a/hydra/core/utils.py +++ b/hydra/core/utils.py @@ -11,8 +11,7 @@ from os.path import splitext from pathlib import Path from textwrap import dedent -from typing import Any, Dict, Optional, Sequence, Union, cast, TYPE_CHECKING -import warnings +from typing import TYPE_CHECKING, Any, Dict, Optional, Sequence, Union, cast from omegaconf import DictConfig, OmegaConf, open_dict, read_write @@ -21,6 +20,7 @@ from hydra.core.singleton import Singleton from hydra.errors import HydraJobException from hydra.types import TaskFunction + if TYPE_CHECKING: from hydra._internal.utils import Callbacks @@ -29,9 +29,8 @@ @dataclass class HydraContext: - config_loader: Optional[ConfigLoader] = None - callbacks: Optional["Callbacks"] = None - + config_loader: ConfigLoader + callbacks: "Callbacks" def simple_stdout_log_config(level: int = logging.INFO) -> None: @@ -253,8 +252,6 @@ def validate_config_path(config_path: Optional[str]) -> None: raise ValueError(msg) - - @contextmanager def env_override(env: Dict[str, str]) -> Any: """Temporarily set environment variables inside the context manager and diff --git a/hydra/plugins/launcher.py b/hydra/plugins/launcher.py index 5cce6fe9a03..04cb8df2c82 100644 --- a/hydra/plugins/launcher.py +++ b/hydra/plugins/launcher.py @@ -7,14 +7,12 @@ from omegaconf import DictConfig -from hydra.core.config_loader import ConfigLoader from hydra.core.utils import JobReturn, HydraContext from hydra.types import TaskFunction from .plugin import Plugin - class Launcher(Plugin): @abstractmethod def setup( diff --git a/hydra/plugins/sweeper.py b/hydra/plugins/sweeper.py index a4dee14000a..68909e55b1e 100644 --- a/hydra/plugins/sweeper.py +++ b/hydra/plugins/sweeper.py @@ -5,13 +5,12 @@ from abc import abstractmethod from typing import Any, List, Sequence, Optional -from hydra.core.config_loader import ConfigLoader from hydra.types import TaskFunction from omegaconf import DictConfig from .launcher import Launcher from .plugin import Plugin -from hydra.core.utils import JobReturn, HydraContext +from hydra.core.utils import HydraContext class Sweeper(Plugin): @@ -21,7 +20,7 @@ class Sweeper(Plugin): (where each job typically takes a different command line arguments) """ - config_loader: Optional[ConfigLoader] + hydra_context: Optional[HydraContext] config: Optional[DictConfig] launcher: Optional[Launcher] @@ -52,9 +51,9 @@ def validate_batch_is_legal(self, batch: Sequence[Sequence[str]]) -> None: This repeat work the launcher will do, but as the launcher may be performing this in a different process/machine it's important to do it here as well to detect failures early. """ - assert self.config_loader is not None + assert self.hydra_context is not None assert self.config is not None for overrides in batch: - self.config_loader.load_sweep_config( + self.hydra_context.config_loader.load_sweep_config( master_config=self.config, sweep_overrides=list(overrides) ) diff --git a/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/_core.py b/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/_core.py index dbc1bbc0c60..af604505515 100644 --- a/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/_core.py +++ b/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/_core.py @@ -6,12 +6,10 @@ from ax.core import types as ax_types # type: ignore from ax.exceptions.core import SearchSpaceExhausted # type: ignore from ax.service.ax_client import AxClient # type: ignore - -from hydra.core.utils import HydraContext -from hydra.core.config_loader import ConfigLoader from hydra.core.override_parser.overrides_parser import OverridesParser from hydra.core.override_parser.types import IntervalSweep, Override, Transformer from hydra.core.plugins import Plugins +from hydra.core.utils import HydraContext from hydra.plugins.launcher import Launcher from hydra.plugins.sweeper import Sweeper from hydra.types import TaskFunction @@ -109,9 +107,9 @@ class CoreAxSweeper(Sweeper): """Class to interface with the Ax Platform""" def __init__(self, ax_config: AxConfig, max_batch_size: Optional[int]): - self.config_loader: Optional[ConfigLoader] = None self.config: Optional[DictConfig] = None self.launcher: Optional[Launcher] = None + self.hydra_context: Optional[HydraContext] = None self.job_results = None self.experiment: ExperimentConfig = ax_config.experiment @@ -138,7 +136,7 @@ def setup( config: DictConfig, ) -> None: self.config = config - self.config_loader = hydra_context.config_loader + self.hydra_context = hydra_context self.launcher = Plugins.instance().instantiate_launcher( config=config, hydra_context=hydra_context, task_function=task_function ) diff --git a/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/ax_sweeper.py b/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/ax_sweeper.py index d41ea2ecb3d..04f4ddd17cf 100644 --- a/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/ax_sweeper.py +++ b/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/ax_sweeper.py @@ -2,7 +2,6 @@ from typing import List, Optional from hydra.core.utils import HydraContext -from hydra.core.config_loader import ConfigLoader from hydra.plugins.sweeper import Sweeper from hydra.types import TaskFunction from omegaconf import DictConfig diff --git a/plugins/hydra_joblib_launcher/hydra_plugins/hydra_joblib_launcher/_core.py b/plugins/hydra_joblib_launcher/hydra_plugins/hydra_joblib_launcher/_core.py index b2b2fdae813..1e74b04fbc2 100644 --- a/plugins/hydra_joblib_launcher/hydra_plugins/hydra_joblib_launcher/_core.py +++ b/plugins/hydra_joblib_launcher/hydra_plugins/hydra_joblib_launcher/_core.py @@ -3,10 +3,10 @@ from pathlib import Path from typing import Any, Dict, List, Sequence -from hydra.core.config_loader import ConfigLoader from hydra.core.hydra_config import HydraConfig from hydra.core.singleton import Singleton from hydra.core.utils import ( + HydraContext, JobReturn, configure_log, filter_overrides, @@ -25,7 +25,7 @@ def execute_job( idx: int, overrides: Sequence[str], - config_loader: ConfigLoader, + hydra_context: HydraContext, config: DictConfig, task_function: TaskFunction, singleton_state: Dict[Any, Any], @@ -34,13 +34,16 @@ def execute_job( setup_globals() Singleton.set_state(singleton_state) - sweep_config = config_loader.load_sweep_config(config, list(overrides)) + sweep_config = hydra_context.config_loader.load_sweep_config( + config, list(overrides) + ) with open_dict(sweep_config): sweep_config.hydra.job.id = "{}_{}".format(sweep_config.hydra.job.name, idx) sweep_config.hydra.job.num = idx HydraConfig.instance().set_config(sweep_config) ret = run_job( + hydra_context=hydra_context, config=sweep_config, task_function=task_function, job_dir_key="hydra.sweep.dir", @@ -73,8 +76,8 @@ def launch( """ setup_globals() assert launcher.config is not None - assert launcher.config_loader is not None assert launcher.task_function is not None + assert launcher.hydra_context is not None configure_log(launcher.config.hydra.hydra_logging, launcher.config.hydra.verbose) sweep_dir = Path(str(launcher.config.hydra.sweep.dir)) @@ -102,7 +105,7 @@ def launch( delayed(execute_job)( initial_job_idx + idx, overrides, - launcher.config_loader, + launcher.hydra_context, launcher.config, launcher.task_function, singleton_state, diff --git a/plugins/hydra_joblib_launcher/hydra_plugins/hydra_joblib_launcher/joblib_launcher.py b/plugins/hydra_joblib_launcher/hydra_plugins/hydra_joblib_launcher/joblib_launcher.py index 93ace1efcd6..32dc1d58750 100644 --- a/plugins/hydra_joblib_launcher/hydra_plugins/hydra_joblib_launcher/joblib_launcher.py +++ b/plugins/hydra_joblib_launcher/hydra_plugins/hydra_joblib_launcher/joblib_launcher.py @@ -2,9 +2,7 @@ import logging from typing import Any, Optional, Sequence -from hydra.core.utils import HydraContext -from hydra.core.config_loader import ConfigLoader -from hydra.core.utils import JobReturn +from hydra.core.utils import HydraContext, JobReturn from hydra.plugins.launcher import Launcher from hydra.types import TaskFunction from omegaconf import DictConfig @@ -23,8 +21,8 @@ def __init__(self, **kwargs: Any) -> None: https://github.com/facebookresearch/hydra/issues/357 """ self.config: Optional[DictConfig] = None - self.config_loader: Optional[ConfigLoader] = None self.task_function: Optional[TaskFunction] = None + self.hydra_context: Optional[HydraContext] = None self.joblib = kwargs @@ -36,8 +34,8 @@ def setup( config: DictConfig, ) -> None: self.config = config - self.config_loader = hydra_context.config_loader self.task_function = task_function + self.hydra_context = hydra_context def launch( self, job_overrides: Sequence[Sequence[str]], initial_job_idx: int diff --git a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py index 6cbd38fe969..12d10e7de0f 100644 --- a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py +++ b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py @@ -12,9 +12,6 @@ ) import nevergrad as ng - -from hydra.core.utils import HydraContext -from hydra.core.config_loader import ConfigLoader from hydra.core.override_parser.overrides_parser import OverridesParser from hydra.core.override_parser.types import ( ChoiceSweep, @@ -23,6 +20,7 @@ Transformer, ) from hydra.core.plugins import Plugins +from hydra.core.utils import HydraContext from hydra.plugins.launcher import Launcher from hydra.plugins.sweeper import Sweeper from hydra.types import TaskFunction @@ -92,6 +90,7 @@ def __init__( self.opt_config = optim self.config: Optional[DictConfig] = None self.launcher: Optional[Launcher] = None + self.hydra_context: Optional[HydraContext] = None self.job_results = None self.parametrization: Dict[str, Any] = {} if parametrization is not None: @@ -111,7 +110,7 @@ def setup( ) -> None: self.job_idx = 0 self.config = config - self.config_loader = hydra_context.config_loader + self.hydra_context = hydra_context self.launcher = Plugins.instance().instantiate_launcher( hydra_context=hydra_context, task_function=task_function, config=config ) diff --git a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/nevergrad_sweeper.py b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/nevergrad_sweeper.py index 335441a5aae..69aaca93730 100644 --- a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/nevergrad_sweeper.py +++ b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/nevergrad_sweeper.py @@ -3,7 +3,6 @@ from hydra import TaskFunction from hydra.core.utils import HydraContext -from hydra.core.config_loader import ConfigLoader from hydra.plugins.sweeper import Sweeper from omegaconf import DictConfig diff --git a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/_impl.py b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/_impl.py index 42e73316559..7fa9d9ff441 100644 --- a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/_impl.py +++ b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/_impl.py @@ -4,9 +4,6 @@ from typing import Any, Dict, List, MutableMapping, MutableSequence, Optional import optuna - -from hydra.core.utils import HydraContext -from hydra.core.config_loader import ConfigLoader from hydra.core.override_parser.overrides_parser import OverridesParser from hydra.core.override_parser.types import ( ChoiceSweep, @@ -16,6 +13,7 @@ Transformer, ) from hydra.core.plugins import Plugins +from hydra.core.utils import HydraContext from hydra.plugins.sweeper import Sweeper from hydra.types import TaskFunction from omegaconf import DictConfig, OmegaConf @@ -144,7 +142,7 @@ def setup( ) -> None: self.job_idx = 0 self.config = config - self.config_loader = hydra_context.config_loader + self.hydra_context = hydra_context self.launcher = Plugins.instance().instantiate_launcher( config=config, hydra_context=hydra_context, task_function=task_function ) @@ -153,6 +151,7 @@ def setup( def sweep(self, arguments: List[str]) -> None: assert self.config is not None assert self.launcher is not None + assert self.hydra_context is not None assert self.job_idx is not None parser = OverridesParser.create() diff --git a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/optuna_sweeper.py b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/optuna_sweeper.py index e0a713ae5f5..f7d558e3863 100644 --- a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/optuna_sweeper.py +++ b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/optuna_sweeper.py @@ -2,7 +2,6 @@ from typing import Any, List, Optional from hydra.core.utils import HydraContext -from hydra.core.config_loader import ConfigLoader from hydra.plugins.sweeper import Sweeper from hydra.types import TaskFunction from omegaconf import DictConfig diff --git a/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_core.py b/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_core.py index ece74e8cba4..a2220df279e 100644 --- a/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_core.py +++ b/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_core.py @@ -24,7 +24,7 @@ def launch( ) -> Sequence[JobReturn]: setup_globals() assert launcher.config is not None - assert launcher.config_loader is not None + assert launcher.hydra_context is not None assert launcher.task_function is not None configure_log(launcher.config.hydra.hydra_logging, launcher.config.hydra.verbose) @@ -42,7 +42,7 @@ def launch( idx = initial_job_idx + idx ostr = " ".join(filter_overrides(overrides)) log.info(f"\t#{idx} : {ostr}") - sweep_config = launcher.config_loader.load_sweep_config( + sweep_config = launcher.hydra_context.config_loader.load_sweep_config( launcher.config, list(overrides) ) with open_dict(sweep_config): @@ -52,6 +52,7 @@ def launch( sweep_config.hydra.job.id = f"job_id_for_{idx}" sweep_config.hydra.job.num = idx ray_obj = launch_job_on_ray( + launcher.hydra_context, launcher.ray_cfg.remote, sweep_config, launcher.task_function, diff --git a/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_core_aws.py b/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_core_aws.py index fcd42b3f73e..cd5c1f3fe0d 100644 --- a/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_core_aws.py +++ b/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_core_aws.py @@ -54,7 +54,7 @@ def launch( ) -> Sequence[JobReturn]: setup_globals() assert launcher.config is not None - assert launcher.config_loader is not None + assert launcher.hydra_context is not None assert launcher.task_function is not None setup_commands = launcher.env_setup.commands @@ -80,7 +80,7 @@ def launch( idx = initial_job_idx + idx ostr = " ".join(filter_overrides(overrides)) log.info(f"\t#{idx} : {ostr}") - sweep_config = launcher.config_loader.load_sweep_config( + sweep_config = launcher.hydra_context.config_loader.load_sweep_config( launcher.config, list(overrides) ) with open_dict(sweep_config): @@ -91,6 +91,7 @@ def launch( _pickle_jobs( tmp_dir=local_tmp_dir, + hydra_context=launcher.hydra_context, sweep_configs=sweep_configs, # type: ignore task_function=launcher.task_function, singleton_state=Singleton.get_state(), diff --git a/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_launcher_util.py b/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_launcher_util.py index 85d0bb1fca3..3b237e11f5a 100644 --- a/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_launcher_util.py +++ b/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_launcher_util.py @@ -8,7 +8,7 @@ import ray from hydra.core.hydra_config import HydraConfig from hydra.core.singleton import Singleton -from hydra.core.utils import JobReturn, run_job, setup_globals +from hydra.core.utils import HydraContext, JobReturn, run_job, setup_globals from hydra.types import TaskFunction from omegaconf import DictConfig, OmegaConf @@ -30,6 +30,7 @@ def start_ray(init_cfg: DictConfig) -> None: def _run_job( + hydra_context: HydraContext, sweep_config: DictConfig, task_function: TaskFunction, singleton_state: Dict[Any, Any], @@ -38,14 +39,16 @@ def _run_job( Singleton.set_state(singleton_state) HydraConfig.instance().set_config(sweep_config) return run_job( - config=sweep_config, + hydra_context=hydra_context, task_function=task_function, + config=sweep_config, job_dir_key="hydra.sweep.dir", job_subdir_key="hydra.sweep.subdir", ) def launch_job_on_ray( + hydra_context: HydraContext, ray_remote: DictConfig, sweep_config: DictConfig, task_function: TaskFunction, @@ -57,6 +60,7 @@ def launch_job_on_ray( run_job_ray = ray.remote(_run_job) ret = run_job_ray.remote( + hydra_context=hydra_context, sweep_config=sweep_config, task_function=task_function, singleton_state=singleton_state, diff --git a/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_remote_invoke.py b/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_remote_invoke.py index 725d04fb959..afe046cbf47 100644 --- a/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_remote_invoke.py +++ b/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_remote_invoke.py @@ -30,6 +30,7 @@ def launch_jobs(temp_dir: str) -> None: runs = [] with open(os.path.join(temp_dir, JOB_SPEC_PICKLE), "rb") as f: job_spec = pickle.load(f) # nosec + hydra_context = job_spec["hydra_context"] singleton_state = job_spec["singleton_state"] sweep_configs = job_spec["sweep_configs"] task_function = job_spec["task_function"] @@ -55,7 +56,7 @@ def launch_jobs(temp_dir: str) -> None: start_ray(ray_init) ray_obj = launch_job_on_ray( - ray_remote, sweep_config, task_function, singleton_state + hydra_context, ray_remote, sweep_config, task_function, singleton_state ) runs.append(ray_obj) diff --git a/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/ray_aws_launcher.py b/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/ray_aws_launcher.py index 9e0e0b80d35..c8987f3cc66 100644 --- a/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/ray_aws_launcher.py +++ b/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/ray_aws_launcher.py @@ -2,9 +2,7 @@ import logging from typing import Optional, Sequence -from hydra.core.utils import HydraContext -from hydra.core.config_loader import ConfigLoader -from hydra.core.utils import JobReturn +from hydra.core.utils import HydraContext, JobReturn from hydra.plugins.launcher import Launcher from hydra.types import TaskFunction from omegaconf import DictConfig @@ -31,7 +29,7 @@ def __init__( self.sync_up = sync_up self.sync_down = sync_down self.config: Optional[DictConfig] = None - self.config_loader: Optional[ConfigLoader] = None + self.hydra_context: Optional[HydraContext] = None self.task_function: Optional[TaskFunction] = None self.ray_yaml_path: Optional[str] = None self.env_setup = env_setup @@ -44,7 +42,7 @@ def setup( config: DictConfig, ) -> None: self.config = config - self.config_loader = hydra_context.config_loader + self.hydra_context = hydra_context self.task_function = task_function def launch( diff --git a/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/ray_launcher.py b/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/ray_launcher.py index 6e161e34321..d27b84eff4d 100644 --- a/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/ray_launcher.py +++ b/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/ray_launcher.py @@ -1,9 +1,7 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved from typing import Optional, Sequence -from hydra.core.utils import HydraContext -from hydra.core.config_loader import ConfigLoader -from hydra.core.utils import JobReturn +from hydra.core.utils import HydraContext, JobReturn from hydra.plugins.launcher import Launcher from hydra.types import TaskFunction from omegaconf import DictConfig @@ -12,9 +10,9 @@ class RayLauncher(Launcher): def __init__(self, ray: DictConfig) -> None: self.ray_cfg = ray - self.config: Optional[DictConfig] = None - self.config_loader: Optional[ConfigLoader] = None + self.hydra_context: Optional[HydraContext] = None self.task_function: Optional[TaskFunction] = None + self.config: Optional[DictConfig] = None def setup( self, @@ -24,7 +22,7 @@ def setup( config: DictConfig, ) -> None: self.config = config - self.config_loader = hydra_context.config_loader + self.hydra_context = hydra_context self.task_function = task_function def launch( diff --git a/plugins/hydra_rq_launcher/hydra_plugins/hydra_rq_launcher/_core.py b/plugins/hydra_rq_launcher/hydra_plugins/hydra_rq_launcher/_core.py index 977ba60b996..588f12582ee 100644 --- a/plugins/hydra_rq_launcher/hydra_plugins/hydra_rq_launcher/_core.py +++ b/plugins/hydra_rq_launcher/hydra_plugins/hydra_rq_launcher/_core.py @@ -10,6 +10,7 @@ from hydra.core.hydra_config import HydraConfig from hydra.core.singleton import Singleton from hydra.core.utils import ( + HydraContext, JobReturn, configure_log, filter_overrides, @@ -27,6 +28,7 @@ def execute_job( + hydra_context: HydraContext, sweep_config: DictConfig, task_function: TaskFunction, singleton_state: Dict[Any, Any], @@ -37,8 +39,9 @@ def execute_job( HydraConfig.instance().set_config(sweep_config) ret = run_job( - config=sweep_config, + hydra_context=hydra_context, task_function=task_function, + config=sweep_config, job_dir_key="hydra.sweep.dir", job_subdir_key="hydra.sweep.subdir", ) @@ -56,8 +59,8 @@ def launch( """ setup_globals() assert launcher.config is not None - assert launcher.config_loader is not None assert launcher.task_function is not None + assert launcher.hydra_context is not None configure_log(launcher.config.hydra.hydra_logging, launcher.config.hydra.verbose) sweep_dir = Path(str(launcher.config.hydra.sweep.dir)) @@ -113,7 +116,7 @@ def launch( if enqueue_keywords["description"] is None: enqueue_keywords["description"] = description - sweep_config = launcher.config_loader.load_sweep_config( + sweep_config = launcher.hydra_context.config_loader.load_sweep_config( launcher.config, list(overrides) ) with open_dict(sweep_config): @@ -122,6 +125,7 @@ def launch( job = queue.enqueue( execute_job, + hydra_context=launcher.hydra_context, sweep_config=sweep_config, task_function=launcher.task_function, singleton_state=singleton_state, diff --git a/plugins/hydra_rq_launcher/hydra_plugins/hydra_rq_launcher/rq_launcher.py b/plugins/hydra_rq_launcher/hydra_plugins/hydra_rq_launcher/rq_launcher.py index 5fd18d88ff7..99bc70391eb 100644 --- a/plugins/hydra_rq_launcher/hydra_plugins/hydra_rq_launcher/rq_launcher.py +++ b/plugins/hydra_rq_launcher/hydra_plugins/hydra_rq_launcher/rq_launcher.py @@ -2,9 +2,7 @@ import logging from typing import Any, Optional, Sequence -from hydra.core.utils import HydraContext -from hydra.core.config_loader import ConfigLoader -from hydra.core.utils import JobReturn +from hydra.core.utils import HydraContext, JobReturn from hydra.plugins.launcher import Launcher from hydra.types import TaskFunction from omegaconf import DictConfig, OmegaConf @@ -22,8 +20,8 @@ def __init__(self, **params: Any) -> None: https://python-rq.org """ self.config: Optional[DictConfig] = None - self.config_loader: Optional[ConfigLoader] = None self.task_function: Optional[TaskFunction] = None + self.hydra_context: Optional[HydraContext] = None self.rq = OmegaConf.structured(RQLauncherConf(**params)) @@ -35,8 +33,8 @@ def setup( config: DictConfig, ) -> None: self.config = config - self.config_loader = hydra_context.config_loader self.task_function = task_function + self.hydra_context = hydra_context def launch( self, job_overrides: Sequence[Sequence[str]], initial_job_idx: int diff --git a/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py b/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py index e0d09057a3f..6b77f2a74f4 100644 --- a/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py +++ b/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py @@ -5,10 +5,14 @@ from typing import Any, Dict, List, Optional, Sequence from hydra import TaskFunction -from hydra.core.utils import HydraContext -from hydra.core.config_loader import ConfigLoader from hydra.core.singleton import Singleton -from hydra.core.utils import JobReturn, filter_overrides, run_job, setup_globals +from hydra.core.utils import ( + HydraContext, + JobReturn, + filter_overrides, + run_job, + setup_globals, +) from hydra.plugins.launcher import Launcher from omegaconf import DictConfig, OmegaConf, open_dict @@ -29,9 +33,9 @@ def __init__(self, **params: Any) -> None: self.params[k] = v self.config: Optional[DictConfig] = None - self.config_loader: Optional[ConfigLoader] = None self.task_function: Optional[TaskFunction] = None self.sweep_configs: Optional[TaskFunction] = None + self.hydra_context: Optional[HydraContext] = None def setup( self, @@ -41,7 +45,7 @@ def setup( config: DictConfig, ) -> None: self.config = config - self.config_loader = hydra_context.config_loader + self.hydra_context = hydra_context self.task_function = task_function def __call__( @@ -55,13 +59,13 @@ def __call__( # lazy import to ensure plugin discovery remains fast import submitit - assert self.config_loader is not None + assert self.hydra_context is not None assert self.config is not None assert self.task_function is not None Singleton.set_state(singleton_state) setup_globals() - sweep_config = self.config_loader.load_sweep_config( + sweep_config = self.hydra_context.config_loader.load_sweep_config( self.config, sweep_overrides ) @@ -71,8 +75,9 @@ def __call__( sweep_config.hydra.job.num = job_num return run_job( - config=sweep_config, + hydra_context=self.hydra_context, task_function=self.task_function, + config=sweep_config, job_dir_key=job_dir_key, job_subdir_key="hydra.sweep.subdir", ) diff --git a/tests/test_callbacks.py b/tests/test_callbacks.py index 6d045c0fff6..e64b503690a 100644 --- a/tests/test_callbacks.py +++ b/tests/test_callbacks.py @@ -23,7 +23,6 @@ """\ [HYDRA] Init custom_callback [HYDRA] custom_callback on_run_start - [HYDRA] Init custom_callback [JOB] custom_callback on_job_start [JOB] foo: bar @@ -42,7 +41,6 @@ [HYDRA] custom_callback on_multirun_start [HYDRA] Launching 1 jobs locally [HYDRA] \t#0 : foo=bar - [HYDRA] Init custom_callback [JOB] custom_callback on_job_start [JOB] foo: bar @@ -61,8 +59,6 @@ [HYDRA] Init callback_2 [HYDRA] callback_1 on_run_start [HYDRA] callback_2 on_run_start - [HYDRA] Init callback_1 - [HYDRA] Init callback_2 [JOB] callback_1 on_job_start [JOB] callback_2 on_job_start [JOB] {}