diff --git a/dvc/utils/hydra.py b/dvc/utils/hydra.py index 274ae2fd0e..5674c9696b 100644 --- a/dvc/utils/hydra.py +++ b/dvc/utils/hydra.py @@ -1,27 +1,9 @@ -import sys from pathlib import Path from typing import TYPE_CHECKING, List -try: - from hydra import compose, initialize_config_dir - from hydra._internal.config_loader_impl import ConfigLoaderImpl - from hydra._internal.core_plugins.basic_sweeper import BasicSweeper - from hydra.core.override_parser.overrides_parser import OverridesParser - from hydra.core.override_parser.types import ValueType - from hydra.errors import ConfigCompositionException, OverrideParseException - from omegaconf import OmegaConf - - hydra_compatible = True -except ValueError: - if sys.version_info >= (3, 11): - hydra_compatible = False - else: - raise - -from dvc.exceptions import DvcException, InvalidArgumentError +from dvc.exceptions import InvalidArgumentError from .collections import merge_dicts, remove_missing_keys, to_omegaconf -from .serialize import DUMPERS, MODIFIERS if TYPE_CHECKING: from dvc.types import StrPath @@ -46,10 +28,10 @@ def compose_and_dump( .. _Hydra Override: https://hydra.cc/docs/advanced/override_grammar/basic/ """ - if not hydra_compatible: - raise DvcException( - "hydra functionality is not supported in Python >= 3.11" - ) + from hydra import compose, initialize_config_dir + from omegaconf import OmegaConf + + from .serialize import DUMPERS with initialize_config_dir(config_dir, version_base=None): cfg = compose(config_name=config_name, overrides=overrides) @@ -71,10 +53,11 @@ def apply_overrides(path: "StrPath", overrides: List[str]) -> None: .. _Hydra Override: https://hydra.cc/docs/next/advanced/override_grammar/basic/ """ - if not hydra_compatible: - raise DvcException( - "hydra functionality is not supported in Python >= 3.11" - ) + from hydra._internal.config_loader_impl import ConfigLoaderImpl + from hydra.errors import ConfigCompositionException, OverrideParseException + from omegaconf import OmegaConf + + from .serialize import MODIFIERS suffix = Path(path).suffix.lower() @@ -101,10 +84,7 @@ def apply_overrides(path: "StrPath", overrides: List[str]) -> None: def to_hydra_overrides(path_overrides): - if not hydra_compatible: - raise DvcException( - "hydra functionality is not supported in Python >= 3.11" - ) + from hydra.core.override_parser.overrides_parser import OverridesParser parser = OverridesParser.create() return parser.parse_overrides(overrides=path_overrides) @@ -117,10 +97,8 @@ def dict_product(dicts): def get_hydra_sweeps(path_overrides): - if not hydra_compatible: - raise DvcException( - "hydra functionality is not supported in Python >= 3.11" - ) + from hydra._internal.core_plugins.basic_sweeper import BasicSweeper + from hydra.core.override_parser.types import ValueType path_sweeps = {} for path, overrides in path_overrides.items(): diff --git a/pyproject.toml b/pyproject.toml index 716b55fdac..78e422fa84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,7 +161,7 @@ line_length = 79 [tool.pytest.ini_options] log_level = "debug" -addopts = "-ra -p no:hydra_pytest --cov-config pyproject.toml" +addopts = "-ra --cov-config pyproject.toml" markers = [ "needs_internet: Might need network access for the tests", "vscode: Tests verifying contract between DVC and VSCode plugin", diff --git a/tests/conftest.py b/tests/conftest.py index 72835f6f67..5337d4ff0e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -250,18 +250,6 @@ def run( return run -@pytest.fixture(autouse=True) -def mock_hydra_conf(mocker): - if sys.version_info < (3, 11): - return - - # `hydra.conf` fails to import in 3.11, it raises ValueError due to changes - # in dataclasses. See https://github.com/python/cpython/pull/29867. - # NOTE: using sentinel here so that any imports from `hydra.conf` - # return a mock. - sys.modules["hydra.conf"] = mocker.sentinel - - @pytest.fixture(autouse=True) def gc_collect_on_dvc_close_on_win_311(mocker): if sys.version_info < (3, 11) and os.name != "nt": diff --git a/tests/func/experiments/test_set_params.py b/tests/func/experiments/test_set_params.py index 5095c666d1..cfa8e25bed 100644 --- a/tests/func/experiments/test_set_params.py +++ b/tests/func/experiments/test_set_params.py @@ -1,5 +1,3 @@ -import sys - import pytest from dvc.exceptions import InvalidArgumentError @@ -21,18 +19,7 @@ def test_modify_params(params_repo, dvc, changes, expected): assert fobj.read().strip() == expected -@pytest.mark.parametrize( - "hydra_enabled", - [ - pytest.param( - True, - marks=pytest.mark.skipif( - sys.version_info >= (3, 11), reason="unsupported on 3.11" - ), - ), - False, - ], -) +@pytest.mark.parametrize("hydra_enabled", [True, False]) @pytest.mark.parametrize( "config_dir,config_name", [ @@ -93,12 +80,7 @@ def test_hydra_compose_and_dump( "hydra_enabled,overrides,expected", [ ( - pytest.param( - True, - marks=pytest.mark.xfail( - sys.version_info >= (3, 11), reason="unsupported on 3.11" - ), - ), + True, ["db=mysql,postgresql"], [ {"params.yaml": ["db=mysql"]}, diff --git a/tests/func/utils/test_hydra.py b/tests/func/utils/test_hydra.py index cfde109a69..d2f7679c45 100644 --- a/tests/func/utils/test_hydra.py +++ b/tests/func/utils/test_hydra.py @@ -1,5 +1,3 @@ -import sys - import pytest from dvc.exceptions import InvalidArgumentError @@ -93,16 +91,16 @@ def test_apply_overrides(tmp_dir, suffix, overrides, expected): from dvc.utils.hydra import apply_overrides - if suffix == "toml": - if overrides in [ - ["foo=baz"], - ["foo.0=bar"], - ["foo=baz", "goo=bar"], - ["lorem=null"], - ]: - # TOML dumper breaks when overriding a list/dict with other type - # or when handling `null` values. - pytest.xfail() + if suffix == "toml" and overrides in [ + ["foo=baz"], + ["foo.0=bar"], + ["foo=baz", "goo=bar"], + ["lorem=null"], + ]: + pytest.skip( + "TOML dumper breaks when overriding a list/dict with other type or" + " when handling `null` values." + ) params_file = tmp_dir / f"params.{suffix}" params_file.dump( @@ -140,7 +138,6 @@ def hydra_setup(tmp_dir, config_dir, config_name): return str(config_dir) -@pytest.mark.skipif(sys.version_info >= (3, 11), reason="unsupported on 3.11") @pytest.mark.parametrize("suffix", ["yaml", "toml", "json"]) @pytest.mark.parametrize( "overrides,expected", @@ -180,7 +177,6 @@ def test_compose_and_dump(tmp_dir, suffix, overrides, expected): assert output_file.parse() == expected -@pytest.mark.skipif(sys.version_info >= (3, 11), reason="unsupported on 3.11") def test_compose_and_dump_yaml_handles_string(tmp_dir): """Regression test for 8583""" from dvc.utils.hydra import compose_and_dump