Skip to content

Commit

Permalink
Update config_dict_generator with queue_config dicts
Browse files Browse the repository at this point in the history
  • Loading branch information
xjules committed Nov 1, 2023
1 parent 1ff6e86 commit f0f2d5c
Show file tree
Hide file tree
Showing 6 changed files with 42 additions and 70 deletions.
9 changes: 8 additions & 1 deletion src/ert/config/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,14 @@
from .observations import EnkfObs
from .parameter_config import ParameterConfig
from .parsing import ConfigValidationError, ConfigWarning
from .queue_config import QueueConfig
from .queue_config import (
QueueConfig,
queue_bool_options,
queue_memory_options,
queue_positive_int_options,
queue_positive_number_options,
queue_string_options,
)
from .queue_system import QueueSystem
from .response_config import ResponseConfig
from .summary_config import SummaryConfig
Expand Down
20 changes: 10 additions & 10 deletions src/ert/config/queue_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,14 +113,14 @@ def generate_dict(option_list: List[Tuple[str, str]]) -> Dict[str, List[str]]:
)


memory_options: Mapping[str, List[str]] = {
queue_memory_options: Mapping[str, List[str]] = {
"LSF": [],
"SLURM": [],
"TORQUE": ["MEMORY_PER_JOB"],
"LOCAL": [],
}

string_options: Mapping[str, List[str]] = {
queue_string_options: Mapping[str, List[str]] = {
"LSF": [
"DEBUG_OUTPUT",
"LSF_RESOURCE",
Expand Down Expand Up @@ -157,7 +157,7 @@ def generate_dict(option_list: List[Tuple[str, str]]) -> Dict[str, List[str]]:
"LOCAL": [],
}

positive_int_options: Mapping[str, List[str]] = {
queue_positive_int_options: Mapping[str, List[str]] = {
"LSF": [
"BJOBS_TIMEOUT",
"MAX_RUNNING",
Expand All @@ -175,7 +175,7 @@ def generate_dict(option_list: List[Tuple[str, str]]) -> Dict[str, List[str]]:
"LOCAL": ["MAX_RUNNING"],
}

positive_number_options: Mapping[str, List[str]] = {
queue_positive_number_options: Mapping[str, List[str]] = {
"LSF": [
"SUBMIT_SLEEP",
],
Expand All @@ -187,7 +187,7 @@ def generate_dict(option_list: List[Tuple[str, str]]) -> Dict[str, List[str]]:
"LOCAL": [],
}

bool_options: Mapping[str, List[str]] = {
queue_bool_options: Mapping[str, List[str]] = {
"LSF": ["DEBUG_OUTPUT"],
"SLURM": [],
"TORQUE": [],
Expand All @@ -204,7 +204,7 @@ def _validate_queue_driver_settings(
if option_value == "": # This is equivalent to the option not being set
continue
if (
option_name in memory_options[queue_type]
option_name in queue_memory_options[queue_type]
and re.match("[0-9]+[mg]b", option_value) is None
):
raise ConfigValidationError.with_context(
Expand All @@ -213,31 +213,31 @@ def _validate_queue_driver_settings(
"the format '<integer>mb' or '<integer>gb'.",
option_value,
)
if option_name in string_options[queue_type] and not isinstance(
if option_name in queue_string_options[queue_type] and not isinstance(
option_value, str
):
raise ConfigValidationError.with_context(
f"'{option_value}' for {option_name} is not a valid string type "
f"for the {queue_type} QUEUE.",
option_value,
)
if option_name in positive_number_options[queue_type] and (
if option_name in queue_positive_number_options[queue_type] and (
re.match(r"^\d+(\.\d+)?$", option_value) is None
):
raise ConfigValidationError.with_context(
f"'{option_value}' for {option_name} is not a valid integer or float"
f" for the {queue_type} QUEUE.",
option_value,
)
if option_name in positive_int_options[queue_type] and (
if option_name in queue_positive_int_options[queue_type] and (
re.match(r"^\d+$", option_value) is None
):
raise ConfigValidationError.with_context(
f"'{option_value}' for {option_name} is not a valid positive integer"
f" for the {queue_type} QUEUE.",
option_value,
)
if option_name in bool_options[queue_type] and not option_value in [
if option_name in queue_bool_options[queue_type] and not option_value in [
"TRUE",
"FALSE",
"0",
Expand Down
71 changes: 18 additions & 53 deletions tests/unit_tests/config/config_dict_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,14 @@
from pydantic import PositiveInt

from ert import _clib
from ert.config import QueueSystem
from ert.config import (
QueueSystem,
queue_bool_options,
queue_memory_options,
queue_positive_int_options,
queue_positive_number_options,
queue_string_options,
)
from ert.config.field import TRANSFORM_FUNCTIONS
from ert.config.parsing import ConfigKeys

Expand Down Expand Up @@ -95,59 +102,17 @@ def valid_queue_options(queue_system: str):
return valids


def valid_queue_values(option_name):
if option_name in [
"QSUB_CMD",
"QSTAT_CMD",
"QSTAT_OPTIONS",
"QDEL_CMD",
"QUEUE",
"CLUSTER_LABEL",
"JOB_PREFIX",
"DEBUG_OUTPUT",
"LSF_RESOURCE",
"LSF_SERVER",
"LSF_QUEUE",
"LSF_LOGIN_SHELL",
"LSF_RSH_CMD",
"BSUB_CMD",
"BJOBS_CMD",
"BKILL_CMD",
"BHIST_CMD",
"BJOBS_TIMEOUT",
"EXCLUDE_HOST",
"PARTITION",
"PROJECT_CODE",
"SBATCH",
"SCANCEL",
"SCONTROL",
"SQUEUE",
"MEMORY",
"MEMORY_PER_CPU",
"EXCLUDE_HOST",
"INCLUDE_HOST",
]:
def valid_queue_values(option_name, queue_system):
if option_name in queue_string_options[queue_system]:
return words
if option_name in [
"SUBMIT_SLEEP",
"SQUEUE_TIMEOUT",
]:
return st.builds(str, small_floats)
if option_name in ["MEMORY_PER_JOB"]:
elif option_name in queue_positive_number_options[queue_system]:
return small_floats
elif option_name in queue_positive_int_options[queue_system]:
return positives
elif option_name in queue_bool_options[queue_system]:
return booleans
elif option_name in queue_memory_options[queue_system]:
return st.builds(str, memory_with_unit())
if option_name in [
"NUM_CPUS_PER_NODE",
"NUM_NODES",
"MAX_RUNNING",
"MAX_RUNTIME",
"QUEUE_QUERY_TIMEOUT",
]:
return st.builds(str, positives)
if option_name in [
"KEEP_QSUB_OUTPUT",
"DEBUG_OUTPUT",
]:
return st.builds(str, booleans)
else:
raise ValueError(
"config_dict_generator does not know how to "
Expand All @@ -161,7 +126,7 @@ def queue_options(draw, systems):
name = draw(st.sampled_from(valid_queue_options(queue_system)))
do_set = draw(booleans)
if do_set:
return [queue_system, name, draw(valid_queue_values(name))]
return [queue_system, name, draw(valid_queue_values(name, queue_system))]
else:
# Missing VALUE means unset
return [queue_system, name]
Expand Down
4 changes: 2 additions & 2 deletions tests/unit_tests/config/test_ert_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -707,8 +707,8 @@ def test_default_ens_path(tmpdir):
"max_running_value, expected_error",
[
(100, None), # positive integer
(-1, "not valid positive integer"), # negative integer
("not_an_integer", "not valid positive integer"), # non-integer
(-1, "is not a valid positive integer"), # negative integer
("not_an_integer", "is not a valid positive integer"), # non-integer
],
)
def test_queue_config_max_running_invalid_values(max_running_value, expected_error):
Expand Down
2 changes: 1 addition & 1 deletion tests/unit_tests/config/test_parser_error_collection.py
Original file line number Diff line number Diff line change
Expand Up @@ -477,7 +477,7 @@ def test_queue_option_max_running_non_int():
line=4,
column=32,
end_column=33,
match=r"'ert' for MAX_RUNNING is not a valid positive integer for the LOCAL QUEUE.",
match=r"'s' for MAX_RUNNING is not a valid positive integer for the LOCAL QUEUE.",
),
)

Expand Down
6 changes: 3 additions & 3 deletions tests/unit_tests/config/test_queue_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,9 +209,9 @@ def test_initializing_empty_config_values(queue_system, queue_system_option):
@pytest.mark.parametrize(
"queue_system, queue_option, queue_value, err_msg",
[
("LSF", "BJOBS_TIMEOUT", "-3", "is not valid positive integer"),
("SLURM", "SQUEUE_TIMEOUT", "5a", "is not valid integer or float"),
("TORQUE", "NUM_NODES", "3.5", "is not valid positive integer"),
("LSF", "BJOBS_TIMEOUT", "-3", "is not a valid positive integer"),
("SLURM", "SQUEUE_TIMEOUT", "5a", "is not a valid integer or float"),
("TORQUE", "NUM_NODES", "3.5", "is not a valid positive integer"),
],
)
def test_wrong_config_option_types(queue_system, queue_option, queue_value, err_msg):
Expand Down

0 comments on commit f0f2d5c

Please sign in to comment.