diff --git a/hydra/grammar/OverrideLexer.g4 b/hydra/grammar/OverrideLexer.g4 index 93dff89326a..82e2670465d 100644 --- a/hydra/grammar/OverrideLexer.g4 +++ b/hydra/grammar/OverrideLexer.g4 @@ -58,7 +58,7 @@ BOOL: NULL: [Nn][Uu][Ll][Ll]; -UNQUOTED_CHAR: [/\-\\+.$%*@]; // other characters allowed in unquoted strings +UNQUOTED_CHAR: [/\-\\+.$%*@?]; // other characters allowed in unquoted strings ID: (CHAR|'_') (CHAR|DIGIT|'_')*; // Note: when adding more characters to the ESC rule below, also add them to // the `_ESC` string in `_internal/grammar/utils.py`. diff --git a/hydra/grammar/OverrideParser.g4 b/hydra/grammar/OverrideParser.g4 index 77ef6f97df0..739c257ad74 100644 --- a/hydra/grammar/OverrideParser.g4 +++ b/hydra/grammar/OverrideParser.g4 @@ -59,7 +59,7 @@ primitive: | FLOAT // 3.14, -20.0, 1e-1, -10e3 | BOOL // true, TrUe, false, False | INTERPOLATION // ${foo.bar}, ${oc.env:USER,me} - | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @ + | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @, ? | COLON // : | ESC // \\, \(, \), \[, \], \{, \}, \:, \=, \ , \\t, \, | WS // whitespaces @@ -73,7 +73,7 @@ dictKey: | INT // 0, 10, -20, 1_000_000 | FLOAT // 3.14, -20.0, 1e-1, -10e3 | BOOL // true, TrUe, false, False - | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @ + | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @, ? | ESC // \\, \(, \), \[, \], \{, \}, \:, \=, \ , \\t, \, | WS // whitespaces )+; diff --git a/news/1597.feature b/news/1597.feature new file mode 100644 index 00000000000..02a4cb121ea --- /dev/null +++ b/news/1597.feature @@ -0,0 +1 @@ +Override grammar: question marks are now allowed in unquoted strings (e.g. "key=abc?def") diff --git a/tests/test_overrides_parser.py b/tests/test_overrides_parser.py index 72c8c75a913..2bdd4a47821 100644 --- a/tests/test_overrides_parser.py +++ b/tests/test_overrides_parser.py @@ -31,6 +31,8 @@ ) from hydra.errors import HydraException +UNQUOTED_SPECIAL = r"/-\+.$%*@?" # special characters allowed in unquoted strings + parser = OverridesParser(create_functions()) @@ -285,7 +287,11 @@ def test_shuffle_sequence(value: str, expected: Any) -> None: param("{123: 1, 0: 2, -1: 3}", {123: 1, 0: 2, -1: 3}, id="dict_int_key"), param("{3.14: 0, 1e3: 1}", {3.14: 0, 1000.0: 1}, id="dict_float_key"), param("{true: 1, fAlSe: 0}", {True: 1, False: 0}, id="dict_bool_key"), - param("{/-\\+.$%*@: 1}", {"/-\\+.$%*@": 1}, id="dict_unquoted_char_key"), + param( + "{%s: 1}" % UNQUOTED_SPECIAL, + {UNQUOTED_SPECIAL: 1}, + id="dict_unquoted_char_key", + ), param( "{\\\\\\(\\)\\[\\]\\{\\}\\:\\=\\ \\\t\\,: 1}", {"\\()[]{}:= \t,": 1}, @@ -497,7 +503,7 @@ def test_key(value: str, expected: Any) -> None: "value,expected", [ param("a", "a", id="a"), - param("/:-\\+.$%*@", "/:-\\+.$%*@", id="accepted_specials"), + param(UNQUOTED_SPECIAL, UNQUOTED_SPECIAL, id="accepted_specials"), param("abc10", "abc10", id="abc10"), param("a.b.c", "a.b.c", id="a.b.c"), param("list.0.bar", "list.0.bar", id="list.0.bar"), @@ -945,8 +951,9 @@ def test_get_key_element(override: str, expected: str) -> None: id="dict_quoted_key", ), param( - "key={/-\\+.$%*@: 1}", - "{/-\\\\+.$%*@: 1}", # note that \ gets escaped + "key={%s: 1}" % UNQUOTED_SPECIAL, + # Note that \ gets escaped. + "{%s: 1}" % UNQUOTED_SPECIAL.replace("\\", "\\\\"), True, id="dict_unquoted_key_special", ), @@ -993,7 +1000,11 @@ def test_override_get_value_element_method( param("key={123id: 0}", {"123id": 0}, id="dict_key_int_plus_id"), param("key={' abc ': 0}", {" abc ": 0}, id="dict_key_quoted_single"), param('key={" abc ": 0}', {" abc ": 0}, id="dict_key_quoted_double"), - param("key={a/-\\+.$%*@: 0}", {"a/-\\+.$%*@": 0}, id="dict_key_noquote"), + param( + "key={a%s: 0}" % UNQUOTED_SPECIAL, + {f"a{UNQUOTED_SPECIAL}": 0}, + id="dict_key_noquote", + ), param("key={w s: 0}", {"w s": 0}, id="dict_key_ws"), param( "key={\\\\\\(\\)\\[\\]\\{\\}\\:\\=\\ \\\t\\,: 0}", diff --git a/website/docs/advanced/override_grammar/basic.md b/website/docs/advanced/override_grammar/basic.md index 384ef016524..2d037944910 100644 --- a/website/docs/advanced/override_grammar/basic.md +++ b/website/docs/advanced/override_grammar/basic.md @@ -85,7 +85,7 @@ primitive: | FLOAT // 3.14, -20.0, 1e-1, -10e3 | BOOL // true, TrUe, false, False | INTERPOLATION // ${foo.bar}, ${oc.env:USER,me} - | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @ + | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @, ? | COLON // : | ESC // \\, \(, \), \[, \], \{, \}, \:, \=, \ , \\t, \, | WS // whitespaces @@ -99,7 +99,7 @@ dictKey: | INT // 0, 10, -20, 1_000_000 | FLOAT // 3.14, -20.0, 1e-1, -10e3 | BOOL // true, TrUe, false, False - | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @ + | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @, ? | ESC // \\, \(, \), \[, \], \{, \}, \:, \=, \ , \\t, \, | WS // whitespaces )+;