Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Self-references and += operator #81

Merged
merged 8 commits into from
May 6, 2016
Merged

Self-references and += operator #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c0afa2a
Implement self referential substitutions
Apr 6, 2016
bde3f44
Add unittest which tests self-references over ConfigTree boundaries
Apr 6, 2016
b2a639c
Update README.md and CHANGELOG.md
Apr 6, 2016
d390422
Fix formatting issues
Apr 7, 2016
4250bf1
Implement += operator
Apr 7, 2016
f867c28
Implement += for strings and make += operator work over parse boundaries
Apr 7, 2016
c692045
Merge remote-tracking branch 'origin/master' into topic/self-reference
gilles-duboscq May 4, 2016
fa7ad25
Fix python 2.6 compatibility issue
gilles-duboscq May 4, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# Changelog

# Version 0.3.xx (TBA)

* Implemented self-referential subsitutions

# Version 0.3.25

* ConfigValue.transform: do not wrap lists. PR [#76]
Expand Down
5 changes: 3 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -300,8 +300,8 @@ Arrays without commas | :white_check_mark:
Path expressions | :white_check_mark:
Paths as keys | :white_check_mark:
Substitutions | :white_check_mark:
Self-referential substitutions | :x:
The `+=` separator | :x:
Self-referential substitutions | :white_check_mark:
The `+=` separator | :white_check_mark:
Includes | :white_check_mark:
Include semantics: merging | :white_check_mark:
Include semantics: substitution | :white_check_mark:
Expand Down Expand Up @@ -335,6 +335,7 @@ Java properties mapping | :x:
- Virgil Palanciuc ([@virgil-palanciuc](https://github.com/virgil-palanciuc))
- Douglas Simon ([@dougxc](https://github.com/dougxc))
- Gilles Duboscq ([@gilles-duboscq](https://github.com/gilles-duboscq))
- Stefan Anzinger ([@sanzinger](https://github.com/sanzinger))

### Thanks

Expand Down
212 changes: 131 additions & 81 deletions pyhocon/config_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import socket
import contextlib
from pyparsing import Forward, Keyword, QuotedString, Word, Literal, Suppress, Regex, Optional, SkipTo, ZeroOrMore, \
Group, lineno, col, TokenConverter, replaceWith, alphanums
Group, lineno, col, TokenConverter, replaceWith, alphanums, ParseSyntaxException
from pyparsing import ParserElement
from pyhocon.config_tree import ConfigTree, ConfigSubstitution, ConfigList, ConfigValues, ConfigUnquotedString, \
ConfigInclude, NoneValue
Expand Down Expand Up @@ -230,27 +230,30 @@ def include_config(token):
(Keyword('url') | Keyword('file')) - Literal('(').suppress() - quoted_string - Literal(')').suppress()))) \
.setParseAction(include_config)

root_dict_expr = Forward()
dict_expr = Forward()
list_expr = Forward()
multi_value_expr = ZeroOrMore((Literal(
'\\') - eol).suppress() | comment_eol | include_expr | substitution_expr | dict_expr | list_expr | value_expr)
# for a dictionary : or = is optional
# last zeroOrMore is because we can have t = {a:4} {b: 6} {c: 7} which is dictionary concatenation
inside_dict_expr = ConfigTreeParser(ZeroOrMore(comment_eol | include_expr | assign_expr | eol_comma))
inside_root_dict_expr = ConfigTreeParser(ZeroOrMore(comment_eol | include_expr | assign_expr | eol_comma), root=True)
dict_expr << Suppress('{') - inside_dict_expr - Suppress('}')
root_dict_expr << Suppress('{') - inside_root_dict_expr - Suppress('}')
list_entry = ConcatenatedValueParser(multi_value_expr)
list_expr << Suppress('[') - ListParser(list_entry - ZeroOrMore(eol_comma - list_entry)) - Suppress(']')

# special case when we have a value assignment where the string can potentially be the remainder of the line
assign_expr << Group(
key -
ZeroOrMore(comment_no_comma_eol) -
(dict_expr | Suppress(Literal('=') | Literal(':')) - ZeroOrMore(
(dict_expr | (Literal('=') | Literal(':') | Literal('+=')) - ZeroOrMore(
comment_no_comma_eol) - ConcatenatedValueParser(multi_value_expr))
)

# the file can be { ... } where {} can be omitted or []
config_expr = ZeroOrMore(comment_eol | eol) + (list_expr | dict_expr | inside_dict_expr) + ZeroOrMore(
config_expr = ZeroOrMore(comment_eol | eol) + (list_expr | root_dict_expr | inside_root_dict_expr) + ZeroOrMore(
comment_eol | eol_comma)
config = config_expr.parseString(content, parseAll=True)[0]
if resolve:
Expand Down Expand Up @@ -290,41 +293,106 @@ def _resolve_variable(config, substitution):
col=col(substitution.loc, substitution.instring)))
return True, value

@staticmethod
def _fixup_self_references(config):
if isinstance(config, ConfigTree) and config.root:
for key in config: # Traverse history of element
history = config.history[key]
previous_item = history[0]
for current_item in history[1:]:
for substitution in ConfigParser._find_substitutions(current_item):
prop_path = ConfigTree.parse_key(substitution.variable)
if len(prop_path) > 1 and config.get(substitution.variable, None) is not None:
continue # If value is present in latest version, don't do anything
if prop_path[0] == key:
if isinstance(previous_item, ConfigValues): # We hit a dead end, we cannot evaluate
raise ConfigSubstitutionException("Property {variable} cannot be substituted. Check for cycles.".format(
variable=substitution.variable))
value = previous_item if len(prop_path) == 1 else previous_item.get(".".join(prop_path[1:]))
(_, _, current_item) = ConfigParser._do_substitute(substitution, value)
previous_item = current_item

if len(history) == 1: # special case, when self optional referencing without existing
for substitution in ConfigParser._find_substitutions(previous_item):
prop_path = ConfigTree.parse_key(substitution.variable)
if len(prop_path) > 1 and config.get(substitution.variable, None) is not None:
continue # If value is present in latest version, don't do anything
if prop_path[0] == key and substitution.optional:
ConfigParser._do_substitute(substitution, None)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sanzinger Really nice implementation for supporting self references! 👍

# traverse config to find all the substitutions
@staticmethod
def _find_substitutions(item):
"""Convert HOCON input into a JSON output

:return: JSON string representation
:type return: basestring
"""
if isinstance(item, ConfigValues):
return item.get_substitutions()

substitutions = []
if isinstance(item, ConfigTree):
for key, child in item.items():
substitutions += ConfigParser._find_substitutions(child)
elif isinstance(item, list):
for child in item:
substitutions += ConfigParser._find_substitutions(child)
return substitutions

@staticmethod
def _do_substitute(substitution, resolved_value, is_optional_resolved=True):
unresolved = False
new_substitutions = []
if isinstance(resolved_value, ConfigValues):
resolved_value = resolved_value.transform()
if isinstance(resolved_value, ConfigValues):
unresolved = True
result = None
else:
# replace token by substitution
config_values = substitution.parent
# if it is a string, then add the extra ws that was present in the original string after the substitution
formatted_resolved_value = resolved_value \
if resolved_value is None \
or isinstance(resolved_value, (dict, list)) \
or substitution.index == len(config_values.tokens) - 1 \
else (str(resolved_value) + substitution.ws)
config_values.put(substitution.index, formatted_resolved_value)
transformation = config_values.transform()
result = None
if transformation is None and not is_optional_resolved:
result = config_values.overriden_value
else:
result = transformation

if result is None:
del config_values.parent[config_values.key]
else:
config_values.parent[config_values.key] = result
s = ConfigParser._find_substitutions(result)
if s:
new_substitutions = s
unresolved = True

return (unresolved, new_substitutions, result)

@staticmethod
def _final_fixup(item):
if isinstance(item, ConfigValues):
return item.transform()
elif isinstance(item, list):
return list([ConfigParser._final_fixup(child) for child in item])
elif isinstance(item, ConfigTree):
items = list(item.items())
for key, child in items:
item[key] = ConfigParser._final_fixup(child)
return item

@staticmethod
def resolve_substitutions(config):
# traverse config to find all the substitutions
def find_substitutions(item):
"""Convert HOCON input into a JSON output

:return: JSON string representation
:type return: basestring
"""
if isinstance(item, ConfigValues):
return item.get_substitutions()

substitutions = []
if isinstance(item, ConfigTree):
for key, child in item.items():
substitutions += find_substitutions(child)
elif isinstance(item, list):
for child in item:
substitutions += find_substitutions(child)

return substitutions

def final_fixup(item):
if isinstance(item, ConfigValues):
return item.transform()
elif isinstance(item, list):
return list([final_fixup(child) for child in item])
elif isinstance(item, ConfigTree):
items = list(item.items())
for key, child in items:
item[key] = final_fixup(child)

return item

substitutions = find_substitutions(config)
ConfigParser._fixup_self_references(config)
substitutions = ConfigParser._find_substitutions(config)

if len(substitutions) > 0:
unresolved = True
Expand All @@ -340,42 +408,11 @@ def final_fixup(item):
if not is_optional_resolved and substitution.optional:
resolved_value = None

if isinstance(resolved_value, ConfigValues):
resolved_value = resolved_value.transform()
if isinstance(resolved_value, ConfigValues):
unresolved = True
else:
# replace token by substitution
config_values = substitution.parent
# if it is a string, then add the extra ws that was present in the original string after the substitution
formatted_resolved_value = resolved_value \
if resolved_value is None \
or isinstance(resolved_value, (dict, list)) \
or substitution.index == len(config_values.tokens) - 1 \
else (str(resolved_value) + substitution.ws)
config_values.put(substitution.index, formatted_resolved_value)
transformation = config_values.transform()
if transformation is None and not is_optional_resolved:
# if it does not override anything remove the key
# otherwise put back old value that it was overriding
if config_values.overriden_value is None:
if config_values.key in config_values.parent:
del config_values.parent[config_values.key]
else:
config_values.parent[config_values.key] = config_values.overriden_value
s = find_substitutions(config_values.overriden_value)
if s:
substitutions.extend(s)
unresolved = True
else:
config_values.parent[config_values.key] = transformation
s = find_substitutions(transformation)
if s:
substitutions.extend(s)
unresolved = True
substitutions.remove(substitution)

final_fixup(config)
unresolved, new_subsitutions, _ = ConfigParser._do_substitute(substitution, resolved_value, is_optional_resolved)
substitutions.extend(new_subsitutions)
substitutions.remove(substitution)

ConfigParser._final_fixup(config)
if unresolved:
raise ConfigSubstitutionException("Cannot resolve {variables}. Check for cycles.".format(
variables=', '.join('${{{variable}}}: (line: {line}, col: {col})'.format(
Expand Down Expand Up @@ -425,8 +462,9 @@ class ConfigTreeParser(TokenConverter):
Parse a config tree from tokens
"""

def __init__(self, expr=None):
def __init__(self, expr=None, root=False):
super(ConfigTreeParser, self).__init__(expr)
self.root = root
self.saveAsList = True

def postParse(self, instring, loc, token_list):
Expand All @@ -437,35 +475,47 @@ def postParse(self, instring, loc, token_list):
:param token_list:
:return:
"""
config_tree = ConfigTree()
config_tree = ConfigTree(root=self.root)
for element in token_list:
expanded_tokens = element.tokens if isinstance(element, ConfigInclude) else [element]

for tokens in expanded_tokens:
# key, value1 (optional), ...
key = tokens[0].strip()
values = tokens[1:]

operator = '='
if len(tokens) == 3 and tokens[1].strip() in [':', '=', '+=']:
operator = tokens[1].strip()
values = tokens[2:]
elif len(tokens) == 2:
values = tokens[1:]
else:
raise ParseSyntaxException("Unknown tokens {} received".format(tokens))
# empty string
if len(values) == 0:
config_tree.put(key, '')
else:
value = values[0]
if isinstance(value, list):
if isinstance(value, list) and operator == "+=":
value = ConfigValues([ConfigSubstitution(key, True, '', False, loc), value], False, loc)
config_tree.put(key, value, False)
elif isinstance(value, str) and operator == "+=":
value = ConfigValues([ConfigSubstitution(key, True, '', True, loc), ' ' + value], True, loc)
config_tree.put(key, value, False)
elif isinstance(value, list):
config_tree.put(key, value, False)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sanzinger Nice implementation of += operator! 👍

else:
if isinstance(value, ConfigTree):
existing_value = config_tree.get(key, None)
if isinstance(value, ConfigTree) and not isinstance(existing_value, list):
# Only Tree has to be merged with tree
config_tree.put(key, value, True)
elif isinstance(value, ConfigValues):
conf_value = value
value.parent = config_tree
value.key = key
existing_value = config_tree.get(key, None)
if isinstance(existing_value, list) or isinstance(existing_value, ConfigTree):
config_tree.put(key, conf_value, True)
else:
config_tree.put(key, conf_value, False)
else:
conf_value = value
config_tree.put(key, conf_value, False)
config_tree.put(key, value, False)
return config_tree
Loading