diff --git a/CHANGELOG.md b/CHANGELOG.md index ce8d31066..2b1fba2f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## Upcoming/Master + +- Add JSON and YAML codecs to file lookup + ## 1.3.0 (2018-05-03) - Support for provisioning stacks in multiple accounts and regions has been added [GH-553], [GH-551] diff --git a/docs/lookups.rst b/docs/lookups.rst index 55badd53f..8c2327ebb 100644 --- a/docs/lookups.rst +++ b/docs/lookups.rst @@ -246,6 +246,35 @@ Supported codecs: - parameterized-b64 - the same as parameterized, with the results additionally wrapped in { "Fn::Base64": ... } , which is what you actually need for EC2 UserData + - json - decode the file as JSON and return the resulting object + - json-parameterized - Same as ``json``, but applying templating rules from + ``parameterized`` to every object *value*. Note that object *keys* are not + modified. Example (an external PolicyDocument):: + + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "some:Action" + ], + "Resource": "{{MyResource}}" + } + ] + } + + - yaml - decode the file as YAML and return the resulting object. All strings + are returned as ``unicode`` even in Python 2. + - yaml-parameterized - Same as ``json-parameterized``, but using YAML. Example:: + + Version: 2012-10-17 + Statement + - Effect: Allow + Action: + - "some:Action" + Resource: "{{MyResource}}" + When using parameterized-b64 for UserData, you should use a local_parameter defined as such:: diff --git a/setup.py b/setup.py index 31ac84a5d..85b84068a 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ "gitpython~=2.0", "schematics~=2.0.1", "formic2", + "python-dateutil~=2.0", ] tests_require = [ diff --git a/stacker/lookups/handlers/file.py b/stacker/lookups/handlers/file.py index e913a1279..2ea88935a 100644 --- a/stacker/lookups/handlers/file.py +++ b/stacker/lookups/handlers/file.py @@ -1,14 +1,27 @@ from __future__ import print_function from __future__ import division from __future__ import absolute_import -import re +from builtins import bytes, str + import base64 +import json +import re +try: + from collections.abc import Mapping, Sequence +except ImportError: + from collections import Mapping, Sequence + +import yaml -from ...util import read_value_from_path from troposphere import GenericHelperFn, Base64 +from ...util import read_value_from_path + + TYPE_NAME = "file" +_PARAMETER_PATTERN = re.compile(r'{{([::|\w]+)}}') + def handler(value, **kwargs): """Translate a filename into the file contents. @@ -99,29 +112,113 @@ def handler(value, **kwargs): return CODECS[codec](value) -def parameterized_codec(raw, b64): - pattern = re.compile(r'{{([::|\w]+)}}') +def _parameterize_string(raw): + """Substitute placeholders in a string using CloudFormation references + + Args: + raw (`str`): String to be processed. Byte strings are not + supported; decode them before passing them to this function. + + Returns: + `str` | :class:`troposphere.GenericHelperFn`: An expression with + placeholders from the input replaced, suitable to be passed to + Troposphere to be included in CloudFormation template. This will + be the input string without modification if no substitutions are + found, and a composition of CloudFormation calls otherwise. + """ parts = [] s_index = 0 - for match in pattern.finditer(raw): + for match in _PARAMETER_PATTERN.finditer(raw): parts.append(raw[s_index:match.start()]) - parts.append({"Ref": match.group(1)}) + parts.append({u"Ref": match.group(1)}) s_index = match.end() + if not parts: + return raw + parts.append(raw[s_index:]) - result = {"Fn::Join": ["", parts]} + return GenericHelperFn({u"Fn::Join": [u"", parts]}) + + +def parameterized_codec(raw, b64): + """Parameterize a string, possibly encoding it as Base64 afterwards + + Args: + raw (`str` | `bytes`): String to be processed. Byte strings will be + interpreted as UTF-8. + b64 (`bool`): Whether to wrap the output in a Base64 CloudFormation + call + + Returns: + :class:`troposphere.GenericHelperFn`: output to be included in a + CloudFormation template. + """ + + if isinstance(raw, bytes): + raw = raw.decode('utf-8') + + result = _parameterize_string(raw) # Note, since we want a raw JSON object (not a string) output in the # template, we wrap the result in GenericHelperFn (not needed if we're # using Base64) - return Base64(result) if b64 else GenericHelperFn(result) + return Base64(result.data) if b64 else result + + +def _parameterize_obj(obj): + """Recursively parameterize all strings contained in an object. + + Parameterizes all values of a Mapping, all items of a Sequence, an + unicode string, or pass other objects through unmodified. + + Byte strings will be interpreted as UTF-8. + + Args: + obj: data to parameterize + + Return: + A parameterized object to be included in a CloudFormation template. + Mappings are converted to `dict`, Sequences are converted to `list`, + and strings possibly replaced by compositions of function calls. + """ + + if isinstance(obj, Mapping): + return dict((key, _parameterize_obj(value)) + for key, value in obj.items()) + elif isinstance(obj, bytes): + return _parameterize_string(obj.decode('utf8')) + elif isinstance(obj, str): + return _parameterize_string(obj) + elif isinstance(obj, Sequence): + return list(_parameterize_obj(item) for item in obj) + else: + return obj + + +class SafeUnicodeLoader(yaml.SafeLoader): + def construct_yaml_str(self, node): + return self.construct_scalar(node) + + +def yaml_codec(raw, parameterized=False): + data = yaml.load(raw, Loader=SafeUnicodeLoader) + return _parameterize_obj(data) if parameterized else data + + +def json_codec(raw, parameterized=False): + data = json.loads(raw) + return _parameterize_obj(data) if parameterized else data CODECS = { "plain": lambda x: x, "base64": lambda x: base64.b64encode(x.encode('utf8')), "parameterized": lambda x: parameterized_codec(x, False), - "parameterized-b64": lambda x: parameterized_codec(x, True) + "parameterized-b64": lambda x: parameterized_codec(x, True), + "yaml": lambda x: yaml_codec(x, parameterized=False), + "yaml-parameterized": lambda x: yaml_codec(x, parameterized=True), + "json": lambda x: json_codec(x, parameterized=False), + "json-parameterized": lambda x: json_codec(x, parameterized=True), } diff --git a/stacker/tests/lookups/handlers/test_file.py b/stacker/tests/lookups/handlers/test_file.py index 79f510f52..c2eb93f97 100644 --- a/stacker/tests/lookups/handlers/test_file.py +++ b/stacker/tests/lookups/handlers/test_file.py @@ -1,88 +1,198 @@ +# encoding: utf-8 + from __future__ import print_function from __future__ import division from __future__ import absolute_import + import unittest import mock import base64 -import troposphere +import yaml +import json +from troposphere import Base64, Join + +from stacker.lookups.handlers.file import (json_codec, handler, + parameterized_codec, yaml_codec) -from stacker.lookups.handlers.file import parameterized_codec, handler + +def to_template_dict(obj): + """Extract the CFN template dict of an object for test comparisons""" + + if hasattr(obj, 'to_dict') and callable(obj.to_dict): + return obj.to_dict() + elif isinstance(obj, dict): + return dict((key, to_template_dict(value)) + for (key, value) in obj.items()) + elif isinstance(obj, (list, tuple)): + return type(obj)(to_template_dict(item) for item in obj) + else: + return obj class TestFileTranslator(unittest.TestCase): + @staticmethod + def assertTemplateEqual(left, right): + """ + Assert that two codec results are equivalent + + Convert both sides to their template representations, since Troposphere + objects are not natively comparable + """ + return to_template_dict(left) == to_template_dict(right) + def test_parameterized_codec_b64(self): - expected = { - 'Fn::Base64': { - 'Fn::Join': [ - '', - ['Test ', {'Ref': 'Interpolation'}, ' Here'] - ] - } - } - self.assertEqual( - expected, - parameterized_codec('Test {{Interpolation}} Here', True).data + expected = Base64( + Join(u'', [u'Test ', {u'Ref': u'Interpolation'}, u' Here']) ) + out = parameterized_codec(u'Test {{Interpolation}} Here', True) + self.assertTemplateEqual(expected, out) + def test_parameterized_codec_plain(self): - expected = { - 'Fn::Join': ['', ['Test ', {'Ref': 'Interpolation'}, ' Here']] + expected = Join(u'', [u'Test ', {u'Ref': u'Interpolation'}, u' Here']) + + out = parameterized_codec(u'Test {{Interpolation}} Here', False) + self.assertTemplateEqual(expected, out) + + def test_yaml_codec_raw(self): + structured = { + u'Test': [1, None, u'unicode ✓', {u'some': u'obj'}] } - self.assertEqual( - expected, - parameterized_codec('Test {{Interpolation}} Here', False).data - ) + # Note: must use safe_dump, since regular dump adds !python/unicode + # tags, which we don't want, or we can't be sure we're correctly + # loading string as unicode. + raw = yaml.safe_dump(structured) - def test_file_loaded(self): - with mock.patch('stacker.lookups.handlers.file.read_value_from_path', - return_value='') as amock: - handler('plain:file://tmp/test') - amock.assert_called_with('file://tmp/test') - - def test_handler_plain(self): - expected = 'Hello, world' - with mock.patch('stacker.lookups.handlers.file.read_value_from_path', - return_value=expected): - out = handler('plain:file://tmp/test') - self.assertEqual(expected, out) - - def test_handler_b64(self): - expected = 'Hello, world' - with mock.patch('stacker.lookups.handlers.file.read_value_from_path', - return_value=expected): - out = handler('base64:file://tmp/test') - self.assertEqual(expected, base64.b64decode(out).decode()) - - def test_handler_parameterized(self): - expected = 'Hello, world' - with mock.patch('stacker.lookups.handlers.file.read_value_from_path', - return_value=expected): - out = handler('parameterized:file://tmp/test') - self.assertEqual(troposphere.GenericHelperFn, type(out)) - - def test_handler_parameterized_b64(self): - expected = 'Regular text {{Psuedo::Var}} {{RegVar}} {{Non-Var}}' + out = yaml_codec(raw, parameterized=False) + self.assertEqual(structured, out) + + def test_yaml_codec_parameterized(self): processed = { - 'Fn::Base64': { - 'Fn::Join': [ - '', - ['Regular text ', - {'Ref': 'Psuedo::Var'}, - ' ', - {'Ref': 'RegVar'}, - ' {{Non-Var}}'] - ] - } + u'Test': Join(u'', [u'Test ', {u'Ref': u'Interpolation'}, + u' Here']) + } + structured = { + u'Test': u'Test {{Interpolation}} Here' + } + raw = yaml.safe_dump(structured) + + out = yaml_codec(raw, parameterized=True) + self.assertTemplateEqual(processed, out) + + def test_json_codec_raw(self): + structured = { + u'Test': [1, None, u'str', {u'some': u'obj'}] } - with mock.patch('stacker.lookups.handlers.file.read_value_from_path', - return_value=expected): - out = handler('parameterized-b64:file://tmp/test') - self.assertEqual(troposphere.Base64, type(out)) - self.assertEqual(processed, out.data) - - def test_unknown_codec(self): - expected = 'Hello, world' - with mock.patch('stacker.lookups.handlers.file.read_value_from_path', - return_value=expected): - with self.assertRaises(KeyError): - handler('bad:file://tmp/test') + raw = json.dumps(structured) + + out = json_codec(raw, parameterized=False) + self.assertEqual(structured, out) + + def test_json_codec_parameterized(self): + processed = { + u'Test': Join(u'', [u'Test ', {u'Ref': u'Interpolation'}, + u' Here']) + } + structured = { + u'Test': u'Test {{Interpolation}} Here' + } + raw = json.dumps(structured) + + out = json_codec(raw, parameterized=True) + self.assertTemplateEqual(processed, out) + + @mock.patch('stacker.lookups.handlers.file.read_value_from_path', + return_value='') + def test_file_loaded(self, content_mock): + handler(u'plain:file://tmp/test') + content_mock.assert_called_with(u'file://tmp/test') + + @mock.patch('stacker.lookups.handlers.file.read_value_from_path', + return_value=u'Hello, world') + def test_handler_plain(self, _): + out = handler(u'plain:file://tmp/test') + self.assertEqual(u'Hello, world', out) + + @mock.patch('stacker.lookups.handlers.file.read_value_from_path') + def test_handler_b64(self, content_mock): + plain = u'Hello, world' + encoded = base64.b64encode(plain.encode('utf8')) + + content_mock.return_value = plain + out = handler(u'base64:file://tmp/test') + self.assertEqual(encoded, out) + + @mock.patch('stacker.lookups.handlers.file.parameterized_codec') + @mock.patch('stacker.lookups.handlers.file.read_value_from_path') + def test_handler_parameterized(self, content_mock, codec_mock): + result = mock.Mock() + codec_mock.return_value = result + + out = handler(u'parameterized:file://tmp/test') + codec_mock.assert_called_once_with(content_mock.return_value, False) + + self.assertEqual(result, out) + + @mock.patch('stacker.lookups.handlers.file.parameterized_codec') + @mock.patch('stacker.lookups.handlers.file.read_value_from_path') + def test_handler_parameterized_b64(self, content_mock, codec_mock): + result = mock.Mock() + codec_mock.return_value = result + + out = handler(u'parameterized-b64:file://tmp/test') + codec_mock.assert_called_once_with(content_mock.return_value, True) + + self.assertEqual(result, out) + + @mock.patch('stacker.lookups.handlers.file.yaml_codec') + @mock.patch('stacker.lookups.handlers.file.read_value_from_path') + def test_handler_yaml(self, content_mock, codec_mock): + result = mock.Mock() + codec_mock.return_value = result + + out = handler(u'yaml:file://tmp/test') + codec_mock.assert_called_once_with(content_mock.return_value, + parameterized=False) + + self.assertEqual(result, out) + + @mock.patch('stacker.lookups.handlers.file.yaml_codec') + @mock.patch('stacker.lookups.handlers.file.read_value_from_path') + def test_handler_yaml_parameterized(self, content_mock, codec_mock): + result = mock.Mock() + codec_mock.return_value = result + + out = handler(u'yaml-parameterized:file://tmp/test') + codec_mock.assert_called_once_with(content_mock.return_value, + parameterized=True) + + self.assertEqual(result, out) + + @mock.patch('stacker.lookups.handlers.file.json_codec') + @mock.patch('stacker.lookups.handlers.file.read_value_from_path') + def test_handler_json(self, content_mock, codec_mock): + result = mock.Mock() + codec_mock.return_value = result + + out = handler(u'json:file://tmp/test') + codec_mock.assert_called_once_with(content_mock.return_value, + parameterized=False) + + self.assertEqual(result, out) + + @mock.patch('stacker.lookups.handlers.file.json_codec') + @mock.patch('stacker.lookups.handlers.file.read_value_from_path') + def test_handler_json_parameterized(self, content_mock, codec_mock): + result = mock.Mock() + codec_mock.return_value = result + + out = handler(u'json-parameterized:file://tmp/test') + codec_mock.assert_called_once_with(content_mock.return_value, + parameterized=True) + + self.assertEqual(result, out) + + @mock.patch('stacker.lookups.handlers.file.read_value_from_path') + def test_unknown_codec(self, _): + with self.assertRaises(KeyError): + handler(u'bad:file://tmp/test')