From 9d99384d4d954c79dc4442fac44a4cb83455e4db Mon Sep 17 00:00:00 2001 From: Austin Weisgrau Date: Wed, 14 Sep 2022 17:37:07 -0700 Subject: [PATCH 1/4] Create adapter for Secrets Manager backend --- db_facts/aws_secrets_manager.py | 38 +++++++++++++++++++++++++++++++++ db_facts/db_facts_types.py | 2 ++ db_facts/db_info.py | 22 +++++++++++++++++++ setup.py | 2 +- 4 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 db_facts/aws_secrets_manager.py diff --git a/db_facts/aws_secrets_manager.py b/db_facts/aws_secrets_manager.py new file mode 100644 index 0000000..45ad106 --- /dev/null +++ b/db_facts/aws_secrets_manager.py @@ -0,0 +1,38 @@ +import json + +import boto3 + +from .db_facts_types import AWSSecret, AWSSecretUsernamePassword +from .db_type import canonicalize_db_type, db_protocol + + +def pull_aws_secrets_manager_secret(sm_entry_name: str) -> AWSSecret: + response = boto3.client("secretsmanager").get_secret_value(SecretId=sm_entry_name) + return json.loads(response["SecretString"]) + + +def pull_aws_secrets_manager_username_password( + sm_entry_name: str, +) -> AWSSecretUsernamePassword: + secret = pull_aws_secrets_manager_secret(sm_entry_name) + return {"user": secret["username"], "password": secret["password"]} + + +def db_info_from_secrets_manager(sm_entry_name: str): + response = pull_aws_secrets_manager_secret(sm_entry_name) + + result = {key.lower(): value for key, value in response.items()} + + result["host"] = result.pop("hostname") + result["user"] = result.pop("username") + + # mypy has issues with `result.get('type')` + # https://stackoverflow.com/questions/70955906/how-to-deal-with-incompatible-type-optionalstr-expected-str + if "type" in result: + result["type"] = canonicalize_db_type(result["type"]) + result["protocol"] = db_protocol(result["type"]) + else: + result["type"] = "" + result["protocol"] = "" + + return result diff --git a/db_facts/db_facts_types.py b/db_facts/db_facts_types.py index 996556a..b613ce7 100644 --- a/db_facts/db_facts_types.py +++ b/db_facts/db_facts_types.py @@ -65,6 +65,8 @@ class DBFacts(TypedDict, total=False): DBFacts = Dict[str, Any] DBName = List[str] LastPassUsernamePassword = Dict[str, str] +AWSSecret = Any +AWSSecretUsernamePassword = Dict[str, str] DBConfig = Dict[str, Any] DBCLIConfig = Any JinjaContext = Dict[str, Any] diff --git a/db_facts/db_info.py b/db_facts/db_info.py index 4d40d07..ad7f29a 100644 --- a/db_facts/db_info.py +++ b/db_facts/db_info.py @@ -5,6 +5,10 @@ from .errors import fail_on_invalid_db_name from .config import load_config from .lpass import pull_lastpass_username_password, db_info_from_lpass +from .aws_secrets_manager import ( + db_info_from_secrets_manager, + pull_aws_secrets_manager_username_password +) from .db_facts_types import DBConfig, DBCLIConfig, DBFacts, DBName from .db_config import db_config @@ -78,6 +82,24 @@ def db(db_name: DBName, dbcli_config: DBCLIConfig = None) -> DBFacts: additional_attributes = \ pull_lastpass_username_password(lastpass_entry_name) db_info['exports'].update(additional_attributes) + elif 'pull_secrets_manager_from' in dbcli_config['exports_from'][method]: + template_for_secrets_manager_entry_name =\ + dbcli_config['exports_from'][method]['pull_secrets_manager_from'] + secrets_manager_entry_name = template(template_for_secrets_manager_entry_name, + (db_info, {})) + additional_attributes = \ + db_info_from_secrets_manager(secrets_manager_entry_name) + db_info['exports'].update(additional_attributes) + elif 'pull_secrets_manager_username_password_from' in \ + dbcli_config['exports_from'][method]: + method = dbcli_config['exports_from'][method] + template_for_secrets_manager_entry_name =\ + method['pull_secrets_manager_username_password_from'] + secrets_manager_entry_name = template(template_for_secrets_manager_entry_name, + (db_info, {})) + additional_attributes = \ + pull_aws_secrets_manager_username_password(secrets_manager_entry_name) + db_info['exports'].update(additional_attributes) else: raise SyntaxError(f'Did not understand exports_from {method}') diff --git a/setup.py b/setup.py index 7b08849..0488fd0 100644 --- a/setup.py +++ b/setup.py @@ -113,7 +113,7 @@ def initialize_options(self) -> None: author_email='opensource@bluelabs.com', packages=find_packages(), package_data={"db_facts": ["py.typed"]}, - install_requires=['jinja2', 'pyyaml'], + install_requires=['jinja2', 'pyyaml', 'boto3'], entry_points={ 'console_scripts': [ 'db_facts = db_facts.__main__:main' From 047db33dd5c1ce8d18d6b24c04d10cd51f73d7ad Mon Sep 17 00:00:00 2001 From: Austin Weisgrau Date: Wed, 14 Sep 2022 17:37:22 -0700 Subject: [PATCH 2/4] Add tests to cover secrets manager backend --- tests/mock_dbcli_config.py | 19 +++++++ tests/test_db_info_secrets_manager.py | 75 +++++++++++++++++++++++++++ tests/test_secrets_manager.py | 29 +++++++++++ 3 files changed, 123 insertions(+) create mode 100644 tests/test_db_info_secrets_manager.py create mode 100644 tests/test_secrets_manager.py diff --git a/tests/mock_dbcli_config.py b/tests/mock_dbcli_config.py index e7f60dd..6d1e825 100644 --- a/tests/mock_dbcli_config.py +++ b/tests/mock_dbcli_config.py @@ -13,6 +13,12 @@ }, 'invalid-method': { }, + 'secrets_manager': { + 'pull_secrets_manager_from': "{{ secrets_manager_entry }}" + }, + 'secrets_manager_user_and_pass_only': { + 'pull_secrets_manager_username_password_from': "{{ secrets_manager_entry }}" + }, }, 'dbs': { 'baz': { @@ -29,6 +35,10 @@ 'exports_from': 'lpass', 'lastpass_entry': 'different lpass entry name' }, + 'fromage': { + 'exports_from': 'secrets_manager', + 'secrets_manager_entry': 'secrets manager entry name' + }, 'frazzle': { 'exports_from': 'lpass', 'lastpass_entry': 'lpass entry name' @@ -42,6 +52,15 @@ 'a_numbered_export': 123 }, }, + 'fronk': { + 'exports_from': 'secrets_manager_user_and_pass_only', + 'secrets_manager_entry': 'secrets manager entry name', + 'jinja_context_name': 'standard', + 'exports': { + 'some_additional': 'export', + 'a_numbered_export': 123 + }, + }, 'gaggle': { 'jinja_context_name': [ 'env', diff --git a/tests/test_db_info_secrets_manager.py b/tests/test_db_info_secrets_manager.py new file mode 100644 index 0000000..00bf50d --- /dev/null +++ b/tests/test_db_info_secrets_manager.py @@ -0,0 +1,75 @@ +from db_facts.db_info import db +import unittest +from unittest.mock import patch +from .mock_dbcli_config import mock_dbcli_config + + +@patch("db_facts.db_info.pull_jinja_context") +@patch("db_facts.db_info.db_info_from_secrets_manager") +@patch("db_facts.aws_secrets_manager.pull_aws_secrets_manager_secret") +class TestDBInfoSecretsManager(unittest.TestCase): + def test_db_info_secrets_manager( + self, + mock_pull_aws_secrets_manager_secret, + mock_db_info_from_secrets_manager, + mock_pull_jinja_context, + ): + + expected_result = { + "database": "database", + "host": "host", + "password": "password", + "port": "port", + "protocol": "protocol", + "type": "type", + "user": "user", + "connection_type": "direct", + } + mock_db_info_from_secrets_manager.return_value = { + "database": "database", + "port": "port", + "host": "host", + "type": "type", + "user": "user", + "protocol": "protocol", + "password": "password", + } + mock_pull_jinja_context.return_value = ({}, {}) + db_facts = db(["fromage"], dbcli_config=mock_dbcli_config) + mock_db_info_from_secrets_manager.assert_called_with( + "secrets manager entry name" + ) + self.assertEqual(expected_result, db_facts) + mock_pull_jinja_context.assert_called_with( + ["fromage"], mock_dbcli_config["dbs"]["fromage"], mock_dbcli_config + ) + + def test_db_info_pull_secrets_manager_user_and_pass_only( + self, + mock_pull_aws_secrets_manager_secret, + mock_db_info_from_secrets_manager, + mock_pull_jinja_context, + ): + + sm_entry = { + "username": "user", + "password": "password", + } + + mock_pull_aws_secrets_manager_secret.return_value = sm_entry + expected_result = { + "password": "password", + "user": "user", + "connection_type": "direct", + "some_additional": "export", + "a_numbered_export": 123, + } + mock_pull_jinja_context.return_value = ({}, {}) + db_facts = db(["fronk"], dbcli_config=mock_dbcli_config) + mock_pull_aws_secrets_manager_secret.assert_called_with( + "secrets manager entry name" + ) + self.assertEqual(expected_result, db_facts) + mock_pull_jinja_context.assert_called_with( + ["fronk"], mock_dbcli_config["dbs"]["fronk"], mock_dbcli_config + ) diff --git a/tests/test_secrets_manager.py b/tests/test_secrets_manager.py new file mode 100644 index 0000000..272fcf2 --- /dev/null +++ b/tests/test_secrets_manager.py @@ -0,0 +1,29 @@ +import unittest +from unittest.mock import patch + +from db_facts import aws_secrets_manager + + +class TestSecretsManager(unittest.TestCase): + @patch("db_facts.aws_secrets_manager.pull_aws_secrets_manager_secret") + def test_db_info_from_secrets_manager(self, mock_pull_from_aws_sm): + mock_pull_from_aws_sm.return_value = { + "Database": "fakedatabase", + "Port": 123, + "Hostname": "fakehost", + "Type": "faketype", + "Username": "fakeuser", + "Password": "fakepassword", + } + db_info = aws_secrets_manager.db_info_from_secrets_manager("my_secret") + expected_db_info = { + "database": "fakedatabase", + "host": "fakehost", + "password": "fakepassword", + "port": 123, + "type": "faketype", + "user": "fakeuser", + "protocol": "faketype", # if we don't know, just pass through + } + + assert db_info == expected_db_info From 01b7e07b0fdafbc422430216ba0d07bc17c81ab1 Mon Sep 17 00:00:00 2001 From: Austin Weisgrau Date: Mon, 19 Sep 2022 10:29:29 -0600 Subject: [PATCH 3/4] Ratchet up quality metrics --- metrics/bigfiles_high_water_mark | 2 +- metrics/flake8_high_water_mark | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metrics/bigfiles_high_water_mark b/metrics/bigfiles_high_water_mark index 446dfcc..f138657 100644 --- a/metrics/bigfiles_high_water_mark +++ b/metrics/bigfiles_high_water_mark @@ -1 +1 @@ -369 +383 diff --git a/metrics/flake8_high_water_mark b/metrics/flake8_high_water_mark index d00491f..0cfbf08 100644 --- a/metrics/flake8_high_water_mark +++ b/metrics/flake8_high_water_mark @@ -1 +1 @@ -1 +2 From 669690349b1a9631e2bda2d8640b352fd9c3175c Mon Sep 17 00:00:00 2001 From: Austin Weisgrau Date: Mon, 19 Sep 2022 10:41:49 -0600 Subject: [PATCH 4/4] Include boto3 type stubs for mypy type check --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 0fdea5c..4c14868 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ mypy lxml # needed for mypy coverage report types-pyyaml # needed for mypy coverage report types-pkg-resources # needed for mypy coverage report +boto3-stubs[essential] # needed for mypy coverage report wheel # needed to publish to PyPI in CircleCI twine # needed to publish to PyPI in CircleCI sphinx>=3 # used to generate and upload docs - sphinx-autodoc-typehints requires 4 or better per https://github.com/agronholm/sphinx-autodoc-typehints/pull/138