Skip to content

Commit

Permalink
[DPE-4413] Use TLS CA chain for backups (#493)
Browse files Browse the repository at this point in the history 
* Use TLS CA chain for backups

Signed-off-by: Marcelo Henrique Neppel <[email protected]>

* Use TLS CA chain in boto3

Signed-off-by: Marcelo Henrique Neppel <[email protected]>

* Add and update unit tests

Signed-off-by: Marcelo Henrique Neppel <[email protected]>

---------

Signed-off-by: Marcelo Henrique Neppel <[email protected]>
  • Loading branch information
@marceloneppel
marceloneppel authored Jun 4, 2024
1 parent 7e8862e commit 0d41111
Show file tree
Hide file tree
Showing 3 changed files with 111 additions and 14 deletions.
29 changes: 27 additions & 2 deletions src/backups.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,14 @@ def stanza_name(self) -> str:
"""Stanza name, composed by model and cluster name."""
return f"{self.model.name}.{self.charm.cluster_name}"

@property
def _tls_ca_chain_filename(self) -> str:
"""Returns the path to the TLS CA chain file."""
s3_parameters, _ = self._retrieve_s3_parameters()
if s3_parameters.get("tls-ca-chain") is not None:
return f"{self.charm._storage_path}/pgbackrest-tls-ca-chain.crt"
return ""

def _are_backup_settings_ok(self) -> Tuple[bool, Optional[str]]:
"""Validates whether backup settings are OK."""
if self.model.get_relation(self.relation_name) is None:
Expand Down Expand Up @@ -199,7 +207,11 @@ def _create_bucket_if_not_exists(self) -> None:
)

try:
s3 = session.resource("s3", endpoint_url=self._construct_endpoint(s3_parameters))
s3 = session.resource(
"s3",
endpoint_url=self._construct_endpoint(s3_parameters),
verify=(self._tls_ca_chain_filename or None),
)
except ValueError as e:
logger.exception("Failed to create a session '%s' in region=%s.", bucket_name, region)
raise e
Expand Down Expand Up @@ -826,6 +838,14 @@ def _render_pgbackrest_conf_file(self) -> bool:
)
return False

if self._tls_ca_chain_filename != "":
self.container.push(
self._tls_ca_chain_filename,
"\n".join(s3_parameters["tls-ca-chain"]),
user=WORKLOAD_OS_USER,
group=WORKLOAD_OS_GROUP,
)

# Open the template pgbackrest.conf file.
with open("templates/pgbackrest.conf.j2", "r") as file:
template = Template(file.read())
Expand All @@ -838,6 +858,7 @@ def _render_pgbackrest_conf_file(self) -> bool:
endpoint=s3_parameters["endpoint"],
bucket=s3_parameters["bucket"],
s3_uri_style=s3_parameters["s3-uri-style"],
tls_ca_chain=self._tls_ca_chain_filename,
access_key=s3_parameters["access-key"],
secret_key=s3_parameters["secret-key"],
stanza=self.stanza_name,
Expand Down Expand Up @@ -958,7 +979,11 @@ def _upload_content_to_s3(
region_name=s3_parameters["region"],
)

s3 = session.resource("s3", endpoint_url=self._construct_endpoint(s3_parameters))
s3 = session.resource(
"s3",
endpoint_url=self._construct_endpoint(s3_parameters),
verify=(self._tls_ca_chain_filename or None),
)
bucket = s3.Bucket(bucket_name)

with tempfile.NamedTemporaryFile() as temp_file:
Expand Down
3 changes: 3 additions & 0 deletions templates/pgbackrest.conf.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ repo1-s3-region={{ region }}
repo1-s3-endpoint={{ endpoint }}
repo1-s3-bucket={{ bucket }}
repo1-s3-uri-style={{ s3_uri_style }}
{%- if tls_ca_chain != '' %}
repo1-s3-ca-file={{ tls_ca_chain }}
{%- endif %}
repo1-s3-key={{ access_key }}
repo1-s3-key-secret={{ secret_key }}
repo1-block=y
Expand Down
93 changes: 81 additions & 12 deletions tests/unit/test_backups.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,33 @@ def test_stanza_name(harness):
)


def test_tls_ca_chain_filename(harness):
# Test when the TLS CA chain is not available.
tc.assertEqual(
harness.charm.backup._tls_ca_chain_filename,
"",
)

# Test when the TLS CA chain is available.
with harness.hooks_disabled():
remote_application = "s3-integrator"
s3_rel_id = harness.add_relation(S3_PARAMETERS_RELATION, remote_application)
harness.update_relation_data(
s3_rel_id,
remote_application,
{
"bucket": "fake-bucket",
"access-key": "fake-access-key",
"secret-key": "fake-secret-key",
"tls-ca-chain": '["fake-tls-ca-chain"]',
},
)
tc.assertEqual(
harness.charm.backup._tls_ca_chain_filename,
"/var/lib/postgresql/data/pgbackrest-tls-ca-chain.crt",
)


def test_are_backup_settings_ok(harness):
# Test without S3 relation.
tc.assertEqual(
Expand Down Expand Up @@ -393,9 +420,16 @@ def test_construct_endpoint(harness):
)


def test_create_bucket_if_not_exists(harness):
@pytest.mark.parametrize(
"tls_ca_chain_filename", ["", "/var/lib/postgresql/data/pgbackrest-tls-ca-chain.crt"]
)
def test_create_bucket_if_not_exists(harness, tls_ca_chain_filename):
with (
patch("boto3.session.Session.resource") as _resource,
patch(
"charm.PostgreSQLBackups._tls_ca_chain_filename",
new_callable=PropertyMock(return_value=tls_ca_chain_filename),
) as _tls_ca_chain_filename,
patch("charm.PostgreSQLBackups._retrieve_s3_parameters") as _retrieve_s3_parameters,
):
# Test when there are missing S3 parameters.
Expand All @@ -419,11 +453,15 @@ def test_create_bucket_if_not_exists(harness):
harness.charm.backup._create_bucket_if_not_exists()

# Test when the bucket already exists.
_resource.reset_mock()
_resource.side_effect = None
head_bucket = _resource.return_value.Bucket.return_value.meta.client.head_bucket
create = _resource.return_value.Bucket.return_value.create
wait_until_exists = _resource.return_value.Bucket.return_value.wait_until_exists
harness.charm.backup._create_bucket_if_not_exists()
_resource.assert_called_once_with(
"s3", endpoint_url="test-endpoint", verify=(tls_ca_chain_filename or None)
)
head_bucket.assert_called_once()
create.assert_not_called()
wait_until_exists.assert_not_called()
Expand Down Expand Up @@ -1482,9 +1520,16 @@ def test_pre_restore_checks(harness):
mock_event.fail.assert_not_called()


def test_render_pgbackrest_conf_file(harness):
@pytest.mark.parametrize(
"tls_ca_chain_filename", ["", "/var/lib/postgresql/data/pgbackrest-tls-ca-chain.crt"]
)
def test_render_pgbackrest_conf_file(harness, tls_ca_chain_filename):
with (
patch("ops.model.Container.push") as _push,
patch(
"charm.PostgreSQLBackups._tls_ca_chain_filename",
new_callable=PropertyMock(return_value=tls_ca_chain_filename),
) as _tls_ca_chain_filename,
patch("charm.PostgreSQLBackups._retrieve_s3_parameters") as _retrieve_s3_parameters,
):
# Set up a mock for the `open` method, set returned data to postgresql.conf template.
Expand Down Expand Up @@ -1513,6 +1558,7 @@ def test_render_pgbackrest_conf_file(harness):
"region": "us-east-1",
"s3-uri-style": "path",
"delete-older-than-days": "30",
"tls-ca-chain": (["fake-tls-ca-chain"] if tls_ca_chain_filename != "" else ""),
},
[],
)
Expand All @@ -1529,6 +1575,7 @@ def test_render_pgbackrest_conf_file(harness):
endpoint="https://storage.googleapis.com",
bucket="test-bucket",
s3_uri_style="path",
tls_ca_chain=(tls_ca_chain_filename or ""),
access_key="test-access-key",
secret_key="test-secret-key",
stanza=harness.charm.backup.stanza_name,
Expand All @@ -1546,12 +1593,15 @@ def test_render_pgbackrest_conf_file(harness):
tc.assertEqual(mock.call_args_list[0][0], ("templates/pgbackrest.conf.j2", "r"))

# Ensure the correct rendered template is sent to _render_file method.
_push.assert_called_once_with(
"/etc/pgbackrest.conf",
expected_content,
user="postgres",
group="postgres",
)
calls = [call("/etc/pgbackrest.conf", expected_content, user="postgres", group="postgres")]
if tls_ca_chain_filename != "":
calls.insert(
0,
call(
tls_ca_chain_filename, "fake-tls-ca-chain", user="postgres", group="postgres"
),
)
_push.assert_has_calls(calls)


def test_restart_database(harness):
Expand Down Expand Up @@ -1735,11 +1785,18 @@ def test_start_stop_pgbackrest_service(harness):
_restart.assert_called_once()


def test_upload_content_to_s3(harness):
@pytest.mark.parametrize(
"tls_ca_chain_filename", ["", "/var/lib/postgresql/data/pgbackrest-tls-ca-chain.crt"]
)
def test_upload_content_to_s3(harness, tls_ca_chain_filename):
with (
patch("tempfile.NamedTemporaryFile") as _named_temporary_file,
patch("charm.PostgreSQLBackups._construct_endpoint") as _construct_endpoint,
patch("boto3.session.Session.resource") as _resource,
patch(
"charm.PostgreSQLBackups._tls_ca_chain_filename",
new_callable=PropertyMock(return_value=tls_ca_chain_filename),
) as _tls_ca_chain_filename,
):
# Set some parameters.
content = "test-content"
Expand All @@ -1762,7 +1819,11 @@ def test_upload_content_to_s3(harness):
harness.charm.backup._upload_content_to_s3(content, s3_path, s3_parameters),
False,
)
_resource.assert_called_once_with("s3", endpoint_url="https://s3.us-east-1.amazonaws.com")
_resource.assert_called_once_with(
"s3",
endpoint_url="https://s3.us-east-1.amazonaws.com",
verify=(tls_ca_chain_filename or None),
)
_named_temporary_file.assert_not_called()
upload_file.assert_not_called()

Expand All @@ -1773,7 +1834,11 @@ def test_upload_content_to_s3(harness):
harness.charm.backup._upload_content_to_s3(content, s3_path, s3_parameters),
False,
)
_resource.assert_called_once_with("s3", endpoint_url="https://s3.us-east-1.amazonaws.com")
_resource.assert_called_once_with(
"s3",
endpoint_url="https://s3.us-east-1.amazonaws.com",
verify=(tls_ca_chain_filename or None),
)
_named_temporary_file.assert_called_once()
upload_file.assert_called_once_with("/tmp/test-file", "test-path/test-file.")

Expand All @@ -1786,6 +1851,10 @@ def test_upload_content_to_s3(harness):
harness.charm.backup._upload_content_to_s3(content, s3_path, s3_parameters),
True,
)
_resource.assert_called_once_with("s3", endpoint_url="https://s3.us-east-1.amazonaws.com")
_resource.assert_called_once_with(
"s3",
endpoint_url="https://s3.us-east-1.amazonaws.com",
verify=(tls_ca_chain_filename or None),
)
_named_temporary_file.assert_called_once()
upload_file.assert_called_once_with("/tmp/test-file", "test-path/test-file.")

0 comments on commit 0d41111

Please sign in to comment.