fix(mgmt_cli_test): configure TLS certificate for manager node

After TLS certificates preparation was reworked in SCT, individual TLS certificates
are created and distributed for loader and cluster nodes.

The change adds the same logic for creating and distributing certificate/key
to a manager node (which is usually a monitor node in the mgmt_cli tests).

sdcm/cluster.py

@@ -70,7 +70,7 @@
 from sdcm.provision.scylla_yaml.scylla_yaml import ScyllaYaml
 from sdcm.provision.helpers.certificate import (
     install_client_certificate, install_encryption_at_rest_files, create_certificate,
-    export_pem_cert_to_pkcs12_keystore, CA_CERT_FILE, CA_KEY_FILE, JKS_TRUSTSTORE_FILE)
+    export_pem_cert_to_pkcs12_keystore, CA_CERT_FILE, CA_KEY_FILE, JKS_TRUSTSTORE_FILE, TLSAssets)
 from sdcm.remote import RemoteCmdRunnerBase, LOCALRUNNER, NETWORK_EXCEPTIONS, shell_script_cmd, RetryableNetworkException
 from sdcm.remote.libssh2_client import UnexpectedExit as Libssh2_UnexpectedExit
 from sdcm.remote.remote_file import remote_file, yaml_file_to_dict, dict_to_yaml_file
@@ -2270,8 +2270,11 @@ def install_mgmt(self, package_url: Optional[str] = None) -> None:
             self.download_scylla_manager_repo(manager_repo_url)
         self.install_package(package_names)
 
-        self.log.debug("Copying TLS files from data_dir to the node")
-        self.remoter.send_files(src="./data_dir/ssl_conf", dst="/tmp/")
+        self.log.debug("Create and send client TLS certificate/key to the node")
+        self.create_node_certificate(self.ssl_conf_dir / TLSAssets.CLIENT_CERT,
+                                     self.ssl_conf_dir / TLSAssets.CLIENT_KEY)
+        self.remoter.send_files(
+            str(self.ssl_conf_dir), dst='/tmp/ssl_conf')
 
         if self.is_docker():
             try:
@@ -3487,9 +3490,9 @@ def _create_session(self, node, keyspace, user, password, compression, protocol_
 
         if ssl_context is None and self.params.get('client_encrypt'):
             if 'db' in node.node_type:
-                cert_name, key_name = "db.crt", "db.key"
+                cert_name, key_name = TLSAssets.DB_CERT, TLSAssets.DB_KEY
             else:
-                cert_name, key_name = "test.crt", "test.key"
+                cert_name, key_name = TLSAssets.CLIENT_CERT, TLSAssets.CLIENT_KEY
             ssl_context = self.create_ssl_context(
                 keyfile=node.ssl_conf_dir / key_name, certfile=node.ssl_conf_dir / cert_name, truststore=CA_CERT_FILE)
         self.log.debug("ssl_context: %s", str(ssl_context))
@@ -4633,9 +4636,10 @@ def node_setup(self, node: BaseNode, verbose: bool = False, timeout: int = 3600)
             SnitchConfig(node=node, datacenters=datacenters).apply()
 
         # Create node certificate for internode communication
-        node.create_node_certificate(node.ssl_conf_dir / 'db.crt', node.ssl_conf_dir / 'db.key')
+        node.create_node_certificate(node.ssl_conf_dir / TLSAssets.DB_CERT, node.ssl_conf_dir / TLSAssets.DB_KEY)
         # Create client facing node certificate, for client-to-node communication
-        node.create_node_certificate(node.ssl_conf_dir / 'client-facing.crt', node.ssl_conf_dir / 'client-facing.key')
+        node.create_node_certificate(
+            node.ssl_conf_dir / TLSAssets.DB_CLIENT_FACING_CERT, node.ssl_conf_dir / TLSAssets.DB_CLIENT_FACING_KEY)
         for src in (CA_CERT_FILE, JKS_TRUSTSTORE_FILE):
             shutil.copy(src, node.ssl_conf_dir)
         node.config_setup(append_scylla_args=self.get_scylla_args())
@@ -5069,12 +5073,14 @@ def node_setup(self, node, verbose=False, **kwargs):  # pylint: disable=unused-a
             self.log.info("Don't install anything because bare loaders requested")
             return
 
-        node.create_node_certificate(node.ssl_conf_dir / 'test.crt', node.ssl_conf_dir / 'test.key')
+        node.create_node_certificate(node.ssl_conf_dir / TLSAssets.CLIENT_CERT,
+                                     node.ssl_conf_dir / TLSAssets.CLIENT_KEY)
         for src in (CA_CERT_FILE, JKS_TRUSTSTORE_FILE):
             shutil.copy(src, node.ssl_conf_dir)
         if self.params.get('client_encrypt'):
-            export_pem_cert_to_pkcs12_keystore(node.ssl_conf_dir / 'test.crt', node.ssl_conf_dir / 'test.key',
-                                               node.ssl_conf_dir / 'keystore.p12')
+            export_pem_cert_to_pkcs12_keystore(
+                node.ssl_conf_dir / TLSAssets.CLIENT_CERT, node.ssl_conf_dir / TLSAssets.CLIENT_KEY,
+                node.ssl_conf_dir / TLSAssets.PKCS12_KEYSTORE)
             if self.params.get('use_prepared_loaders'):
                 node.config_client_encrypt()
 

sdcm/mgmt/cli.py

@@ -16,6 +16,7 @@
 import time
 import logging
 import datetime
+from pathlib import Path
 from re import findall
 from textwrap import dedent
 from statistics import mean
@@ -24,21 +25,22 @@
 
 import requests
 from invoke.exceptions import Failure as InvokeFailure
-from sdcm.remote.libssh2_client.exceptions import Failure as Libssh2Failure
 
+from sdcm.remote.libssh2_client.exceptions import Failure as Libssh2Failure
 from sdcm import wait
 from sdcm.mgmt.common import \
     TaskStatus, ScyllaManagerError, HostStatus, HostSsl, HostRestStatus, duration_to_timedelta, DEFAULT_TASK_TIMEOUT
+from sdcm.provision.helpers.certificate import TLSAssets
 from sdcm.utils.distro import Distro
 from sdcm.wait import WaitForTimeoutError
 
 LOGGER = logging.getLogger(__name__)
 
 STATUS_DONE = 'done'
 STATUS_ERROR = 'error'
-SSL_CONF_DIR = '/tmp/ssl_conf'
-SSL_USER_CERT_FILE = SSL_CONF_DIR + '/db.crt'
-SSL_USER_KEY_FILE = SSL_CONF_DIR + '/db.key'
+SSL_CONF_DIR = Path('/tmp/ssl_conf')
+SSL_USER_CERT_FILE = SSL_CONF_DIR / TLSAssets.CLIENT_CERT
+SSL_USER_KEY_FILE = SSL_CONF_DIR / TLSAssets.CLIENT_KEY
 REPAIR_TIMEOUT_SEC = 7200  # 2 hours
 
 

sdcm/nemesis.py

@@ -71,7 +71,7 @@
 from sdcm.paths import SCYLLA_YAML_PATH
 from sdcm.prometheus import nemesis_metrics_obj
 from sdcm.provision.scylla_yaml import SeedProvider
-from sdcm.provision.helpers.certificate import update_certificate
+from sdcm.provision.helpers.certificate import update_certificate, TLSAssets
 from sdcm.remote.libssh2_client.exceptions import UnexpectedExit as Libssh2UnexpectedExit
 from sdcm.sct_events import Severity
 from sdcm.sct_events.database import DatabaseLogEvent
@@ -4258,8 +4258,8 @@ def check_ssl_reload_log(node_system_log):
             for node in self.cluster.nodes:
                 node_system_logs[node] = node.follow_system_log(
                     patterns=f'messaging_service - Reloaded {{{ssl_files_location}}}')
-                node.remoter.send_files(src='data_dir/ssl_conf/db.crt', dst='/tmp')
-                node.remoter.run(f"sudo cp -f /tmp/db.crt {ssl_files_location}")
+                node.remoter.send_files(src=f'data_dir/ssl_conf/{TLSAssets.DB_CERT}', dst='/tmp')
+                node.remoter.run(f"sudo cp -f /tmp/{TLSAssets.DB_CERT} {ssl_files_location}")
                 new_crt = node.remoter.run(f"cat {ssl_files_location}").stdout
                 if in_place_crt == new_crt:
                     raise Exception('The CRT file was not replaced')

sdcm/provision/helpers/certificate.py

@@ -14,6 +14,7 @@
 import ipaddress
 import shutil
 import tarfile
+from dataclasses import dataclass
 from datetime import datetime, timedelta
 from pathlib import Path
 from textwrap import dedent
@@ -29,21 +30,37 @@
 from sdcm.utils.common import get_data_dir_path
 from sdcm.utils.docker_utils import ContainerManager, DockerException
 
+
+@dataclass(frozen=True)
+class TLSAssets:
+    CA_CERT: str = 'ca.pem'
+    CA_KEY: str = 'ca.key'
+    DB_CERT: str = 'db.crt'
+    DB_KEY: str = 'db.key'
+    DB_CSR: str = 'db.csr'
+    DB_CLIENT_FACING_CERT: str = 'client-facing.crt'
+    DB_CLIENT_FACING_KEY: str = 'client-facing.key'
+    CLIENT_CERT: str = 'test.crt'
+    CLIENT_KEY: str = 'test.key'
+    JKS_TRUSTSTORE: str = 'truststore.jks'
+    PKCS12_KEYSTORE: str = 'keystore.p12'
+
+
 SCYLLA_SSL_CONF_DIR = Path('/etc/scylla/ssl_conf')
-CA_CERT_FILE = Path(get_data_dir_path('ssl_conf', 'ca.pem'))
-CA_KEY_FILE = Path(get_data_dir_path('ssl_conf', 'ca.key'))
+CA_CERT_FILE = Path(get_data_dir_path('ssl_conf', TLSAssets.CA_CERT))
+CA_KEY_FILE = Path(get_data_dir_path('ssl_conf', TLSAssets.CA_KEY))
 
 # Cluster artifacts
-SERVER_KEY_FILE = Path(get_data_dir_path('ssl_conf', 'db.key'))
-SERVER_CERT_FILE = Path(get_data_dir_path('ssl_conf', 'db.crt'))
-SERVER_CSR_FILE = Path(get_data_dir_path('ssl_conf', 'db.csr'))
-CLIENT_FACING_KEYFILE = Path(get_data_dir_path('ssl_conf', 'client-facing.key'))
-CLIENT_FACING_CERTFILE = Path(get_data_dir_path('ssl_conf', 'client-facing.crt'))
+SERVER_KEY_FILE = Path(get_data_dir_path('ssl_conf', TLSAssets.DB_KEY))
+SERVER_CERT_FILE = Path(get_data_dir_path('ssl_conf', TLSAssets.DB_CERT))
+SERVER_CSR_FILE = Path(get_data_dir_path('ssl_conf', TLSAssets.DB_CSR))
+CLIENT_FACING_KEYFILE = Path(get_data_dir_path('ssl_conf', TLSAssets.DB_CLIENT_FACING_KEY))
+CLIENT_FACING_CERTFILE = Path(get_data_dir_path('ssl_conf', TLSAssets.DB_CLIENT_FACING_CERT))
 
 # Client artifacts
-CLIENT_KEY_FILE = Path(get_data_dir_path('ssl_conf', 'test.key'))
-CLIENT_CERT_FILE = Path(get_data_dir_path('ssl_conf', 'test.crt'))
-JKS_TRUSTSTORE_FILE = Path(get_data_dir_path('ssl_conf', 'truststore.jks'))
+CLIENT_KEY_FILE = Path(get_data_dir_path('ssl_conf', TLSAssets.CLIENT_KEY))
+CLIENT_CERT_FILE = Path(get_data_dir_path('ssl_conf', TLSAssets.CLIENT_CERT))
+JKS_TRUSTSTORE_FILE = Path(get_data_dir_path('ssl_conf', TLSAssets.JKS_TRUSTSTORE))
 
 
 def install_client_certificate(remoter, node_identifier):
@@ -195,15 +212,15 @@ def import_ca_to_jks_truststore(
     # Create tar archive with CA and put it to container
     tar_stream = io.BytesIO()
     with tarfile.open(fileobj=tar_stream, mode='w') as tar:
-        tarinfo = tarfile.TarInfo(name='ca.pem')
+        tarinfo = tarfile.TarInfo(name=TLSAssets.CA_CERT)
         tarinfo.size = len(cert)
         tar.addfile(tarinfo, io.BytesIO(cert.encode('utf-8')))
     tar_stream.seek(0)
     java.put_archive('/tmp', tar_stream)
 
     # Import CA to Java truststore
-    tmp_cert_path = Path('/tmp/ca.pem')
-    tmp_truststore_path = Path('/tmp/truststore.jks')
+    tmp_cert_path = Path('/tmp') / TLSAssets.CA_CERT
+    tmp_truststore_path = Path('/tmp') / TLSAssets.JKS_TRUSTSTORE
     _ = java.exec_run(f'rm {tmp_truststore_path}')  # delete truststore if it exists (needed for local SCT runs)
     exit_code, output = java.exec_run(
         f'keytool -importcert -noprompt -file {tmp_cert_path} -keystore {tmp_truststore_path} '
@@ -289,8 +306,8 @@ def update_certificate(
 
 def c_s_transport_str(client_mtls: bool) -> str:
     """Build transport string for cassandra-stress."""
-    transport_str = f'truststore={SCYLLA_SSL_CONF_DIR}/truststore.jks truststore-password=cassandra'
+    transport_str = f'truststore={SCYLLA_SSL_CONF_DIR}/{TLSAssets.JKS_TRUSTSTORE} truststore-password=cassandra'
     if client_mtls:
         transport_str = (
-            f'{transport_str} keystore={SCYLLA_SSL_CONF_DIR}/keystore.p12 keystore-password=cassandra')
+            f'{transport_str} keystore={SCYLLA_SSL_CONF_DIR}/{TLSAssets.PKCS12_KEYSTORE} keystore-password=cassandra')
     return transport_str

sdcm/scylla_bench_thread.py

@@ -21,7 +21,7 @@
 
 from sdcm.loader import ScyllaBenchStressExporter
 from sdcm.prometheus import nemesis_metrics_obj
-from sdcm.provision.helpers.certificate import SCYLLA_SSL_CONF_DIR
+from sdcm.provision.helpers.certificate import SCYLLA_SSL_CONF_DIR, TLSAssets
 from sdcm.sct_events.loaders import ScyllaBenchEvent, SCYLLA_BENCH_ERROR_EVENTS_PATTERNS
 from sdcm.utils.common import FileFollowerThread, convert_metric_to_ms
 from sdcm.stress_thread import DockerBasedStressThread
@@ -160,11 +160,12 @@ def create_stress_cmd(self, stress_cmd, loader, cmd_runner):
                         cmd_runner.send_files(str(ssl_file),
                                               str(SCYLLA_SSL_CONF_DIR / ssl_file.name),
                                               verbose=True)
-                stress_cmd = f'{stress_cmd.strip()} -tls -tls-ca-cert-file {SCYLLA_SSL_CONF_DIR}/ca.pem'
+                stress_cmd = f'{stress_cmd.strip()} -tls -tls-ca-cert-file {SCYLLA_SSL_CONF_DIR}/{TLSAssets.CA_CERT}'
 
                 if self.params.get("client_encrypt_mtls"):
-                    stress_cmd = (f'{stress_cmd.strip()} -tls-client-key-file {SCYLLA_SSL_CONF_DIR}/test.key '
-                                  f'-tls-client-cert-file {SCYLLA_SSL_CONF_DIR}/test.crt')
+                    stress_cmd = (
+                        f'{stress_cmd.strip()} -tls-client-key-file {SCYLLA_SSL_CONF_DIR}/{TLSAssets.CLIENT_KEY} '
+                        f'-tls-client-cert-file {SCYLLA_SSL_CONF_DIR}/{TLSAssets.CLIENT_CERT}')
 
                     # TBD: update after https://github.com/scylladb/scylla-bench/issues/140 is resolved
                     # server_names = ' '.join(f'-tls-server-name {ip}' for ip in ips.split(","))

sdcm/stress/latte_thread.py

@@ -19,7 +19,7 @@
 from pathlib import Path
 
 from sdcm.prometheus import nemesis_metrics_obj
-from sdcm.provision.helpers.certificate import SCYLLA_SSL_CONF_DIR
+from sdcm.provision.helpers.certificate import SCYLLA_SSL_CONF_DIR, TLSAssets
 from sdcm.sct_events.loaders import LatteStressEvent
 from sdcm.utils.common import (
     FileFollowerThread,
@@ -116,9 +116,9 @@ def build_stress_cmd(self, cmd_runner, loader):
                                           str(SCYLLA_SSL_CONF_DIR / ssl_file.name),
                                           verbose=True)
 
-            ssl_config += (f' --ssl --ssl-ca {SCYLLA_SSL_CONF_DIR}/ca.pem '
-                           f'--ssl-cert {SCYLLA_SSL_CONF_DIR}/test.crt '
-                           f'--ssl-key {SCYLLA_SSL_CONF_DIR}/test.key')
+            ssl_config += (f' --ssl --ssl-ca {SCYLLA_SSL_CONF_DIR}/{TLSAssets.CA_CERT} '
+                           f'--ssl-cert {SCYLLA_SSL_CONF_DIR}/{TLSAssets.CLIENT_CERT} '
+                           f'--ssl-key {SCYLLA_SSL_CONF_DIR}/{TLSAssets.CLIENT_KEY}')
         datacenter = ""
         if self.loader_set.test_config.MULTI_REGION:
             # The datacenter name can be received from "nodetool status" output. It's possible for DB nodes only,