diff --git a/exasol_transformers_extension/deployment/deployment_utils.py b/exasol_transformers_extension/deployment/deployment_utils.py index b1b04e5b..d761d7fa 100644 --- a/exasol_transformers_extension/deployment/deployment_utils.py +++ b/exasol_transformers_extension/deployment/deployment_utils.py @@ -14,8 +14,6 @@ DB_PASSWORD_ENVIRONMENT_VARIABLE = f"TE_DB_PASSWORD" BUCKETFS_PASSWORD_ENVIRONMENT_VARIABLE = f"TE_BUCKETFS_PASSWORD" -SLC_NAME = "exasol_transformers_extension_container_release.tar.gz" -GH_RELEASE_URL = "https://github.com/exasol/transformers-extension/releases/download" def load_and_render_statement(template_name, **kwargs) -> str: @@ -27,16 +25,6 @@ def load_and_render_statement(template_name, **kwargs) -> str: return statement -def _download_slc(tmp_dir: Path, version: str) -> Path: - url = "/".join((GH_RELEASE_URL, version, SLC_NAME)) - response = requests.get(url, stream=True) - response.raise_for_status() - slc_path = Path(tmp_dir, SLC_NAME) - with open(slc_path, 'wb') as f: - f.write(response.content) - return slc_path - - def get_websocket_ssl_options(use_ssl_cert_validation: bool, ssl_cert_path: str): websocket_sslopt = { "cert_reqs": ssl.CERT_REQUIRED, @@ -47,11 +35,3 @@ def get_websocket_ssl_options(use_ssl_cert_validation: bool, ssl_cert_path: str) if ssl_cert_path is not None: websocket_sslopt["ca_certs"] = ssl_cert_path return websocket_sslopt - - -@contextmanager -def get_container_file_from_github_release(version: str): - with tempfile.TemporaryDirectory() as tmp_dir: - container_file_path = _download_slc(tmp_dir, version) - yield container_file_path - diff --git a/exasol_transformers_extension/deployment/language_container_deployer.py b/exasol_transformers_extension/deployment/language_container_deployer.py index 8da2a82a..c874dacb 100644 --- a/exasol_transformers_extension/deployment/language_container_deployer.py +++ b/exasol_transformers_extension/deployment/language_container_deployer.py @@ -1,16 +1,56 @@ +######################################################### +# To be migrated to the script-languages-container-tool # +######################################################### from enum import Enum -import pyexasol +from textwrap import dedent from typing import List, Optional from pathlib import Path, PurePosixPath -from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation import logging -from exasol_transformers_extension.utils.bucketfs_operations import \ - create_bucketfs_location -from exasol_transformers_extension.deployment.deployment_utils import get_websocket_ssl_options +import tempfile +import requests +import ssl +import pyexasol +from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation +from exasol_transformers_extension.utils.bucketfs_operations import create_bucketfs_location logger = logging.getLogger(__name__) +def get_websocket_sslopt(use_ssl_cert_validation: bool = True, + ssl_trusted_ca: Optional[str] = None, + ssl_client_certificate: Optional[str] = None, + ssl_private_key: Optional[str] = None) -> dict: + """ + Returns a dictionary in the winsocket-client format + (see https://websocket-client.readthedocs.io/en/latest/faq.html#what-else-can-i-do-with-sslopts) + """ + + # Is server certificate validation required? + sslopt: dict[str, object] = {"cert_reqs": ssl.CERT_REQUIRED if use_ssl_cert_validation else ssl.CERT_NONE} + + # Is a bundle with trusted CAs provided? + if ssl_trusted_ca: + trusted_ca_path = Path(ssl_trusted_ca) + if trusted_ca_path.is_dir(): + sslopt["ca_cert_path"] = ssl_trusted_ca + elif trusted_ca_path.is_file(): + sslopt["ca_certs"] = ssl_trusted_ca + else: + raise ValueError(f"Trusted CA location {ssl_trusted_ca} doesn't exist.") + + # Is client's own certificate provided? + if ssl_client_certificate: + if not Path(ssl_client_certificate).is_file(): + raise ValueError(f"Certificate file {ssl_client_certificate} doesn't exist.") + sslopt["certfile"] = ssl_client_certificate + if ssl_private_key: + if not Path(ssl_private_key).is_file(): + raise ValueError(f"Private key file {ssl_private_key} doesn't exist.") + sslopt["keyfile"] = ssl_private_key + + return sslopt + + class LanguageActivationLevel(Enum): f""" Language activation level, i.e. @@ -21,82 +61,149 @@ class LanguageActivationLevel(Enum): class LanguageContainerDeployer: + def __init__(self, pyexasol_connection: pyexasol.ExaConnection, language_alias: str, - bucketfs_location: BucketFSLocation, - container_file: Path): - self._container_file = container_file + bucketfs_location: BucketFSLocation) -> None: + self._bucketfs_location = bucketfs_location self._language_alias = language_alias self._pyexasol_conn = pyexasol_connection logger.debug(f"Init {LanguageContainerDeployer.__name__}") - def deploy_container(self, allow_override: bool = False) -> None: + def download_and_run(self, url: str, + bucket_file_path: str, + alter_system: bool = True, + allow_override: bool = False) -> None: """ - Uploads the SLC and activates it at the SYSTEM level. + Downloads language container from the provided url to a temporary file and then deploys it. + See docstring on the `run` method for details on what is involved in the deployment. - allow_override - If True the activation of a language container with the same alias will be overriden, - otherwise a RuntimeException will be thrown. + url - Address where the container will be downloaded from. + bucket_file_path - Path within the designated bucket where the container should be uploaded. + alter_system - If True will try to activate the container at the System level. + allow_override - If True the activation of a language container with the same alias will be + overriden, otherwise a RuntimeException will be thrown. """ - path_in_udf = self.upload_container() - self.activate_container(LanguageActivationLevel.System, allow_override, path_in_udf) - def upload_container(self) -> PurePosixPath: + with tempfile.NamedTemporaryFile() as tmp_file: + response = requests.get(url, stream=True) + response.raise_for_status() + tmp_file.write(response.content) + + self.run(Path(tmp_file.name), bucket_file_path, alter_system, allow_override) + + def run(self, container_file: Optional[Path] = None, + bucket_file_path: Optional[str] = None, + alter_system: bool = True, + allow_override: bool = False) -> None: """ - Uploads the SLC. - Returns the path where the container is uploaded as it's seen by a UDF. + Deploys the language container. This includes two steps, both of which are optional: + - Uploading the container into the database. This step can be skipped if the container + has already been uploaded. + - Activating the container. This step may have to be skipped if the user does not have + System Privileges in the database. In that case two alternative activation SQL commands + will be printed on the console. + + container_file - Path of the container tar.gz file in a local file system. + If not provided the container is assumed to be uploaded already. + bucket_file_path - Path within the designated bucket where the container should be uploaded. + If not specified the name of the container file will be used instead. + alter_system - If True will try to activate the container at the System level. + allow_override - If True the activation of a language container with the same alias will be + overriden, otherwise a RuntimeException will be thrown. """ - if not self._container_file.is_file(): - raise RuntimeError(f"Container file {self._container_file} " + + if not bucket_file_path: + if not container_file: + raise ValueError('Either a container file or a bucket file path must be specified.') + bucket_file_path = container_file.name + + if container_file: + self.upload_container(container_file, bucket_file_path) + + if alter_system: + self.activate_container(bucket_file_path, LanguageActivationLevel.System, allow_override) + else: + message = dedent(f""" + In SQL, you can activate the SLC of the Transformers Extension + by using the following statements: + + To activate the SLC only for the current session: + {self.generate_activation_command(bucket_file_path, LanguageActivationLevel.Session, True)} + + To activate the SLC on the system: + {self.generate_activation_command(bucket_file_path, LanguageActivationLevel.System, True)} + """) + print(message) + + def upload_container(self, container_file: Path, + bucket_file_path: Optional[str] = None) -> None: + """ + Uploads SLC to the BucketFS. + + container_file - Path of the container tar.gz file in a local file system. + bucket_file_path - Path within the designated bucket where the container should be uploaded. + """ + if not container_file.is_file(): + raise RuntimeError(f"Container file {container_file} " f"is not a file.") - with open(self._container_file, "br") as f: - upload_uri, path_in_udf = \ - self._bucketfs_location.upload_fileobj_to_bucketfs( - fileobj=f, bucket_file_path=self._container_file.name) + with open(container_file, "br") as f: + self._bucketfs_location.upload_fileobj_to_bucketfs( + fileobj=f, bucket_file_path=bucket_file_path) logging.debug("Container is uploaded to bucketfs") - return PurePosixPath(path_in_udf) - def activate_container(self, alter_type: LanguageActivationLevel = LanguageActivationLevel.Session, - allow_override: bool = False, - path_in_udf: Optional[PurePosixPath] = None) -> None: + def activate_container(self, bucket_file_path: str, + alter_type: LanguageActivationLevel = LanguageActivationLevel.Session, + allow_override: bool = False) -> None: """ Activates the SLC container at the required level. - alter_type - Language activation level, defaults to the SESSION. - allow_override - If True the activation of a language container with the same alias will be overriden, - otherwise a RuntimeException will be thrown. - path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF. + bucket_file_path - Path within the designated bucket where the container is uploaded. + alter_type - Language activation level, defaults to the SESSION. + allow_override - If True the activation of a language container with the same alias will be overriden, + otherwise a RuntimeException will be thrown. """ - alter_command = self.generate_activation_command(alter_type, allow_override, path_in_udf) + alter_command = self.generate_activation_command(bucket_file_path, alter_type, allow_override) self._pyexasol_conn.execute(alter_command) logging.debug(alter_command) - def generate_activation_command(self, alter_type: LanguageActivationLevel, - allow_override: bool = False, - path_in_udf: Optional[PurePosixPath] = None) -> str: + def generate_activation_command(self, bucket_file_path: str, + alter_type: LanguageActivationLevel, + allow_override: bool = False) -> str: """ Generates an SQL command to activate the SLC container at the required level. The command will preserve existing activations of other containers identified by different language aliases. Activation of a container with the same alias, if exists, will be overwritten. - alter_type - Activation level - SYSTEM or SESSION. - allow_override - If True the activation of a language container with the same alias will be overriden, - otherwise a RuntimeException will be thrown. - path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF. + bucket_file_path - Path within the designated bucket where the container is uploaded. + alter_type - Activation level - SYSTEM or SESSION. + allow_override - If True the activation of a language container with the same alias will be overriden, + otherwise a RuntimeException will be thrown. """ - if path_in_udf is None: - path_in_udf = self._bucketfs_location.generate_bucket_udf_path(self._container_file.name) + path_in_udf = self._bucketfs_location.generate_bucket_udf_path(bucket_file_path) new_settings = \ self._update_previous_language_settings(alter_type, allow_override, path_in_udf) alter_command = \ f"ALTER {alter_type.value} SET SCRIPT_LANGUAGES='{new_settings}';" return alter_command + def get_language_settings(self, alter_type: LanguageActivationLevel) -> str: + """ + Reads the current language settings at the specified level. + + alter_type - Activation level - SYSTEM or SESSION. + """ + result = self._pyexasol_conn.execute( + f"""SELECT "{alter_type.value}_VALUE" FROM SYS.EXA_PARAMETERS WHERE + PARAMETER_NAME='SCRIPT_LANGUAGES'""").fetchall() + return result[0][0] + def _update_previous_language_settings(self, alter_type: LanguageActivationLevel, allow_override: bool, path_in_udf: PurePosixPath) -> str: - prev_lang_settings = self._get_previous_language_settings(alter_type) + prev_lang_settings = self.get_language_settings(alter_type) prev_lang_aliases = prev_lang_settings.split(" ") self._check_if_requested_language_alias_already_exists( allow_override, prev_lang_aliases) @@ -131,20 +238,17 @@ def _check_if_requested_language_alias_already_exists( else: raise RuntimeError(warning_message) - def _get_previous_language_settings(self, alter_type: LanguageActivationLevel) -> str: - result = self._pyexasol_conn.execute( - f"""SELECT "{alter_type.value}_VALUE" FROM SYS.EXA_PARAMETERS WHERE - PARAMETER_NAME='SCRIPT_LANGUAGES'""").fetchall() - return result[0][0] - @classmethod def create(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int, - bucketfs_use_https: bool, bucketfs_user: str, container_file: Path, - bucketfs_password: str, bucket: str, path_in_bucket: str, - dsn: str, db_user: str, db_password: str, language_alias: str, - ssl_cert_path: str = None, use_ssl_cert_validation: bool = True) -> "LanguageContainerDeployer": + bucketfs_use_https: bool, bucketfs_user: str, + bucketfs_password: str, bucket: str, path_in_bucket: str, + dsn: str, db_user: str, db_password: str, language_alias: str, + use_ssl_cert_validation: bool = True, ssl_trusted_ca: Optional[str] = None, + ssl_client_certificate: Optional[str] = None, + ssl_private_key: Optional[str] = None) -> "LanguageContainerDeployer": - websocket_sslopt = get_websocket_ssl_options(use_ssl_cert_validation, ssl_cert_path) + websocket_sslopt = get_websocket_sslopt(use_ssl_cert_validation, ssl_trusted_ca, + ssl_client_certificate, ssl_private_key) pyexasol_conn = pyexasol.connect( dsn=dsn, @@ -158,4 +262,4 @@ def create(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int, bucketfs_name, bucketfs_host, bucketfs_port, bucketfs_use_https, bucketfs_user, bucketfs_password, bucket, path_in_bucket) - return cls(pyexasol_conn, language_alias, bucketfs_location, container_file) + return cls(pyexasol_conn, language_alias, bucketfs_location) diff --git a/exasol_transformers_extension/deployment/language_container_deployer_cli.py b/exasol_transformers_extension/deployment/language_container_deployer_cli.py index e8224c53..971c1c05 100644 --- a/exasol_transformers_extension/deployment/language_container_deployer_cli.py +++ b/exasol_transformers_extension/deployment/language_container_deployer_cli.py @@ -1,34 +1,8 @@ import os import click from pathlib import Path -from textwrap import dedent from exasol_transformers_extension.deployment import deployment_utils as utils -from exasol_transformers_extension.deployment.language_container_deployer import \ - LanguageContainerDeployer, LanguageActivationLevel - - -def run_deployer(deployer, upload_container: bool = True, - alter_system: bool = True, - allow_override: bool = False) -> None: - if upload_container and alter_system: - deployer.deploy_container(allow_override) - elif upload_container: - deployer.upload_container() - elif alter_system: - deployer.activate_container(LanguageActivationLevel.System, allow_override) - - if not alter_system: - message = dedent(f""" - In SQL, you can activate the SLC of the Transformers Extension - by using the following statements: - - To activate the SLC only for the current session: - {deployer.generate_activation_command(LanguageActivationLevel.Session, True)} - - To activate the SLC on the system: - {deployer.generate_activation_command(LanguageActivationLevel.System, True)} - """) - print(message) +from exasol_transformers_extension.deployment.te_language_container_deployer import TeLanguageContainerDeployer @click.command(name="language-container") @@ -52,6 +26,8 @@ def run_deployer(deployer, upload_container: bool = True, utils.DB_PASSWORD_ENVIRONMENT_VARIABLE, "")) @click.option('--language-alias', type=str, default="PYTHON3_TE") @click.option('--ssl-cert-path', type=str, default="") +@click.option('--ssl-client-cert-path', type=str, default="") +@click.option('--ssl-client-private-key', type=str, default="") @click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True) @click.option('--upload-container/--no-upload_container', type=bool, default=True) @click.option('--alter-system/--no-alter-system', type=bool, default=True) @@ -72,41 +48,40 @@ def language_container_deployer_main( db_pass: str, language_alias: str, ssl_cert_path: str, + ssl_client_cert_path: str, + ssl_client_private_key: str, use_ssl_cert_validation: bool, upload_container: bool, alter_system: bool, allow_override: bool): - def call_runner(): - deployer = LanguageContainerDeployer.create( - bucketfs_name=bucketfs_name, - bucketfs_host=bucketfs_host, - bucketfs_port=bucketfs_port, - bucketfs_use_https=bucketfs_use_https, - bucketfs_user=bucketfs_user, - bucketfs_password=bucketfs_password, - bucket=bucket, - path_in_bucket=path_in_bucket, - container_file=Path(container_file), - dsn=dsn, - db_user=db_user, - db_password=db_pass, - language_alias=language_alias, - ssl_cert_path=ssl_cert_path, - use_ssl_cert_validation=use_ssl_cert_validation) - run_deployer(deployer, upload_container=upload_container, alter_system=alter_system, - allow_override=allow_override) + deployer = TeLanguageContainerDeployer.create( + bucketfs_name=bucketfs_name, + bucketfs_host=bucketfs_host, + bucketfs_port=bucketfs_port, + bucketfs_use_https=bucketfs_use_https, + bucketfs_user=bucketfs_user, + bucketfs_password=bucketfs_password, + bucket=bucket, + path_in_bucket=path_in_bucket, + dsn=dsn, + db_user=db_user, + db_password=db_pass, + language_alias=language_alias, + ssl_trusted_ca=ssl_cert_path, + ssl_client_certificate=ssl_client_cert_path, + ssl_private_key=ssl_client_private_key, + use_ssl_cert_validation=use_ssl_cert_validation) - if container_file: - call_runner() + if not upload_container: + deployer.run(alter_system=alter_system, allow_override=allow_override) + elif container_file: + deployer.run(container_file=Path(container_file), alter_system=alter_system, allow_override=allow_override) elif version: - with utils.get_container_file_from_github_release(version) as container: - container_file = container - call_runner() + deployer.download_from_git_and_run(version, alter_system=alter_system, allow_override=allow_override) else: - raise ValueError("You should specify either the release version to " - "download container file or the path of the already " - "downloaded container file.") + raise ValueError("To upload a language container you should specify either its " + "release version or a path of the already downloaded container file.") if __name__ == '__main__': diff --git a/exasol_transformers_extension/deployment/te_language_container_deployer.py b/exasol_transformers_extension/deployment/te_language_container_deployer.py new file mode 100644 index 00000000..e652cd29 --- /dev/null +++ b/exasol_transformers_extension/deployment/te_language_container_deployer.py @@ -0,0 +1,25 @@ +from typing import Optional +from pathlib import Path +from exasol_transformers_extension.deployment.language_container_deployer import LanguageContainerDeployer + + +class TeLanguageContainerDeployer(LanguageContainerDeployer): + + SLC_NAME = "exasol_transformers_extension_container_release.tar.gz" + GH_RELEASE_URL = "https://github.com/exasol/transformers-extension/releases/download" + + def download_from_git_and_run(self, version: str, + alter_system: bool = True, + allow_override: bool = False) -> None: + + url = "/".join((self.GH_RELEASE_URL, version, self.SLC_NAME)) + self.download_and_run(url, self.SLC_NAME, alter_system=alter_system, allow_override=allow_override) + + def run(self, container_file: Optional[Path] = None, + bucket_file_path: Optional[str] = None, + alter_system: bool = True, + allow_override: bool = False) -> None: + + if not bucket_file_path: + bucket_file_path = self.SLC_NAME + super().run(container_file, bucket_file_path, alter_system, allow_override) diff --git a/tests/integration_tests/with_db/deployment/test_language_container_deployer.py b/tests/integration_tests/with_db/deployment/test_language_container_deployer.py index 21183a59..f9d3af3b 100644 --- a/tests/integration_tests/with_db/deployment/test_language_container_deployer.py +++ b/tests/integration_tests/with_db/deployment/test_language_container_deployer.py @@ -1,15 +1,19 @@ +######################################################### +# To be migrated to the script-languages-container-tool # +######################################################### import textwrap from typing import Callable from pathlib import Path - import pytest from _pytest.fixtures import FixtureRequest -from tests.fixtures.language_container_fixture import export_slc, flavor_path -from tests.fixtures.database_connection_fixture import pyexasol_connection -from exasol_bucketfs_utils_python.bucketfs_factory import BucketFSFactory -from exasol_script_languages_container_tool.lib.tasks.export.export_info import ExportInfo + from pyexasol import ExaConnection from pytest_itde import config +from exasol_bucketfs_utils_python.bucketfs_factory import BucketFSFactory +from exasol_script_languages_container_tool.lib.tasks.export.export_info import ExportInfo + +from tests.fixtures.language_container_fixture import export_slc, flavor_path +from tests.fixtures.database_connection_fixture import pyexasol_connection from exasol_transformers_extension.deployment.language_container_deployer \ import LanguageContainerDeployer, LanguageActivationLevel @@ -23,19 +27,20 @@ def test_language_container_deployer( pyexasol_connection: ExaConnection, connection_factory: Callable[[config.Exasol], ExaConnection], exasol_config: config.Exasol, - bucketfs_config: config.BucketFs, -): + bucketfs_config: config.BucketFs): + """ + Tests the deployment of a container in one call, including the activation at the System level. + """ test_name: str = request.node.name schema = test_name language_alias = f"PYTHON3_TE_{test_name.upper()}" container_path = Path(export_slc.cache_file) with revert_language_settings(pyexasol_connection): create_schema(pyexasol_connection, schema) - deployer = create_container_deployer(container_path=container_path, - language_alias=language_alias, + deployer = create_container_deployer(language_alias=language_alias, pyexasol_connection=pyexasol_connection, bucketfs_config=bucketfs_config) - deployer.deploy_container(True) + deployer.run(container_file=container_path, alter_system=True, allow_override=True) with connection_factory(exasol_config) as new_connection: assert_udf_running(new_connection, language_alias, schema) @@ -46,25 +51,26 @@ def test_language_container_deployer_alter_session( pyexasol_connection: ExaConnection, connection_factory: Callable[[config.Exasol], ExaConnection], exasol_config: config.Exasol, - bucketfs_config: config.BucketFs, -): + bucketfs_config: config.BucketFs): + """ + Tests the deployment of a container in two stages - uploading the container + followed by activation at the Session level. + """ test_name: str = request.node.name schema = test_name language_alias = f"PYTHON3_TE_{test_name.upper()}" container_path = Path(export_slc.cache_file) with revert_language_settings(pyexasol_connection): create_schema(pyexasol_connection, schema) - deployer = create_container_deployer(container_path=container_path, - language_alias=language_alias, + deployer = create_container_deployer(language_alias=language_alias, pyexasol_connection=pyexasol_connection, bucketfs_config=bucketfs_config) - deployer.upload_container() + deployer.run(container_file=container_path, alter_system=False) with connection_factory(exasol_config) as new_connection: - deployer = create_container_deployer(container_path=container_path, - language_alias=language_alias, + deployer = create_container_deployer(language_alias=language_alias, pyexasol_connection=new_connection, bucketfs_config=bucketfs_config) - deployer.activate_container(LanguageActivationLevel.Session, True) + deployer.activate_container(container_path.name, LanguageActivationLevel.Session, True) assert_udf_running(new_connection, language_alias, schema) @@ -74,26 +80,27 @@ def test_language_container_deployer_activation_fail( pyexasol_connection: ExaConnection, connection_factory: Callable[[config.Exasol], ExaConnection], exasol_config: config.Exasol, - bucketfs_config: config.BucketFs, -): + bucketfs_config: config.BucketFs): + """ + Tests that an attempt to activate a container using alias that already exists + causes exception if overriding is disallowed. + """ test_name: str = request.node.name schema = test_name language_alias = f"PYTHON3_TE_{test_name.upper()}" container_path = Path(export_slc.cache_file) with revert_language_settings(pyexasol_connection): create_schema(pyexasol_connection, schema) - deployer = create_container_deployer(container_path=container_path, - language_alias=language_alias, + deployer = create_container_deployer(language_alias=language_alias, pyexasol_connection=pyexasol_connection, bucketfs_config=bucketfs_config) - deployer.deploy_container(True) + deployer.run(container_file=container_path, alter_system=True, allow_override=True) with connection_factory(exasol_config) as new_connection: - deployer = create_container_deployer(container_path=container_path, - language_alias=language_alias, + deployer = create_container_deployer(language_alias=language_alias, pyexasol_connection=new_connection, bucketfs_config=bucketfs_config) with pytest.raises(RuntimeError): - deployer.activate_container(LanguageActivationLevel.System, False) + deployer.activate_container(container_path.name, LanguageActivationLevel.System, False) def create_schema(pyexasol_connection: ExaConnection, schema: str): @@ -113,8 +120,7 @@ def run(ctx): assert result[0][0] == True -def create_container_deployer(container_path: Path, - language_alias: str, +def create_container_deployer(language_alias: str, pyexasol_connection: ExaConnection, bucketfs_config: config.BucketFs) -> LanguageContainerDeployer: bucket_fs_factory = BucketFSFactory() @@ -126,5 +132,5 @@ def create_container_deployer(container_path: Path, pwd=f"{bucketfs_config.password}", base_path=None) return LanguageContainerDeployer( - pyexasol_connection, language_alias, bucketfs_location, container_path) + pyexasol_connection, language_alias, bucketfs_location) diff --git a/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py b/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py index 21c7de5c..c6b229a5 100644 --- a/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py +++ b/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py @@ -162,12 +162,7 @@ def test_language_container_deployer_cli_with_missing_container_option( bucketfs_config=bucketfs_config, exasol_config=exasol_config ) - expected_exception_message = "You should specify either the release version to " \ - "download container file or the path of the already " \ - "downloaded container file." - assert result.exit_code == 1 \ - and result.exception.args[0] == expected_exception_message \ - and type(result.exception) == ValueError + assert result.exit_code == 1 and type(result.exception) == ValueError def test_language_container_deployer_cli_with_check_cert( diff --git a/tests/unit_tests/deployment/test_language_container_deployer.py b/tests/unit_tests/deployment/test_language_container_deployer.py new file mode 100644 index 00000000..49f99605 --- /dev/null +++ b/tests/unit_tests/deployment/test_language_container_deployer.py @@ -0,0 +1,122 @@ +######################################################### +# To be migrated to the script-languages-container-tool # +######################################################### +from pathlib import Path, PurePosixPath +from unittest.mock import create_autospec, MagicMock +import pytest +from pyexasol import ExaConnection +from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation +from exasol_transformers_extension.deployment.language_container_deployer import ( + LanguageContainerDeployer, LanguageActivationLevel) + + +@pytest.fixture(scope='module') +def container_file_name() -> str: + return 'container_xyz.tag.gz' + + +@pytest.fixture(scope='module') +def container_file_path(container_file_name) -> Path: + return Path(container_file_name) + + +@pytest.fixture(scope='module') +def language_alias() -> str: + return 'PYTHON3_TEST' + + +@pytest.fixture(scope='module') +def container_bfs_path(container_file_name) -> str: + return f'bfsdefault/default/container/{container_file_name[:-7]}' + + +@pytest.fixture(scope='module') +def mock_pyexasol_conn() -> ExaConnection: + return create_autospec(ExaConnection) + + +@pytest.fixture(scope='module') +def mock_bfs_location(container_bfs_path) -> BucketFSLocation: + mock_loc = create_autospec(BucketFSLocation) + mock_loc.generate_bucket_udf_path.return_value = PurePosixPath(f'/buckets/{container_bfs_path}') + return mock_loc + + +@pytest.fixture +def container_deployer(mock_pyexasol_conn, mock_bfs_location, language_alias) -> LanguageContainerDeployer: + deployer = LanguageContainerDeployer(pyexasol_connection=mock_pyexasol_conn, + language_alias=language_alias, + bucketfs_location=mock_bfs_location) + + deployer.upload_container = MagicMock() + deployer.activate_container = MagicMock() + deployer.get_language_settings = MagicMock() + return deployer + + +def test_slc_deployer_deploy(container_deployer, container_file_name, container_file_path): + container_deployer.run(container_file=container_file_path, bucket_file_path=container_file_name, alter_system=True, + allow_override=True) + container_deployer.upload_container.assert_called_once_with(container_file_path, container_file_name) + container_deployer.activate_container.assert_called_once_with(container_file_name, LanguageActivationLevel.System, + True) + + +def test_slc_deployer_upload(container_deployer, container_file_name, container_file_path): + container_deployer.run(container_file=container_file_path, alter_system=False) + container_deployer.upload_container.assert_called_once_with(container_file_path, container_file_name) + container_deployer.activate_container.assert_not_called() + + +def test_slc_deployer_activate(container_deployer, container_file_name, container_file_path): + container_deployer.run(bucket_file_path=container_file_path, alter_system=True, allow_override=True) + container_deployer.upload_container.assert_not_called() + container_deployer.activate_container.assert_called_once_with(container_file_name, LanguageActivationLevel.System, + True) + + +def test_slc_deployer_generate_activation_command(container_deployer, language_alias, container_file_name, + container_bfs_path): + + container_deployer.get_language_settings.return_value = 'R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3' + + alter_type = LanguageActivationLevel.Session + expected_command = f"ALTER {alter_type.value.upper()} SET SCRIPT_LANGUAGES='" \ + "R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3 " \ + f"{language_alias}=localzmq+protobuf:///{container_bfs_path}?" \ + f"lang=python#/buckets/{container_bfs_path}/exaudf/exaudfclient_py3';" + + command = container_deployer.generate_activation_command(container_file_name, alter_type) + assert command == expected_command + + +def test_slc_deployer_generate_activation_command_override(container_deployer, language_alias, container_file_name, + container_bfs_path): + + current_bfs_path = 'bfsdefault/default/container_abc' + container_deployer.get_language_settings.return_value = \ + 'R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3 ' \ + f'{language_alias}=localzmq+protobuf:///{current_bfs_path}?' \ + f'lang=python#/buckets/{current_bfs_path}/exaudf/exaudfclient_py3' + + alter_type = LanguageActivationLevel.Session + expected_command = f"ALTER {alter_type.value.upper()} SET SCRIPT_LANGUAGES='" \ + "R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3 " \ + f"{language_alias}=localzmq+protobuf:///{container_bfs_path}?" \ + f"lang=python#/buckets/{container_bfs_path}/exaudf/exaudfclient_py3';" + + command = container_deployer.generate_activation_command(container_file_name, alter_type, allow_override=True) + assert command == expected_command + + +def test_slc_deployer_generate_activation_command_failure(container_deployer, language_alias, container_file_name): + + current_bfs_path = 'bfsdefault/default/container_abc' + container_deployer.get_language_settings.return_value = \ + 'R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3 ' \ + f'{language_alias}=localzmq+protobuf:///{current_bfs_path}?' \ + f'lang=python#/buckets/{current_bfs_path}/exaudf/exaudfclient_py3' + + with pytest.raises(RuntimeError): + container_deployer.generate_activation_command(container_file_name, LanguageActivationLevel.Session, + allow_override=False) diff --git a/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py b/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py deleted file mode 100644 index dc026ec1..00000000 --- a/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py +++ /dev/null @@ -1,48 +0,0 @@ -from pathlib import Path -from unittest.mock import create_autospec, MagicMock -import pytest -from pyexasol import ExaConnection -from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation -from exasol_transformers_extension.deployment.language_container_deployer import ( - LanguageContainerDeployer, LanguageActivationLevel) -from exasol_transformers_extension.deployment.language_container_deployer_cli import run_deployer - - -@pytest.fixture(scope='module') -def mock_pyexasol_conn() -> ExaConnection: - return create_autospec(ExaConnection) - - -@pytest.fixture(scope='module') -def mock_bfs_location() -> BucketFSLocation: - return create_autospec(BucketFSLocation) - - -@pytest.fixture -def container_deployer(mock_pyexasol_conn, mock_bfs_location) -> LanguageContainerDeployer: - return LanguageContainerDeployer(pyexasol_connection=mock_pyexasol_conn, - language_alias='alias', - bucketfs_location=mock_bfs_location, - container_file=Path('container_file')) - - -def test_language_container_deployer_cli_deploy(container_deployer): - container_deployer.deploy_container = MagicMock() - run_deployer(container_deployer, True, True, False) - container_deployer.deploy_container.assert_called_once_with(False) - - -def test_language_container_deployer_cli_upload(container_deployer): - container_deployer.upload_container = MagicMock() - container_deployer.activate_container = MagicMock() - run_deployer(container_deployer, True, False, False) - container_deployer.upload_container.assert_called_once() - container_deployer.activate_container.assert_not_called() - - -def test_language_container_deployer_cli_register(container_deployer): - container_deployer.upload_container = MagicMock() - container_deployer.activate_container = MagicMock() - run_deployer(container_deployer, False, True, True) - container_deployer.upload_container.assert_not_called() - container_deployer.activate_container.assert_called_once_with(LanguageActivationLevel.System, True)