From 8fae58e6baef6723b0588dda60cc8c98cdee9b39 Mon Sep 17 00:00:00 2001 From: mibe Date: Wed, 22 Nov 2023 11:27:04 +0000 Subject: [PATCH 1/6] Made the container uploading and language registration two separate actions --- .../deployment/language_container_deployer.py | 72 ++++++++++++++----- .../language_container_deployer_cli.py | 29 ++++++-- .../test_language_container_deployer.py | 65 +++++++++++++---- .../test_language_container_deployer_cli.py | 6 +- .../deployment/test_scripts_deployer_cli.py | 5 +- ...est_language_container_deployer_cli_run.py | 48 +++++++++++++ 6 files changed, 179 insertions(+), 46 deletions(-) create mode 100644 tests/unit_tests/deployment/test_language_container_deployer_cli_run.py diff --git a/exasol_transformers_extension/deployment/language_container_deployer.py b/exasol_transformers_extension/deployment/language_container_deployer.py index 5fecd024..34df7f01 100644 --- a/exasol_transformers_extension/deployment/language_container_deployer.py +++ b/exasol_transformers_extension/deployment/language_container_deployer.py @@ -1,5 +1,6 @@ +from enum import Enum import pyexasol -from typing import List +from typing import List, Optional from pathlib import Path, PurePosixPath from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation import logging @@ -10,6 +11,15 @@ logger = logging.getLogger(__name__) +class LanguageRegLevel(Enum): + f""" + Language alias registration level, i.e. + ALTER SET SCRIPT_LANGUAGES=... + """ + Session = 'SESSION' + System = 'SYSTEM' + + class LanguageContainerDeployer: def __init__(self, pyexasol_connection: pyexasol.ExaConnection, @@ -22,14 +32,18 @@ def __init__(self, self._pyexasol_conn = pyexasol_connection logger.debug(f"Init {LanguageContainerDeployer.__name__}") - def deploy_container(self): - path_in_udf = self._upload_container() - for alter in ["SESSION", "SYSTEM"]: - alter_command = self._generate_alter_command(alter, path_in_udf) - self._pyexasol_conn.execute(alter_command) - logging.debug(alter_command) + def deploy_container(self) -> None: + """ + Uploads the SLC and registers it at the SYSTEM level. + """ + path_in_udf = self.upload_container() + self.register_container(LanguageRegLevel.System, path_in_udf) - def _upload_container(self) -> PurePosixPath: + def upload_container(self) -> PurePosixPath: + """ + Uploads the SLC. + Returns the path where the container is uploaded as it's seen by a UDF. + """ if not self._container_file.is_file(): raise RuntimeError(f"Container file {self._container_file} " f"is not a file.") @@ -40,15 +54,37 @@ def _upload_container(self) -> PurePosixPath: logging.debug("Container is uploaded to bucketfs") return PurePosixPath(path_in_udf) - def _generate_alter_command(self, alter_type: str, - path_in_udf: PurePosixPath) -> str: + def register_container(self, alter_type: LanguageRegLevel = LanguageRegLevel.Session, + path_in_udf: Optional[PurePosixPath] = None) -> None: + """ + Registers the SLC container at the required level. + + alter_type - Language registration level, defaults to the SESSION. + path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF. + """ + alter_command = self.generate_alter_command(alter_type, path_in_udf) + self._pyexasol_conn.execute(alter_command) + logging.debug(alter_command) + + def generate_alter_command(self, alter_type: LanguageRegLevel, + path_in_udf: Optional[PurePosixPath] = None) -> str: + """ + Generates an SQL command to register the SLC container at the required level. The command will + preserve existing registrations of other containers identified by different language aliases. + Registration of a container with the same alias, if exists, will be overwritten. + + alter_type - Registration level - SYSTEM or SESSION. + path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF. + """ + if path_in_udf is None: + path_in_udf = self._bucketfs_location.generate_bucket_udf_path(self._container_file.name) new_settings = \ self._update_previous_language_settings(alter_type, path_in_udf) alter_command = \ - f"ALTER {alter_type} SET SCRIPT_LANGUAGES='{new_settings}';" + f"ALTER {alter_type.value} SET SCRIPT_LANGUAGES='{new_settings}';" return alter_command - def _update_previous_language_settings(self, alter_type: str, + def _update_previous_language_settings(self, alter_type: LanguageRegLevel, path_in_udf: PurePosixPath) -> str: prev_lang_settings = self._get_previous_language_settings(alter_type) prev_lang_aliases = prev_lang_settings.split(" ") @@ -81,18 +117,18 @@ def _check_if_requested_language_alias_already_exists( logging.warning(f"The requested language alias " f"{self._language_alias} is already in use.") - def _get_previous_language_settings(self, alter_type: str) -> str: + def _get_previous_language_settings(self, alter_type: LanguageRegLevel) -> str: result = self._pyexasol_conn.execute( - f"""SELECT "{alter_type}_VALUE" FROM SYS.EXA_PARAMETERS WHERE + f"""SELECT "{alter_type.value}_VALUE" FROM SYS.EXA_PARAMETERS WHERE PARAMETER_NAME='SCRIPT_LANGUAGES'""").fetchall() return result[0][0] @classmethod - def run(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int, + def create(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int, bucketfs_use_https: bool, bucketfs_user: str, container_file: Path, bucketfs_password: str, bucket: str, path_in_bucket: str, dsn: str, db_user: str, db_password: str, language_alias: str, - ssl_cert_path: str = None, use_ssl_cert_validation: bool = True): + ssl_cert_path: str = None, use_ssl_cert_validation: bool = True) -> "LanguageContainerDeployer": websocket_sslopt = get_websocket_ssl_options(use_ssl_cert_validation, ssl_cert_path) @@ -108,6 +144,4 @@ def run(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int, bucketfs_name, bucketfs_host, bucketfs_port, bucketfs_use_https, bucketfs_user, bucketfs_password, bucket, path_in_bucket) - language_container_deployer = cls( - pyexasol_conn, language_alias, bucketfs_location, container_file) - language_container_deployer.deploy_container() + return cls(pyexasol_conn, language_alias, bucketfs_location, container_file) diff --git a/exasol_transformers_extension/deployment/language_container_deployer_cli.py b/exasol_transformers_extension/deployment/language_container_deployer_cli.py index b8500828..1084234d 100644 --- a/exasol_transformers_extension/deployment/language_container_deployer_cli.py +++ b/exasol_transformers_extension/deployment/language_container_deployer_cli.py @@ -3,7 +3,20 @@ from pathlib import Path from exasol_transformers_extension.deployment import deployment_utils as utils from exasol_transformers_extension.deployment.language_container_deployer import \ - LanguageContainerDeployer + LanguageContainerDeployer, LanguageRegLevel + + +def run_deployer(deployer, upload_container: bool = True, alter_system: bool = True) -> None: + if upload_container and alter_system: + deployer.deploy_container() + elif upload_container: + deployer.upload_container() + elif alter_system: + deployer.register_container(LanguageRegLevel.System) + + if not alter_system: + print('Use the following command to register the SLC at the SESSION level:\n' + + deployer.generate_alter_command(LanguageRegLevel.Session)) @click.command(name="language-container") @@ -28,6 +41,8 @@ @click.option('--language-alias', type=str, default="PYTHON3_TE") @click.option('--ssl-cert-path', type=str, default="") @click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True) +@click.option('--upload-container/--no-upload_container', type=bool, default=True) +@click.option('--alter-system/--no-alter-system', type=bool, default=True) def language_container_deployer_main( bucketfs_name: str, bucketfs_host: str, @@ -44,9 +59,12 @@ def language_container_deployer_main( db_pass: str, language_alias: str, ssl_cert_path: str, - use_ssl_cert_validation: bool): + use_ssl_cert_validation: bool, + upload_container: bool, + alter_system: bool): + def call_runner(): - LanguageContainerDeployer.run( + deployer = LanguageContainerDeployer.create( bucketfs_name=bucketfs_name, bucketfs_host=bucketfs_host, bucketfs_port=bucketfs_port, @@ -61,8 +79,9 @@ def call_runner(): db_password=db_pass, language_alias=language_alias, ssl_cert_path=ssl_cert_path, - use_ssl_cert_validation=use_ssl_cert_validation - ) + use_ssl_cert_validation=use_ssl_cert_validation) + run_deployer(deployer, upload_container=upload_container, alter_system=alter_system) + if container_file: call_runner() elif version: diff --git a/tests/integration_tests/with_db/deployment/test_language_container_deployer.py b/tests/integration_tests/with_db/deployment/test_language_container_deployer.py index 12c94247..3e003d81 100644 --- a/tests/integration_tests/with_db/deployment/test_language_container_deployer.py +++ b/tests/integration_tests/with_db/deployment/test_language_container_deployer.py @@ -1,14 +1,17 @@ import textwrap +from typing import Callable from pathlib import Path from _pytest.fixtures import FixtureRequest +from tests.fixtures.language_container_fixture import export_slc, flavor_path +from tests.fixtures.database_connection_fixture import pyexasol_connection from exasol_bucketfs_utils_python.bucketfs_factory import BucketFSFactory from exasol_script_languages_container_tool.lib.tasks.export.export_info import ExportInfo from pyexasol import ExaConnection from pytest_itde import config from exasol_transformers_extension.deployment.language_container_deployer \ - import LanguageContainerDeployer + import LanguageContainerDeployer, LanguageRegLevel from tests.utils.parameters import bucketfs_params from tests.utils.revert_language_settings import revert_language_settings @@ -17,6 +20,8 @@ def test_language_container_deployer( request: FixtureRequest, export_slc: ExportInfo, pyexasol_connection: ExaConnection, + connection_factory: Callable[[config.Exasol], ExaConnection], + exasol_config: config.Exasol, bucketfs_config: config.BucketFs, ): test_name: str = request.node.name @@ -25,11 +30,41 @@ def test_language_container_deployer( container_path = Path(export_slc.cache_file) with revert_language_settings(pyexasol_connection): create_schema(pyexasol_connection, schema) - call_language_container_deployer(container_path=container_path, - language_alias=language_alias, - pyexasol_connection=pyexasol_connection, - bucketfs_config=bucketfs_config) - assert_udf_running(pyexasol_connection, language_alias) + deployer = create_container_deployer(container_path=container_path, + language_alias=language_alias, + pyexasol_connection=pyexasol_connection, + bucketfs_config=bucketfs_config) + deployer.deploy_container() + with connection_factory(exasol_config) as new_connection: + assert_udf_running(new_connection, language_alias, schema) + + +def test_language_container_deployer_alter_session( + request: FixtureRequest, + export_slc: ExportInfo, + pyexasol_connection: ExaConnection, + connection_factory: Callable[[config.Exasol], ExaConnection], + exasol_config: config.Exasol, + bucketfs_config: config.BucketFs, +): + test_name: str = request.node.name + schema = test_name + language_alias = f"PYTHON3_TE_{test_name.upper()}" + container_path = Path(export_slc.cache_file) + with revert_language_settings(pyexasol_connection): + create_schema(pyexasol_connection, schema) + deployer = create_container_deployer(container_path=container_path, + language_alias=language_alias, + pyexasol_connection=pyexasol_connection, + bucketfs_config=bucketfs_config) + deployer.upload_container() + with connection_factory(exasol_config) as new_connection: + deployer = create_container_deployer(container_path=container_path, + language_alias=language_alias, + pyexasol_connection=new_connection, + bucketfs_config=bucketfs_config) + deployer.register_container(LanguageRegLevel.Session) + assert_udf_running(new_connection, language_alias, schema) def create_schema(pyexasol_connection: ExaConnection, schema: str): @@ -37,22 +72,22 @@ def create_schema(pyexasol_connection: ExaConnection, schema: str): pyexasol_connection.execute(f"CREATE SCHEMA IF NOT EXISTS {schema};") -def assert_udf_running(pyexasol_connection: ExaConnection, language_alias: str): +def assert_udf_running(pyexasol_connection: ExaConnection, language_alias: str, schema: str): pyexasol_connection.execute(textwrap.dedent(f""" - CREATE OR REPLACE {language_alias} SCALAR SCRIPT "TEST_UDF"() + CREATE OR REPLACE {language_alias} SCALAR SCRIPT {schema}."TEST_UDF"() RETURNS BOOLEAN AS def run(ctx): return True / """)) - result = pyexasol_connection.execute('SELECT "TEST_UDF"()').fetchall() + result = pyexasol_connection.execute(f'SELECT {schema}."TEST_UDF"()').fetchall() assert result[0][0] == True -def call_language_container_deployer(container_path: Path, - language_alias: str, - pyexasol_connection: ExaConnection, - bucketfs_config: config.BucketFs): +def create_container_deployer(container_path: Path, + language_alias: str, + pyexasol_connection: ExaConnection, + bucketfs_config: config.BucketFs) -> LanguageContainerDeployer: bucket_fs_factory = BucketFSFactory() bucketfs_location = bucket_fs_factory.create_bucketfs_location( url=f"{bucketfs_config.url}/" @@ -61,6 +96,6 @@ def call_language_container_deployer(container_path: Path, user=f"{bucketfs_config.username}", pwd=f"{bucketfs_config.password}", base_path=None) - language_container_deployer = LanguageContainerDeployer( + return LanguageContainerDeployer( pyexasol_connection, language_alias, bucketfs_location, container_path) - language_container_deployer.deploy_container() + diff --git a/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py b/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py index 3370c896..21c7de5c 100644 --- a/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py +++ b/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py @@ -174,13 +174,11 @@ def test_language_container_deployer_cli_with_check_cert( request: FixtureRequest, export_slc: ExportInfo, pyexasol_connection: ExaConnection, - connection_factory: Callable[[config.Exasol], ExaConnection], exasol_config: config.Exasol, bucketfs_config: config.BucketFs ): use_ssl_cert_validation = True - expected_exception_message = 'Could not connect to Exasol: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify ' \ - 'failed: self signed certificate in certificate chain (_ssl.c:1131)' + expected_exception_message = '[SSL: CERTIFICATE_VERIFY_FAILED]' test_name: str = request.node.name schema = test_name language_alias = f"PYTHON3_TE_{test_name.upper()}" @@ -198,5 +196,5 @@ def test_language_container_deployer_cli_with_check_cert( use_ssl_cert_validation=use_ssl_cert_validation) assert result.exit_code == 1 \ - and result.exception.args[0].message in expected_exception_message \ + and expected_exception_message in result.exception.args[0].message \ and type(result.exception) == ExaConnectionFailedError diff --git a/tests/integration_tests/with_db/deployment/test_scripts_deployer_cli.py b/tests/integration_tests/with_db/deployment/test_scripts_deployer_cli.py index 34ca0814..79ac0fa4 100644 --- a/tests/integration_tests/with_db/deployment/test_scripts_deployer_cli.py +++ b/tests/integration_tests/with_db/deployment/test_scripts_deployer_cli.py @@ -49,11 +49,10 @@ def test_scripts_deployer_cli_with_encryption_verify(language_alias: str, "--language-alias", language_alias, "--use-ssl-cert-validation" ] - expected_exception_message = 'Could not connect to Exasol: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify ' \ - 'failed: self signed certificate in certificate chain (_ssl.c:1131)' + expected_exception_message = '[SSL: CERTIFICATE_VERIFY_FAILED]' runner = CliRunner() result = runner.invoke(deploy.main, args_list) assert result.exit_code == 1 \ - and result.exception.args[0].message in expected_exception_message \ + and expected_exception_message in result.exception.args[0].message \ and type(result.exception) == ExaConnectionFailedError diff --git a/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py b/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py new file mode 100644 index 00000000..04d0604d --- /dev/null +++ b/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py @@ -0,0 +1,48 @@ +from pathlib import Path +from unittest.mock import create_autospec, MagicMock +import pytest +from pyexasol import ExaConnection +from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation +from exasol_transformers_extension.deployment.language_container_deployer import ( + LanguageContainerDeployer, LanguageRegLevel) +from exasol_transformers_extension.deployment.language_container_deployer_cli import run_deployer + + +@pytest.fixture(scope='module') +def mock_pyexasol_conn() -> ExaConnection: + return create_autospec(ExaConnection) + + +@pytest.fixture(scope='module') +def mock_bfs_location() -> BucketFSLocation: + return create_autospec(BucketFSLocation) + + +@pytest.fixture +def container_deployer(mock_pyexasol_conn, mock_bfs_location) -> LanguageContainerDeployer: + return LanguageContainerDeployer(pyexasol_connection=mock_pyexasol_conn, + language_alias='alias', + bucketfs_location=mock_bfs_location, + container_file=Path('container_file')) + + +def test_language_container_deployer_cli_deploy(container_deployer): + container_deployer.deploy_container = MagicMock() + run_deployer(container_deployer, True, True) + container_deployer.deploy_container.assert_called_once() + + +def test_language_container_deployer_cli_upload(container_deployer): + container_deployer.upload_container = MagicMock() + container_deployer.register_container = MagicMock() + run_deployer(container_deployer, True, False) + container_deployer.upload_container.assert_called_once() + container_deployer.register_container.assert_not_called() + + +def test_language_container_deployer_cli_register(container_deployer): + container_deployer.upload_container = MagicMock() + container_deployer.register_container = MagicMock() + run_deployer(container_deployer, False, True) + container_deployer.upload_container.assert_not_called() + container_deployer.register_container.assert_called_once_with(LanguageRegLevel.System) From 6594eca96879112215965739e853803220386531 Mon Sep 17 00:00:00 2001 From: mibe Date: Wed, 22 Nov 2023 17:16:35 +0000 Subject: [PATCH 2/6] [CodeBuild] From c4e99e6924e27194b5bbcf4efd294d44ba5aba13 Mon Sep 17 00:00:00 2001 From: mibe Date: Fri, 24 Nov 2023 16:19:39 +0000 Subject: [PATCH 3/6] Addressed review comments [CodeBuild] --- doc/user_guide/user_guide.md | 28 ++++++-- .../deployment/language_container_deployer.py | 66 +++++++++++-------- .../language_container_deployer_cli.py | 21 +++--- .../test_language_container_deployer.py | 35 +++++++++- ...est_language_container_deployer_cli_run.py | 18 ++--- 5 files changed, 118 insertions(+), 50 deletions(-) diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index 1b65a4a4..4e04bd13 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -109,13 +109,30 @@ Transformers Extension Package. See [the latest release](https://github.com/exas --language-alias \ --version \ --ssl-cert-path \ - --use-ssl-cert-validation + --use-ssl-cert-validation \ ``` The `--ssl-cert-path` is optional if your certificate is not in the OS truststore. The option `--use-ssl-cert-validation`is the default, you can disable it with `--no-use-ssl-cert-validation`. Use caution if you want to turn certificate validation off as it potentially lowers the security of your Database connection. +By default, the above command will upload and activate the language container at the System level. +The latter requires you to have the System Privileges, as it will attempt to change DB system settings. +If such privileges cannot be granted the activation can be skipped by using the `--no-alter-system` option. +The command will then print the language activation SQL query, which looks like this: +```sql +ALTER SESSION SET SCRIPT_LANGUAGES=... +``` +This query activates the language container at the Session level. It doesn't require System Privileges. +However, it must be run every time a new session starts. + +It is also possible to activate the language without repeatedly uploading the container. If the container +has already been uploaded one can use the `--no-upload_container` option to skip this step. + +By default, overriding language activation is not permitted. If a language with the same alias has already +been activated the command will result in an error. To override the activation, you can use the +`--allow_override` option. + #### Customized Installation In this installation, you can install the desired or customized language container. In the following steps, it is explained how to install the @@ -132,8 +149,8 @@ There are two ways to install the language container: (1) using a python script 1. *Installation with Python Script* To install the language container, it is necessary to load the container - into the BucketFS and register it to the database. The following command - provides this setup using the python script provided with this library: + into the BucketFS and activate it in the database. The following command + performs this setup using the python script provided with this library: ```buildoutcfg python -m exasol_transformers_extension.deploy language-container @@ -150,6 +167,9 @@ There are two ways to install the language container: (1) using a python script --language-alias \ --container-file ``` + Please note, that all considerations described in the Quick Installation + section are still applicable. + 2. *Manual Installation* @@ -171,7 +191,7 @@ There are two ways to install the language container: (1) using a python script ``` The uploaded container should be secondly activated through adjusting - the session parameter `SCRIPT_LANGUAGES`. The activation can be scoped + the session parameter `SCRIPT_LANGUAGES`. As it was mentioned before, the activation can be scoped either session-wide (`ALTER SESSION`) or system-wide (`ALTER SYSTEM`). The following example query activates the container session-wide: diff --git a/exasol_transformers_extension/deployment/language_container_deployer.py b/exasol_transformers_extension/deployment/language_container_deployer.py index 34df7f01..39ca167a 100644 --- a/exasol_transformers_extension/deployment/language_container_deployer.py +++ b/exasol_transformers_extension/deployment/language_container_deployer.py @@ -11,10 +11,10 @@ logger = logging.getLogger(__name__) -class LanguageRegLevel(Enum): +class LanguageActiveLevel(Enum): f""" - Language alias registration level, i.e. - ALTER SET SCRIPT_LANGUAGES=... + Language activation level, i.e. + ALTER SET SCRIPT_LANGUAGES=... """ Session = 'SESSION' System = 'SYSTEM' @@ -32,12 +32,15 @@ def __init__(self, self._pyexasol_conn = pyexasol_connection logger.debug(f"Init {LanguageContainerDeployer.__name__}") - def deploy_container(self) -> None: + def deploy_container(self, allow_override: bool = False) -> None: """ - Uploads the SLC and registers it at the SYSTEM level. + Uploads the SLC and activates it at the SYSTEM level. + + allow_override - If True the activation of a language container with the same alias will be overriden, + otherwise a RuntimeException will be thrown. """ path_in_udf = self.upload_container() - self.register_container(LanguageRegLevel.System, path_in_udf) + self.activate_container(LanguageActiveLevel.System, allow_override, path_in_udf) def upload_container(self) -> PurePosixPath: """ @@ -54,42 +57,49 @@ def upload_container(self) -> PurePosixPath: logging.debug("Container is uploaded to bucketfs") return PurePosixPath(path_in_udf) - def register_container(self, alter_type: LanguageRegLevel = LanguageRegLevel.Session, + def activate_container(self, alter_type: LanguageActiveLevel = LanguageActiveLevel.Session, + allow_override: bool = False, path_in_udf: Optional[PurePosixPath] = None) -> None: """ - Registers the SLC container at the required level. + Activates the SLC container at the required level. - alter_type - Language registration level, defaults to the SESSION. - path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF. + alter_type - Language activation level, defaults to the SESSION. + allow_override - If True the activation of a language container with the same alias will be overriden, + otherwise a RuntimeException will be thrown. + path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF. """ - alter_command = self.generate_alter_command(alter_type, path_in_udf) + alter_command = self.generate_activation_command(alter_type, allow_override, path_in_udf) self._pyexasol_conn.execute(alter_command) logging.debug(alter_command) - def generate_alter_command(self, alter_type: LanguageRegLevel, - path_in_udf: Optional[PurePosixPath] = None) -> str: + def generate_activation_command(self, alter_type: LanguageActiveLevel, + allow_override: bool = False, + path_in_udf: Optional[PurePosixPath] = None) -> str: """ - Generates an SQL command to register the SLC container at the required level. The command will - preserve existing registrations of other containers identified by different language aliases. - Registration of a container with the same alias, if exists, will be overwritten. - - alter_type - Registration level - SYSTEM or SESSION. - path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF. + Generates an SQL command to activate the SLC container at the required level. The command will + preserve existing activations of other containers identified by different language aliases. + Activation of a container with the same alias, if exists, will be overwritten. + + alter_type - Activation level - SYSTEM or SESSION. + allow_override - If True the activation of a language container with the same alias will be overriden, + otherwise a RuntimeException will be thrown. + path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF. """ if path_in_udf is None: path_in_udf = self._bucketfs_location.generate_bucket_udf_path(self._container_file.name) new_settings = \ - self._update_previous_language_settings(alter_type, path_in_udf) + self._update_previous_language_settings(alter_type, allow_override, path_in_udf) alter_command = \ f"ALTER {alter_type.value} SET SCRIPT_LANGUAGES='{new_settings}';" return alter_command - def _update_previous_language_settings(self, alter_type: LanguageRegLevel, + def _update_previous_language_settings(self, alter_type: LanguageActiveLevel, + allow_override: bool, path_in_udf: PurePosixPath) -> str: prev_lang_settings = self._get_previous_language_settings(alter_type) prev_lang_aliases = prev_lang_settings.split(" ") self._check_if_requested_language_alias_already_exists( - prev_lang_aliases) + allow_override, prev_lang_aliases) new_definitions_str = self._generate_new_language_settings( path_in_udf, prev_lang_aliases) return new_definitions_str @@ -109,15 +119,19 @@ def _generate_new_language_settings(self, path_in_udf: PurePosixPath, return new_definitions_str def _check_if_requested_language_alias_already_exists( - self, prev_lang_aliases: List[str]) -> None: + self, allow_override: bool, + prev_lang_aliases: List[str]) -> None: definition_for_requested_alias = [ alias_definition for alias_definition in prev_lang_aliases if alias_definition.startswith(self._language_alias + "=")] if not len(definition_for_requested_alias) == 0: - logging.warning(f"The requested language alias " - f"{self._language_alias} is already in use.") + warning_message = f"The requested language alias {self._language_alias} is already in use." + if allow_override: + logging.warning(warning_message) + else: + raise RuntimeError(warning_message) - def _get_previous_language_settings(self, alter_type: LanguageRegLevel) -> str: + def _get_previous_language_settings(self, alter_type: LanguageActiveLevel) -> str: result = self._pyexasol_conn.execute( f"""SELECT "{alter_type.value}_VALUE" FROM SYS.EXA_PARAMETERS WHERE PARAMETER_NAME='SCRIPT_LANGUAGES'""").fetchall() diff --git a/exasol_transformers_extension/deployment/language_container_deployer_cli.py b/exasol_transformers_extension/deployment/language_container_deployer_cli.py index 1084234d..1e7a772a 100644 --- a/exasol_transformers_extension/deployment/language_container_deployer_cli.py +++ b/exasol_transformers_extension/deployment/language_container_deployer_cli.py @@ -3,20 +3,22 @@ from pathlib import Path from exasol_transformers_extension.deployment import deployment_utils as utils from exasol_transformers_extension.deployment.language_container_deployer import \ - LanguageContainerDeployer, LanguageRegLevel + LanguageContainerDeployer, LanguageActiveLevel -def run_deployer(deployer, upload_container: bool = True, alter_system: bool = True) -> None: +def run_deployer(deployer, upload_container: bool = True, + alter_system: bool = True, + allow_override: bool = False) -> None: if upload_container and alter_system: - deployer.deploy_container() + deployer.deploy_container(allow_override) elif upload_container: deployer.upload_container() elif alter_system: - deployer.register_container(LanguageRegLevel.System) + deployer.activate_container(LanguageActiveLevel.System, allow_override) if not alter_system: - print('Use the following command to register the SLC at the SESSION level:\n' + - deployer.generate_alter_command(LanguageRegLevel.Session)) + print('Use the following command to activate the SLC at the SESSION level:\n' + + deployer.generate_activation_command(LanguageActiveLevel.Session, True)) @click.command(name="language-container") @@ -43,6 +45,7 @@ def run_deployer(deployer, upload_container: bool = True, alter_system: bool = T @click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True) @click.option('--upload-container/--no-upload_container', type=bool, default=True) @click.option('--alter-system/--no-alter-system', type=bool, default=True) +@click.option('--allow-override/--disallow-override', type=bool, default=False) def language_container_deployer_main( bucketfs_name: str, bucketfs_host: str, @@ -61,7 +64,8 @@ def language_container_deployer_main( ssl_cert_path: str, use_ssl_cert_validation: bool, upload_container: bool, - alter_system: bool): + alter_system: bool, + allow_override: bool): def call_runner(): deployer = LanguageContainerDeployer.create( @@ -80,7 +84,8 @@ def call_runner(): language_alias=language_alias, ssl_cert_path=ssl_cert_path, use_ssl_cert_validation=use_ssl_cert_validation) - run_deployer(deployer, upload_container=upload_container, alter_system=alter_system) + run_deployer(deployer, upload_container=upload_container, alter_system=alter_system, + allow_override=allow_override) if container_file: call_runner() diff --git a/tests/integration_tests/with_db/deployment/test_language_container_deployer.py b/tests/integration_tests/with_db/deployment/test_language_container_deployer.py index 3e003d81..597118c7 100644 --- a/tests/integration_tests/with_db/deployment/test_language_container_deployer.py +++ b/tests/integration_tests/with_db/deployment/test_language_container_deployer.py @@ -2,6 +2,7 @@ from typing import Callable from pathlib import Path +import pytest from _pytest.fixtures import FixtureRequest from tests.fixtures.language_container_fixture import export_slc, flavor_path from tests.fixtures.database_connection_fixture import pyexasol_connection @@ -11,7 +12,7 @@ from pytest_itde import config from exasol_transformers_extension.deployment.language_container_deployer \ - import LanguageContainerDeployer, LanguageRegLevel + import LanguageContainerDeployer, LanguageActiveLevel from tests.utils.parameters import bucketfs_params from tests.utils.revert_language_settings import revert_language_settings @@ -34,7 +35,7 @@ def test_language_container_deployer( language_alias=language_alias, pyexasol_connection=pyexasol_connection, bucketfs_config=bucketfs_config) - deployer.deploy_container() + deployer.deploy_container(True) with connection_factory(exasol_config) as new_connection: assert_udf_running(new_connection, language_alias, schema) @@ -63,10 +64,38 @@ def test_language_container_deployer_alter_session( language_alias=language_alias, pyexasol_connection=new_connection, bucketfs_config=bucketfs_config) - deployer.register_container(LanguageRegLevel.Session) + deployer.activate_container(LanguageActiveLevel.Session, True) assert_udf_running(new_connection, language_alias, schema) +def test_language_container_deployer_activation_fail( + request: FixtureRequest, + export_slc: ExportInfo, + pyexasol_connection: ExaConnection, + connection_factory: Callable[[config.Exasol], ExaConnection], + exasol_config: config.Exasol, + bucketfs_config: config.BucketFs, +): + test_name: str = request.node.name + schema = test_name + language_alias = f"PYTHON3_TE_{test_name.upper()}" + container_path = Path(export_slc.cache_file) + with revert_language_settings(pyexasol_connection): + create_schema(pyexasol_connection, schema) + deployer = create_container_deployer(container_path=container_path, + language_alias=language_alias, + pyexasol_connection=pyexasol_connection, + bucketfs_config=bucketfs_config) + deployer.deploy_container(True) + with connection_factory(exasol_config) as new_connection: + deployer = create_container_deployer(container_path=container_path, + language_alias=language_alias, + pyexasol_connection=new_connection, + bucketfs_config=bucketfs_config) + with pytest.raises(RuntimeError): + deployer.activate_container(LanguageActiveLevel.System, False) + + def create_schema(pyexasol_connection: ExaConnection, schema: str): pyexasol_connection.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE;") pyexasol_connection.execute(f"CREATE SCHEMA IF NOT EXISTS {schema};") diff --git a/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py b/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py index 04d0604d..8d87bcff 100644 --- a/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py +++ b/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py @@ -4,7 +4,7 @@ from pyexasol import ExaConnection from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation from exasol_transformers_extension.deployment.language_container_deployer import ( - LanguageContainerDeployer, LanguageRegLevel) + LanguageContainerDeployer, LanguageActiveLevel) from exasol_transformers_extension.deployment.language_container_deployer_cli import run_deployer @@ -28,21 +28,21 @@ def container_deployer(mock_pyexasol_conn, mock_bfs_location) -> LanguageContain def test_language_container_deployer_cli_deploy(container_deployer): container_deployer.deploy_container = MagicMock() - run_deployer(container_deployer, True, True) - container_deployer.deploy_container.assert_called_once() + run_deployer(container_deployer, True, True, False) + container_deployer.deploy_container.assert_called_once_with(False) def test_language_container_deployer_cli_upload(container_deployer): container_deployer.upload_container = MagicMock() - container_deployer.register_container = MagicMock() - run_deployer(container_deployer, True, False) + container_deployer.activate_container = MagicMock() + run_deployer(container_deployer, True, False, False) container_deployer.upload_container.assert_called_once() - container_deployer.register_container.assert_not_called() + container_deployer.activate_container.assert_not_called() def test_language_container_deployer_cli_register(container_deployer): container_deployer.upload_container = MagicMock() - container_deployer.register_container = MagicMock() - run_deployer(container_deployer, False, True) + container_deployer.activate_container = MagicMock() + run_deployer(container_deployer, False, True, True) container_deployer.upload_container.assert_not_called() - container_deployer.register_container.assert_called_once_with(LanguageRegLevel.System) + container_deployer.activate_container.assert_called_once_with(LanguageActiveLevel.System, True) From 625c23abc22ef47e50953b97c4ded7167903f679 Mon Sep 17 00:00:00 2001 From: mibe Date: Tue, 28 Nov 2023 12:42:13 +0000 Subject: [PATCH 4/6] More review comments [CodeBuild] --- doc/user_guide/user_guide.md | 15 +++++++++------ .../deployment/language_container_deployer.py | 14 +++++++------- .../language_container_deployer_cli.py | 18 ++++++++++++++---- .../test_language_container_deployer.py | 6 +++--- ...test_language_container_deployer_cli_run.py | 4 ++-- 5 files changed, 35 insertions(+), 22 deletions(-) diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index 4e04bd13..ccf80661 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -109,7 +109,7 @@ Transformers Extension Package. See [the latest release](https://github.com/exas --language-alias \ --version \ --ssl-cert-path \ - --use-ssl-cert-validation \ + --use-ssl-cert-validation ``` The `--ssl-cert-path` is optional if your certificate is not in the OS truststore. The option `--use-ssl-cert-validation`is the default, you can disable it with `--no-use-ssl-cert-validation`. @@ -119,19 +119,22 @@ Database connection. By default, the above command will upload and activate the language container at the System level. The latter requires you to have the System Privileges, as it will attempt to change DB system settings. If such privileges cannot be granted the activation can be skipped by using the `--no-alter-system` option. -The command will then print the language activation SQL query, which looks like this: +The command will then print two possible language activation SQL queries, which look like the following: ```sql ALTER SESSION SET SCRIPT_LANGUAGES=... +ALTER SYSTEM SET SCRIPT_LANGUAGES=... ``` -This query activates the language container at the Session level. It doesn't require System Privileges. -However, it must be run every time a new session starts. +These quires represent two alternative ways of activating a language container. The first one activates the +container at the Session level. It doesn't require System Privileges. However, it must be run every time a +new session starts. The second one activates the container at the System level. It needs to be run just once, +but it does require System Privileges. It may be executed by a database administrator. It is also possible to activate the language without repeatedly uploading the container. If the container -has already been uploaded one can use the `--no-upload_container` option to skip this step. +has already been uploaded one can use the `--no-upload-container` option to skip this step. By default, overriding language activation is not permitted. If a language with the same alias has already been activated the command will result in an error. To override the activation, you can use the -`--allow_override` option. +`--allow-override` option. #### Customized Installation In this installation, you can install the desired or customized language diff --git a/exasol_transformers_extension/deployment/language_container_deployer.py b/exasol_transformers_extension/deployment/language_container_deployer.py index 39ca167a..8da2a82a 100644 --- a/exasol_transformers_extension/deployment/language_container_deployer.py +++ b/exasol_transformers_extension/deployment/language_container_deployer.py @@ -11,10 +11,10 @@ logger = logging.getLogger(__name__) -class LanguageActiveLevel(Enum): +class LanguageActivationLevel(Enum): f""" Language activation level, i.e. - ALTER SET SCRIPT_LANGUAGES=... + ALTER SET SCRIPT_LANGUAGES=... """ Session = 'SESSION' System = 'SYSTEM' @@ -40,7 +40,7 @@ def deploy_container(self, allow_override: bool = False) -> None: otherwise a RuntimeException will be thrown. """ path_in_udf = self.upload_container() - self.activate_container(LanguageActiveLevel.System, allow_override, path_in_udf) + self.activate_container(LanguageActivationLevel.System, allow_override, path_in_udf) def upload_container(self) -> PurePosixPath: """ @@ -57,7 +57,7 @@ def upload_container(self) -> PurePosixPath: logging.debug("Container is uploaded to bucketfs") return PurePosixPath(path_in_udf) - def activate_container(self, alter_type: LanguageActiveLevel = LanguageActiveLevel.Session, + def activate_container(self, alter_type: LanguageActivationLevel = LanguageActivationLevel.Session, allow_override: bool = False, path_in_udf: Optional[PurePosixPath] = None) -> None: """ @@ -72,7 +72,7 @@ def activate_container(self, alter_type: LanguageActiveLevel = LanguageActiveLev self._pyexasol_conn.execute(alter_command) logging.debug(alter_command) - def generate_activation_command(self, alter_type: LanguageActiveLevel, + def generate_activation_command(self, alter_type: LanguageActivationLevel, allow_override: bool = False, path_in_udf: Optional[PurePosixPath] = None) -> str: """ @@ -93,7 +93,7 @@ def generate_activation_command(self, alter_type: LanguageActiveLevel, f"ALTER {alter_type.value} SET SCRIPT_LANGUAGES='{new_settings}';" return alter_command - def _update_previous_language_settings(self, alter_type: LanguageActiveLevel, + def _update_previous_language_settings(self, alter_type: LanguageActivationLevel, allow_override: bool, path_in_udf: PurePosixPath) -> str: prev_lang_settings = self._get_previous_language_settings(alter_type) @@ -131,7 +131,7 @@ def _check_if_requested_language_alias_already_exists( else: raise RuntimeError(warning_message) - def _get_previous_language_settings(self, alter_type: LanguageActiveLevel) -> str: + def _get_previous_language_settings(self, alter_type: LanguageActivationLevel) -> str: result = self._pyexasol_conn.execute( f"""SELECT "{alter_type.value}_VALUE" FROM SYS.EXA_PARAMETERS WHERE PARAMETER_NAME='SCRIPT_LANGUAGES'""").fetchall() diff --git a/exasol_transformers_extension/deployment/language_container_deployer_cli.py b/exasol_transformers_extension/deployment/language_container_deployer_cli.py index 1e7a772a..de511212 100644 --- a/exasol_transformers_extension/deployment/language_container_deployer_cli.py +++ b/exasol_transformers_extension/deployment/language_container_deployer_cli.py @@ -1,9 +1,10 @@ import os import click from pathlib import Path +from textwrap import dedent from exasol_transformers_extension.deployment import deployment_utils as utils from exasol_transformers_extension.deployment.language_container_deployer import \ - LanguageContainerDeployer, LanguageActiveLevel + LanguageContainerDeployer, LanguageActivationLevel def run_deployer(deployer, upload_container: bool = True, @@ -14,11 +15,20 @@ def run_deployer(deployer, upload_container: bool = True, elif upload_container: deployer.upload_container() elif alter_system: - deployer.activate_container(LanguageActiveLevel.System, allow_override) + deployer.activate_container(LanguageActivationLevel.System, allow_override) if not alter_system: - print('Use the following command to activate the SLC at the SESSION level:\n' + - deployer.generate_activation_command(LanguageActiveLevel.Session, True)) + message = dedent(f""" + In SQL, you can activate the SLC of the Transformer Extension + by using the following statements: + + To activate the SLC only for the current session: + {deployer.generate_activation_command(LanguageActivationLevel.Session, True)} + + To activate the SLC on the system: + {deployer.generate_activation_command(LanguageActivationLevel.System, True)} + """) + print(message) @click.command(name="language-container") diff --git a/tests/integration_tests/with_db/deployment/test_language_container_deployer.py b/tests/integration_tests/with_db/deployment/test_language_container_deployer.py index 597118c7..21183a59 100644 --- a/tests/integration_tests/with_db/deployment/test_language_container_deployer.py +++ b/tests/integration_tests/with_db/deployment/test_language_container_deployer.py @@ -12,7 +12,7 @@ from pytest_itde import config from exasol_transformers_extension.deployment.language_container_deployer \ - import LanguageContainerDeployer, LanguageActiveLevel + import LanguageContainerDeployer, LanguageActivationLevel from tests.utils.parameters import bucketfs_params from tests.utils.revert_language_settings import revert_language_settings @@ -64,7 +64,7 @@ def test_language_container_deployer_alter_session( language_alias=language_alias, pyexasol_connection=new_connection, bucketfs_config=bucketfs_config) - deployer.activate_container(LanguageActiveLevel.Session, True) + deployer.activate_container(LanguageActivationLevel.Session, True) assert_udf_running(new_connection, language_alias, schema) @@ -93,7 +93,7 @@ def test_language_container_deployer_activation_fail( pyexasol_connection=new_connection, bucketfs_config=bucketfs_config) with pytest.raises(RuntimeError): - deployer.activate_container(LanguageActiveLevel.System, False) + deployer.activate_container(LanguageActivationLevel.System, False) def create_schema(pyexasol_connection: ExaConnection, schema: str): diff --git a/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py b/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py index 8d87bcff..dc026ec1 100644 --- a/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py +++ b/tests/unit_tests/deployment/test_language_container_deployer_cli_run.py @@ -4,7 +4,7 @@ from pyexasol import ExaConnection from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation from exasol_transformers_extension.deployment.language_container_deployer import ( - LanguageContainerDeployer, LanguageActiveLevel) + LanguageContainerDeployer, LanguageActivationLevel) from exasol_transformers_extension.deployment.language_container_deployer_cli import run_deployer @@ -45,4 +45,4 @@ def test_language_container_deployer_cli_register(container_deployer): container_deployer.activate_container = MagicMock() run_deployer(container_deployer, False, True, True) container_deployer.upload_container.assert_not_called() - container_deployer.activate_container.assert_called_once_with(LanguageActiveLevel.System, True) + container_deployer.activate_container.assert_called_once_with(LanguageActivationLevel.System, True) From c4648c669def677a999e2b9dfdca9fdbcb0e7fea Mon Sep 17 00:00:00 2001 From: mibe Date: Tue, 28 Nov 2023 15:23:59 +0000 Subject: [PATCH 5/6] Some User Guide updates [CodeBuild] --- doc/user_guide/user_guide.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index ccf80661..f34c6599 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -124,10 +124,13 @@ The command will then print two possible language activation SQL queries, which ALTER SESSION SET SCRIPT_LANGUAGES=... ALTER SYSTEM SET SCRIPT_LANGUAGES=... ``` -These quires represent two alternative ways of activating a language container. The first one activates the -container at the Session level. It doesn't require System Privileges. However, it must be run every time a -new session starts. The second one activates the container at the System level. It needs to be run just once, -but it does require System Privileges. It may be executed by a database administrator. +These queries represent two alternative ways of activating a language container. The first one activates the +container at the [Session level](https://docs.exasol.com/db/latest/sql/alter_session.htm). It doesn't require +System Privileges. However, it must be run every time a new session starts. The second one activates the container +at the [System level](https://docs.exasol.com/db/latest/sql/alter_system.htm). It needs to be run just once, +but it does require System Privileges. It may be executed by a database administrator. Please note, that changes +made at the system level only become effective in new sessions, as described +[here](https://docs.exasol.com/db/latest/sql/alter_system.htm#microcontent1). It is also possible to activate the language without repeatedly uploading the container. If the container has already been uploaded one can use the `--no-upload-container` option to skip this step. From 20368f6d80fd35ddb3aa995e48639289afe4266e Mon Sep 17 00:00:00 2001 From: mibe Date: Tue, 28 Nov 2023 17:17:01 +0000 Subject: [PATCH 6/6] Some User Guide updates [CodeBuild] --- .../deployment/language_container_deployer_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol_transformers_extension/deployment/language_container_deployer_cli.py b/exasol_transformers_extension/deployment/language_container_deployer_cli.py index de511212..e8224c53 100644 --- a/exasol_transformers_extension/deployment/language_container_deployer_cli.py +++ b/exasol_transformers_extension/deployment/language_container_deployer_cli.py @@ -19,7 +19,7 @@ def run_deployer(deployer, upload_container: bool = True, if not alter_system: message = dedent(f""" - In SQL, you can activate the SLC of the Transformer Extension + In SQL, you can activate the SLC of the Transformers Extension by using the following statements: To activate the SLC only for the current session: