Made the container uploading and language registration two separate a…

…ctions

exasol_transformers_extension/deployment/language_container_deployer.py

@@ -1,5 +1,6 @@
+from enum import Enum
 import pyexasol
-from typing import List
+from typing import List, Optional
 from pathlib import Path, PurePosixPath
 from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation
 import logging
@@ -10,6 +11,15 @@
 logger = logging.getLogger(__name__)
 
 
+class LanguageRegLevel(Enum):
+    f"""
+    Language alias registration level, i.e.
+    ALTER <LanguageRegLevel> SET SCRIPT_LANGUAGES=...
+    """
+    Session = 'SESSION'
+    System = 'SYSTEM'
+
+
 class LanguageContainerDeployer:
     def __init__(self,
                  pyexasol_connection: pyexasol.ExaConnection,
@@ -22,14 +32,18 @@ def __init__(self,
         self._pyexasol_conn = pyexasol_connection
         logger.debug(f"Init {LanguageContainerDeployer.__name__}")
 
-    def deploy_container(self):
-        path_in_udf = self._upload_container()
-        for alter in ["SESSION", "SYSTEM"]:
-            alter_command = self._generate_alter_command(alter, path_in_udf)
-            self._pyexasol_conn.execute(alter_command)
-            logging.debug(alter_command)
+    def deploy_container(self) -> None:
+        """
+        Uploads the SLC and registers it at the SYSTEM level.
+        """
+        path_in_udf = self.upload_container()
+        self.register_container(LanguageRegLevel.System, path_in_udf)
 
-    def _upload_container(self) -> PurePosixPath:
+    def upload_container(self) -> PurePosixPath:
+        """
+        Uploads the SLC.
+        Returns the path where the container is uploaded as it's seen by a UDF.
+        """
         if not self._container_file.is_file():
             raise RuntimeError(f"Container file {self._container_file} "
                                f"is not a file.")
@@ -40,15 +54,37 @@ def _upload_container(self) -> PurePosixPath:
         logging.debug("Container is uploaded to bucketfs")
         return PurePosixPath(path_in_udf)
 
-    def _generate_alter_command(self, alter_type: str,
-                                path_in_udf: PurePosixPath) -> str:
+    def register_container(self, alter_type: LanguageRegLevel = LanguageRegLevel.Session,
+                           path_in_udf: Optional[PurePosixPath] = None) -> None:
+        """
+        Registers the SLC container at the required level.
+
+        alter_type  - Language registration level, defaults to the SESSION.
+        path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF.
+        """
+        alter_command = self.generate_alter_command(alter_type, path_in_udf)
+        self._pyexasol_conn.execute(alter_command)
+        logging.debug(alter_command)
+
+    def generate_alter_command(self, alter_type: LanguageRegLevel,
+                               path_in_udf: Optional[PurePosixPath] = None) -> str:
+        """
+        Generates an SQL command to register the SLC container at the required level. The command will
+        preserve existing registrations of other containers identified by different language aliases.
+        Registration of a container with the same alias, if exists, will be overwritten.
+
+        alter_type  - Registration level - SYSTEM or SESSION.
+        path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF.
+        """
+        if path_in_udf is None:
+            path_in_udf = self._bucketfs_location.generate_bucket_udf_path(self._container_file.name)
         new_settings = \
             self._update_previous_language_settings(alter_type, path_in_udf)
         alter_command = \
-            f"ALTER {alter_type} SET SCRIPT_LANGUAGES='{new_settings}';"
+            f"ALTER {alter_type.value} SET SCRIPT_LANGUAGES='{new_settings}';"
         return alter_command
 
-    def _update_previous_language_settings(self, alter_type: str,
+    def _update_previous_language_settings(self, alter_type: LanguageRegLevel,
                                            path_in_udf: PurePosixPath) -> str:
         prev_lang_settings = self._get_previous_language_settings(alter_type)
         prev_lang_aliases = prev_lang_settings.split(" ")
@@ -81,18 +117,18 @@ def _check_if_requested_language_alias_already_exists(
             logging.warning(f"The requested language alias "
                             f"{self._language_alias} is already in use.")
 
-    def _get_previous_language_settings(self, alter_type: str) -> str:
+    def _get_previous_language_settings(self, alter_type: LanguageRegLevel) -> str:
         result = self._pyexasol_conn.execute(
-            f"""SELECT "{alter_type}_VALUE" FROM SYS.EXA_PARAMETERS WHERE 
+            f"""SELECT "{alter_type.value}_VALUE" FROM SYS.EXA_PARAMETERS WHERE 
             PARAMETER_NAME='SCRIPT_LANGUAGES'""").fetchall()
         return result[0][0]
 
     @classmethod
-    def run(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int,
+    def create(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int,
             bucketfs_use_https: bool, bucketfs_user: str, container_file: Path,
             bucketfs_password: str, bucket: str, path_in_bucket: str,
             dsn: str, db_user: str, db_password: str, language_alias: str,
-            ssl_cert_path: str = None, use_ssl_cert_validation: bool = True):
+            ssl_cert_path: str = None, use_ssl_cert_validation: bool = True) -> "LanguageContainerDeployer":
 
         websocket_sslopt = get_websocket_ssl_options(use_ssl_cert_validation, ssl_cert_path)
 
@@ -108,6 +144,4 @@ def run(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int,
             bucketfs_name, bucketfs_host, bucketfs_port, bucketfs_use_https,
             bucketfs_user, bucketfs_password, bucket, path_in_bucket)
 
-        language_container_deployer = cls(
-            pyexasol_conn, language_alias, bucketfs_location, container_file)
-        language_container_deployer.deploy_container()
+        return cls(pyexasol_conn, language_alias, bucketfs_location, container_file)

exasol_transformers_extension/deployment/language_container_deployer_cli.py

@@ -3,7 +3,20 @@
 from pathlib import Path
 from exasol_transformers_extension.deployment import deployment_utils as utils
 from exasol_transformers_extension.deployment.language_container_deployer import \
-    LanguageContainerDeployer
+    LanguageContainerDeployer, LanguageRegLevel
+
+
+def run_deployer(deployer, upload_container: bool = True, alter_system: bool = True) -> None:
+    if upload_container and alter_system:
+        deployer.deploy_container()
+    elif upload_container:
+        deployer.upload_container()
+    elif alter_system:
+        deployer.register_container(LanguageRegLevel.System)
+
+    if not alter_system:
+        print('Use the following command to register the SLC at the SESSION level:\n' +
+              deployer.generate_alter_command(LanguageRegLevel.Session))
 
 
 @click.command(name="language-container")
@@ -28,6 +41,8 @@
 @click.option('--language-alias', type=str, default="PYTHON3_TE")
 @click.option('--ssl-cert-path', type=str, default="")
 @click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True)
+@click.option('--upload-container/--no-upload_container', type=bool, default=True)
+@click.option('--alter-system/--no-alter-system', type=bool, default=True)
 def language_container_deployer_main(
         bucketfs_name: str,
         bucketfs_host: str,
@@ -44,9 +59,12 @@ def language_container_deployer_main(
         db_pass: str,
         language_alias: str,
         ssl_cert_path: str,
-        use_ssl_cert_validation: bool):
+        use_ssl_cert_validation: bool,
+        upload_container: bool,
+        alter_system: bool):
+
     def call_runner():
-        LanguageContainerDeployer.run(
+        deployer = LanguageContainerDeployer.create(
             bucketfs_name=bucketfs_name,
             bucketfs_host=bucketfs_host,
             bucketfs_port=bucketfs_port,
@@ -61,8 +79,9 @@ def call_runner():
             db_password=db_pass,
             language_alias=language_alias,
             ssl_cert_path=ssl_cert_path,
-            use_ssl_cert_validation=use_ssl_cert_validation
-        )
+            use_ssl_cert_validation=use_ssl_cert_validation)
+        run_deployer(deployer, upload_container=upload_container, alter_system=alter_system)
+
     if container_file:
         call_runner()
     elif version:

tests/integration_tests/with_db/deployment/test_language_container_deployer.py

@@ -1,14 +1,17 @@
 import textwrap
+from typing import Callable
 from pathlib import Path
 
 from _pytest.fixtures import FixtureRequest
+from tests.fixtures.language_container_fixture import export_slc, flavor_path
+from tests.fixtures.database_connection_fixture import pyexasol_connection
 from exasol_bucketfs_utils_python.bucketfs_factory import BucketFSFactory
 from exasol_script_languages_container_tool.lib.tasks.export.export_info import ExportInfo
 from pyexasol import ExaConnection
 from pytest_itde import config
 
 from exasol_transformers_extension.deployment.language_container_deployer \
-    import LanguageContainerDeployer
+    import LanguageContainerDeployer, LanguageRegLevel
 from tests.utils.parameters import bucketfs_params
 from tests.utils.revert_language_settings import revert_language_settings
 
@@ -17,6 +20,8 @@ def test_language_container_deployer(
         request: FixtureRequest,
         export_slc: ExportInfo,
         pyexasol_connection: ExaConnection,
+        connection_factory: Callable[[config.Exasol], ExaConnection],
+        exasol_config: config.Exasol,
         bucketfs_config: config.BucketFs,
 ):
     test_name: str = request.node.name
@@ -25,34 +30,64 @@ def test_language_container_deployer(
     container_path = Path(export_slc.cache_file)
     with revert_language_settings(pyexasol_connection):
         create_schema(pyexasol_connection, schema)
-        call_language_container_deployer(container_path=container_path,
-                                         language_alias=language_alias,
-                                         pyexasol_connection=pyexasol_connection,
-                                         bucketfs_config=bucketfs_config)
-        assert_udf_running(pyexasol_connection, language_alias)
+        deployer = create_container_deployer(container_path=container_path,
+                                             language_alias=language_alias,
+                                             pyexasol_connection=pyexasol_connection,
+                                             bucketfs_config=bucketfs_config)
+        deployer.deploy_container()
+        with connection_factory(exasol_config) as new_connection:
+            assert_udf_running(new_connection, language_alias, schema)
+
+
+def test_language_container_deployer_alter_session(
+        request: FixtureRequest,
+        export_slc: ExportInfo,
+        pyexasol_connection: ExaConnection,
+        connection_factory: Callable[[config.Exasol], ExaConnection],
+        exasol_config: config.Exasol,
+        bucketfs_config: config.BucketFs,
+):
+    test_name: str = request.node.name
+    schema = test_name
+    language_alias = f"PYTHON3_TE_{test_name.upper()}"
+    container_path = Path(export_slc.cache_file)
+    with revert_language_settings(pyexasol_connection):
+        create_schema(pyexasol_connection, schema)
+        deployer = create_container_deployer(container_path=container_path,
+                                             language_alias=language_alias,
+                                             pyexasol_connection=pyexasol_connection,
+                                             bucketfs_config=bucketfs_config)
+        deployer.upload_container()
+        with connection_factory(exasol_config) as new_connection:
+            deployer = create_container_deployer(container_path=container_path,
+                                                 language_alias=language_alias,
+                                                 pyexasol_connection=new_connection,
+                                                 bucketfs_config=bucketfs_config)
+            deployer.register_container(LanguageRegLevel.Session)
+            assert_udf_running(new_connection, language_alias, schema)
 
 
 def create_schema(pyexasol_connection: ExaConnection, schema: str):
     pyexasol_connection.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE;")
     pyexasol_connection.execute(f"CREATE SCHEMA IF NOT EXISTS {schema};")
 
 
-def assert_udf_running(pyexasol_connection: ExaConnection, language_alias: str):
+def assert_udf_running(pyexasol_connection: ExaConnection, language_alias: str, schema: str):
     pyexasol_connection.execute(textwrap.dedent(f"""
-        CREATE OR REPLACE {language_alias} SCALAR SCRIPT "TEST_UDF"()
+        CREATE OR REPLACE {language_alias} SCALAR SCRIPT {schema}."TEST_UDF"()
         RETURNS BOOLEAN AS
         def run(ctx):
             return True
         /
         """))
-    result = pyexasol_connection.execute('SELECT "TEST_UDF"()').fetchall()
+    result = pyexasol_connection.execute(f'SELECT {schema}."TEST_UDF"()').fetchall()
     assert result[0][0] == True
 
 
-def call_language_container_deployer(container_path: Path,
-                                     language_alias: str,
-                                     pyexasol_connection: ExaConnection,
-                                     bucketfs_config: config.BucketFs):
+def create_container_deployer(container_path: Path,
+                              language_alias: str,
+                              pyexasol_connection: ExaConnection,
+                              bucketfs_config: config.BucketFs) -> LanguageContainerDeployer:
     bucket_fs_factory = BucketFSFactory()
     bucketfs_location = bucket_fs_factory.create_bucketfs_location(
         url=f"{bucketfs_config.url}/"
@@ -61,6 +96,6 @@ def call_language_container_deployer(container_path: Path,
         user=f"{bucketfs_config.username}",
         pwd=f"{bucketfs_config.password}",
         base_path=None)
-    language_container_deployer = LanguageContainerDeployer(
+    return LanguageContainerDeployer(
         pyexasol_connection, language_alias, bucketfs_location, container_path)
-    language_container_deployer.deploy_container()
+

tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py

@@ -174,13 +174,11 @@ def test_language_container_deployer_cli_with_check_cert(
         request: FixtureRequest,
         export_slc: ExportInfo,
         pyexasol_connection: ExaConnection,
-        connection_factory: Callable[[config.Exasol], ExaConnection],
         exasol_config: config.Exasol,
         bucketfs_config: config.BucketFs
 ):
     use_ssl_cert_validation = True
-    expected_exception_message = 'Could not connect to Exasol: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify ' \
-                                 'failed: self signed certificate in certificate chain (_ssl.c:1131)'
+    expected_exception_message = '[SSL: CERTIFICATE_VERIFY_FAILED]'
     test_name: str = request.node.name
     schema = test_name
     language_alias = f"PYTHON3_TE_{test_name.upper()}"
@@ -198,5 +196,5 @@ def test_language_container_deployer_cli_with_check_cert(
                                                        use_ssl_cert_validation=use_ssl_cert_validation)
 
         assert result.exit_code == 1 \
-            and result.exception.args[0].message in expected_exception_message \
+            and expected_exception_message in result.exception.args[0].message \
             and type(result.exception) == ExaConnectionFailedError

tests/integration_tests/with_db/deployment/test_scripts_deployer_cli.py

@@ -49,11 +49,10 @@ def test_scripts_deployer_cli_with_encryption_verify(language_alias: str,
         "--language-alias", language_alias,
         "--use-ssl-cert-validation"
     ]
-    expected_exception_message = 'Could not connect to Exasol: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify ' \
-                                 'failed: self signed certificate in certificate chain (_ssl.c:1131)'
+    expected_exception_message = '[SSL: CERTIFICATE_VERIFY_FAILED]'
     runner = CliRunner()
     result = runner.invoke(deploy.main, args_list)
     assert result.exit_code == 1 \
-           and result.exception.args[0].message in expected_exception_message \
+           and expected_exception_message in result.exception.args[0].message \
            and type(result.exception) == ExaConnectionFailedError
 

tests/unit_tests/deployment/test_language_container_deployer_cli_run.py

@@ -0,0 +1,48 @@
+from pathlib import Path
+from unittest.mock import create_autospec, MagicMock
+import pytest
+from pyexasol import ExaConnection
+from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation
+from exasol_transformers_extension.deployment.language_container_deployer import (
+    LanguageContainerDeployer, LanguageRegLevel)
+from exasol_transformers_extension.deployment.language_container_deployer_cli import run_deployer
+
+
+@pytest.fixture(scope='module')
+def mock_pyexasol_conn() -> ExaConnection:
+    return create_autospec(ExaConnection)
+
+
+@pytest.fixture(scope='module')
+def mock_bfs_location() -> BucketFSLocation:
+    return create_autospec(BucketFSLocation)
+
+
+@pytest.fixture
+def container_deployer(mock_pyexasol_conn, mock_bfs_location) -> LanguageContainerDeployer:
+    return LanguageContainerDeployer(pyexasol_connection=mock_pyexasol_conn,
+                                     language_alias='alias',
+                                     bucketfs_location=mock_bfs_location,
+                                     container_file=Path('container_file'))
+
+
+def test_language_container_deployer_cli_deploy(container_deployer):
+    container_deployer.deploy_container = MagicMock()
+    run_deployer(container_deployer, True, True)
+    container_deployer.deploy_container.assert_called_once()
+
+
+def test_language_container_deployer_cli_upload(container_deployer):
+    container_deployer.upload_container = MagicMock()
+    container_deployer.register_container = MagicMock()
+    run_deployer(container_deployer, True, False)
+    container_deployer.upload_container.assert_called_once()
+    container_deployer.register_container.assert_not_called()
+
+
+def test_language_container_deployer_cli_register(container_deployer):
+    container_deployer.upload_container = MagicMock()
+    container_deployer.register_container = MagicMock()
+    run_deployer(container_deployer, False, True)
+    container_deployer.upload_container.assert_not_called()
+    container_deployer.register_container.assert_called_once_with(LanguageRegLevel.System)