Merge remote-tracking branch 'origin/feature/#143_Add_model_transfer_…

…with_save_pretrained' into feature/#143_Add_model_transfer_with_save_pretrained

doc/user_guide/user_guide.md

@@ -116,6 +116,29 @@ The option `--use-ssl-cert-validation`is the default, you can disable it with `-
 Use caution if you want to turn certificate validation off as it potentially lowers the security of your 
 Database connection.
 
+By default, the above command will upload and activate the language container at the System level.
+The latter requires you to have the System Privileges, as it will attempt to change DB system settings.
+If such privileges cannot be granted the activation can be skipped by using the `--no-alter-system` option.
+The command will then print two possible language activation SQL queries, which look like the following:
+```sql
+ALTER SESSION SET SCRIPT_LANGUAGES=...
+ALTER SYSTEM SET SCRIPT_LANGUAGES=...
+```
+These queries represent two alternative ways of activating a language container. The first one activates the
+container at the [Session level](https://docs.exasol.com/db/latest/sql/alter_session.htm). It doesn't require 
+System Privileges. However, it must be run every time a new session starts. The second one activates the container
+at the [System level](https://docs.exasol.com/db/latest/sql/alter_system.htm). It  needs to be run just once,
+but it does require System Privileges. It may be executed by a database administrator. Please note, that changes 
+made at the system level only become effective in new sessions, as described
+[here](https://docs.exasol.com/db/latest/sql/alter_system.htm#microcontent1).
+
+It is also possible to activate the language without repeatedly uploading the container. If the container
+has already been uploaded one can use the `--no-upload-container` option to skip this step.
+
+By default, overriding language activation is not permitted. If a language with the same alias has already 
+been activated the command will result in an error. To override the activation, you can use the
+`--allow-override` option.
+
 #### Customized Installation
 In this installation, you can install the desired or customized language 
 container. In the following steps,  it is explained how to install the 
@@ -132,8 +155,8 @@ There are two ways to install the language container: (1) using a python script
   1. *Installation with Python Script*
 
      To install the language container, it is necessary to load the container 
-     into the BucketFS and register it to the database. The following command 
-     provides this setup using the python script provided with this library:
+     into the BucketFS and activate it in the database. The following command 
+     performs this setup using the python script provided with this library:
 
       ```buildoutcfg
       python -m exasol_transformers_extension.deploy language-container
@@ -150,6 +173,9 @@ There are two ways to install the language container: (1) using a python script
           --language-alias <LANGUAGE_ALIAS> \ 
           --container-file <path/to/language_container_name.tar.gz>       
       ```
+     Please note, that all considerations described in the Quick Installation 
+     section are still applicable.
+
 
   2. *Manual Installation*
 
@@ -171,7 +197,7 @@ There are two ways to install the language container: (1) using a python script
       ```
 
       The uploaded container should be secondly activated through adjusting 
-      the session parameter `SCRIPT_LANGUAGES`. The activation can be scoped
+      the session parameter `SCRIPT_LANGUAGES`. As it was mentioned before, the activation can be scoped
       either session-wide (`ALTER SESSION`) or system-wide (`ALTER SYSTEM`). 
       The following example query activates the container session-wide:
 

exasol_transformers_extension/deployment/language_container_deployer.py

@@ -1,5 +1,6 @@
+from enum import Enum
 import pyexasol
-from typing import List
+from typing import List, Optional
 from pathlib import Path, PurePosixPath
 from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation
 import logging
@@ -10,6 +11,15 @@
 logger = logging.getLogger(__name__)
 
 
+class LanguageActivationLevel(Enum):
+    f"""
+    Language activation level, i.e.
+    ALTER <LanguageActivationLevel> SET SCRIPT_LANGUAGES=...
+    """
+    Session = 'SESSION'
+    System = 'SYSTEM'
+
+
 class LanguageContainerDeployer:
     def __init__(self,
                  pyexasol_connection: pyexasol.ExaConnection,
@@ -22,14 +32,21 @@ def __init__(self,
         self._pyexasol_conn = pyexasol_connection
         logger.debug(f"Init {LanguageContainerDeployer.__name__}")
 
-    def deploy_container(self):
-        path_in_udf = self._upload_container()
-        for alter in ["SESSION", "SYSTEM"]:
-            alter_command = self._generate_alter_command(alter, path_in_udf)
-            self._pyexasol_conn.execute(alter_command)
-            logging.debug(alter_command)
+    def deploy_container(self, allow_override: bool = False) -> None:
+        """
+        Uploads the SLC and activates it at the SYSTEM level.
+
+        allow_override - If True the activation of a language container with the same alias will be overriden,
+                         otherwise a RuntimeException will be thrown.
+        """
+        path_in_udf = self.upload_container()
+        self.activate_container(LanguageActivationLevel.System, allow_override, path_in_udf)
 
-    def _upload_container(self) -> PurePosixPath:
+    def upload_container(self) -> PurePosixPath:
+        """
+        Uploads the SLC.
+        Returns the path where the container is uploaded as it's seen by a UDF.
+        """
         if not self._container_file.is_file():
             raise RuntimeError(f"Container file {self._container_file} "
                                f"is not a file.")
@@ -40,20 +57,49 @@ def _upload_container(self) -> PurePosixPath:
         logging.debug("Container is uploaded to bucketfs")
         return PurePosixPath(path_in_udf)
 
-    def _generate_alter_command(self, alter_type: str,
-                                path_in_udf: PurePosixPath) -> str:
+    def activate_container(self, alter_type: LanguageActivationLevel = LanguageActivationLevel.Session,
+                           allow_override: bool = False,
+                           path_in_udf: Optional[PurePosixPath] = None) -> None:
+        """
+        Activates the SLC container at the required level.
+
+        alter_type     - Language activation level, defaults to the SESSION.
+        allow_override - If True the activation of a language container with the same alias will be overriden,
+                         otherwise a RuntimeException will be thrown.
+        path_in_udf    - If known, a path where the container is uploaded as it's seen by a UDF.
+        """
+        alter_command = self.generate_activation_command(alter_type, allow_override, path_in_udf)
+        self._pyexasol_conn.execute(alter_command)
+        logging.debug(alter_command)
+
+    def generate_activation_command(self, alter_type: LanguageActivationLevel,
+                                    allow_override: bool = False,
+                                    path_in_udf: Optional[PurePosixPath] = None) -> str:
+        """
+        Generates an SQL command to activate the SLC container at the required level. The command will
+        preserve existing activations of other containers identified by different language aliases.
+        Activation of a container with the same alias, if exists, will be overwritten.
+
+        alter_type     - Activation level - SYSTEM or SESSION.
+        allow_override - If True the activation of a language container with the same alias will be overriden,
+                         otherwise a RuntimeException will be thrown.
+        path_in_udf    - If known, a path where the container is uploaded as it's seen by a UDF.
+        """
+        if path_in_udf is None:
+            path_in_udf = self._bucketfs_location.generate_bucket_udf_path(self._container_file.name)
         new_settings = \
-            self._update_previous_language_settings(alter_type, path_in_udf)
+            self._update_previous_language_settings(alter_type, allow_override, path_in_udf)
         alter_command = \
-            f"ALTER {alter_type} SET SCRIPT_LANGUAGES='{new_settings}';"
+            f"ALTER {alter_type.value} SET SCRIPT_LANGUAGES='{new_settings}';"
         return alter_command
 
-    def _update_previous_language_settings(self, alter_type: str,
+    def _update_previous_language_settings(self, alter_type: LanguageActivationLevel,
+                                           allow_override: bool,
                                            path_in_udf: PurePosixPath) -> str:
         prev_lang_settings = self._get_previous_language_settings(alter_type)
         prev_lang_aliases = prev_lang_settings.split(" ")
         self._check_if_requested_language_alias_already_exists(
-            prev_lang_aliases)
+            allow_override, prev_lang_aliases)
         new_definitions_str = self._generate_new_language_settings(
             path_in_udf, prev_lang_aliases)
         return new_definitions_str
@@ -73,26 +119,30 @@ def _generate_new_language_settings(self, path_in_udf: PurePosixPath,
         return new_definitions_str
 
     def _check_if_requested_language_alias_already_exists(
-            self, prev_lang_aliases: List[str]) -> None:
+            self, allow_override: bool,
+            prev_lang_aliases: List[str]) -> None:
         definition_for_requested_alias = [
             alias_definition for alias_definition in prev_lang_aliases
             if alias_definition.startswith(self._language_alias + "=")]
         if not len(definition_for_requested_alias) == 0:
-            logging.warning(f"The requested language alias "
-                            f"{self._language_alias} is already in use.")
+            warning_message = f"The requested language alias {self._language_alias} is already in use."
+            if allow_override:
+                logging.warning(warning_message)
+            else:
+                raise RuntimeError(warning_message)
 
-    def _get_previous_language_settings(self, alter_type: str) -> str:
+    def _get_previous_language_settings(self, alter_type: LanguageActivationLevel) -> str:
         result = self._pyexasol_conn.execute(
-            f"""SELECT "{alter_type}_VALUE" FROM SYS.EXA_PARAMETERS WHERE 
+            f"""SELECT "{alter_type.value}_VALUE" FROM SYS.EXA_PARAMETERS WHERE 
             PARAMETER_NAME='SCRIPT_LANGUAGES'""").fetchall()
         return result[0][0]
 
     @classmethod
-    def run(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int,
+    def create(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int,
             bucketfs_use_https: bool, bucketfs_user: str, container_file: Path,
             bucketfs_password: str, bucket: str, path_in_bucket: str,
             dsn: str, db_user: str, db_password: str, language_alias: str,
-            ssl_cert_path: str = None, use_ssl_cert_validation: bool = True):
+            ssl_cert_path: str = None, use_ssl_cert_validation: bool = True) -> "LanguageContainerDeployer":
 
         websocket_sslopt = get_websocket_ssl_options(use_ssl_cert_validation, ssl_cert_path)
 
@@ -108,6 +158,4 @@ def run(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int,
             bucketfs_name, bucketfs_host, bucketfs_port, bucketfs_use_https,
             bucketfs_user, bucketfs_password, bucket, path_in_bucket)
 
-        language_container_deployer = cls(
-            pyexasol_conn, language_alias, bucketfs_location, container_file)
-        language_container_deployer.deploy_container()
+        return cls(pyexasol_conn, language_alias, bucketfs_location, container_file)

exasol_transformers_extension/deployment/language_container_deployer_cli.py

@@ -1,9 +1,34 @@
 import os
 import click
 from pathlib import Path
+from textwrap import dedent
 from exasol_transformers_extension.deployment import deployment_utils as utils
 from exasol_transformers_extension.deployment.language_container_deployer import \
-    LanguageContainerDeployer
+    LanguageContainerDeployer, LanguageActivationLevel
+
+
+def run_deployer(deployer, upload_container: bool = True,
+                 alter_system: bool = True,
+                 allow_override: bool = False) -> None:
+    if upload_container and alter_system:
+        deployer.deploy_container(allow_override)
+    elif upload_container:
+        deployer.upload_container()
+    elif alter_system:
+        deployer.activate_container(LanguageActivationLevel.System, allow_override)
+
+    if not alter_system:
+        message = dedent(f"""
+        In SQL, you can activate the SLC of the Transformers Extension
+        by using the following statements:
+
+        To activate the SLC only for the current session:
+        {deployer.generate_activation_command(LanguageActivationLevel.Session, True)}
+
+        To activate the SLC on the system:
+        {deployer.generate_activation_command(LanguageActivationLevel.System, True)}
+        """)
+        print(message)
 
 
 @click.command(name="language-container")
@@ -28,6 +53,9 @@
 @click.option('--language-alias', type=str, default="PYTHON3_TE")
 @click.option('--ssl-cert-path', type=str, default="")
 @click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True)
+@click.option('--upload-container/--no-upload_container', type=bool, default=True)
+@click.option('--alter-system/--no-alter-system', type=bool, default=True)
+@click.option('--allow-override/--disallow-override', type=bool, default=False)
 def language_container_deployer_main(
         bucketfs_name: str,
         bucketfs_host: str,
@@ -44,9 +72,13 @@ def language_container_deployer_main(
         db_pass: str,
         language_alias: str,
         ssl_cert_path: str,
-        use_ssl_cert_validation: bool):
+        use_ssl_cert_validation: bool,
+        upload_container: bool,
+        alter_system: bool,
+        allow_override: bool):
+
     def call_runner():
-        LanguageContainerDeployer.run(
+        deployer = LanguageContainerDeployer.create(
             bucketfs_name=bucketfs_name,
             bucketfs_host=bucketfs_host,
             bucketfs_port=bucketfs_port,
@@ -61,8 +93,10 @@ def call_runner():
             db_password=db_pass,
             language_alias=language_alias,
             ssl_cert_path=ssl_cert_path,
-            use_ssl_cert_validation=use_ssl_cert_validation
-        )
+            use_ssl_cert_validation=use_ssl_cert_validation)
+        run_deployer(deployer, upload_container=upload_container, alter_system=alter_system,
+                     allow_override=allow_override)
+
     if container_file:
         call_runner()
     elif version: