Skip to content

Commit

Permalink
Addressed review comments [CodeBuild]
Browse files Browse the repository at this point in the history
  • Loading branch information
ahsimb committed Nov 24, 2023
1 parent 6594eca commit c4e99e6
Show file tree
Hide file tree
Showing 5 changed files with 118 additions and 50 deletions.
28 changes: 24 additions & 4 deletions doc/user_guide/user_guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,13 +109,30 @@ Transformers Extension Package. See [the latest release](https://github.com/exas
--language-alias <LANGUAGE_ALIAS> \
--version <RELEASE_VERSION> \
--ssl-cert-path <ssl-cert-path> \
--use-ssl-cert-validation
--use-ssl-cert-validation \
```
The `--ssl-cert-path` is optional if your certificate is not in the OS truststore.
The option `--use-ssl-cert-validation`is the default, you can disable it with `--no-use-ssl-cert-validation`.
Use caution if you want to turn certificate validation off as it potentially lowers the security of your
Database connection.

By default, the above command will upload and activate the language container at the System level.
The latter requires you to have the System Privileges, as it will attempt to change DB system settings.
If such privileges cannot be granted the activation can be skipped by using the `--no-alter-system` option.
The command will then print the language activation SQL query, which looks like this:
```sql
ALTER SESSION SET SCRIPT_LANGUAGES=...
```
This query activates the language container at the Session level. It doesn't require System Privileges.
However, it must be run every time a new session starts.

It is also possible to activate the language without repeatedly uploading the container. If the container
has already been uploaded one can use the `--no-upload_container` option to skip this step.

By default, overriding language activation is not permitted. If a language with the same alias has already
been activated the command will result in an error. To override the activation, you can use the
`--allow_override` option.

#### Customized Installation
In this installation, you can install the desired or customized language
container. In the following steps, it is explained how to install the
Expand All @@ -132,8 +149,8 @@ There are two ways to install the language container: (1) using a python script
1. *Installation with Python Script*

To install the language container, it is necessary to load the container
into the BucketFS and register it to the database. The following command
provides this setup using the python script provided with this library:
into the BucketFS and activate it in the database. The following command
performs this setup using the python script provided with this library:

```buildoutcfg
python -m exasol_transformers_extension.deploy language-container
Expand All @@ -150,6 +167,9 @@ There are two ways to install the language container: (1) using a python script
--language-alias <LANGUAGE_ALIAS> \
--container-file <path/to/language_container_name.tar.gz>
```
Please note, that all considerations described in the Quick Installation
section are still applicable.
2. *Manual Installation*
Expand All @@ -171,7 +191,7 @@ There are two ways to install the language container: (1) using a python script
```
The uploaded container should be secondly activated through adjusting
the session parameter `SCRIPT_LANGUAGES`. The activation can be scoped
the session parameter `SCRIPT_LANGUAGES`. As it was mentioned before, the activation can be scoped
either session-wide (`ALTER SESSION`) or system-wide (`ALTER SYSTEM`).
The following example query activates the container session-wide:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
logger = logging.getLogger(__name__)


class LanguageRegLevel(Enum):
class LanguageActiveLevel(Enum):
f"""
Language alias registration level, i.e.
ALTER <LanguageRegLevel> SET SCRIPT_LANGUAGES=...
Language activation level, i.e.
ALTER <LanguageActiveLevel> SET SCRIPT_LANGUAGES=...
"""
Session = 'SESSION'
System = 'SYSTEM'
Expand All @@ -32,12 +32,15 @@ def __init__(self,
self._pyexasol_conn = pyexasol_connection
logger.debug(f"Init {LanguageContainerDeployer.__name__}")

def deploy_container(self) -> None:
def deploy_container(self, allow_override: bool = False) -> None:
"""
Uploads the SLC and registers it at the SYSTEM level.
Uploads the SLC and activates it at the SYSTEM level.
allow_override - If True the activation of a language container with the same alias will be overriden,
otherwise a RuntimeException will be thrown.
"""
path_in_udf = self.upload_container()
self.register_container(LanguageRegLevel.System, path_in_udf)
self.activate_container(LanguageActiveLevel.System, allow_override, path_in_udf)

def upload_container(self) -> PurePosixPath:
"""
Expand All @@ -54,42 +57,49 @@ def upload_container(self) -> PurePosixPath:
logging.debug("Container is uploaded to bucketfs")
return PurePosixPath(path_in_udf)

def register_container(self, alter_type: LanguageRegLevel = LanguageRegLevel.Session,
def activate_container(self, alter_type: LanguageActiveLevel = LanguageActiveLevel.Session,
allow_override: bool = False,
path_in_udf: Optional[PurePosixPath] = None) -> None:
"""
Registers the SLC container at the required level.
Activates the SLC container at the required level.
alter_type - Language registration level, defaults to the SESSION.
path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF.
alter_type - Language activation level, defaults to the SESSION.
allow_override - If True the activation of a language container with the same alias will be overriden,
otherwise a RuntimeException will be thrown.
path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF.
"""
alter_command = self.generate_alter_command(alter_type, path_in_udf)
alter_command = self.generate_activation_command(alter_type, allow_override, path_in_udf)
self._pyexasol_conn.execute(alter_command)
logging.debug(alter_command)

def generate_alter_command(self, alter_type: LanguageRegLevel,
path_in_udf: Optional[PurePosixPath] = None) -> str:
def generate_activation_command(self, alter_type: LanguageActiveLevel,
allow_override: bool = False,
path_in_udf: Optional[PurePosixPath] = None) -> str:
"""
Generates an SQL command to register the SLC container at the required level. The command will
preserve existing registrations of other containers identified by different language aliases.
Registration of a container with the same alias, if exists, will be overwritten.
alter_type - Registration level - SYSTEM or SESSION.
path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF.
Generates an SQL command to activate the SLC container at the required level. The command will
preserve existing activations of other containers identified by different language aliases.
Activation of a container with the same alias, if exists, will be overwritten.
alter_type - Activation level - SYSTEM or SESSION.
allow_override - If True the activation of a language container with the same alias will be overriden,
otherwise a RuntimeException will be thrown.
path_in_udf - If known, a path where the container is uploaded as it's seen by a UDF.
"""
if path_in_udf is None:
path_in_udf = self._bucketfs_location.generate_bucket_udf_path(self._container_file.name)
new_settings = \
self._update_previous_language_settings(alter_type, path_in_udf)
self._update_previous_language_settings(alter_type, allow_override, path_in_udf)
alter_command = \
f"ALTER {alter_type.value} SET SCRIPT_LANGUAGES='{new_settings}';"
return alter_command

def _update_previous_language_settings(self, alter_type: LanguageRegLevel,
def _update_previous_language_settings(self, alter_type: LanguageActiveLevel,
allow_override: bool,
path_in_udf: PurePosixPath) -> str:
prev_lang_settings = self._get_previous_language_settings(alter_type)
prev_lang_aliases = prev_lang_settings.split(" ")
self._check_if_requested_language_alias_already_exists(
prev_lang_aliases)
allow_override, prev_lang_aliases)
new_definitions_str = self._generate_new_language_settings(
path_in_udf, prev_lang_aliases)
return new_definitions_str
Expand All @@ -109,15 +119,19 @@ def _generate_new_language_settings(self, path_in_udf: PurePosixPath,
return new_definitions_str

def _check_if_requested_language_alias_already_exists(
self, prev_lang_aliases: List[str]) -> None:
self, allow_override: bool,
prev_lang_aliases: List[str]) -> None:
definition_for_requested_alias = [
alias_definition for alias_definition in prev_lang_aliases
if alias_definition.startswith(self._language_alias + "=")]
if not len(definition_for_requested_alias) == 0:
logging.warning(f"The requested language alias "
f"{self._language_alias} is already in use.")
warning_message = f"The requested language alias {self._language_alias} is already in use."
if allow_override:
logging.warning(warning_message)
else:
raise RuntimeError(warning_message)

def _get_previous_language_settings(self, alter_type: LanguageRegLevel) -> str:
def _get_previous_language_settings(self, alter_type: LanguageActiveLevel) -> str:
result = self._pyexasol_conn.execute(
f"""SELECT "{alter_type.value}_VALUE" FROM SYS.EXA_PARAMETERS WHERE
PARAMETER_NAME='SCRIPT_LANGUAGES'""").fetchall()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,22 @@
from pathlib import Path
from exasol_transformers_extension.deployment import deployment_utils as utils
from exasol_transformers_extension.deployment.language_container_deployer import \
LanguageContainerDeployer, LanguageRegLevel
LanguageContainerDeployer, LanguageActiveLevel


def run_deployer(deployer, upload_container: bool = True, alter_system: bool = True) -> None:
def run_deployer(deployer, upload_container: bool = True,
alter_system: bool = True,
allow_override: bool = False) -> None:
if upload_container and alter_system:
deployer.deploy_container()
deployer.deploy_container(allow_override)
elif upload_container:
deployer.upload_container()
elif alter_system:
deployer.register_container(LanguageRegLevel.System)
deployer.activate_container(LanguageActiveLevel.System, allow_override)

if not alter_system:
print('Use the following command to register the SLC at the SESSION level:\n' +
deployer.generate_alter_command(LanguageRegLevel.Session))
print('Use the following command to activate the SLC at the SESSION level:\n' +
deployer.generate_activation_command(LanguageActiveLevel.Session, True))


@click.command(name="language-container")
Expand All @@ -43,6 +45,7 @@ def run_deployer(deployer, upload_container: bool = True, alter_system: bool = T
@click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True)
@click.option('--upload-container/--no-upload_container', type=bool, default=True)
@click.option('--alter-system/--no-alter-system', type=bool, default=True)
@click.option('--allow-override/--disallow-override', type=bool, default=False)
def language_container_deployer_main(
bucketfs_name: str,
bucketfs_host: str,
Expand All @@ -61,7 +64,8 @@ def language_container_deployer_main(
ssl_cert_path: str,
use_ssl_cert_validation: bool,
upload_container: bool,
alter_system: bool):
alter_system: bool,
allow_override: bool):

def call_runner():
deployer = LanguageContainerDeployer.create(
Expand All @@ -80,7 +84,8 @@ def call_runner():
language_alias=language_alias,
ssl_cert_path=ssl_cert_path,
use_ssl_cert_validation=use_ssl_cert_validation)
run_deployer(deployer, upload_container=upload_container, alter_system=alter_system)
run_deployer(deployer, upload_container=upload_container, alter_system=alter_system,
allow_override=allow_override)

if container_file:
call_runner()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from typing import Callable
from pathlib import Path

import pytest
from _pytest.fixtures import FixtureRequest
from tests.fixtures.language_container_fixture import export_slc, flavor_path
from tests.fixtures.database_connection_fixture import pyexasol_connection
Expand All @@ -11,7 +12,7 @@
from pytest_itde import config

from exasol_transformers_extension.deployment.language_container_deployer \
import LanguageContainerDeployer, LanguageRegLevel
import LanguageContainerDeployer, LanguageActiveLevel
from tests.utils.parameters import bucketfs_params
from tests.utils.revert_language_settings import revert_language_settings

Expand All @@ -34,7 +35,7 @@ def test_language_container_deployer(
language_alias=language_alias,
pyexasol_connection=pyexasol_connection,
bucketfs_config=bucketfs_config)
deployer.deploy_container()
deployer.deploy_container(True)
with connection_factory(exasol_config) as new_connection:
assert_udf_running(new_connection, language_alias, schema)

Expand Down Expand Up @@ -63,10 +64,38 @@ def test_language_container_deployer_alter_session(
language_alias=language_alias,
pyexasol_connection=new_connection,
bucketfs_config=bucketfs_config)
deployer.register_container(LanguageRegLevel.Session)
deployer.activate_container(LanguageActiveLevel.Session, True)
assert_udf_running(new_connection, language_alias, schema)


def test_language_container_deployer_activation_fail(
request: FixtureRequest,
export_slc: ExportInfo,
pyexasol_connection: ExaConnection,
connection_factory: Callable[[config.Exasol], ExaConnection],
exasol_config: config.Exasol,
bucketfs_config: config.BucketFs,
):
test_name: str = request.node.name
schema = test_name
language_alias = f"PYTHON3_TE_{test_name.upper()}"
container_path = Path(export_slc.cache_file)
with revert_language_settings(pyexasol_connection):
create_schema(pyexasol_connection, schema)
deployer = create_container_deployer(container_path=container_path,
language_alias=language_alias,
pyexasol_connection=pyexasol_connection,
bucketfs_config=bucketfs_config)
deployer.deploy_container(True)
with connection_factory(exasol_config) as new_connection:
deployer = create_container_deployer(container_path=container_path,
language_alias=language_alias,
pyexasol_connection=new_connection,
bucketfs_config=bucketfs_config)
with pytest.raises(RuntimeError):
deployer.activate_container(LanguageActiveLevel.System, False)


def create_schema(pyexasol_connection: ExaConnection, schema: str):
pyexasol_connection.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE;")
pyexasol_connection.execute(f"CREATE SCHEMA IF NOT EXISTS {schema};")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from pyexasol import ExaConnection
from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation
from exasol_transformers_extension.deployment.language_container_deployer import (
LanguageContainerDeployer, LanguageRegLevel)
LanguageContainerDeployer, LanguageActiveLevel)
from exasol_transformers_extension.deployment.language_container_deployer_cli import run_deployer


Expand All @@ -28,21 +28,21 @@ def container_deployer(mock_pyexasol_conn, mock_bfs_location) -> LanguageContain

def test_language_container_deployer_cli_deploy(container_deployer):
container_deployer.deploy_container = MagicMock()
run_deployer(container_deployer, True, True)
container_deployer.deploy_container.assert_called_once()
run_deployer(container_deployer, True, True, False)
container_deployer.deploy_container.assert_called_once_with(False)


def test_language_container_deployer_cli_upload(container_deployer):
container_deployer.upload_container = MagicMock()
container_deployer.register_container = MagicMock()
run_deployer(container_deployer, True, False)
container_deployer.activate_container = MagicMock()
run_deployer(container_deployer, True, False, False)
container_deployer.upload_container.assert_called_once()
container_deployer.register_container.assert_not_called()
container_deployer.activate_container.assert_not_called()


def test_language_container_deployer_cli_register(container_deployer):
container_deployer.upload_container = MagicMock()
container_deployer.register_container = MagicMock()
run_deployer(container_deployer, False, True)
container_deployer.activate_container = MagicMock()
run_deployer(container_deployer, False, True, True)
container_deployer.upload_container.assert_not_called()
container_deployer.register_container.assert_called_once_with(LanguageRegLevel.System)
container_deployer.activate_container.assert_called_once_with(LanguageActiveLevel.System, True)

0 comments on commit c4e99e6

Please sign in to comment.