#103: Added option to toggle TLS cert validation (#123)

*  added SSL verify toggle option

doc/changes/changes_0.5.0.md

@@ -10,6 +10,7 @@ TBD
 ### Features
 
  - #88: Added custom matcher functions for unit tests
+ - #103: Added option to toggle use of TLS certificate validation for Database connection
  - #42: Update transformers to 4.31 and adapt the model uploader
 
 ### Bug Fixes

doc/user_guide/user_guide.md

@@ -101,11 +101,19 @@ deployment script below with the desired version. (see GitHub Releases
       --bucketfs-port <BUCKETFS_PORT> \
       --bucketfs-user <BUCKETFS_USER> \
       --bucketfs-password <BUCKETFS_PASSWORD> \
+      --bucketfs-use-https <USE_HTTPS_BOOL> \
       --bucket <BUCKETFS_NAME> \
       --path-in-bucket <PATH_IN_BUCKET> \
       --language-alias <LANGUAGE_ALIAS> \ 
-      --version <RELEASE_VERSION>       
+      --version <RELEASE_VERSION> \
+      --ssl-cert-path <ssl-cert-path> \
+      --use-ssl-cert-validation \
+      --no-use-ssl-cert-valiation
   ```
+The `--ssl-cert-path` is optional if your certificate is not in the OS truststore. 
+The option `--use-ssl-cert-validation`is the default, you can disable it with `--no-use-ssl-cert-validation`.
+Use caution if you want to turn certificate validation off as it potentially lowers the security of your 
+Database connection.
 
 #### Customized Installation
 In this installation, you can install the desired or customized language 

exasol_transformers_extension/deployment/deployment_utils.py

@@ -4,6 +4,7 @@
 import requests
 import tempfile
 import subprocess
+import ssl
 from pathlib import Path
 from getpass import getpass
 from contextlib import contextmanager
@@ -51,6 +52,18 @@ def _concatenate_slc_parts(tmp_dir):
     return slc_final_path
 
 
+def get_websocket_ssl_options(use_ssl_cert_validation: bool, ssl_cert_path: str):
+    websocket_sslopt = {
+        "cert_reqs": ssl.CERT_REQUIRED,
+    }
+    if not use_ssl_cert_validation:
+        websocket_sslopt["cert_reqs"] = ssl.CERT_NONE
+
+    if ssl_cert_path is not None:
+        websocket_sslopt["ca_certs"] = ssl_cert_path
+    return websocket_sslopt
+
+
 @contextmanager
 def get_container_file_from_github_release(version):
     with tempfile.TemporaryDirectory() as tmp_dir:

exasol_transformers_extension/deployment/language_container_deployer.py

@@ -5,7 +5,7 @@
 import logging
 from exasol_transformers_extension.utils.bucketfs_operations import \
     create_bucketfs_location
-import ssl
+from exasol_transformers_extension.deployment.deployment_utils import get_websocket_ssl_options
 
 logger = logging.getLogger(__name__)
 
@@ -91,16 +91,17 @@ def _get_previous_language_settings(self, alter_type: str) -> str:
     def run(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int,
             bucketfs_use_https: bool, bucketfs_user: str, container_file: Path,
             bucketfs_password: str, bucket: str, path_in_bucket: str,
-            dsn: str, db_user: str, db_password: str, language_alias: str):
+            dsn: str, db_user: str, db_password: str, language_alias: str,
+            ssl_cert_path: str = None, use_ssl_cert_validation: bool = True):
+
+        websocket_sslopt = get_websocket_ssl_options(use_ssl_cert_validation, ssl_cert_path)
 
         pyexasol_conn = pyexasol.connect(
             dsn=dsn,
             user=db_user,
             password=db_password,
             encryption=True,
-            websocket_sslopt={
-                "cert_reqs": ssl.CERT_NONE,
-            }
+            websocket_sslopt=websocket_sslopt
         )
 
         bucketfs_location = create_bucketfs_location(

exasol_transformers_extension/deployment/language_container_deployer_cli.py

@@ -10,7 +10,7 @@
 @click.option('--bucketfs-name', type=str, required=True)
 @click.option('--bucketfs-host', type=str, required=True)
 @click.option('--bucketfs-port', type=int, required=True)
-@click.option('--bucketfs_use-https', type=bool, default=False)
+@click.option('--bucketfs-use-https', type=bool, default=False)
 @click.option('--bucketfs-user', type=str, required=True, default="w")
 @click.option('--bucketfs-password', prompt='bucketFS password', hide_input=True,
               default=lambda: os.environ.get(
@@ -26,6 +26,8 @@
               default=lambda: os.environ.get(
                   utils.DB_PASSWORD_ENVIRONMENT_VARIABLE, ""))
 @click.option('--language-alias', type=str, default="PYTHON3_TE")
+@click.option('--ssl-cert-path', type=str, default="")
+@click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True)
 def language_container_deployer_main(
         bucketfs_name: str,
         bucketfs_host: str,
@@ -40,7 +42,9 @@ def language_container_deployer_main(
         dsn: str,
         db_user: str,
         db_pass: str,
-        language_alias: str):
+        language_alias: str,
+        ssl_cert_path: str,
+        use_ssl_cert_validation: bool):
     def call_runner():
         LanguageContainerDeployer.run(
             bucketfs_name=bucketfs_name,
@@ -55,7 +59,9 @@ def call_runner():
             dsn=dsn,
             db_user=db_user,
             db_password=db_pass,
-            language_alias=language_alias
+            language_alias=language_alias,
+            ssl_cert_path=ssl_cert_path,
+            use_ssl_cert_validation=use_ssl_cert_validation
         )
     if container_file:
         call_runner()

exasol_transformers_extension/deployment/scripts_deployer.py

@@ -5,7 +5,6 @@
 
 logger = logging.getLogger(__name__)
 
-import ssl
 
 class ScriptsDeployer:
     def __init__(self, language_alias: str, schema: str,
@@ -47,16 +46,17 @@ def deploy_scripts(self) -> None:
 
     @classmethod
     def run(cls, dsn: str, user: str, password: str,
-            schema: str, language_alias: str):
+            schema: str, language_alias: str,
+            ssl_cert_path: str, use_ssl_cert_validation: bool = True):
+        websocket_sslopt = utils.get_websocket_ssl_options(use_ssl_cert_validation, ssl_cert_path)
 
         pyexasol_conn = pyexasol.connect(
             dsn=dsn,
             user=user,
             password=password,
             encryption=True,
-            websocket_sslopt={
-                "cert_reqs": ssl.CERT_NONE,
-            }
+            websocket_sslopt=websocket_sslopt
         )
+
         scripts_deployer = cls(language_alias, schema, pyexasol_conn)
         scripts_deployer.deploy_scripts()

exasol_transformers_extension/deployment/scripts_deployer_cli.py

@@ -13,15 +13,20 @@
                   utils.DB_PASSWORD_ENVIRONMENT_VARIABLE, ""))
 @click.option('--schema', type=str, required=True)
 @click.option('--language-alias', type=str, default="PYTHON3_TE")
+@click.option('--ssl-cert-path', type=str, default="")
+@click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True)
 def scripts_deployer_main(
-        dsn: str, db_user: str, db_pass: str, schema: str, language_alias: str):
+        dsn: str, db_user: str, db_pass: str, schema: str, language_alias: str,
+        ssl_cert_path: str, use_ssl_cert_validation: bool):
 
     ScriptsDeployer.run(
         dsn=dsn,
         user=db_user,
         password=db_pass,
         schema=schema,
-        language_alias=language_alias
+        language_alias=language_alias,
+        ssl_cert_path=ssl_cert_path,
+        use_ssl_cert_validation=use_ssl_cert_validation
     )
 
 

exasol_transformers_extension/upload_model.py

@@ -14,7 +14,7 @@
 @click.option('--bucketfs-name', type=str, required=True)
 @click.option('--bucketfs-host', type=str, required=True)
 @click.option('--bucketfs-port', type=int, required=True)
-@click.option('--bucketfs_use-https', type=bool, default=False)
+@click.option('--bucketfs-use-https', type=bool, default=False)
 @click.option('--bucketfs-user', type=str, required=True, default="w")
 @click.option('--bucketfs-password', prompt='bucketFS password', hide_input=True,
               default=lambda: os.environ.get(

githooks/pre-commit

@@ -7,5 +7,4 @@ REPO_DIR=$(git rev-parse --show-toplevel)
 GITHOOKS_PATH="$REPO_DIR/githooks"
 pushd "$REPO_DIR"
 bash "$GITHOOKS_PATH/prohibit_commit_to_main.sh"
-bash "$GITHOOKS_PATH/update_setup_py.sh"
 popd

tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py

@@ -3,10 +3,12 @@
 from urllib.parse import urlparse
 
 import pytest
+from tests.fixtures.language_container_fixture import export_slc, flavor_path
+from tests.fixtures.database_connection_fixture import pyexasol_connection
 from _pytest.fixtures import FixtureRequest
 from click.testing import CliRunner
 from exasol_script_languages_container_tool.lib.tasks.export.export_info import ExportInfo
-from pyexasol import ExaConnection
+from pyexasol import ExaConnection, ExaConnectionFailedError
 from pytest_itde import config
 
 from exasol_transformers_extension import deploy
@@ -39,14 +41,15 @@ def call_language_definition_deployer_cli(dsn: str,
                                           language_alias: str,
                                           version: Optional[str],
                                           exasol_config: config.Exasol,
-                                          bucketfs_config: config.BucketFs):
+                                          bucketfs_config: config.BucketFs,
+                                          use_ssl_cert_validation: bool = False):
     parsed_url = urlparse(bucketfs_config.url)
     args_list = [
         "language-container",
         "--bucketfs-name", bucketfs_params.name,
         "--bucketfs-host", parsed_url.hostname,
         "--bucketfs-port", parsed_url.port,
-        "--bucketfs_use-https", False,
+        "--bucketfs-use-https", False,
         "--bucketfs-user", bucketfs_config.username,
         "--bucketfs-password", bucketfs_config.password,
         "--bucket", bucketfs_params.bucket,
@@ -56,6 +59,14 @@ def call_language_definition_deployer_cli(dsn: str,
         "--db-pass", exasol_config.password,
         "--language-alias", language_alias
     ]
+    if use_ssl_cert_validation:
+        args_list += [
+            "--use-ssl-cert-validation"
+        ]
+    else:
+        args_list += [
+            "--no-use-ssl-cert-validation"
+        ]
     if version is not None:
         args_list += [
             "--version", version,
@@ -158,3 +169,35 @@ def test_language_container_deployer_cli_with_missing_container_option(
                and result.exception.args[0] == expected_exception_message \
                and type(result.exception) == ValueError
 
+
+def test_language_container_deployer_cli_with_check_cert(
+        request: FixtureRequest,
+        export_slc: ExportInfo,
+        pyexasol_connection: ExaConnection,
+        connection_factory: Callable[[config.Exasol], ExaConnection],
+        exasol_config: config.Exasol,
+        bucketfs_config: config.BucketFs
+):
+    use_ssl_cert_validation = True
+    expected_exception_message = 'Could not connect to Exasol: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify ' \
+                                 'failed: self signed certificate in certificate chain (_ssl.c:1131)'
+    test_name: str = request.node.name
+    schema = test_name
+    language_alias = f"PYTHON3_TE_{test_name.upper()}"
+    container_path = export_slc.cache_file
+    version = None
+    create_schema(pyexasol_connection, schema)
+    dsn = f"{exasol_config.host}:{exasol_config.port}"
+    with revert_language_settings(pyexasol_connection):
+        result = call_language_definition_deployer_cli(dsn=dsn,
+                                                       container_path=container_path,
+                                                       language_alias=language_alias,
+                                                       version=version,
+                                                       exasol_config=exasol_config,
+                                                       bucketfs_config=bucketfs_config,
+                                                       use_ssl_cert_validation=use_ssl_cert_validation)
+
+        assert result.exit_code == 1 \
+            and result.exception.args[0].message in expected_exception_message \
+            and type(result.exception) == ExaConnectionFailedError
+

tests/integration_tests/with_db/deployment/test_scripts_deployer.py

@@ -20,7 +20,9 @@ def test_scripts_deployer(
         user=exasol_config.username,
         password=exasol_config.password,
         schema=schema_name,
-        language_alias=language_alias
+        language_alias=language_alias,
+        ssl_cert_path="",
+        use_ssl_cert_validation=False
     )
     assert DBQueries.check_all_scripts_deployed(
         pyexasol_connection, schema_name)
@@ -49,7 +51,9 @@ def test_scripts_deployer_no_schema_creation_permission(
         user=limited_user,
         password=limited_user_password,
         schema=schema_name,
-        language_alias=language_alias
+        language_alias=language_alias,
+        ssl_cert_path="",
+        use_ssl_cert_validation=False
     )
     assert DBQueries.check_all_scripts_deployed(
         pyexasol_connection, schema_name)

tests/integration_tests/with_db/deployment/test_scripts_deployer_cli.py

@@ -1,7 +1,11 @@
 from click.testing import CliRunner
-from pyexasol import ExaConnection
+from pyexasol import ExaConnection, ExaConnectionFailedError
 from pytest_itde import config
 
+import pytest
+from tests.fixtures.language_container_fixture import export_slc, flavor_path, language_alias
+from tests.fixtures.database_connection_fixture import pyexasol_connection
+
 from exasol_transformers_extension import deploy
 from tests.utils.db_queries import DBQueries
 
@@ -19,10 +23,37 @@ def test_scripts_deployer_cli(language_alias: str,
         "--db-user", exasol_config.username,
         "--db-pass", exasol_config.password,
         "--schema", schema_name,
-        "--language-alias", language_alias
+        "--language-alias", language_alias,
+        "--no-use-ssl-cert-validation"
     ]
     runner = CliRunner()
     result = runner.invoke(deploy.main, args_list)
     assert result.exit_code == 0
     assert DBQueries.check_all_scripts_deployed(
         pyexasol_connection, schema_name)
+
+
+def test_scripts_deployer_cli_with_encryption_verify(language_alias: str,
+                              pyexasol_connection: ExaConnection,
+                              exasol_config: config.Exasol,
+                              request):
+    schema_name = request.node.name
+    pyexasol_connection.execute(f"DROP SCHEMA IF EXISTS {schema_name} CASCADE;")
+
+    args_list = [
+        "scripts",
+        "--dsn", f"{exasol_config.host}:{exasol_config.port}",
+        "--db-user", exasol_config.username,
+        "--db-pass", exasol_config.password,
+        "--schema", schema_name,
+        "--language-alias", language_alias,
+        "--use-ssl-cert-validation"
+    ]
+    expected_exception_message = 'Could not connect to Exasol: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify ' \
+                                 'failed: self signed certificate in certificate chain (_ssl.c:1131)'
+    runner = CliRunner()
+    result = runner.invoke(deploy.main, args_list)
+    assert result.exit_code == 1 \
+           and result.exception.args[0].message in expected_exception_message \
+           and type(result.exception) == ExaConnectionFailedError
+

tests/integration_tests/with_db/test_upload_model.py

@@ -46,7 +46,7 @@ def test_model_upload(setup_database, pyexasol_connection, download_sample_model
         "--bucketfs-name", bucketfs_params.name,
         "--bucketfs-host", host,
         "--bucketfs-port", port,
-        "--bucketfs_use-https", False,
+        "--bucketfs-use-https", False,
         "--bucketfs-user", bucketfs_config.username,
         "--bucketfs-password", bucketfs_config.password,
         "--bucket", bucketfs_params.bucket,