From 8b57c763202a06d52c5b64187a62f97d04001bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marlene=20Kre=C3=9F?= Date: Wed, 6 Sep 2023 14:55:47 +0200 Subject: [PATCH] #103: Added option to toggle TLS cert validation (#123) * added SSL verify toggle option --- doc/changes/changes_0.5.0.md | 1 + doc/user_guide/user_guide.md | 10 +++- .../deployment/deployment_utils.py | 13 +++++ .../deployment/language_container_deployer.py | 11 +++-- .../language_container_deployer_cli.py | 12 +++-- .../deployment/scripts_deployer.py | 10 ++-- .../deployment/scripts_deployer_cli.py | 9 +++- exasol_transformers_extension/upload_model.py | 2 +- githooks/pre-commit | 1 - .../test_language_container_deployer_cli.py | 49 +++++++++++++++++-- .../deployment/test_scripts_deployer.py | 8 ++- .../deployment/test_scripts_deployer_cli.py | 35 ++++++++++++- .../with_db/test_upload_model.py | 2 +- 13 files changed, 137 insertions(+), 26 deletions(-) diff --git a/doc/changes/changes_0.5.0.md b/doc/changes/changes_0.5.0.md index 1d3ef21c..a83c0221 100644 --- a/doc/changes/changes_0.5.0.md +++ b/doc/changes/changes_0.5.0.md @@ -10,6 +10,7 @@ TBD ### Features - #88: Added custom matcher functions for unit tests + - #103: Added option to toggle use of TLS certificate validation for Database connection - #42: Update transformers to 4.31 and adapt the model uploader ### Bug Fixes diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index 71ab0772..b8156ed0 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -101,11 +101,19 @@ deployment script below with the desired version. (see GitHub Releases --bucketfs-port \ --bucketfs-user \ --bucketfs-password \ + --bucketfs-use-https \ --bucket \ --path-in-bucket \ --language-alias \ - --version + --version \ + --ssl-cert-path \ + --use-ssl-cert-validation \ + --no-use-ssl-cert-valiation ``` +The `--ssl-cert-path` is optional if your certificate is not in the OS truststore. +The option `--use-ssl-cert-validation`is the default, you can disable it with `--no-use-ssl-cert-validation`. +Use caution if you want to turn certificate validation off as it potentially lowers the security of your +Database connection. #### Customized Installation In this installation, you can install the desired or customized language diff --git a/exasol_transformers_extension/deployment/deployment_utils.py b/exasol_transformers_extension/deployment/deployment_utils.py index f5df47d6..910c352e 100644 --- a/exasol_transformers_extension/deployment/deployment_utils.py +++ b/exasol_transformers_extension/deployment/deployment_utils.py @@ -4,6 +4,7 @@ import requests import tempfile import subprocess +import ssl from pathlib import Path from getpass import getpass from contextlib import contextmanager @@ -51,6 +52,18 @@ def _concatenate_slc_parts(tmp_dir): return slc_final_path +def get_websocket_ssl_options(use_ssl_cert_validation: bool, ssl_cert_path: str): + websocket_sslopt = { + "cert_reqs": ssl.CERT_REQUIRED, + } + if not use_ssl_cert_validation: + websocket_sslopt["cert_reqs"] = ssl.CERT_NONE + + if ssl_cert_path is not None: + websocket_sslopt["ca_certs"] = ssl_cert_path + return websocket_sslopt + + @contextmanager def get_container_file_from_github_release(version): with tempfile.TemporaryDirectory() as tmp_dir: diff --git a/exasol_transformers_extension/deployment/language_container_deployer.py b/exasol_transformers_extension/deployment/language_container_deployer.py index df360e32..1f13faa3 100644 --- a/exasol_transformers_extension/deployment/language_container_deployer.py +++ b/exasol_transformers_extension/deployment/language_container_deployer.py @@ -5,7 +5,7 @@ import logging from exasol_transformers_extension.utils.bucketfs_operations import \ create_bucketfs_location -import ssl +from exasol_transformers_extension.deployment.deployment_utils import get_websocket_ssl_options logger = logging.getLogger(__name__) @@ -91,16 +91,17 @@ def _get_previous_language_settings(self, alter_type: str) -> str: def run(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int, bucketfs_use_https: bool, bucketfs_user: str, container_file: Path, bucketfs_password: str, bucket: str, path_in_bucket: str, - dsn: str, db_user: str, db_password: str, language_alias: str): + dsn: str, db_user: str, db_password: str, language_alias: str, + ssl_cert_path: str = None, use_ssl_cert_validation: bool = True): + + websocket_sslopt = get_websocket_ssl_options(use_ssl_cert_validation, ssl_cert_path) pyexasol_conn = pyexasol.connect( dsn=dsn, user=db_user, password=db_password, encryption=True, - websocket_sslopt={ - "cert_reqs": ssl.CERT_NONE, - } + websocket_sslopt=websocket_sslopt ) bucketfs_location = create_bucketfs_location( diff --git a/exasol_transformers_extension/deployment/language_container_deployer_cli.py b/exasol_transformers_extension/deployment/language_container_deployer_cli.py index de8bfe19..b8500828 100644 --- a/exasol_transformers_extension/deployment/language_container_deployer_cli.py +++ b/exasol_transformers_extension/deployment/language_container_deployer_cli.py @@ -10,7 +10,7 @@ @click.option('--bucketfs-name', type=str, required=True) @click.option('--bucketfs-host', type=str, required=True) @click.option('--bucketfs-port', type=int, required=True) -@click.option('--bucketfs_use-https', type=bool, default=False) +@click.option('--bucketfs-use-https', type=bool, default=False) @click.option('--bucketfs-user', type=str, required=True, default="w") @click.option('--bucketfs-password', prompt='bucketFS password', hide_input=True, default=lambda: os.environ.get( @@ -26,6 +26,8 @@ default=lambda: os.environ.get( utils.DB_PASSWORD_ENVIRONMENT_VARIABLE, "")) @click.option('--language-alias', type=str, default="PYTHON3_TE") +@click.option('--ssl-cert-path', type=str, default="") +@click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True) def language_container_deployer_main( bucketfs_name: str, bucketfs_host: str, @@ -40,7 +42,9 @@ def language_container_deployer_main( dsn: str, db_user: str, db_pass: str, - language_alias: str): + language_alias: str, + ssl_cert_path: str, + use_ssl_cert_validation: bool): def call_runner(): LanguageContainerDeployer.run( bucketfs_name=bucketfs_name, @@ -55,7 +59,9 @@ def call_runner(): dsn=dsn, db_user=db_user, db_password=db_pass, - language_alias=language_alias + language_alias=language_alias, + ssl_cert_path=ssl_cert_path, + use_ssl_cert_validation=use_ssl_cert_validation ) if container_file: call_runner() diff --git a/exasol_transformers_extension/deployment/scripts_deployer.py b/exasol_transformers_extension/deployment/scripts_deployer.py index c53be9d2..79575031 100644 --- a/exasol_transformers_extension/deployment/scripts_deployer.py +++ b/exasol_transformers_extension/deployment/scripts_deployer.py @@ -5,7 +5,6 @@ logger = logging.getLogger(__name__) -import ssl class ScriptsDeployer: def __init__(self, language_alias: str, schema: str, @@ -47,16 +46,17 @@ def deploy_scripts(self) -> None: @classmethod def run(cls, dsn: str, user: str, password: str, - schema: str, language_alias: str): + schema: str, language_alias: str, + ssl_cert_path: str, use_ssl_cert_validation: bool = True): + websocket_sslopt = utils.get_websocket_ssl_options(use_ssl_cert_validation, ssl_cert_path) pyexasol_conn = pyexasol.connect( dsn=dsn, user=user, password=password, encryption=True, - websocket_sslopt={ - "cert_reqs": ssl.CERT_NONE, - } + websocket_sslopt=websocket_sslopt ) + scripts_deployer = cls(language_alias, schema, pyexasol_conn) scripts_deployer.deploy_scripts() diff --git a/exasol_transformers_extension/deployment/scripts_deployer_cli.py b/exasol_transformers_extension/deployment/scripts_deployer_cli.py index 5879993c..34250598 100644 --- a/exasol_transformers_extension/deployment/scripts_deployer_cli.py +++ b/exasol_transformers_extension/deployment/scripts_deployer_cli.py @@ -13,15 +13,20 @@ utils.DB_PASSWORD_ENVIRONMENT_VARIABLE, "")) @click.option('--schema', type=str, required=True) @click.option('--language-alias', type=str, default="PYTHON3_TE") +@click.option('--ssl-cert-path', type=str, default="") +@click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True) def scripts_deployer_main( - dsn: str, db_user: str, db_pass: str, schema: str, language_alias: str): + dsn: str, db_user: str, db_pass: str, schema: str, language_alias: str, + ssl_cert_path: str, use_ssl_cert_validation: bool): ScriptsDeployer.run( dsn=dsn, user=db_user, password=db_pass, schema=schema, - language_alias=language_alias + language_alias=language_alias, + ssl_cert_path=ssl_cert_path, + use_ssl_cert_validation=use_ssl_cert_validation ) diff --git a/exasol_transformers_extension/upload_model.py b/exasol_transformers_extension/upload_model.py index 11e673d3..f6f6f27a 100644 --- a/exasol_transformers_extension/upload_model.py +++ b/exasol_transformers_extension/upload_model.py @@ -14,7 +14,7 @@ @click.option('--bucketfs-name', type=str, required=True) @click.option('--bucketfs-host', type=str, required=True) @click.option('--bucketfs-port', type=int, required=True) -@click.option('--bucketfs_use-https', type=bool, default=False) +@click.option('--bucketfs-use-https', type=bool, default=False) @click.option('--bucketfs-user', type=str, required=True, default="w") @click.option('--bucketfs-password', prompt='bucketFS password', hide_input=True, default=lambda: os.environ.get( diff --git a/githooks/pre-commit b/githooks/pre-commit index 68b00192..907b5996 100755 --- a/githooks/pre-commit +++ b/githooks/pre-commit @@ -7,5 +7,4 @@ REPO_DIR=$(git rev-parse --show-toplevel) GITHOOKS_PATH="$REPO_DIR/githooks" pushd "$REPO_DIR" bash "$GITHOOKS_PATH/prohibit_commit_to_main.sh" -bash "$GITHOOKS_PATH/update_setup_py.sh" popd diff --git a/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py b/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py index e6c4b47b..474705d7 100644 --- a/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py +++ b/tests/integration_tests/with_db/deployment/test_language_container_deployer_cli.py @@ -3,10 +3,12 @@ from urllib.parse import urlparse import pytest +from tests.fixtures.language_container_fixture import export_slc, flavor_path +from tests.fixtures.database_connection_fixture import pyexasol_connection from _pytest.fixtures import FixtureRequest from click.testing import CliRunner from exasol_script_languages_container_tool.lib.tasks.export.export_info import ExportInfo -from pyexasol import ExaConnection +from pyexasol import ExaConnection, ExaConnectionFailedError from pytest_itde import config from exasol_transformers_extension import deploy @@ -39,14 +41,15 @@ def call_language_definition_deployer_cli(dsn: str, language_alias: str, version: Optional[str], exasol_config: config.Exasol, - bucketfs_config: config.BucketFs): + bucketfs_config: config.BucketFs, + use_ssl_cert_validation: bool = False): parsed_url = urlparse(bucketfs_config.url) args_list = [ "language-container", "--bucketfs-name", bucketfs_params.name, "--bucketfs-host", parsed_url.hostname, "--bucketfs-port", parsed_url.port, - "--bucketfs_use-https", False, + "--bucketfs-use-https", False, "--bucketfs-user", bucketfs_config.username, "--bucketfs-password", bucketfs_config.password, "--bucket", bucketfs_params.bucket, @@ -56,6 +59,14 @@ def call_language_definition_deployer_cli(dsn: str, "--db-pass", exasol_config.password, "--language-alias", language_alias ] + if use_ssl_cert_validation: + args_list += [ + "--use-ssl-cert-validation" + ] + else: + args_list += [ + "--no-use-ssl-cert-validation" + ] if version is not None: args_list += [ "--version", version, @@ -158,3 +169,35 @@ def test_language_container_deployer_cli_with_missing_container_option( and result.exception.args[0] == expected_exception_message \ and type(result.exception) == ValueError + +def test_language_container_deployer_cli_with_check_cert( + request: FixtureRequest, + export_slc: ExportInfo, + pyexasol_connection: ExaConnection, + connection_factory: Callable[[config.Exasol], ExaConnection], + exasol_config: config.Exasol, + bucketfs_config: config.BucketFs +): + use_ssl_cert_validation = True + expected_exception_message = 'Could not connect to Exasol: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify ' \ + 'failed: self signed certificate in certificate chain (_ssl.c:1131)' + test_name: str = request.node.name + schema = test_name + language_alias = f"PYTHON3_TE_{test_name.upper()}" + container_path = export_slc.cache_file + version = None + create_schema(pyexasol_connection, schema) + dsn = f"{exasol_config.host}:{exasol_config.port}" + with revert_language_settings(pyexasol_connection): + result = call_language_definition_deployer_cli(dsn=dsn, + container_path=container_path, + language_alias=language_alias, + version=version, + exasol_config=exasol_config, + bucketfs_config=bucketfs_config, + use_ssl_cert_validation=use_ssl_cert_validation) + + assert result.exit_code == 1 \ + and result.exception.args[0].message in expected_exception_message \ + and type(result.exception) == ExaConnectionFailedError + diff --git a/tests/integration_tests/with_db/deployment/test_scripts_deployer.py b/tests/integration_tests/with_db/deployment/test_scripts_deployer.py index dfeef150..75f1f307 100644 --- a/tests/integration_tests/with_db/deployment/test_scripts_deployer.py +++ b/tests/integration_tests/with_db/deployment/test_scripts_deployer.py @@ -20,7 +20,9 @@ def test_scripts_deployer( user=exasol_config.username, password=exasol_config.password, schema=schema_name, - language_alias=language_alias + language_alias=language_alias, + ssl_cert_path="", + use_ssl_cert_validation=False ) assert DBQueries.check_all_scripts_deployed( pyexasol_connection, schema_name) @@ -49,7 +51,9 @@ def test_scripts_deployer_no_schema_creation_permission( user=limited_user, password=limited_user_password, schema=schema_name, - language_alias=language_alias + language_alias=language_alias, + ssl_cert_path="", + use_ssl_cert_validation=False ) assert DBQueries.check_all_scripts_deployed( pyexasol_connection, schema_name) diff --git a/tests/integration_tests/with_db/deployment/test_scripts_deployer_cli.py b/tests/integration_tests/with_db/deployment/test_scripts_deployer_cli.py index 13c4ddf2..34ca0814 100644 --- a/tests/integration_tests/with_db/deployment/test_scripts_deployer_cli.py +++ b/tests/integration_tests/with_db/deployment/test_scripts_deployer_cli.py @@ -1,7 +1,11 @@ from click.testing import CliRunner -from pyexasol import ExaConnection +from pyexasol import ExaConnection, ExaConnectionFailedError from pytest_itde import config +import pytest +from tests.fixtures.language_container_fixture import export_slc, flavor_path, language_alias +from tests.fixtures.database_connection_fixture import pyexasol_connection + from exasol_transformers_extension import deploy from tests.utils.db_queries import DBQueries @@ -19,10 +23,37 @@ def test_scripts_deployer_cli(language_alias: str, "--db-user", exasol_config.username, "--db-pass", exasol_config.password, "--schema", schema_name, - "--language-alias", language_alias + "--language-alias", language_alias, + "--no-use-ssl-cert-validation" ] runner = CliRunner() result = runner.invoke(deploy.main, args_list) assert result.exit_code == 0 assert DBQueries.check_all_scripts_deployed( pyexasol_connection, schema_name) + + +def test_scripts_deployer_cli_with_encryption_verify(language_alias: str, + pyexasol_connection: ExaConnection, + exasol_config: config.Exasol, + request): + schema_name = request.node.name + pyexasol_connection.execute(f"DROP SCHEMA IF EXISTS {schema_name} CASCADE;") + + args_list = [ + "scripts", + "--dsn", f"{exasol_config.host}:{exasol_config.port}", + "--db-user", exasol_config.username, + "--db-pass", exasol_config.password, + "--schema", schema_name, + "--language-alias", language_alias, + "--use-ssl-cert-validation" + ] + expected_exception_message = 'Could not connect to Exasol: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify ' \ + 'failed: self signed certificate in certificate chain (_ssl.c:1131)' + runner = CliRunner() + result = runner.invoke(deploy.main, args_list) + assert result.exit_code == 1 \ + and result.exception.args[0].message in expected_exception_message \ + and type(result.exception) == ExaConnectionFailedError + diff --git a/tests/integration_tests/with_db/test_upload_model.py b/tests/integration_tests/with_db/test_upload_model.py index 2809e78a..52c9a81e 100644 --- a/tests/integration_tests/with_db/test_upload_model.py +++ b/tests/integration_tests/with_db/test_upload_model.py @@ -46,7 +46,7 @@ def test_model_upload(setup_database, pyexasol_connection, download_sample_model "--bucketfs-name", bucketfs_params.name, "--bucketfs-host", host, "--bucketfs-port", port, - "--bucketfs_use-https", False, + "--bucketfs-use-https", False, "--bucketfs-user", bucketfs_config.username, "--bucketfs-password", bucketfs_config.password, "--bucket", bucketfs_params.bucket,