Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#134: Fixed language container download #140

Merged
merged 2 commits into from
Oct 19, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions doc/changes/changes_0.6.0.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# Transformers Extension 0.6.0, released T.B.D

Code name: T.B.D


## Summary

T.B.D

### Features

- n/a

### Bug Fixes

- # 134: Fixed slc download still assuming existence of slc-parts

### Refactorings

- n/a

### Documentation

- n/a


4 changes: 2 additions & 2 deletions doc/user_guide/user_guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ pip install transformers_extension.whl --extra-index-url https://download.pytorc
### The Pre-built Language Container

This extension requires the installation of the language container for this
extension to run. It can be installed in two ways:Quick and Customized
extension to run. It can be installed in two ways: Quick and Customized
installations

#### Quick Installation
Expand Down Expand Up @@ -147,7 +147,7 @@ There are two ways to install the language container: (1) using a python script
--bucket <BUCKETFS_NAME> \
--path-in-bucket <PATH_IN_BUCKET> \
--language-alias <LANGUAGE_ALIAS> \
--container-file <path/to/language_container.tar.gz>
--container-file <path/to/language_container_name.tar.gz>
```

2. *Manual Installation*
Expand Down
42 changes: 13 additions & 29 deletions exasol_transformers_extension/deployment/deployment_utils.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,20 @@
import os
import glob

import logging
import requests
import tempfile
import subprocess
import ssl
from pathlib import Path
from getpass import getpass
from contextlib import contextmanager
from jinja2 import Environment, PackageLoader, select_autoescape
from exasol_transformers_extension.deployment import constants


logger = logging.getLogger(__name__)


DB_PASSWORD_ENVIRONMENT_VARIABLE = f"TE_DB_PASSWORD"
BUCKETFS_PASSWORD_ENVIRONMENT_VARIABLE = f"TE_BUCKETFS_PASSWORD"
SLC_PARTS_PREFIX_NAME = 'language_container_part_0'
SLC_FINAL_NAME = "language_container.tar.gz"
N_SLC_PARTS = 2
SLC_NAME = "exasol_transformers_extension_container_release.tar.gz"
GH_RELEASE_URL = "https://github.com/exasol/transformers-extension/releases/download"


Expand All @@ -31,25 +27,14 @@ def load_and_render_statement(template_name, **kwargs) -> str:
return statement


def _download_slc_parts(tmp_dir, version):
for i in range(N_SLC_PARTS):
slc_part_name = f"{SLC_PARTS_PREFIX_NAME}" + str(i)
url = "/".join((GH_RELEASE_URL, version, slc_part_name))
response = requests.get(url, stream=True)
response.raise_for_status()
with open(Path(tmp_dir, slc_part_name), 'wb') as f:
f.write(response.content)


def _concatenate_slc_parts(tmp_dir):
slc_final_path = Path(tmp_dir, SLC_FINAL_NAME)
slc_parts_path = glob.glob(str(Path(tmp_dir, SLC_PARTS_PREFIX_NAME)) + "*")
cmd = f"cat {' '.join(slc_parts_path)} > {slc_final_path}"
subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True)
return slc_final_path
def _download_slc(tmp_dir: Path, version: str) -> Path:
url = "/".join((GH_RELEASE_URL, version, SLC_NAME))
response = requests.get(url, stream=True)
response.raise_for_status()
slc_path = Path(tmp_dir, SLC_NAME)
with open(slc_path, 'wb') as f:
f.write(response.content)
return slc_path


def get_websocket_ssl_options(use_ssl_cert_validation: bool, ssl_cert_path: str):
Expand All @@ -65,9 +50,8 @@ def get_websocket_ssl_options(use_ssl_cert_validation: bool, ssl_cert_path: str)


@contextmanager
def get_container_file_from_github_release(version):
def get_container_file_from_github_release(version: str):
with tempfile.TemporaryDirectory() as tmp_dir:
_download_slc_parts(tmp_dir, version)
container_file_path = _concatenate_slc_parts(tmp_dir)
container_file_path = _download_slc(tmp_dir, version)
yield container_file_path

Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ def _generate_alter_command(self, alter_type: str,
f"ALTER {alter_type} SET SCRIPT_LANGUAGES='{new_settings}';"
return alter_command

def _update_previous_language_settings(
self, alter_type: str, path_in_udf: PurePosixPath) -> str:
def _update_previous_language_settings(self, alter_type: str,
path_in_udf: PurePosixPath) -> str:
prev_lang_settings = self._get_previous_language_settings(alter_type)
prev_lang_aliases = prev_lang_settings.split(" ")
self._check_if_requested_language_alias_already_exists(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ def get_unique_values(
Filter given dataframe and return the unique values. Sorts the unique values
if the sort parameter is set to True

:param df: Dataframe from which the uniques values are extracted
:param df: Dataframe from which the unique values are extracted
:param columns: List of columns to be filtered and sorted accordingly
:param sort: Sort the unique values by given columns if True
"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ def test_language_container_deployer_cli_by_downloading_container(
schema = test_name
language_alias = f"PYTHON3_TE_{test_name.upper()}"
container_path = None
version = "0.2.0"
version = "0.5.0"
create_schema(pyexasol_connection, schema)
dsn = f"{exasol_config.host}:{exasol_config.port}"
with revert_language_settings(pyexasol_connection):
Expand Down Expand Up @@ -200,4 +200,3 @@ def test_language_container_deployer_cli_with_check_cert(
assert result.exit_code == 1 \
and result.exception.args[0].message in expected_exception_message \
and type(result.exception) == ExaConnectionFailedError

Loading