#181: Remove torch package index from pyproject.toml

doc/changes/changes_0.9.0.md

@@ -17,7 +17,7 @@ T.B.D
 
 ### Refactorings
 
- - n/a
+ - #182: Removed torch package index from pyproject.toml
 
 ### Documentation
 

exasol_transformers_extension/deployment/language_container.py

@@ -95,6 +95,11 @@ def add_wheel_to_flavor(flavor_base_path):
 
 def add_requirements_to_flavor(flavor_base_path: Path):
     project_directory = find_project_directory()
-    requirements = subprocess.check_output(["poetry", "export"], cwd=project_directory)
+    requirements_bytes = subprocess.check_output(["poetry", "export", "--without-hashes", "--without-urls"],
+                                                 cwd=project_directory)
+    requirements = requirements_bytes.decode("UTF-8")
+    requirements_without_cuda = "\n".join(line
+                                          for line in requirements.splitlines()
+                                          if not line.startswith("nvidia"))
     requirements_file = flavor_base_path / "dependencies" / "requirements.txt"
-    requirements_file.write_bytes(requirements)
+    requirements_file.write_text(requirements_without_cuda)

language_container/exasol_transformers_extension_container/flavor_base/dependencies/.gitignore

@@ -0,0 +1 @@
+requirements.txt

language_container/exasol_transformers_extension_container/flavor_base/dependencies/Dockerfile

@@ -2,4 +2,4 @@ FROM exasol/script-language-container:python-3.8-minimal-EXASOL-6.2.0-build_run_
 
 Run mkdir /project
 COPY dependencies/requirements.txt /project/requirements.txt
-RUN python3.8 -m pip install -r /project/requirements.txt
+RUN python3.8 -m pip install --extra-index-url https://download.pytorch.org/whl/cpu -r /project/requirements.txt

language_container/exasol_transformers_extension_container/flavor_base/release/.gitignore

@@ -0,0 +1 @@
+dist

pyproject.toml

@@ -18,7 +18,7 @@ keywords = ['exasol']
 [tool.poetry.dependencies]
 python = "^3.8.0"
 pandas = ">=1.4.2,<2.0.0"
-torch = { version="2.0.1+cpu", source = "torch"} # we use cpu version until GPUs are supported from Exasol
+torch = "^2.0.1"
 transformers = {extras = ["torch"], version = "^4.36.2"}
 Jinja2 = "^3.0.3"
 importlib-resources = "^5.4.0"
@@ -35,12 +35,6 @@ exasol-udf-mock-python = "^0.1.0"
 exasol-script-languages-container-tool = "^0.18.1"
 toml = "^0.10.2"
 
-[[tool.poetry.source]]
-name = "torch"
-url = "https://download.pytorch.org/whl/cpu"
-secondary = true
-
-
 [tool.poetry.group.dev.dependencies]
 nox = "^2023.4.22"
 

tests/integration_tests/without_db/test_slc_size.py

@@ -0,0 +1,9 @@
+from pathlib import Path
+
+import humanfriendly
+from exasol_script_languages_container_tool.lib.tasks.export.export_info import ExportInfo
+
+
+def test_slc_size_below_2gb(export_slc: ExportInfo):
+    stat_result = Path(export_slc.cache_file).resolve().stat()
+    assert stat_result.st_size < humanfriendly.parse_size(size="2GB")