From 035774e9208c5aeebf8f2491e6cdd2f613a382f0 Mon Sep 17 00:00:00 2001
From: Nicola Coretti <nicola.coretti@exasol.com>
Date: Wed, 26 Jun 2024 13:03:47 +0200
Subject: [PATCH 1/7] Add basic test setup for ETL related functionality
---
pyproject.toml | 1 +
test/integration/etl_test.py | 90 +++++++++++++++++++++++++++++++++
test/integration/export_test.py | 26 ++++++++++
test/integration/import_test.py | 21 ++++++++
4 files changed, 138 insertions(+)
create mode 100644 test/integration/etl_test.py
create mode 100644 test/integration/export_test.py
create mode 100644 test/integration/import_test.py
diff --git a/pyproject.toml b/pyproject.toml
index 506b36a..ab2513b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -89,6 +89,7 @@ markers = [
"tls: tests related to tls.",
"udf: tests related to user defined functions.",
"pandas: tests related to support of pandas library.",
+ "etl: etl related tests, export, load (import) and transform data.",
"misc: miscellaneous tests which did not fit in the other categories."
]
diff --git a/test/integration/etl_test.py b/test/integration/etl_test.py
new file mode 100644
index 0000000..17bf4e9
--- /dev/null
+++ b/test/integration/etl_test.py
@@ -0,0 +1,90 @@
+import pytest
+import pyexasol
+from inspect import cleandoc
+
+
+@pytest.fixture
+def connection(dsn, user, password, schema):
+ with pyexasol.connect(
+ dsn=dsn, user=user, password=password, schema=schema, compression=True
+ ) as con:
+ yield con
+
+
+@pytest.fixture
+def export_file(tmp_path):
+ yield tmp_path / "export"
+
+
+@pytest.fixture
+def table(connection):
+ yield "USERS"
+
+
+@pytest.fixture
+def import_table(connection, table):
+ name = f"{table}_IMPORT"
+ ddl = cleandoc(f"""
+ CREATE TABLE IF NOT EXISTS {name}
+ (
+ user_id DECIMAL(18,0),
+ user_name VARCHAR(255),
+ register_dt DATE,
+ last_visit_ts TIMESTAMP,
+ is_female BOOLEAN,
+ user_rating DECIMAL(10,5),
+ user_score DOUBLE,
+ status VARCHAR(50)
+ );
+ """)
+ connection.execute(ddl)
+ connection.commit()
+
+ yield name
+
+ ddl = f"DROP TABLE IF EXISTS {name};"
+ connection.execute(ddl)
+ connection.commit()
+
+
+@pytest.mark.etl
+@pytest.mark.parametrize(
+ "params,expected",
+ [
+ ({}, []),
+ ({"format": "gz"}, []),
+ ({"encoding": "WINDOWS-1251"}, []),
+ ({"columns": ["register_dt", "user_id", "status", "user_name"]}, []),
+ (
+ {
+ "csv_cols": [
+ "1",
+ "2",
+ "3 FORMAT='DD-MM-YYYY'",
+ "4..6",
+ "7 FORMAT='999.99999'",
+ "8",
+ ]
+ },
+ [],
+ ),
+ ],
+)
+def test_export_import_round_trip_to_and_from_file(
+ connection, export_file, table, import_table, params, expected
+):
+ connection.export_to_file(export_file, table, export_params=params)
+ connection.import_from_file(export_file, import_table, import_params=params)
+
+ query = cleandoc(
+ f"""
+ SELECT * FROM {table}
+ WHERE USER_ID NOT IN (SELECT USER_ID from {import_table});
+ """
+ )
+ result = connection.execute(query)
+
+ expected = []
+ actual = result.fetchall()
+
+ assert actual == expected
diff --git a/test/integration/export_test.py b/test/integration/export_test.py
new file mode 100644
index 0000000..46ae787
--- /dev/null
+++ b/test/integration/export_test.py
@@ -0,0 +1,26 @@
+import pytest
+
+
+@pytest.mark.etl
+def test_export_with_column_names():
+ assert False
+
+
+@pytest.mark.etl
+def test_skip_rows_in_export():
+ assert False
+
+
+@pytest.mark.etl
+def test_custom_export_callback():
+ assert False
+
+
+@pytest.mark.etl
+def test_export_with_reodered_columns():
+ assert False
+
+
+@pytest.mark.etl
+def test_export_with_custom_csv_format():
+ assert False
diff --git a/test/integration/import_test.py b/test/integration/import_test.py
new file mode 100644
index 0000000..658ea30
--- /dev/null
+++ b/test/integration/import_test.py
@@ -0,0 +1,21 @@
+import pytest
+
+
+@pytest.mark.etl
+def test_import_with_custom_csv_format():
+ assert False
+
+
+@pytest.mark.etl
+def test_import_with_reordered_columns():
+ assert False
+
+
+@pytest.mark.etl
+def test_custom_import_callback():
+ assert False
+
+
+@pytest.mark.etl
+def test_skip_rows_in_import():
+ assert False
From 27306dfbd159f38a8331feb1d17ce063f42c9542 Mon Sep 17 00:00:00 2001
From: Nicola Coretti <nicola.coretti@exasol.com>
Date: Fri, 28 Jun 2024 11:24:27 +0200
Subject: [PATCH 2/7] Add import tests
---
test/integration/import_test.py | 101 +++++++++++++++++++++++++++++---
1 file changed, 93 insertions(+), 8 deletions(-)
diff --git a/test/integration/import_test.py b/test/integration/import_test.py
index 658ea30..23c4637 100644
--- a/test/integration/import_test.py
+++ b/test/integration/import_test.py
@@ -1,21 +1,106 @@
+import csv
import pytest
+import pyexasol
+from inspect import cleandoc
+
+
+@pytest.fixture
+def connection(dsn, user, password, schema):
+ with pyexasol.connect(
+ dsn=dsn, user=user, password=password, schema=schema, compression=True
+ ) as con:
+ yield con
+
+
+@pytest.fixture
+def table_name():
+ yield "CLIENT_NAMES"
+
+
+@pytest.fixture
+def empty_table(connection, table_name):
+ ddl = cleandoc(f"""
+ CREATE TABLE IF NOT EXISTS {table_name}
+ (
+ FIRSTNAME VARCHAR(255),
+ LASTNAME VARCHAR(255)
+ );
+ """)
+ connection.execute(ddl)
+ connection.commit()
+
+ yield table_name
+
+ ddl = f"DROP TABLE IF EXISTS {table_name};"
+ connection.execute(ddl)
+ connection.commit()
+
+
+@pytest.fixture
+def data(faker):
+ yield [(faker.first_name(), faker.last_name()) for _ in range(0, 10)]
+
+
+@pytest.fixture
+def swaped_data(data):
+ yield [(lastname, firstname) for firstname, lastname in data]
+
+
+@pytest.fixture
+def csv_file(tmp_path, data):
+ file = tmp_path / "names.csv"
+ with open(file, "w", newline="") as csvfile:
+ writer = csv.writer(csvfile, dialect="unix")
+ for row in data:
+ writer.writerow(row)
+ yield file
@pytest.mark.etl
-def test_import_with_custom_csv_format():
- assert False
+def test_import_with_custom_csv_format(connection, empty_table, csv_file, data):
+ connection.import_from_file(csv_file, empty_table)
+ result = connection.execute(f"SELECT * FROM {empty_table};")
+
+ expected = set(data)
+ actual = set(result.fetchall())
+
+ assert actual == expected
@pytest.mark.etl
-def test_import_with_reordered_columns():
- assert False
+def test_import_with_reordered_columns(connection, empty_table, csv_file, swaped_data):
+ params = {"columns": ["LASTNAME", "FIRSTNAME"]}
+ connection.import_from_file(csv_file, empty_table, import_params=params)
+ result = connection.execute(f"SELECT * FROM {empty_table};")
+
+ expected = set(swaped_data)
+ actual = set(result.fetchall())
+
+ assert actual == expected
@pytest.mark.etl
-def test_custom_import_callback():
- assert False
+def test_custom_import_callback(connection, empty_table, csv_file, data):
+ def import_cb(pipe, src):
+ pipe.write(src.read_bytes())
+
+ connection.import_from_callback(import_cb, csv_file, empty_table)
+ result = connection.execute(f"SELECT * FROM {empty_table};")
+
+ expected = set(data)
+ actual = set(result.fetchall())
+
+ assert actual == expected
@pytest.mark.etl
-def test_skip_rows_in_import():
- assert False
+def test_skip_rows_in_import(connection, empty_table, csv_file, data):
+ offset = 2
+ params = {"skip": offset}
+ connection.import_from_file(csv_file, empty_table, import_params=params)
+ result = connection.execute(f"SELECT * FROM {empty_table};")
+
+ expected = set(data[offset::])
+ actual = set(result.fetchall())
+
+ assert actual == expected
From 24a55150d28ed0926b245c287065c2f80f7dd0cb Mon Sep 17 00:00:00 2001
From: Nicola Coretti <nicola.coretti@exasol.com>
Date: Mon, 1 Jul 2024 08:36:23 +0200
Subject: [PATCH 3/7] WIP
---
test/integration/export_test.py | 72 ++++++++++++++++++++++++++++++++-
1 file changed, 70 insertions(+), 2 deletions(-)
diff --git a/test/integration/export_test.py b/test/integration/export_test.py
index 46ae787..cc9f9a0 100644
--- a/test/integration/export_test.py
+++ b/test/integration/export_test.py
@@ -1,4 +1,64 @@
+import csv
import pytest
+import pyexasol
+from inspect import cleandoc
+
+
+@pytest.fixture
+def connection(dsn, user, password, schema):
+ with pyexasol.connect(
+ dsn=dsn, user=user, password=password, schema=schema, compression=True
+ ) as con:
+ yield con
+
+
+@pytest.fixture
+def table_name():
+ yield "CLIENT_NAMES"
+
+
+@pytest.fixture
+def empty_table(connection, table_name):
+ ddl = cleandoc(f"""
+ CREATE TABLE IF NOT EXISTS {table_name}
+ (
+ FIRSTNAME VARCHAR(255),
+ LASTNAME VARCHAR(255)
+ );
+ """)
+ connection.execute(ddl)
+ connection.commit()
+
+ yield table_name
+
+ ddl = f"DROP TABLE IF EXISTS {table_name};"
+ connection.execute(ddl)
+ connection.commit()
+
+
+@pytest.fixture
+def data(faker):
+ yield [(faker.first_name(), faker.last_name()) for _ in range(0, 10)]
+
+
+@pytest.fixture
+def swaped_data(data):
+ yield [(lastname, firstname) for firstname, lastname in data]
+
+
+@pytest.fixture
+def table(connection, empty_table, data):
+ name = empty_table
+ stmt = "INSERT INTO {table} VALUES ({{col1}}, {{col2}});"
+ for col1, col2 in data:
+ connection.execute(stmt.format(table=name), {"col1": col1, "col2":col2})
+ connection.commit()
+ yield name
+
+
+@pytest.fixture
+def export_file(tmp_path, data):
+ yield tmp_path / "names.csv"
@pytest.mark.etl
@@ -12,8 +72,16 @@ def test_skip_rows_in_export():
@pytest.mark.etl
-def test_custom_export_callback():
- assert False
+def test_custom_export_callback(connection, table, data, export_file):
+ def export_cb(pipe, dst):
+ dst.write_bytes(pipe.read())
+
+ connection.export_to_callback(export_cb, export_file, table)
+
+ expected = 0 # TBD
+ actual = None # TBD
+
+ assert actual == expected
@pytest.mark.etl
From e62dfae510a7e857e17e16189d98cb896fb4f039 Mon Sep 17 00:00:00 2001
From: Nicola Coretti <nicola.coretti@exasol.com>
Date: Mon, 8 Jul 2024 07:50:57 +0200
Subject: [PATCH 4/7] Relock dependencies
---
poetry.lock | 121 +++++++++++++++++++++++++++-------------------------
1 file changed, 63 insertions(+), 58 deletions(-)
diff --git a/poetry.lock b/poetry.lock
index a2d36b3..af9078b 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -97,13 +97,13 @@ typecheck = ["mypy"]
[[package]]
name = "certifi"
-version = "2024.6.2"
+version = "2024.7.4"
description = "Python package for providing Mozilla's CA Bundle."
optional = false
python-versions = ">=3.6"
files = [
- {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"},
- {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"},
+ {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"},
+ {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"},
]
[[package]]
@@ -1009,57 +1009,62 @@ files = [
[[package]]
name = "orjson"
-version = "3.10.5"
+version = "3.10.6"
description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy"
optional = true
python-versions = ">=3.8"
files = [
- {file = "orjson-3.10.5-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:545d493c1f560d5ccfc134803ceb8955a14c3fcb47bbb4b2fee0232646d0b932"},
- {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4324929c2dd917598212bfd554757feca3e5e0fa60da08be11b4aa8b90013c1"},
- {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c13ca5e2ddded0ce6a927ea5a9f27cae77eee4c75547b4297252cb20c4d30e6"},
- {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b6c8e30adfa52c025f042a87f450a6b9ea29649d828e0fec4858ed5e6caecf63"},
- {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:338fd4f071b242f26e9ca802f443edc588fa4ab60bfa81f38beaedf42eda226c"},
- {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6970ed7a3126cfed873c5d21ece1cd5d6f83ca6c9afb71bbae21a0b034588d96"},
- {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:235dadefb793ad12f7fa11e98a480db1f7c6469ff9e3da5e73c7809c700d746b"},
- {file = "orjson-3.10.5-cp310-none-win32.whl", hash = "sha256:be79e2393679eda6a590638abda16d167754393f5d0850dcbca2d0c3735cebe2"},
- {file = "orjson-3.10.5-cp310-none-win_amd64.whl", hash = "sha256:c4a65310ccb5c9910c47b078ba78e2787cb3878cdded1702ac3d0da71ddc5228"},
- {file = "orjson-3.10.5-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:cdf7365063e80899ae3a697def1277c17a7df7ccfc979990a403dfe77bb54d40"},
- {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b68742c469745d0e6ca5724506858f75e2f1e5b59a4315861f9e2b1df77775a"},
- {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7d10cc1b594951522e35a3463da19e899abe6ca95f3c84c69e9e901e0bd93d38"},
- {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dcbe82b35d1ac43b0d84072408330fd3295c2896973112d495e7234f7e3da2e1"},
- {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c0eb7e0c75e1e486c7563fe231b40fdd658a035ae125c6ba651ca3b07936f5"},
- {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:53ed1c879b10de56f35daf06dbc4a0d9a5db98f6ee853c2dbd3ee9d13e6f302f"},
- {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:099e81a5975237fda3100f918839af95f42f981447ba8f47adb7b6a3cdb078fa"},
- {file = "orjson-3.10.5-cp311-none-win32.whl", hash = "sha256:1146bf85ea37ac421594107195db8bc77104f74bc83e8ee21a2e58596bfb2f04"},
- {file = "orjson-3.10.5-cp311-none-win_amd64.whl", hash = "sha256:36a10f43c5f3a55c2f680efe07aa93ef4a342d2960dd2b1b7ea2dd764fe4a37c"},
- {file = "orjson-3.10.5-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:68f85ecae7af14a585a563ac741b0547a3f291de81cd1e20903e79f25170458f"},
- {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28afa96f496474ce60d3340fe8d9a263aa93ea01201cd2bad844c45cd21f5268"},
- {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9cd684927af3e11b6e754df80b9ffafd9fb6adcaa9d3e8fdd5891be5a5cad51e"},
- {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d21b9983da032505f7050795e98b5d9eee0df903258951566ecc358f6696969"},
- {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ad1de7fef79736dde8c3554e75361ec351158a906d747bd901a52a5c9c8d24b"},
- {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2d97531cdfe9bdd76d492e69800afd97e5930cb0da6a825646667b2c6c6c0211"},
- {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d69858c32f09c3e1ce44b617b3ebba1aba030e777000ebdf72b0d8e365d0b2b3"},
- {file = "orjson-3.10.5-cp312-none-win32.whl", hash = "sha256:64c9cc089f127e5875901ac05e5c25aa13cfa5dbbbd9602bda51e5c611d6e3e2"},
- {file = "orjson-3.10.5-cp312-none-win_amd64.whl", hash = "sha256:b2efbd67feff8c1f7728937c0d7f6ca8c25ec81373dc8db4ef394c1d93d13dc5"},
- {file = "orjson-3.10.5-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:03b565c3b93f5d6e001db48b747d31ea3819b89abf041ee10ac6988886d18e01"},
- {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:584c902ec19ab7928fd5add1783c909094cc53f31ac7acfada817b0847975f26"},
- {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a35455cc0b0b3a1eaf67224035f5388591ec72b9b6136d66b49a553ce9eb1e6"},
- {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1670fe88b116c2745a3a30b0f099b699a02bb3482c2591514baf5433819e4f4d"},
- {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:185c394ef45b18b9a7d8e8f333606e2e8194a50c6e3c664215aae8cf42c5385e"},
- {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ca0b3a94ac8d3886c9581b9f9de3ce858263865fdaa383fbc31c310b9eac07c9"},
- {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dfc91d4720d48e2a709e9c368d5125b4b5899dced34b5400c3837dadc7d6271b"},
- {file = "orjson-3.10.5-cp38-none-win32.whl", hash = "sha256:c05f16701ab2a4ca146d0bca950af254cb7c02f3c01fca8efbbad82d23b3d9d4"},
- {file = "orjson-3.10.5-cp38-none-win_amd64.whl", hash = "sha256:8a11d459338f96a9aa7f232ba95679fc0c7cedbd1b990d736467894210205c09"},
- {file = "orjson-3.10.5-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:85c89131d7b3218db1b24c4abecea92fd6c7f9fab87441cfc342d3acc725d807"},
- {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66215277a230c456f9038d5e2d84778141643207f85336ef8d2a9da26bd7ca"},
- {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:51bbcdea96cdefa4a9b4461e690c75ad4e33796530d182bdd5c38980202c134a"},
- {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbead71dbe65f959b7bd8cf91e0e11d5338033eba34c114f69078d59827ee139"},
- {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5df58d206e78c40da118a8c14fc189207fffdcb1f21b3b4c9c0c18e839b5a214"},
- {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c4057c3b511bb8aef605616bd3f1f002a697c7e4da6adf095ca5b84c0fd43595"},
- {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b39e006b00c57125ab974362e740c14a0c6a66ff695bff44615dcf4a70ce2b86"},
- {file = "orjson-3.10.5-cp39-none-win32.whl", hash = "sha256:eded5138cc565a9d618e111c6d5c2547bbdd951114eb822f7f6309e04db0fb47"},
- {file = "orjson-3.10.5-cp39-none-win_amd64.whl", hash = "sha256:cc28e90a7cae7fcba2493953cff61da5a52950e78dc2dacfe931a317ee3d8de7"},
- {file = "orjson-3.10.5.tar.gz", hash = "sha256:7a5baef8a4284405d96c90c7c62b755e9ef1ada84c2406c24a9ebec86b89f46d"},
+ {file = "orjson-3.10.6-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:fb0ee33124db6eaa517d00890fc1a55c3bfe1cf78ba4a8899d71a06f2d6ff5c7"},
+ {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c1c4b53b24a4c06547ce43e5fee6ec4e0d8fe2d597f4647fc033fd205707365"},
+ {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eadc8fd310edb4bdbd333374f2c8fec6794bbbae99b592f448d8214a5e4050c0"},
+ {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61272a5aec2b2661f4fa2b37c907ce9701e821b2c1285d5c3ab0207ebd358d38"},
+ {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57985ee7e91d6214c837936dc1608f40f330a6b88bb13f5a57ce5257807da143"},
+ {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:633a3b31d9d7c9f02d49c4ab4d0a86065c4a6f6adc297d63d272e043472acab5"},
+ {file = "orjson-3.10.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1c680b269d33ec444afe2bdc647c9eb73166fa47a16d9a75ee56a374f4a45f43"},
+ {file = "orjson-3.10.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f759503a97a6ace19e55461395ab0d618b5a117e8d0fbb20e70cfd68a47327f2"},
+ {file = "orjson-3.10.6-cp310-none-win32.whl", hash = "sha256:95a0cce17f969fb5391762e5719575217bd10ac5a189d1979442ee54456393f3"},
+ {file = "orjson-3.10.6-cp310-none-win_amd64.whl", hash = "sha256:df25d9271270ba2133cc88ee83c318372bdc0f2cd6f32e7a450809a111efc45c"},
+ {file = "orjson-3.10.6-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b1ec490e10d2a77c345def52599311849fc063ae0e67cf4f84528073152bb2ba"},
+ {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55d43d3feb8f19d07e9f01e5b9be4f28801cf7c60d0fa0d279951b18fae1932b"},
+ {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac3045267e98fe749408eee1593a142e02357c5c99be0802185ef2170086a863"},
+ {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c27bc6a28ae95923350ab382c57113abd38f3928af3c80be6f2ba7eb8d8db0b0"},
+ {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d27456491ca79532d11e507cadca37fb8c9324a3976294f68fb1eff2dc6ced5a"},
+ {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05ac3d3916023745aa3b3b388e91b9166be1ca02b7c7e41045da6d12985685f0"},
+ {file = "orjson-3.10.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1335d4ef59ab85cab66fe73fd7a4e881c298ee7f63ede918b7faa1b27cbe5212"},
+ {file = "orjson-3.10.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4bbc6d0af24c1575edc79994c20e1b29e6fb3c6a570371306db0993ecf144dc5"},
+ {file = "orjson-3.10.6-cp311-none-win32.whl", hash = "sha256:450e39ab1f7694465060a0550b3f6d328d20297bf2e06aa947b97c21e5241fbd"},
+ {file = "orjson-3.10.6-cp311-none-win_amd64.whl", hash = "sha256:227df19441372610b20e05bdb906e1742ec2ad7a66ac8350dcfd29a63014a83b"},
+ {file = "orjson-3.10.6-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:ea2977b21f8d5d9b758bb3f344a75e55ca78e3ff85595d248eee813ae23ecdfb"},
+ {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b6f3d167d13a16ed263b52dbfedff52c962bfd3d270b46b7518365bcc2121eed"},
+ {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f710f346e4c44a4e8bdf23daa974faede58f83334289df80bc9cd12fe82573c7"},
+ {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7275664f84e027dcb1ad5200b8b18373e9c669b2a9ec33d410c40f5ccf4b257e"},
+ {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0943e4c701196b23c240b3d10ed8ecd674f03089198cf503105b474a4f77f21f"},
+ {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:446dee5a491b5bc7d8f825d80d9637e7af43f86a331207b9c9610e2f93fee22a"},
+ {file = "orjson-3.10.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:64c81456d2a050d380786413786b057983892db105516639cb5d3ee3c7fd5148"},
+ {file = "orjson-3.10.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:960db0e31c4e52fa0fc3ecbaea5b2d3b58f379e32a95ae6b0ebeaa25b93dfd34"},
+ {file = "orjson-3.10.6-cp312-none-win32.whl", hash = "sha256:a6ea7afb5b30b2317e0bee03c8d34c8181bc5a36f2afd4d0952f378972c4efd5"},
+ {file = "orjson-3.10.6-cp312-none-win_amd64.whl", hash = "sha256:874ce88264b7e655dde4aeaacdc8fd772a7962faadfb41abe63e2a4861abc3dc"},
+ {file = "orjson-3.10.6-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:66680eae4c4e7fc193d91cfc1353ad6d01b4801ae9b5314f17e11ba55e934183"},
+ {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caff75b425db5ef8e8f23af93c80f072f97b4fb3afd4af44482905c9f588da28"},
+ {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3722fddb821b6036fd2a3c814f6bd9b57a89dc6337b9924ecd614ebce3271394"},
+ {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2c116072a8533f2fec435fde4d134610f806bdac20188c7bd2081f3e9e0133f"},
+ {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6eeb13218c8cf34c61912e9df2de2853f1d009de0e46ea09ccdf3d757896af0a"},
+ {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:965a916373382674e323c957d560b953d81d7a8603fbeee26f7b8248638bd48b"},
+ {file = "orjson-3.10.6-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:03c95484d53ed8e479cade8628c9cea00fd9d67f5554764a1110e0d5aa2de96e"},
+ {file = "orjson-3.10.6-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:e060748a04cccf1e0a6f2358dffea9c080b849a4a68c28b1b907f272b5127e9b"},
+ {file = "orjson-3.10.6-cp38-none-win32.whl", hash = "sha256:738dbe3ef909c4b019d69afc19caf6b5ed0e2f1c786b5d6215fbb7539246e4c6"},
+ {file = "orjson-3.10.6-cp38-none-win_amd64.whl", hash = "sha256:d40f839dddf6a7d77114fe6b8a70218556408c71d4d6e29413bb5f150a692ff7"},
+ {file = "orjson-3.10.6-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:697a35a083c4f834807a6232b3e62c8b280f7a44ad0b759fd4dce748951e70db"},
+ {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd502f96bf5ea9a61cbc0b2b5900d0dd68aa0da197179042bdd2be67e51a1e4b"},
+ {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f215789fb1667cdc874c1b8af6a84dc939fd802bf293a8334fce185c79cd359b"},
+ {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2debd8ddce948a8c0938c8c93ade191d2f4ba4649a54302a7da905a81f00b56"},
+ {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5410111d7b6681d4b0d65e0f58a13be588d01b473822483f77f513c7f93bd3b2"},
+ {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb1f28a137337fdc18384079fa5726810681055b32b92253fa15ae5656e1dddb"},
+ {file = "orjson-3.10.6-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:bf2fbbce5fe7cd1aa177ea3eab2b8e6a6bc6e8592e4279ed3db2d62e57c0e1b2"},
+ {file = "orjson-3.10.6-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:79b9b9e33bd4c517445a62b90ca0cc279b0f1f3970655c3df9e608bc3f91741a"},
+ {file = "orjson-3.10.6-cp39-none-win32.whl", hash = "sha256:30b0a09a2014e621b1adf66a4f705f0809358350a757508ee80209b2d8dae219"},
+ {file = "orjson-3.10.6-cp39-none-win_amd64.whl", hash = "sha256:49e3bc615652617d463069f91b867a4458114c5b104e13b7ae6872e5f79d0844"},
+ {file = "orjson-3.10.6.tar.gz", hash = "sha256:e54b63d0a7c6c54a5f5f726bc93a2078111ef060fec4ecbf34c5db800ca3b3a7"},
]
[[package]]
@@ -1552,18 +1557,18 @@ pyasn1 = ">=0.1.3"
[[package]]
name = "setuptools"
-version = "70.1.1"
+version = "70.2.0"
description = "Easily download, build, install, upgrade, and uninstall Python packages"
optional = false
python-versions = ">=3.8"
files = [
- {file = "setuptools-70.1.1-py3-none-any.whl", hash = "sha256:a58a8fde0541dab0419750bcc521fbdf8585f6e5cb41909df3a472ef7b81ca95"},
- {file = "setuptools-70.1.1.tar.gz", hash = "sha256:937a48c7cdb7a21eb53cd7f9b59e525503aa8abaf3584c730dc5f7a5bec3a650"},
+ {file = "setuptools-70.2.0-py3-none-any.whl", hash = "sha256:b8b8060bb426838fbe942479c90296ce976249451118ef566a5a0b7d8b78fb05"},
+ {file = "setuptools-70.2.0.tar.gz", hash = "sha256:bd63e505105011b25c3c11f753f7e3b8465ea739efddaccef8f0efac2137bac1"},
]
[package.extras]
-docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
-testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
+doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
+test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
[[package]]
name = "simplejson"
@@ -1718,13 +1723,13 @@ files = [
[[package]]
name = "tenacity"
-version = "8.4.2"
+version = "8.5.0"
description = "Retry code until it succeeds"
optional = false
python-versions = ">=3.8"
files = [
- {file = "tenacity-8.4.2-py3-none-any.whl", hash = "sha256:9e6f7cf7da729125c7437222f8a522279751cdfbe6b67bfe64f75d3a348661b2"},
- {file = "tenacity-8.4.2.tar.gz", hash = "sha256:cd80a53a79336edba8489e767f729e4f391c896956b57140b5d7511a64bbd3ef"},
+ {file = "tenacity-8.5.0-py3-none-any.whl", hash = "sha256:b594c2a5945830c267ce6b79a166228323ed52718f30302c1359836112346687"},
+ {file = "tenacity-8.5.0.tar.gz", hash = "sha256:8bc6c0c8a09b31e6cad13c47afbed1a567518250a9a171418582ed8d9c20ca78"},
]
[package.extras]
From 0e7cd5f374dff12bd366cc371c2c2c9d58c5c6d4 Mon Sep 17 00:00:00 2001
From: Nicola Coretti <nicola.coretti@exasol.com>
Date: Mon, 8 Jul 2024 12:07:00 +0200
Subject: [PATCH 5/7] Update export tests
---
test/integration/export_test.py | 55 ++++++++++++++++++++++++---------
1 file changed, 41 insertions(+), 14 deletions(-)
diff --git a/test/integration/export_test.py b/test/integration/export_test.py
index cc9f9a0..11bc812 100644
--- a/test/integration/export_test.py
+++ b/test/integration/export_test.py
@@ -1,3 +1,4 @@
+import io
import csv
import pytest
import pyexasol
@@ -51,7 +52,7 @@ def table(connection, empty_table, data):
name = empty_table
stmt = "INSERT INTO {table} VALUES ({{col1}}, {{col2}});"
for col1, col2 in data:
- connection.execute(stmt.format(table=name), {"col1": col1, "col2":col2})
+ connection.execute(stmt.format(table=name), {"col1": col1, "col2": col2})
connection.commit()
yield name
@@ -61,34 +62,60 @@ def export_file(tmp_path, data):
yield tmp_path / "names.csv"
-@pytest.mark.etl
-def test_export_with_column_names():
- assert False
+@pytest.fixture
+def csv_dialect():
+ class PyexasolCsvDialect(csv.Dialect):
+ lineterminator = "\n"
+ delimiter = ","
+ quoting = csv.QUOTE_MINIMAL
+ quotechar = '"'
+
+ yield PyexasolCsvDialect()
+
+
+@pytest.fixture
+def expected_csv(csv_dialect):
+ def create_csv(table, data, **params):
+ csvfile = io.StringIO()
+ if "with_column_names" in params and params["with_column_names"]:
+ data = [("FIRSTNAME", "LASTNAME")] + data
+ writer = csv.writer(csvfile, csv_dialect)
+ writer.writerows(data)
+ return csvfile.getvalue()
+
+ yield create_csv
@pytest.mark.etl
-def test_skip_rows_in_export():
- assert False
+def test_export_with_column_names(connection, table, data, export_file, expected_csv):
+ params = {"with_column_names": True}
+ connection.export_to_file(export_file, table, export_params=params)
+
+ expected = expected_csv(table, data, **params)
+ actual = export_file.read_text()
+
+ assert actual == expected
@pytest.mark.etl
-def test_custom_export_callback(connection, table, data, export_file):
+def test_custom_export_callback(connection, table, data, export_file, expected_csv):
def export_cb(pipe, dst):
dst.write_bytes(pipe.read())
connection.export_to_callback(export_cb, export_file, table)
- expected = 0 # TBD
- actual = None # TBD
+ expected = expected_csv(table, data)
+ actual = export_file.read_text()
assert actual == expected
@pytest.mark.etl
-def test_export_with_reodered_columns():
- assert False
+def test_export_csv_cols(connection, table, data, export_file, expected_csv):
+ params = {"csv_cols": ["1..2", "2 foo_LASTNAME"]}
+ connection.export_to_file(export_file, table, export_params=params)
+ expected = expected_csv(table, data, **params)
+ actual = export_file.read_text()
-@pytest.mark.etl
-def test_export_with_custom_csv_format():
- assert False
+ assert actual == expected
From 6a8b50face368f91a29525a2b40b5c3dbbba4057 Mon Sep 17 00:00:00 2001
From: Nicola Coretti <nicola.coretti@exasol.com>
Date: Tue, 9 Jul 2024 07:57:40 +0200
Subject: [PATCH 6/7] Fix csv_cols test
---
test/integration/export_test.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/test/integration/export_test.py b/test/integration/export_test.py
index 11bc812..6d16302 100644
--- a/test/integration/export_test.py
+++ b/test/integration/export_test.py
@@ -112,7 +112,7 @@ def export_cb(pipe, dst):
@pytest.mark.etl
def test_export_csv_cols(connection, table, data, export_file, expected_csv):
- params = {"csv_cols": ["1..2", "2 foo_LASTNAME"]}
+ params = {"csv_cols": ["1..2"]}
connection.export_to_file(export_file, table, export_params=params)
expected = expected_csv(table, data, **params)
From 26e9211fe4a0073adbbd7719c48d5494f84837fd Mon Sep 17 00:00:00 2001
From: Nicola Coretti <nicola.coretti@exasol.com>
Date: Tue, 9 Jul 2024 08:10:56 +0200
Subject: [PATCH 7/7] Apply review feedback
---
test/integration/etl_test.py | 35 +++++++++++++++------------------
test/integration/export_test.py | 5 -----
test/integration/import_test.py | 2 +-
3 files changed, 17 insertions(+), 25 deletions(-)
diff --git a/test/integration/etl_test.py b/test/integration/etl_test.py
index 17bf4e9..1d192f9 100644
--- a/test/integration/etl_test.py
+++ b/test/integration/etl_test.py
@@ -49,29 +49,26 @@ def import_table(connection, table):
@pytest.mark.etl
@pytest.mark.parametrize(
- "params,expected",
+ "params",
[
- ({}, []),
- ({"format": "gz"}, []),
- ({"encoding": "WINDOWS-1251"}, []),
- ({"columns": ["register_dt", "user_id", "status", "user_name"]}, []),
- (
- {
- "csv_cols": [
- "1",
- "2",
- "3 FORMAT='DD-MM-YYYY'",
- "4..6",
- "7 FORMAT='999.99999'",
- "8",
- ]
- },
- [],
- ),
+ {},
+ {"format": "gz"},
+ {"encoding": "WINDOWS-1251"},
+ {"columns": ["register_dt", "user_id", "status", "user_name"]},
+ {
+ "csv_cols": [
+ "1",
+ "2",
+ "3 FORMAT='DD-MM-YYYY'",
+ "4..6",
+ "7 FORMAT='999.99999'",
+ "8",
+ ]
+ },
],
)
def test_export_import_round_trip_to_and_from_file(
- connection, export_file, table, import_table, params, expected
+ connection, export_file, table, import_table, params
):
connection.export_to_file(export_file, table, export_params=params)
connection.import_from_file(export_file, import_table, import_params=params)
diff --git a/test/integration/export_test.py b/test/integration/export_test.py
index 6d16302..91d1f30 100644
--- a/test/integration/export_test.py
+++ b/test/integration/export_test.py
@@ -42,11 +42,6 @@ def data(faker):
yield [(faker.first_name(), faker.last_name()) for _ in range(0, 10)]
-@pytest.fixture
-def swaped_data(data):
- yield [(lastname, firstname) for firstname, lastname in data]
-
-
@pytest.fixture
def table(connection, empty_table, data):
name = empty_table
diff --git a/test/integration/import_test.py b/test/integration/import_test.py
index 23c4637..ac90466 100644
--- a/test/integration/import_test.py
+++ b/test/integration/import_test.py
@@ -57,7 +57,7 @@ def csv_file(tmp_path, data):
@pytest.mark.etl
-def test_import_with_custom_csv_format(connection, empty_table, csv_file, data):
+def test_import_csv(connection, empty_table, csv_file, data):
connection.import_from_file(csv_file, empty_table)
result = connection.execute(f"SELECT * FROM {empty_table};")