From 2e127a1d820fc2293e3f789c58482137c6980682 Mon Sep 17 00:00:00 2001 From: dave Date: Wed, 4 Sep 2024 10:00:25 +0200 Subject: [PATCH] separate common from load tests properly --- .github/workflows/test_common.yml | 4 +- .github/workflows/test_local_sources.yml | 23 ++++++------ dlt/sources/rest_api_pipeline.py | 8 ++-- tests/load/sources/sql_database/conftest.py | 37 ++++++++++++++++++- .../sources/sql_database/sql_source.py | 0 .../sources/sql_database/test_helpers.py | 0 .../sql_database/test_sql_database_source.py | 0 ...t_sql_database_source_all_destinations.py} | 0 ...y => test_filesystem_pipeline_template.py} | 0 ....py => test_rest_api_pipeline_template.py} | 0 tests/sources/sql_database/__init__.py | 1 + tests/sources/sql_database/conftest.py | 36 ------------------ 12 files changed, 55 insertions(+), 54 deletions(-) rename tests/{ => load}/sources/sql_database/sql_source.py (100%) rename tests/{ => load}/sources/sql_database/test_helpers.py (100%) rename tests/{ => load}/sources/sql_database/test_sql_database_source.py (100%) rename tests/load/sources/sql_database/{test_sql_database.py => test_sql_database_source_all_destinations.py} (100%) rename tests/sources/filesystem/{test_filesystem_source.py => test_filesystem_pipeline_template.py} (100%) rename tests/sources/rest_api/{integration/test_rest_api_source.py => test_rest_api_pipeline_template.py} (100%) delete mode 100644 tests/sources/sql_database/conftest.py diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml index 18c7e4bfde..bd76f87c11 100644 --- a/.github/workflows/test_common.yml +++ b/.github/workflows/test_common.yml @@ -126,11 +126,11 @@ jobs: run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk --with pipeline -E deltalake - run: | - poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations + poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources if: runner.os != 'Windows' name: Run extract and pipeline tests Linux/MAC - run: | - poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations -m "not forked" + poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources -m "not forked" if: runner.os == 'Windows' name: Run extract tests Windows shell: cmd diff --git a/.github/workflows/test_local_sources.yml b/.github/workflows/test_local_sources.yml index a04ca4586d..7577b908d0 100644 --- a/.github/workflows/test_local_sources.yml +++ b/.github/workflows/test_local_sources.yml @@ -1,5 +1,4 @@ -# Tests destinations that can run without credentials. -# i.e. local postgres, duckdb, filesystem (with local fs/memory bucket) +# Tests sources against a couple of local destinations name: src | rest_api, sql_database, filesystem @@ -23,6 +22,10 @@ env: ACTIVE_DESTINATIONS: "[\"duckdb\", \"postgres\", \"filesystem\"]" ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\"]" + # we need the secrets to inject the github token for the rest_api template tests + # we should not use it for anything else here + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + jobs: get_docs_changes: name: docs changes @@ -82,22 +85,20 @@ jobs: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-local-sources + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + # TODO: which deps should we enable? - name: Install dependencies run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E sql_database --with sentry-sdk --with pipeline - - # we need sqlalchemy 2 for the sql_database tests + + # we need sqlalchemy 2 for the sql_database tests, TODO: make this all work with sqlalchemy 1.4 - name: Upgrade sql alchemy run: poetry run pip install sqlalchemy==2.0.32 - # run sources tests - - run: poetry run pytest tests/sources - name: Run tests Linux - env: - DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader:loader@localhost:5432/dlt_data - # run sources tests in load against configured destinations - run: poetry run pytest tests/load/sources name: Run tests Linux env: - DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader:loader@localhost:5432/dlt_data \ No newline at end of file + DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader:loader@localhost:5432/dlt_data + diff --git a/dlt/sources/rest_api_pipeline.py b/dlt/sources/rest_api_pipeline.py index ba2a83859f..8c373bc517 100644 --- a/dlt/sources/rest_api_pipeline.py +++ b/dlt/sources/rest_api_pipeline.py @@ -17,10 +17,10 @@ def github_source(access_token: str = dlt.secrets.value) -> Any: config: RESTAPIConfig = { "client": { "base_url": "https://api.github.com/repos/dlt-hub/dlt/", - "auth": { - "type": "bearer", - "token": access_token, - }, + # "auth": { + # "type": "bearer", + # "token": access_token, + # }, }, # The default configuration for all resources and their endpoints "resource_defaults": { diff --git a/tests/load/sources/sql_database/conftest.py b/tests/load/sources/sql_database/conftest.py index 1372663663..d107216f1c 100644 --- a/tests/load/sources/sql_database/conftest.py +++ b/tests/load/sources/sql_database/conftest.py @@ -1 +1,36 @@ -from tests.sources.sql_database.conftest import * # noqa: F403 +from typing import Iterator + +import pytest + +import dlt +from dlt.sources.credentials import ConnectionStringCredentials +from tests.sources.sql_database.sql_source import SQLAlchemySourceDB + + +def _create_db(**kwargs) -> Iterator[SQLAlchemySourceDB]: + # TODO: parametrize the fixture so it takes the credentials for all destinations + credentials = dlt.secrets.get( + "destination.postgres.credentials", expected_type=ConnectionStringCredentials + ) + + db = SQLAlchemySourceDB(credentials, **kwargs) + db.create_schema() + try: + db.create_tables() + db.insert_data() + yield db + finally: + db.drop_schema() + + +@pytest.fixture(scope="package") +def sql_source_db(request: pytest.FixtureRequest) -> Iterator[SQLAlchemySourceDB]: + # Without unsupported types so we can test full schema load with connector-x + yield from _create_db(with_unsupported_types=False) + + +@pytest.fixture(scope="package") +def sql_source_db_unsupported_types( + request: pytest.FixtureRequest, +) -> Iterator[SQLAlchemySourceDB]: + yield from _create_db(with_unsupported_types=True) diff --git a/tests/sources/sql_database/sql_source.py b/tests/load/sources/sql_database/sql_source.py similarity index 100% rename from tests/sources/sql_database/sql_source.py rename to tests/load/sources/sql_database/sql_source.py diff --git a/tests/sources/sql_database/test_helpers.py b/tests/load/sources/sql_database/test_helpers.py similarity index 100% rename from tests/sources/sql_database/test_helpers.py rename to tests/load/sources/sql_database/test_helpers.py diff --git a/tests/sources/sql_database/test_sql_database_source.py b/tests/load/sources/sql_database/test_sql_database_source.py similarity index 100% rename from tests/sources/sql_database/test_sql_database_source.py rename to tests/load/sources/sql_database/test_sql_database_source.py diff --git a/tests/load/sources/sql_database/test_sql_database.py b/tests/load/sources/sql_database/test_sql_database_source_all_destinations.py similarity index 100% rename from tests/load/sources/sql_database/test_sql_database.py rename to tests/load/sources/sql_database/test_sql_database_source_all_destinations.py diff --git a/tests/sources/filesystem/test_filesystem_source.py b/tests/sources/filesystem/test_filesystem_pipeline_template.py similarity index 100% rename from tests/sources/filesystem/test_filesystem_source.py rename to tests/sources/filesystem/test_filesystem_pipeline_template.py diff --git a/tests/sources/rest_api/integration/test_rest_api_source.py b/tests/sources/rest_api/test_rest_api_pipeline_template.py similarity index 100% rename from tests/sources/rest_api/integration/test_rest_api_source.py rename to tests/sources/rest_api/test_rest_api_pipeline_template.py diff --git a/tests/sources/sql_database/__init__.py b/tests/sources/sql_database/__init__.py index e69de29bb2..f10ab98368 100644 --- a/tests/sources/sql_database/__init__.py +++ b/tests/sources/sql_database/__init__.py @@ -0,0 +1 @@ +# almost all tests are in tests/load since a postgres instance is required for this to work diff --git a/tests/sources/sql_database/conftest.py b/tests/sources/sql_database/conftest.py deleted file mode 100644 index d107216f1c..0000000000 --- a/tests/sources/sql_database/conftest.py +++ /dev/null @@ -1,36 +0,0 @@ -from typing import Iterator - -import pytest - -import dlt -from dlt.sources.credentials import ConnectionStringCredentials -from tests.sources.sql_database.sql_source import SQLAlchemySourceDB - - -def _create_db(**kwargs) -> Iterator[SQLAlchemySourceDB]: - # TODO: parametrize the fixture so it takes the credentials for all destinations - credentials = dlt.secrets.get( - "destination.postgres.credentials", expected_type=ConnectionStringCredentials - ) - - db = SQLAlchemySourceDB(credentials, **kwargs) - db.create_schema() - try: - db.create_tables() - db.insert_data() - yield db - finally: - db.drop_schema() - - -@pytest.fixture(scope="package") -def sql_source_db(request: pytest.FixtureRequest) -> Iterator[SQLAlchemySourceDB]: - # Without unsupported types so we can test full schema load with connector-x - yield from _create_db(with_unsupported_types=False) - - -@pytest.fixture(scope="package") -def sql_source_db_unsupported_types( - request: pytest.FixtureRequest, -) -> Iterator[SQLAlchemySourceDB]: - yield from _create_db(with_unsupported_types=True)