separate common from load tests properly

.github/workflows/test_common.yml

@@ -126,11 +126,11 @@ jobs:
         run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk --with pipeline -E deltalake
 
       - run: |
-          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations
+          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources
         if: runner.os != 'Windows'
         name: Run extract and pipeline tests Linux/MAC
       - run: |
-          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations  -m "not forked"
+          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources -m "not forked"
         if: runner.os == 'Windows'
         name: Run extract tests Windows
         shell: cmd

.github/workflows/test_local_sources.yml

@@ -1,5 +1,4 @@
-# Tests destinations that can run without credentials.
-# i.e. local postgres, duckdb, filesystem (with local fs/memory bucket)
+# Tests sources against a couple of local destinations
 
 name: src | rest_api, sql_database, filesystem
 
@@ -23,6 +22,10 @@ env:
   ACTIVE_DESTINATIONS: "[\"duckdb\", \"postgres\", \"filesystem\"]"
   ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\"]"
 
+  # we need the secrets to inject the github token for the rest_api template tests
+  # we should not use it for anything else here 
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+
 jobs:
   get_docs_changes:
     name: docs changes
@@ -82,22 +85,20 @@ jobs:
           path: .venv
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-local-sources
 
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
         # TODO: which deps should we enable? 
       - name: Install dependencies
         run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E sql_database --with sentry-sdk --with pipeline 
-        
-        # we need sqlalchemy 2 for the sql_database tests
+
+        # we need sqlalchemy 2 for the sql_database tests, TODO: make this all work with sqlalchemy 1.4
       - name: Upgrade sql alchemy
         run: poetry run pip install sqlalchemy==2.0.32
 
-        # run sources tests 
-      - run: poetry run pytest tests/sources 
-        name: Run tests Linux
-        env:
-          DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader:loader@localhost:5432/dlt_data
-
         # run sources tests in load against configured destinations
       - run: poetry run pytest tests/load/sources 
         name: Run tests Linux
         env:
-          DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader:loader@localhost:5432/dlt_data
+          DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader:loader@localhost:5432/dlt_data
+

dlt/sources/rest_api_pipeline.py

@@ -17,10 +17,10 @@ def github_source(access_token: str = dlt.secrets.value) -> Any:
     config: RESTAPIConfig = {
         "client": {
             "base_url": "https://api.github.com/repos/dlt-hub/dlt/",
-            "auth": {
-                "type": "bearer",
-                "token": access_token,
-            },
+            # "auth": {
+            #     "type": "bearer",
+            #     "token": access_token,
+            # },
         },
         # The default configuration for all resources and their endpoints
         "resource_defaults": {

tests/load/sources/sql_database/conftest.py

@@ -1 +1,36 @@
-from tests.sources.sql_database.conftest import *  # noqa: F403
+from typing import Iterator
+
+import pytest
+
+import dlt
+from dlt.sources.credentials import ConnectionStringCredentials
+from tests.sources.sql_database.sql_source import SQLAlchemySourceDB
+
+
+def _create_db(**kwargs) -> Iterator[SQLAlchemySourceDB]:
+    # TODO: parametrize the fixture so it takes the credentials for all destinations
+    credentials = dlt.secrets.get(
+        "destination.postgres.credentials", expected_type=ConnectionStringCredentials
+    )
+
+    db = SQLAlchemySourceDB(credentials, **kwargs)
+    db.create_schema()
+    try:
+        db.create_tables()
+        db.insert_data()
+        yield db
+    finally:
+        db.drop_schema()
+
+
+@pytest.fixture(scope="package")
+def sql_source_db(request: pytest.FixtureRequest) -> Iterator[SQLAlchemySourceDB]:
+    # Without unsupported types so we can test full schema load with connector-x
+    yield from _create_db(with_unsupported_types=False)
+
+
+@pytest.fixture(scope="package")
+def sql_source_db_unsupported_types(
+    request: pytest.FixtureRequest,
+) -> Iterator[SQLAlchemySourceDB]:
+    yield from _create_db(with_unsupported_types=True)

tests/sources/sql_database/__init__.py

@@ -0,0 +1 @@
+# almost all tests are in tests/load since a postgres instance is required for this to work