move rest_api, sql_database and filesystem sources to dlt core

.github/workflows/test_common.yml

@@ -15,6 +15,10 @@ env:
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 
+  # we need the secrets only for the rest_api_pipeline tests which are in tests/sources
+  # so we inject them only at the end
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+
 jobs:
   get_docs_changes:
     name: docs changes
@@ -87,11 +91,11 @@ jobs:
         run: poetry install --no-interaction --with sentry-sdk
 
       - run: |
-          poetry run pytest tests/common tests/normalize tests/reflection tests/sources tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py
+          poetry run pytest tests/common tests/normalize tests/reflection tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py
         if: runner.os != 'Windows'
         name: Run common tests with minimum dependencies Linux/MAC
       - run: |
-          poetry run pytest tests/common tests/normalize tests/reflection tests/sources tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py -m "not forked"
+          poetry run pytest tests/common tests/normalize tests/reflection tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py -m "not forked"
         if: runner.os == 'Windows'
         name: Run common tests with minimum dependencies Windows
         shell: cmd
@@ -122,15 +126,36 @@ jobs:
         name: Run pipeline tests with pyarrow but no pandas installed Windows
         shell: cmd
 
-      - name: Install pipeline dependencies
-        run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk --with pipeline -E deltalake
+      - name: create secrets.toml for examples
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
+      - name: Install pipeline and sources dependencies
+        run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk --with pipeline -E deltalake -E sql_database
+
+      # TODO: this is needed for the filesystem tests, not sure if this should be in an extra?
+      - name: Install openpyxl for excel tests
+        run: pip install openpyxl
+
+      - run: |
+          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources
+        if: runner.os != 'Windows'
+        name: Run extract and pipeline tests Linux/MAC
+      - run: |
+          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources -m "not forked"
+        if: runner.os == 'Windows'
+        name: Run extract tests Windows
+        shell: cmd
+
+      # here we upgrade sql alchemy to 2 an run the sql_database tests again
+      - name: Upgrade sql alchemy
+        run: poetry run pip install sqlalchemy==2.0.32
 
       - run: |
-          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations
+          poetry run tests/sources/sql_database
         if: runner.os != 'Windows'
         name: Run extract and pipeline tests Linux/MAC
       - run: |
-          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations  -m "not forked"
+          poetry run tests/sources/sql_database
         if: runner.os == 'Windows'
         name: Run extract tests Windows
         shell: cmd