Prepare IBM-DB2 and Snowflake tests.

Snowflake is already deactivated since it requires pandas < 2 and this is not supported by transform (min/max behave strangely with NULLs).

.github/workflows/test.yml

@@ -0,0 +1,64 @@
+on:
+  workflow_call:
+    inputs:
+      os:
+        required: true
+        type: string
+      environment:
+        required: true
+        type: string
+      docker-services:
+        required: false
+        type: string
+      pytest-arguments:
+        required: false
+        type: string
+      timeout-minutes:
+        default: 10
+        type: number
+    secrets:
+      SNOWFLAKE_PASSWORD:
+        required: false
+      SNOWFLAKE_ACCOUNT:
+        required: false
+      SNOWFLAKE_USER:
+        required: false
+
+jobs:
+  test:
+    name: pytest
+    runs-on: ${{ inputs.os }}
+    timeout-minutes: ${{ inputs.timeout-minutes }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Pixi
+        uses: prefix-dev/[email protected]
+        with:
+          environments: ${{ inputs.environment }}
+
+      - name: Start Docker Compose
+        if: ${{ inputs.docker-services != '' }}
+        uses: isbang/compose-action@e5813a5909aca4ae36058edae58f6e52b9c971f8
+        with:
+          compose-file: docker-compose.yaml
+          services: ${{ inputs.docker-services }}
+
+      - name: Install Microsoft ODBC
+        if: ${{ contains(inputs.docker-services, 'mssql') }}
+        run: sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
+
+      - name: Wait for Docker Servers
+        if: ${{ inputs.docker-services != '' }}
+        run: |
+          until bash ./.github/scripts/docker_compose_ready.sh; do
+            sleep 1
+          done
+
+      - name: Run tests
+        env:
+          SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+          SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+          SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}        
+        run: |
+          pixi run -e ${{ inputs.environment }} pytest tests -ra ${RUNNER_DEBUG:+-v} --color=yes ${{ inputs.pytest-arguments }}

.github/workflows/tests.yml

@@ -25,9 +25,22 @@ jobs:
       - name: Linting - Run pre-commit checks
         uses: pre-commit/[email protected]
 
-  test:
-    name: pytest
-    runs-on: ${{ matrix.os }}
+  smoke_test:
+    name: Smoke Test
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+        environment:
+          - py312
+    uses: ./.github/workflows/test.yml
+    with:
+      os: ${{ matrix.os }}
+      environment: ${{ matrix.environment }}
+
+  postgres_test:
+    name: Postgres Tests
+    needs: [smoke_test]
     strategy:
       matrix:
         os:
@@ -36,34 +49,73 @@ jobs:
           - py39
           - py310
           - py311
-    timeout-minutes: 20
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Setup Pixi
-        uses: prefix-dev/[email protected]
-        with:
-          environments: ${{ matrix.environment }}
+          - py312
+    uses: ./.github/workflows/test.yml
+    with:
+      os: ${{ matrix.os }}
+      environment: ${{ matrix.environment }}
+      docker-services: |
+        postgres
+      pytest-arguments: --postgres
 
-      - name: Start Docker Compose
-        uses: isbang/compose-action@e5813a5909aca4ae36058edae58f6e52b9c971f8
-        with:
-          compose-file: docker-compose.yaml
-
-      - name: Install Microsoft ODBC
-        run: sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
+  mssql_test:
+    name: MSSql Tests
+    needs: [smoke_test]
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+        environment:
+          - py39
+          - py311
+          - py312
+    uses: ./.github/workflows/test.yml
+    with:
+      os: ${{ matrix.os }}
+      environment: ${{ matrix.environment }}
+      docker-services: |
+        mssql
+      pytest-arguments: --mssql
 
-      - name: Wait for Docker Servers
-        run: |
-          until bash ./.github/scripts/docker_compose_ready.sh; do
-            sleep 1
-          done
+  db2_test:
+    name: DB2 Tests
+    needs: [smoke_test]
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+        environment:
+          - py39ibm
+          - py312ibm
+    uses: ./.github/workflows/test.yml
+    with:
+      os: ${{ matrix.os }}
+      environment: ${{ matrix.environment }}
+      docker-services: |
+        ibm_db2
+      pytest-arguments: --ibm_db2
 
-      - name: Run tests
-        env:
-          SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
-          SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
-          SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}        
-        run: |
-          pixi run -e ${{ matrix.environment }} pytest tests -ra ${RUNNER_DEBUG:+-v} --color=yes --postgres --mssql
-  
+# ## Deactivated since snowflake requires pandas < 2 and pydiverse transform does not support it
+#  snowflake_test:
+#    name: Snowflake Tests
+#    needs: [smoke_test]
+#    strategy:
+#      matrix:
+#        os:
+#          - ubuntu-latest
+#        environment:
+#          - py311pdsa1
+#    uses: ./.github/workflows/test.yml
+#    secrets:
+#      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+#      SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+#      SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
+#    with:
+#      os: ${{ matrix.os }}
+#      environment: ${{ matrix.environment }}
+#      workers: 1  # tests are often waiting on snowflake
+#      timeout-minutes: 40
+#      docker-services: |
+#        postgres
+#        zoo
+#      pytest-arguments: --snowflake

docker-compose.yaml

@@ -1,4 +1,3 @@
-version: "3.9"
 services:
   postgres:
     image: postgres
@@ -14,3 +13,16 @@ services:
       SA_PASSWORD: PydiQuant27
     ports:
       - "1433:1433"
+  ibm_db2:
+    platform: linux/x86_64
+    image: icr.io/db2_community/db2
+    privileged: true
+    environment:
+      LICENSE: accept
+      DB2INSTANCE: db2inst1
+      DB2INST1_PASSWORD: password
+      DBNAME: testdb
+      UPDATEAVAIL: NO
+    ports:
+      - 50000:50000
+