diff --git a/.github/cluster-upstream.yml b/.github/cluster-upstream.yml
new file mode 100644
index 000000000..a56eb5249
--- /dev/null
+++ b/.github/cluster-upstream.yml
@@ -0,0 +1,21 @@
+# Docker-compose setup used during tests
+version: '3'
+services:
+    dask-scheduler:
+        container_name: dask-scheduler
+        image: daskdev/dask:dev
+        command: dask-scheduler
+        environment:
+            USE_MAMBA: "true"
+            EXTRA_CONDA_PACKAGES: "dask/label/dev::dask"
+        ports:
+            - "8786:8786"
+    dask-worker:
+        container_name: dask-worker
+        image: daskdev/dask:dev
+        command: dask-worker dask-scheduler:8786
+        environment:
+            USE_MAMBA: "true"
+            EXTRA_CONDA_PACKAGES: "dask/label/dev::dask pyarrow>1.0.0"  # required for parquet IO
+        volumes:
+            - /tmp:/tmp
diff --git a/.github/docker-compose.yaml b/.github/cluster.yml
similarity index 100%
rename from .github/docker-compose.yaml
rename to .github/cluster.yml
diff --git a/.github/workflows/test-upstream.yml b/.github/workflows/test-upstream.yml
index a515395a0..95d657d3c 100644
--- a/.github/workflows/test-upstream.yml
+++ b/.github/workflows/test-upstream.yml
@@ -70,6 +70,7 @@ jobs:
           use-mamba: true
           python-version: ${{ matrix.python }}
           channel-priority: strict
+          channels: dask/label/dev,conda-forge,nodefaults
           activate-environment: dask-sql
           environment-file: ${{ env.CONDA_FILE }}
       - name: Download the pre-build jar
@@ -89,19 +90,109 @@ jobs:
           echo "JAVA_HOME=${{ env.CONDA }}\envs\dask-sql\Library" >> $GITHUB_ENV
       - name: Install upstream dev Dask / dask-ml
         run: |
-          python -m pip install --no-deps git+https://github.com/dask/dask
-          python -m pip install --no-deps git+https://github.com/dask/distributed
+          mamba update dask
           python -m pip install --no-deps git+https://github.com/dask/dask-ml
       - name: Test with pytest
         run: |
           pytest --junitxml=junit/test-results.xml --cov-report=xml -n auto tests --dist loadfile
 
+  cluster-dev:
+    name: "Test upstream dev in a dask cluster"
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }}
+      - name: Set up Python
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          use-mamba: true
+          python-version: "3.8"
+          channel-priority: strict
+          channels: dask/label/dev,conda-forge,nodefaults
+          activate-environment: dask-sql
+          environment-file: continuous_integration/environment-3.8-jdk11-dev.yaml
+      - name: Download the pre-build jar
+        uses: actions/download-artifact@v1
+        with:
+          name: jar
+          path: dask_sql/jar/
+      - name: Install cluster dependencies
+        run: |
+          mamba install python-blosc lz4 -c conda-forge
+
+          which python
+          pip list
+          mamba list
+      - name: Install upstream dev dask-ml
+        run: |
+          mamba update dask
+          python -m pip install --no-deps git+https://github.com/dask/dask-ml
+      - name: run a dask cluster
+        run: |
+          docker-compose -f .github/cluster-upstream.yml up -d
+
+          # periodically ping logs until a connection has been established; assume failure after 2 minutes
+          timeout 2m bash -c 'until docker logs dask-worker 2>&1 | grep -q "Starting established connection"; do sleep 1; done'
+
+          docker logs dask-scheduler
+          docker logs dask-worker
+      - name: Test with pytest while running an independent dask cluster
+        run: |
+          DASK_SQL_TEST_SCHEDULER="tcp://127.0.0.1:8786" pytest --junitxml=junit/test-cluster-results.xml --cov-report=xml -n auto tests --dist loadfile
+
+  import-dev:
+    name: "Test importing with bare requirements and upstream dev"
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }}
+      - name: Set up Python
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          python-version: "3.8"
+          mamba-version: "*"
+          channels: dask/label/dev,conda-forge,nodefaults
+          channel-priority: strict
+      - name: Download the pre-build jar
+        uses: actions/download-artifact@v1
+        with:
+          name: jar
+          path: dask_sql/jar/
+      - name: Install upstream dev Dask / dask-ml
+        if: needs.detect-ci-trigger.outputs.triggered == 'true'
+        run: |
+          mamba update dask
+          python -m pip install --no-deps git+https://github.com/dask/dask-ml
+      - name: Install dependencies and nothing else
+        run: |
+          pip install -e .
+
+          which python
+          pip list
+          mamba list
+      - name: Try to import dask-sql
+        run: |
+          python -c "import dask_sql; print('ok')"
+
   report-failures:
     name: Open issue for upstream dev failures
-    needs: test-dev
+    needs: [test-dev, cluster-dev]
     if: |
       always()
-      && needs.test-dev.result == 'failure'
+      && (
+        needs.test-dev.result == 'failure' || needs.cluster-dev.result == 'failure'
+      )
     runs-on: ubuntu-latest
     defaults:
       run:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6201afec8..693471e15 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -92,6 +92,7 @@ jobs:
           use-mamba: true
           python-version: ${{ matrix.python }}
           channel-priority: strict
+          channels: ${{ needs.detect-ci-trigger.outputs.triggered == 'true' && 'dask/label/dev,conda-forge,nodefaults' || 'conda-forge,nodefaults' }}
           activate-environment: dask-sql
           environment-file: ${{ env.CONDA_FILE }}
       - name: Download the pre-build jar
@@ -112,8 +113,7 @@ jobs:
       - name: Optionally install upstream dev Dask / dask-ml
         if: needs.detect-ci-trigger.outputs.triggered == 'true'
         run: |
-          python -m pip install --no-deps git+https://github.com/dask/dask
-          python -m pip install --no-deps git+https://github.com/dask/distributed
+          mamba update dask
           python -m pip install --no-deps git+https://github.com/dask/dask-ml
       - name: Test with pytest
         run: |
@@ -149,6 +149,7 @@ jobs:
           use-mamba: true
           python-version: "3.8"
           channel-priority: strict
+          channels: ${{ needs.detect-ci-trigger.outputs.triggered == 'true' && 'dask/label/dev,conda-forge,nodefaults' || 'conda-forge,nodefaults' }}
           activate-environment: dask-sql
           environment-file: continuous_integration/environment-3.8-jdk11-dev.yaml
       - name: Download the pre-build jar
@@ -156,22 +157,27 @@ jobs:
         with:
           name: jar
           path: dask_sql/jar/
-      - name: Install dependencies
+      - name: Install cluster dependencies
         run: |
           mamba install python-blosc lz4 -c conda-forge
 
           which python
           pip list
           mamba list
-      - name: Optionally install upstream dev Dask / dask-ml
+      - name: Optionally install upstream dev dask-ml
         if: needs.detect-ci-trigger.outputs.triggered == 'true'
         run: |
-          python -m pip install --no-deps git+https://github.com/dask/dask
-          python -m pip install --no-deps git+https://github.com/dask/distributed
+          mamba update dask
           python -m pip install --no-deps git+https://github.com/dask/dask-ml
       - name: run a dask cluster
+        env:
+          UPSTREAM: ${{ needs.detect-ci-trigger.outputs.triggered }}
         run: |
-          docker-compose -f .github/docker-compose.yaml up -d
+          if [[ $UPSTREAM == "true" ]]; then
+            docker-compose -f .github/cluster-upstream.yml up -d
+          else
+            docker-compose -f .github/cluster.yml up -d
+          fi
 
           # periodically ping logs until a connection has been established; assume failure after 2 minutes
           timeout 2m bash -c 'until docker logs dask-worker 2>&1 | grep -q "Starting established connection"; do sleep 1; done'
@@ -198,13 +204,18 @@ jobs:
         with:
           python-version: "3.8"
           mamba-version: "*"
-          channels: conda-forge,defaults
+          channels: ${{ needs.detect-ci-trigger.outputs.triggered == 'true' && 'dask/label/dev,conda-forge,nodefaults' || 'conda-forge,nodefaults' }}
           channel-priority: strict
       - name: Download the pre-build jar
         uses: actions/download-artifact@v1
         with:
           name: jar
           path: dask_sql/jar/
+      - name: Optionally install upstream dev Dask / dask-ml
+        if: needs.detect-ci-trigger.outputs.triggered == 'true'
+        run: |
+          mamba update dask
+          python -m pip install --no-deps git+https://github.com/dask/dask-ml
       - name: Install dependencies and nothing else
         run: |
           pip install -e .
@@ -212,12 +223,6 @@ jobs:
           which python
           pip list
           mamba list
-      - name: Optionally install upstream dev Dask / dask-ml
-        if: needs.detect-ci-trigger.outputs.triggered == 'true'
-        run: |
-          python -m pip install --no-deps git+https://github.com/dask/dask
-          python -m pip install --no-deps git+https://github.com/dask/distributed
-          python -m pip install --no-deps git+https://github.com/dask/dask-ml
       - name: Try to import dask-sql
         run: |
           python -c "import dask_sql; print('ok')"