Use dev images for independent cluster testing (#518)

* Switch to dask dev images

* Use mamba for conda installs in images

* Remove sleep call for installation

* Use timeout / until to wait for cluster to be initialized

.github/docker-compose.yaml

@@ -3,15 +3,16 @@ version: '3'
 services:
     dask-scheduler:
         container_name: dask-scheduler
-        image: daskdev/dask:latest
+        image: daskdev/dask:dev
         command: dask-scheduler
         ports:
             - "8786:8786"
     dask-worker:
         container_name: dask-worker
-        image: daskdev/dask:latest
+        image: daskdev/dask:dev
         command: dask-worker dask-scheduler:8786
         environment:
+            USE_MAMBA: "true"
             EXTRA_CONDA_PACKAGES: "pyarrow>1.0.0"  # required for parquet IO
         volumes:
             - /tmp:/tmp

.github/workflows/test.yml

@@ -173,8 +173,8 @@ jobs:
         run: |
           docker-compose -f .github/docker-compose.yaml up -d
 
-          # Wait for installation
-          sleep 40
+          # periodically ping logs until a connection has been established; assume failure after 2 minutes
+          timeout 2m bash -c 'until docker logs dask-worker 2>&1 | grep -q "Starting established connection"; do sleep 1; done'
 
           docker logs dask-scheduler
           docker logs dask-worker