From f9ce83a466707b09cefa9a42bb19ca02eb065329 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Tue, 9 Feb 2021 20:07:20 +0100 Subject: [PATCH] Fix device synchronization in local_cupy benchmark (#518) The way the code was written would pass the client's device to each worker and synchronize on that, which is wrong. This was catched due to recent changes to Distributed, where the following was raised: ```python Task exception was never retrieved future: .send_message() done, defined at /datasets/pentschev/miniconda3/envs/gdf/lib/python3.7/site-packages/distributed/scheduler.py:4965> exception=Exception("unhashable type: 'cupy.cuda.device.Device'")> Traceback (most recent call last): File "/datasets/pentschev/miniconda3/envs/gdf/lib/python3.7/site-packages/distributed/scheduler.py", line 4969, in send_message resp = await send_recv(comm, close=True, serializers=serializers, **msg) File "/datasets/pentschev/miniconda3/envs/gdf/lib/python3.7/site-packages/distributed/core.py", line 662, in send_recv raise Exception(response["text"]) Exception: unhashable type: 'cupy.cuda.device.Device' ``` Authors: - Peter Andreas Entschev (@pentschev) Approvers: - Benjamin Zaitlen (@quasiben) URL: https://github.com/rapidsai/dask-cuda/pull/518 --- dask_cuda/benchmarks/local_cupy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dask_cuda/benchmarks/local_cupy.py b/dask_cuda/benchmarks/local_cupy.py index 9c6273dee..382721eb4 100644 --- a/dask_cuda/benchmarks/local_cupy.py +++ b/dask_cuda/benchmarks/local_cupy.py @@ -75,7 +75,7 @@ async def _run(client, args): res = client.compute(func(*func_args)) await client.gather(res) if args.type == "gpu": - await client.run(xp.cuda.Device().synchronize) + await client.run(lambda xp: xp.cuda.Device().synchronize(), xp) took = clock() - t1 return {