From 199ece64c4699583401fdb879b714e652f2bea6f Mon Sep 17 00:00:00 2001 From: Rehan Durrani Date: Tue, 14 Jun 2022 11:45:56 -0700 Subject: [PATCH] FEAT-#4574: Warn when pre-initialized Ray cluster is not using all available memory Signed-off-by: Rehan Durrani --- modin/core/execution/ray/common/utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/modin/core/execution/ray/common/utils.py b/modin/core/execution/ray/common/utils.py index b08b30c7c9b..dde04e30cbe 100644 --- a/modin/core/execution/ray/common/utils.py +++ b/modin/core/execution/ray/common/utils.py @@ -219,6 +219,17 @@ def initialize_ray( if not GPU_MANAGERS: for i in range(GpuCount.get()): GPU_MANAGERS.append(GPUManager.remote(i)) + else: + ray_obj_store_mem = ray.available_resources()["object_store_memory"] + virtual_memory = psutil.virtual_memory().total + if (ray_obj_store_mem // 1e9) < (0.6 * virtual_memory) // 1e9: + warnings.warn( + "Modin has detected that it is running on a pre-initialized Ray cluster. " + + f"This cluster has currently allocated {ray_obj_store_mem // 1e9} GB for its " + + f"object store, but the device has {virtual_memory // 1e9} GB of RAM available. " + + "Modin recommends initializing Ray with at least 60% of available RAM to prevent " + + "Out Of Memory errors." + ) _move_stdlib_ahead_of_site_packages() ray.worker.global_worker.run_function_on_all_workers( _move_stdlib_ahead_of_site_packages