From e5842dc503960e1d6c5023c3ac9dba91195c4439 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Tue, 25 Apr 2023 22:00:13 -0700 Subject: [PATCH 01/24] update --- sky/exceptions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sky/exceptions.py b/sky/exceptions.py index 0211cb81efa..91a7ea317ff 100644 --- a/sky/exceptions.py +++ b/sky/exceptions.py @@ -84,6 +84,7 @@ def __init__(self, returncode: int, command: str, error_msg: str) -> None: self.returncode = returncode self.command = command self.error_msg = error_msg + error_msg += 'dogdogdogdogdgodog' message = (f'Command {command} failed with return code {returncode}.' f'\n{error_msg}') super().__init__(message) From 35b966ae3b963be1ddce492f9d6c9087b6d10f88 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Tue, 25 Apr 2023 22:15:14 -0700 Subject: [PATCH 02/24] error check --- sky/exceptions.py | 1 - sky/utils/subprocess_utils.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sky/exceptions.py b/sky/exceptions.py index 91a7ea317ff..0211cb81efa 100644 --- a/sky/exceptions.py +++ b/sky/exceptions.py @@ -84,7 +84,6 @@ def __init__(self, returncode: int, command: str, error_msg: str) -> None: self.returncode = returncode self.command = command self.error_msg = error_msg - error_msg += 'dogdogdogdogdgodog' message = (f'Command {command} failed with return code {returncode}.' f'\n{error_msg}') super().__init__(message) diff --git a/sky/utils/subprocess_utils.py b/sky/utils/subprocess_utils.py index 8e39254b73a..ae34a8ee6cf 100644 --- a/sky/utils/subprocess_utils.py +++ b/sky/utils/subprocess_utils.py @@ -67,6 +67,8 @@ def handle_returncode(returncode: int, echo = logger.error if stream_logs else lambda _: None if returncode != 0: if stderr is not None: + logger.info('##############################') + logger.info(f'\n this is error: {stderr}') echo(stderr) if callable(error_msg): From dcfd0f8a8312de180b20042ec5e31ade67c51a8d Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Wed, 26 Apr 2023 20:19:03 -0700 Subject: [PATCH 03/24] removed logger in handle_returncode --- sky/utils/subprocess_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sky/utils/subprocess_utils.py b/sky/utils/subprocess_utils.py index ae34a8ee6cf..8e39254b73a 100644 --- a/sky/utils/subprocess_utils.py +++ b/sky/utils/subprocess_utils.py @@ -67,8 +67,6 @@ def handle_returncode(returncode: int, echo = logger.error if stream_logs else lambda _: None if returncode != 0: if stderr is not None: - logger.info('##############################') - logger.info(f'\n this is error: {stderr}') echo(stderr) if callable(error_msg): From 215b05301c2a5606025674a1d2356061ebca3537 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Wed, 26 Apr 2023 20:58:55 -0700 Subject: [PATCH 04/24] test mounting failed echo --- sky/data/mounting_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index 1bde951f8b7..8a710316e65 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -85,6 +85,6 @@ def get_mounting_command( command = (f'{first_line}' f'{script}' f') && chmod +x {script_path}' - f' && bash {script_path}' + f' && bash {script_path} || echo "Mounting failed !@#!$#@$!@$' f' && rm {script_path}') return command From ad0818cc0f57094e7e15682d382bb9a4788794ea Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Wed, 26 Apr 2023 21:02:55 -0700 Subject: [PATCH 05/24] fix --- sky/data/mounting_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index 8a710316e65..858145525d2 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -85,6 +85,6 @@ def get_mounting_command( command = (f'{first_line}' f'{script}' f') && chmod +x {script_path}' - f' && bash {script_path} || echo "Mounting failed !@#!$#@$!@$' + f' && bash {script_path} || echo "Mounting failed !@#!$#@$!@$"' f' && rm {script_path}') return command From ac8bc8d9f88e8df13c7b65d64f1b1eac86d1bfb8 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Wed, 26 Apr 2023 21:27:35 -0700 Subject: [PATCH 06/24] fix --- sky/data/mounting_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index 858145525d2..aea18b3b815 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -85,6 +85,6 @@ def get_mounting_command( command = (f'{first_line}' f'{script}' f') && chmod +x {script_path}' - f' && bash {script_path} || echo "Mounting failed !@#!$#@$!@$"' + f' && {{ bash {script_path} 2> >(error_output=$(cat); echo "Mounting failed, detailed error: $error_output") || true; }}' f' && rm {script_path}') return command From 87f1f787f74d1fb04052a8f862fea47665f92be7 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Wed, 26 Apr 2023 21:32:27 -0700 Subject: [PATCH 07/24] fix --- sky/data/mounting_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index aea18b3b815..90edb00fa3f 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -85,6 +85,6 @@ def get_mounting_command( command = (f'{first_line}' f'{script}' f') && chmod +x {script_path}' - f' && {{ bash {script_path} 2> >(error_output=$(cat); echo "Mounting failed, detailed error: $error_output") || true; }}' + f' && bash {script_path} 2> >(error_output=$(cat); echo "Mounting failed, detailed error: $error_output") || true;' f' && rm {script_path}') return command From bec9c9e1217a85a4d13fc3cd024a6a3fc55e81ea Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Wed, 26 Apr 2023 21:38:14 -0700 Subject: [PATCH 08/24] fix --- sky/data/mounting_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index 90edb00fa3f..2fd52abcbc7 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -85,6 +85,6 @@ def get_mounting_command( command = (f'{first_line}' f'{script}' f') && chmod +x {script_path}' - f' && bash {script_path} 2> >(error_output=$(cat); echo "Mounting failed, detailed error: $error_output") || true;' + f' && bash {script_path} 2> >(error_output=$(cat); echo "Mounting failed, detailed error: $error_output");' f' && rm {script_path}') return command From 9a73e03eaa2bc1dcf949e004279ab17168116dda Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Wed, 26 Apr 2023 21:43:57 -0700 Subject: [PATCH 09/24] fix --- sky/data/mounting_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index 2fd52abcbc7..d360509afb9 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -85,6 +85,6 @@ def get_mounting_command( command = (f'{first_line}' f'{script}' f') && chmod +x {script_path}' - f' && bash {script_path} 2> >(error_output=$(cat); echo "Mounting failed, detailed error: $error_output");' + f' && (bash {script_path} 2> >(error_output=$(cat); echo "Mounting failed, detailed error: $error_output");)' f' && rm {script_path}') return command From 2192cfa8d73e465093c14d09f9e3eaffcd7fece7 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Thu, 27 Apr 2023 18:23:15 -0700 Subject: [PATCH 10/24] CommandError handling --- sky/backends/cloud_vm_ray_backend.py | 8 +++++++- sky/data/mounting_utils.py | 2 +- sky/exceptions.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index 67163a78fe2..9ae7c06b2d7 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -3558,7 +3558,8 @@ def _execute_storage_mounts(self, handle: CloudVmRayResourceHandle, if storage_obj.source else storage_obj.name) if isinstance(src_print, list): src_print = ', '.join(src_print) - backend_utils.parallel_data_transfer_to_nodes( + try: + backend_utils.parallel_data_transfer_to_nodes( runners, source=src_print, target=dst, @@ -3567,6 +3568,11 @@ def _execute_storage_mounts(self, handle: CloudVmRayResourceHandle, action_message='Mounting', log_path=log_path, ) + except exceptions.CommandError as e: + if e.returncode == exceptions.MOUNT_PATH_NON_EMPTY_CODE: + raise exceptions.StorageUploadError(f'Mount path {dst} is non-empty') + + end = time.time() logger.debug(f'Storage mount sync took {end - start} seconds.') diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index d360509afb9..d5e0cece7ed 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -85,6 +85,6 @@ def get_mounting_command( command = (f'{first_line}' f'{script}' f') && chmod +x {script_path}' - f' && (bash {script_path} 2> >(error_output=$(cat); echo "Mounting failed, detailed error: $error_output");)' + f' && (bash {script_path}' f' && rm {script_path}') return command diff --git a/sky/exceptions.py b/sky/exceptions.py index 0211cb81efa..0dc819d4b5e 100644 --- a/sky/exceptions.py +++ b/sky/exceptions.py @@ -10,7 +10,7 @@ KEYBOARD_INTERRUPT_CODE = 130 SIGTSTP_CODE = 146 RSYNC_FILE_NOT_FOUND_CODE = 23 - +MOUNT_PATH_NON_EMPTY_CODE = 1 class ResourcesUnavailableError(Exception): """Raised when resources are unavailable. From f87b4785c674691e790180b88bce7e281f81447a Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Thu, 27 Apr 2023 18:42:37 -0700 Subject: [PATCH 11/24] adding from None --- sky/backends/cloud_vm_ray_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index 9ae7c06b2d7..9c4d355fe30 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -3570,7 +3570,7 @@ def _execute_storage_mounts(self, handle: CloudVmRayResourceHandle, ) except exceptions.CommandError as e: if e.returncode == exceptions.MOUNT_PATH_NON_EMPTY_CODE: - raise exceptions.StorageUploadError(f'Mount path {dst} is non-empty') + raise exceptions.StorageUploadError(f'Mount path {dst} is non-empty') from None end = time.time() From 8b4f8b1a8e296dbf6d31c7794da047abec2c19a6 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Thu, 27 Apr 2023 19:20:39 -0700 Subject: [PATCH 12/24] fix --- sky/data/mounting_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index d5e0cece7ed..1bde951f8b7 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -85,6 +85,6 @@ def get_mounting_command( command = (f'{first_line}' f'{script}' f') && chmod +x {script_path}' - f' && (bash {script_path}' + f' && bash {script_path}' f' && rm {script_path}') return command From 847498133b5a78467fb36686bad3555651082e80 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Thu, 27 Apr 2023 19:31:33 -0700 Subject: [PATCH 13/24] fix --- sky/backends/cloud_vm_ray_backend.py | 2 +- sky/exceptions.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index 9c4d355fe30..2a822350a96 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -3570,7 +3570,7 @@ def _execute_storage_mounts(self, handle: CloudVmRayResourceHandle, ) except exceptions.CommandError as e: if e.returncode == exceptions.MOUNT_PATH_NON_EMPTY_CODE: - raise exceptions.StorageUploadError(f'Mount path {dst} is non-empty') from None + raise exceptions.StorageMountPathError(f'Mount path {dst} is non-empty. {dst} may be already taken by the Kernel. Please set the mount path to another name.') from None end = time.time() diff --git a/sky/exceptions.py b/sky/exceptions.py index 0dc819d4b5e..401d609dbe9 100644 --- a/sky/exceptions.py +++ b/sky/exceptions.py @@ -164,6 +164,12 @@ class StorageModeError(StorageSpecError): pass +class StorageMountPathError(StorageSpecError): + # Error raised when bucket is successfully initialized, but mounting fails + # due to non-empty mount path + pass + + class FetchIPError(Exception): """Raised when fetching the IP fails.""" From 51ab5223c2351114089cbc8c689a4e3a683dfc38 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Thu, 27 Apr 2023 20:10:43 -0700 Subject: [PATCH 14/24] message font color update --- sky/backends/cloud_vm_ray_backend.py | 3 ++- sky/data/mounting_utils.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index 2a822350a96..159e9af50ee 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -3570,7 +3570,8 @@ def _execute_storage_mounts(self, handle: CloudVmRayResourceHandle, ) except exceptions.CommandError as e: if e.returncode == exceptions.MOUNT_PATH_NON_EMPTY_CODE: - raise exceptions.StorageMountPathError(f'Mount path {dst} is non-empty. {dst} may be already taken by the Kernel. Please set the mount path to another name.') from None + mount_path = f'{colorama.Fore.RED}{colorama.Style.BRIGHT}{dst}{colorama.Style.RESET_ALL}' + raise exceptions.StorageMountPathError(f'Mount path {mount_path} is non-empty. {mount_path} may have been already taken by the Kernel. Please set the mount path to another name.') from None end = time.time() diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index 1bde951f8b7..bd240c5bb4e 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -88,3 +88,4 @@ def get_mounting_command( f' && bash {script_path}' f' && rm {script_path}') return command + \ No newline at end of file From 0bcddd03cd3e946d2b5cf5f7661c5b33196e3c9e Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Thu, 27 Apr 2023 20:24:58 -0700 Subject: [PATCH 15/24] format --- sky/backends/cloud_vm_ray_backend.py | 28 +++++++++++++++++----------- sky/data/mounting_utils.py | 1 - sky/exceptions.py | 1 + 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index 159e9af50ee..03d43c0a49c 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -3560,19 +3560,25 @@ def _execute_storage_mounts(self, handle: CloudVmRayResourceHandle, src_print = ', '.join(src_print) try: backend_utils.parallel_data_transfer_to_nodes( - runners, - source=src_print, - target=dst, - cmd=mount_cmd, - run_rsync=False, - action_message='Mounting', - log_path=log_path, - ) + runners, + source=src_print, + target=dst, + cmd=mount_cmd, + run_rsync=False, + action_message='Mounting', + log_path=log_path, + ) except exceptions.CommandError as e: if e.returncode == exceptions.MOUNT_PATH_NON_EMPTY_CODE: - mount_path = f'{colorama.Fore.RED}{colorama.Style.BRIGHT}{dst}{colorama.Style.RESET_ALL}' - raise exceptions.StorageMountPathError(f'Mount path {mount_path} is non-empty. {mount_path} may have been already taken by the Kernel. Please set the mount path to another name.') from None - + mount_path = (f'{colorama.Fore.RED}' + f'{colorama.Style.BRIGHT}{dst}' + f'{colorama.Style.RESET_ALL}') + raise exceptions.StorageMountPathError( + f'Mount path {mount_path} you set in ENTRYPOINT' + f'is non-empty. {mount_path} may have been already' + f' taken by the Kernel. Please set the mount path ' + f'to another name.' + ) from None end = time.time() logger.debug(f'Storage mount sync took {end - start} seconds.') diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index bd240c5bb4e..1bde951f8b7 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -88,4 +88,3 @@ def get_mounting_command( f' && bash {script_path}' f' && rm {script_path}') return command - \ No newline at end of file diff --git a/sky/exceptions.py b/sky/exceptions.py index 401d609dbe9..95a80b65010 100644 --- a/sky/exceptions.py +++ b/sky/exceptions.py @@ -12,6 +12,7 @@ RSYNC_FILE_NOT_FOUND_CODE = 23 MOUNT_PATH_NON_EMPTY_CODE = 1 + class ResourcesUnavailableError(Exception): """Raised when resources are unavailable. From f0defc2c1befe65ae051bc97152ed5fee70cc668 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Thu, 27 Apr 2023 20:52:15 -0700 Subject: [PATCH 16/24] format --- sky/backends/cloud_vm_ray_backend.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index 03d43c0a49c..8c3b7bfacc8 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -3574,11 +3574,10 @@ def _execute_storage_mounts(self, handle: CloudVmRayResourceHandle, f'{colorama.Style.BRIGHT}{dst}' f'{colorama.Style.RESET_ALL}') raise exceptions.StorageMountPathError( - f'Mount path {mount_path} you set in ENTRYPOINT' + f'Mount path {mount_path} you set in ENTRYPOINT ' f'is non-empty. {mount_path} may have been already' f' taken by the Kernel. Please set the mount path ' - f'to another name.' - ) from None + f'to another name.') from None end = time.time() logger.debug(f'Storage mount sync took {end - start} seconds.') From 82f210f348f28af2cd0f67a81530cff3406973c7 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Thu, 27 Apr 2023 23:52:54 -0700 Subject: [PATCH 17/24] format --- sky/backends/cloud_vm_ray_backend.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index 8c3b7bfacc8..de2fe9f1f9b 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -3570,14 +3570,10 @@ def _execute_storage_mounts(self, handle: CloudVmRayResourceHandle, ) except exceptions.CommandError as e: if e.returncode == exceptions.MOUNT_PATH_NON_EMPTY_CODE: - mount_path = (f'{colorama.Fore.RED}' - f'{colorama.Style.BRIGHT}{dst}' - f'{colorama.Style.RESET_ALL}') + mount_path = f'{colorama.Fore.RED}{colorama.Style.BRIGHT}{dst}{colorama.Style.RESET_ALL}' raise exceptions.StorageMountPathError( - f'Mount path {mount_path} you set in ENTRYPOINT ' - f'is non-empty. {mount_path} may have been already' - f' taken by the Kernel. Please set the mount path ' - f'to another name.') from None + f'Mount path {mount_path} is non-empty. {mount_path} may have been already taken by the Kernel. Please set the mount path to another name.' + ) from None end = time.time() logger.debug(f'Storage mount sync took {end - start} seconds.') From c56a411518a146abd11bcfd4f1766373c35bb593 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Fri, 28 Apr 2023 00:04:50 -0700 Subject: [PATCH 18/24] format --- sky/backends/cloud_vm_ray_backend.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index de2fe9f1f9b..d9ada02f728 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -3570,10 +3570,15 @@ def _execute_storage_mounts(self, handle: CloudVmRayResourceHandle, ) except exceptions.CommandError as e: if e.returncode == exceptions.MOUNT_PATH_NON_EMPTY_CODE: - mount_path = f'{colorama.Fore.RED}{colorama.Style.BRIGHT}{dst}{colorama.Style.RESET_ALL}' + mount_path = (f'{colorama.Fore.RED}' + f'{colorama.Style.BRIGHT}{dst}' + f'{colorama.Style.RESET_ALL}') + error_msg = (f'Mount path {mount_path} is non-empty.' + f' {mount_path} may have been already ' + f'taken by the Kernel. Please set the ' + f'mount path to another name.') raise exceptions.StorageMountPathError( - f'Mount path {mount_path} is non-empty. {mount_path} may have been already taken by the Kernel. Please set the mount path to another name.' - ) from None + error_msg) from None end = time.time() logger.debug(f'Storage mount sync took {end - start} seconds.') From a0b1eae82574d9ff4c1dea21048cd649cce94f78 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Fri, 28 Apr 2023 00:06:59 -0700 Subject: [PATCH 19/24] format --- sky/backends/cloud_vm_ray_backend.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index d9ada02f728..d216329259d 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -3577,8 +3577,7 @@ def _execute_storage_mounts(self, handle: CloudVmRayResourceHandle, f' {mount_path} may have been already ' f'taken by the Kernel. Please set the ' f'mount path to another name.') - raise exceptions.StorageMountPathError( - error_msg) from None + raise exceptions.StorageMountPathError(error_msg) from None end = time.time() logger.debug(f'Storage mount sync took {end - start} seconds.') From c4099f84ab2add6ea88b343d0d73d4b6ad1e2aa3 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Thu, 4 May 2023 19:20:41 -0700 Subject: [PATCH 20/24] testing echo with wrong exit code --- sky/backends/cloud_vm_ray_backend.py | 7 ++++--- sky/data/mounting_utils.py | 4 ++-- sky/exceptions.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index d216329259d..377c9dc4573 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -3574,9 +3574,10 @@ def _execute_storage_mounts(self, handle: CloudVmRayResourceHandle, f'{colorama.Style.BRIGHT}{dst}' f'{colorama.Style.RESET_ALL}') error_msg = (f'Mount path {mount_path} is non-empty.' - f' {mount_path} may have been already ' - f'taken by the Kernel. Please set the ' - f'mount path to another name.') + f' {mount_path} may be a standard unix ' + f'path or may contain files from a previous' + f' task. To fix, change the mount path' + f' to an empty or non-existent path.') raise exceptions.StorageMountPathError(error_msg) from None end = time.time() diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index 1bde951f8b7..73c5a857575 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -60,7 +60,7 @@ def get_mounting_command( # Check if mount path contains files if [ "$(ls -A $MOUNT_PATH)" ]; then echo "Mount path $MOUNT_PATH is not empty. Please make sure its empty." - exit 1 + exit 43 fi fi echo "Mounting $SOURCE_BUCKET to $MOUNT_PATH with $MOUNT_BINARY..." @@ -85,6 +85,6 @@ def get_mounting_command( command = (f'{first_line}' f'{script}' f') && chmod +x {script_path}' - f' && bash {script_path}' + f' && bash {script_path} || echo "Mounting failed, detailed error: ' f' && rm {script_path}') return command diff --git a/sky/exceptions.py b/sky/exceptions.py index 95a80b65010..db8742f97ef 100644 --- a/sky/exceptions.py +++ b/sky/exceptions.py @@ -10,7 +10,7 @@ KEYBOARD_INTERRUPT_CODE = 130 SIGTSTP_CODE = 146 RSYNC_FILE_NOT_FOUND_CODE = 23 -MOUNT_PATH_NON_EMPTY_CODE = 1 +MOUNT_PATH_NON_EMPTY_CODE = 42 class ResourcesUnavailableError(Exception): From 5c5023c3d6323c033186e233ada683a387ab5510 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Thu, 4 May 2023 20:38:35 -0700 Subject: [PATCH 21/24] format --- sky/data/mounting_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index 73c5a857575..d7726c35205 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -60,7 +60,7 @@ def get_mounting_command( # Check if mount path contains files if [ "$(ls -A $MOUNT_PATH)" ]; then echo "Mount path $MOUNT_PATH is not empty. Please make sure its empty." - exit 43 + exit 42 fi fi echo "Mounting $SOURCE_BUCKET to $MOUNT_PATH with $MOUNT_BINARY..." @@ -85,6 +85,7 @@ def get_mounting_command( command = (f'{first_line}' f'{script}' f') && chmod +x {script_path}' - f' && bash {script_path} || echo "Mounting failed, detailed error: ' + f' && bash {script_path}' f' && rm {script_path}') return command + \ No newline at end of file From f5869c1624833b9764fe36a692dd017b2ac6ec2a Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Thu, 4 May 2023 20:52:55 -0700 Subject: [PATCH 22/24] nit --- sky/data/mounting_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index d7726c35205..8d4132f1326 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -88,4 +88,3 @@ def get_mounting_command( f' && bash {script_path}' f' && rm {script_path}') return command - \ No newline at end of file From 8882e038cbe8480fed74bec1a30d1a280dcfbbbb Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Thu, 18 May 2023 04:15:10 +0000 Subject: [PATCH 23/24] update exceptions and message --- sky/backends/cloud_vm_ray_backend.py | 2 +- sky/data/mounting_utils.py | 2 +- sky/exceptions.py | 8 +------- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index 377c9dc4573..ded17f89854 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -3578,7 +3578,7 @@ def _execute_storage_mounts(self, handle: CloudVmRayResourceHandle, f'path or may contain files from a previous' f' task. To fix, change the mount path' f' to an empty or non-existent path.') - raise exceptions.StorageMountPathError(error_msg) from None + raise RuntimeError(error_msg) from None end = time.time() logger.debug(f'Storage mount sync took {end - start} seconds.') diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index 8d4132f1326..e1e6a6c3cdf 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -59,7 +59,7 @@ def get_mounting_command( else # Check if mount path contains files if [ "$(ls -A $MOUNT_PATH)" ]; then - echo "Mount path $MOUNT_PATH is not empty. Please make sure its empty." + echo "Mount path $MOUNT_PATH is not empty. Please mount to another path or remove it first." exit 42 fi fi diff --git a/sky/exceptions.py b/sky/exceptions.py index db8742f97ef..d868ecf271e 100644 --- a/sky/exceptions.py +++ b/sky/exceptions.py @@ -10,7 +10,7 @@ KEYBOARD_INTERRUPT_CODE = 130 SIGTSTP_CODE = 146 RSYNC_FILE_NOT_FOUND_CODE = 23 -MOUNT_PATH_NON_EMPTY_CODE = 42 +MOUNT_PATH_NON_EMPTY_CODE = 42 # Arbitrarily chosen. class ResourcesUnavailableError(Exception): @@ -165,12 +165,6 @@ class StorageModeError(StorageSpecError): pass -class StorageMountPathError(StorageSpecError): - # Error raised when bucket is successfully initialized, but mounting fails - # due to non-empty mount path - pass - - class FetchIPError(Exception): """Raised when fetching the IP fails.""" From 1491b6198eb92d8d28a3039f2a3761f6caaa2fd5 Mon Sep 17 00:00:00 2001 From: Doyoung Kim Date: Sat, 20 May 2023 03:30:40 +0000 Subject: [PATCH 24/24] update --- sky/data/mounting_utils.py | 4 +++- sky/exceptions.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sky/data/mounting_utils.py b/sky/data/mounting_utils.py index e1e6a6c3cdf..1f071669912 100644 --- a/sky/data/mounting_utils.py +++ b/sky/data/mounting_utils.py @@ -3,6 +3,8 @@ import textwrap from typing import Optional +from sky import exceptions + def get_mounting_command( mount_path: str, @@ -60,7 +62,7 @@ def get_mounting_command( # Check if mount path contains files if [ "$(ls -A $MOUNT_PATH)" ]; then echo "Mount path $MOUNT_PATH is not empty. Please mount to another path or remove it first." - exit 42 + exit {exceptions.MOUNT_PATH_NON_EMPTY_CODE} fi fi echo "Mounting $SOURCE_BUCKET to $MOUNT_PATH with $MOUNT_BINARY..." diff --git a/sky/exceptions.py b/sky/exceptions.py index d868ecf271e..3dfb7172d17 100644 --- a/sky/exceptions.py +++ b/sky/exceptions.py @@ -10,7 +10,8 @@ KEYBOARD_INTERRUPT_CODE = 130 SIGTSTP_CODE = 146 RSYNC_FILE_NOT_FOUND_CODE = 23 -MOUNT_PATH_NON_EMPTY_CODE = 42 # Arbitrarily chosen. +# Arbitrarily chosen value. Used in SkyPilot's storage mounting scripts +MOUNT_PATH_NON_EMPTY_CODE = 42 class ResourcesUnavailableError(Exception):