From eed2663d86c013d9e3f4ce38edd0c1372629c02e Mon Sep 17 00:00:00 2001 From: HysunHe Date: Sun, 22 Dec 2024 12:44:44 +0800 Subject: [PATCH] Resolve dependancy conflict between oci-cli and runpod --- sky/adaptors/oci.py | 18 +++++++++++++ sky/cloud_stores.py | 12 +++------ sky/data/storage.py | 47 ++++++++++++++++++++++++++------- sky/setup_files/dependencies.py | 2 +- 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/sky/adaptors/oci.py b/sky/adaptors/oci.py index 7ce02d30765..6650994fb6c 100644 --- a/sky/adaptors/oci.py +++ b/sky/adaptors/oci.py @@ -2,6 +2,7 @@ import logging import os +from typing import List from sky.adaptors import common from sky.clouds.utils import oci_utils @@ -66,6 +67,23 @@ def get_object_storage_client(region=None, profile='DEFAULT'): get_oci_config(region, profile)) +def goto_oci_cli_venv() -> List: + # Create a specfic venv for oci-cli due to its dependancy conflict + # with runpod (on 'click' version) + # pylint: disable=line-too-long + cmds = [ + 'conda info --envs | grep "sky-oci-cli-env" || conda create -n sky-oci-cli-env python=3.10 -y', + '. $(conda init | grep conda.sh | awk \'{print $NF}\')', + 'conda activate sky-oci-cli-env', 'pip install oci-cli', + 'export OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING=True' + ] + return cmds + + +def leave_oci_cli_venv() -> str: + return 'conda deactivate' + + def service_exception(): """OCI service exception.""" return oci.exceptions.ServiceError diff --git a/sky/cloud_stores.py b/sky/cloud_stores.py index 7e2ffc30b23..f1302105be7 100644 --- a/sky/cloud_stores.py +++ b/sky/cloud_stores.py @@ -475,12 +475,6 @@ def make_sync_file_command(self, source: str, destination: str) -> str: class OciCloudStorage(CloudStorage): """OCI Cloud Storage.""" - # List of commands to install OCI CLI - _GET_OCICLI = [ - 'oci --version >/dev/null 2>&1 || pip3 install oci-cli', - 'export OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING=True' - ] - def is_directory(self, url: str) -> bool: """Returns whether OCI 'url' is a directory. In cloud object stores, a "directory" refers to a regular object whose @@ -520,8 +514,9 @@ def make_sync_dir_command(self, source: str, destination: str) -> str: f'--bucket-name {bucket_name} ' f'--prefix "{path}" --dest-dir "{destination}"') - all_commands = list(self._GET_OCICLI) + all_commands = oci.goto_oci_cli_venv() all_commands.append(download_via_ocicli) + all_commands.append(oci.leave_oci_cli_venv()) return ' && '.join(all_commands) def make_sync_file_command(self, source: str, destination: str) -> str: @@ -537,8 +532,9 @@ def make_sync_file_command(self, source: str, destination: str) -> str: download_via_ocicli = (f'oci os object get --bucket-name {bucket_name} ' f'--name "{path}" --file "{destination}"') - all_commands = list(self._GET_OCICLI) + all_commands = oci.goto_oci_cli_venv() all_commands.append(download_via_ocicli) + all_commands.append(oci.leave_oci_cli_venv()) return ' && '.join(all_commands) diff --git a/sky/data/storage.py b/sky/data/storage.py index f58a3c53388..9941e35c99c 100644 --- a/sky/data/storage.py +++ b/sky/data/storage.py @@ -3615,7 +3615,8 @@ class OciStore(AbstractStore): """ _ACCESS_DENIED_MESSAGE = 'AccessDeniedException' - RCLONE_VERSION = 'v1.61.1' + RCLONE_VERSION_DEB = 'v1.61.1' + RCLONE_VERSION_RPM = 'v1.68.2' def __init__(self, name: str, @@ -3809,7 +3810,12 @@ def get_file_sync_command(base_dir_path, file_names): 'oci os object bulk-upload --no-follow-symlinks --overwrite ' f'--bucket-name {self.name} --namespace-name {self.namespace} ' f'--src-dir "{base_dir_path}" {includes}') - return sync_command + + all_commands = oci.goto_oci_cli_venv() + all_commands.append(sync_command) + all_commands.append(oci.leave_oci_cli_venv()) + + return ' && '.join(all_commands) def get_dir_sync_command(src_dir_path, dest_dir_name): if dest_dir_name and not str(dest_dir_name).endswith('/'): @@ -3821,7 +3827,12 @@ def get_dir_sync_command(src_dir_path, dest_dir_name): f'--bucket-name {self.name} --namespace-name {self.namespace} ' f'--object-prefix "{dest_dir_name}" --src-dir "{src_dir_path}" ' f'--exclude ".git/*" ') - return sync_command + + all_commands = oci.goto_oci_cli_venv() + all_commands.append(sync_command) + all_commands.append(oci.leave_oci_cli_venv()) + + return ' && '.join(all_commands) # Generate message for upload if len(source_path_list) > 1: @@ -3915,13 +3926,13 @@ def mount_command(self, mount_path: str) -> str: # pylint: disable=line-too-long install_cmd = ( f'(which dpkg > /dev/null 2>&1 && (which rclone > /dev/null || (cd ~ > /dev/null' - f' && curl -O https://downloads.rclone.org/{self.RCLONE_VERSION}/rclone-{self.RCLONE_VERSION}-linux-amd64.deb' - f' && sudo dpkg -i rclone-{self.RCLONE_VERSION}-linux-amd64.deb' - f' && rm -f rclone-{self.RCLONE_VERSION}-linux-amd64.deb)))' + f' && curl -O https://downloads.rclone.org/{self.RCLONE_VERSION_DEB}/rclone-{self.RCLONE_VERSION_DEB}-linux-amd64.deb' + f' && sudo dpkg -i rclone-{self.RCLONE_VERSION_DEB}-linux-amd64.deb' + f' && rm -f rclone-{self.RCLONE_VERSION_DEB}-linux-amd64.deb)))' f' || (which rclone > /dev/null || (cd ~ > /dev/null' - f' && curl -O https://downloads.rclone.org/{self.RCLONE_VERSION}/rclone-{self.RCLONE_VERSION}-linux-amd64.rpm' - f' && sudo yum --nogpgcheck install rclone-{self.RCLONE_VERSION}-linux-amd64.rpm -y' - f' && rm -f rclone-{self.RCLONE_VERSION}-linux-amd64.rpm))') + f' && curl -O https://downloads.rclone.org/{self.RCLONE_VERSION_RPM}/rclone-{self.RCLONE_VERSION_RPM}-linux-amd64.rpm' + f' && sudo yum --nogpgcheck install rclone-{self.RCLONE_VERSION_RPM}-linux-amd64.rpm -y' + f' && rm -f rclone-{self.RCLONE_VERSION_RPM}-linux-amd64.rpm))') # pylint: disable=line-too-long mount_cmd = ( @@ -3936,8 +3947,10 @@ def mount_command(self, mount_path: str) -> str: f' && rclone mount oos_{self.name}:{self.name} {mount_path} --daemon --allow-non-empty' ) + # pylint: disable=line-too-long version_check_cmd = ( - f'rclone --version | grep -q {self.RCLONE_VERSION}') + f'(which dpkg > /dev/null 2>&1 && (rclone --version | grep -q {self.RCLONE_VERSION_DEB}))' + f' || (rclone --version | grep -q {self.RCLONE_VERSION_RPM})') return mounting_utils.get_mounting_command(mount_path, install_cmd, mount_cmd, version_check_cmd) @@ -3963,6 +3976,13 @@ def _download_file(self, remote_path: str, local_path: str) -> None: download_command = (f'oci os object get --bucket-name {self.name} ' f'--namespace-name {self.namespace} ' f'--name {remote_path} --file {local_path}') + + all_commands = oci.goto_oci_cli_venv() + all_commands.append(download_command) + all_commands.append(oci.leave_oci_cli_venv()) + + download_command = ' && '.join(all_commands) + try: with rich_utils.safe_status( f'[bold cyan]Downloading: {remote_path} -> {local_path}[/]' @@ -4012,6 +4032,13 @@ def _delete_oci_bucket(self, bucket_name: str) -> bool: logger.debug(f'_delete_oci_bucket: {bucket_name}') remove_command = (f'oci os bucket delete --bucket-name ' f'{bucket_name} --empty --force') + + all_commands = oci.goto_oci_cli_venv() + all_commands.append(remove_command) + all_commands.append(oci.leave_oci_cli_venv()) + + remove_command = ' && '.join(all_commands) + try: with rich_utils.safe_status( f'[bold cyan]Deleting OCI bucket {bucket_name}[/]'): diff --git a/sky/setup_files/dependencies.py b/sky/setup_files/dependencies.py index 39c69d1d9f5..18d2f5cdc08 100644 --- a/sky/setup_files/dependencies.py +++ b/sky/setup_files/dependencies.py @@ -120,7 +120,7 @@ 'lambda': local_ray, 'cloudflare': aws_dependencies, 'scp': local_ray, - 'oci': ['oci', 'oci-cli'] + local_ray, + 'oci': ['oci'] + local_ray, 'kubernetes': ['kubernetes>=20.0.0'], 'remote': remote, 'runpod': ['runpod>=1.5.1'],