Skip to content

Commit

Permalink
Resolve dependancy conflict between oci-cli and runpod
Browse files Browse the repository at this point in the history
  • Loading branch information
@HysunHe
HysunHe committed Dec 22, 2024
1 parent 6c5d7ac commit eed2663
Show file tree
Hide file tree
Showing 4 changed files with 60 additions and 19 deletions.
18 changes: 18 additions & 0 deletions sky/adaptors/oci.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import logging
import os
from typing import List

from sky.adaptors import common
from sky.clouds.utils import oci_utils
Expand Down Expand Up @@ -66,6 +67,23 @@ def get_object_storage_client(region=None, profile='DEFAULT'):
get_oci_config(region, profile))


def goto_oci_cli_venv() -> List:
# Create a specfic venv for oci-cli due to its dependancy conflict
# with runpod (on 'click' version)
# pylint: disable=line-too-long
cmds = [
'conda info --envs | grep "sky-oci-cli-env" || conda create -n sky-oci-cli-env python=3.10 -y',
'. $(conda init | grep conda.sh | awk \'{print $NF}\')',
'conda activate sky-oci-cli-env', 'pip install oci-cli',
'export OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING=True'
]
return cmds


def leave_oci_cli_venv() -> str:
return 'conda deactivate'


def service_exception():
"""OCI service exception."""
return oci.exceptions.ServiceError
12 changes: 4 additions & 8 deletions sky/cloud_stores.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -475,12 +475,6 @@ def make_sync_file_command(self, source: str, destination: str) -> str:
class OciCloudStorage(CloudStorage):
"""OCI Cloud Storage."""

# List of commands to install OCI CLI
_GET_OCICLI = [
'oci --version >/dev/null 2>&1 || pip3 install oci-cli',
'export OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING=True'
]

def is_directory(self, url: str) -> bool:
"""Returns whether OCI 'url' is a directory.
In cloud object stores, a "directory" refers to a regular object whose
Expand Down Expand Up @@ -520,8 +514,9 @@ def make_sync_dir_command(self, source: str, destination: str) -> str:
f'--bucket-name {bucket_name} '
f'--prefix "{path}" --dest-dir "{destination}"')

all_commands = list(self._GET_OCICLI)
all_commands = oci.goto_oci_cli_venv()
all_commands.append(download_via_ocicli)
all_commands.append(oci.leave_oci_cli_venv())
return ' && '.join(all_commands)

def make_sync_file_command(self, source: str, destination: str) -> str:
Expand All @@ -537,8 +532,9 @@ def make_sync_file_command(self, source: str, destination: str) -> str:
download_via_ocicli = (f'oci os object get --bucket-name {bucket_name} '
f'--name "{path}" --file "{destination}"')

all_commands = list(self._GET_OCICLI)
all_commands = oci.goto_oci_cli_venv()
all_commands.append(download_via_ocicli)
all_commands.append(oci.leave_oci_cli_venv())
return ' && '.join(all_commands)


Expand Down
47 changes: 37 additions & 10 deletions sky/data/storage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3615,7 +3615,8 @@ class OciStore(AbstractStore):
"""

_ACCESS_DENIED_MESSAGE = 'AccessDeniedException'
RCLONE_VERSION = 'v1.61.1'
RCLONE_VERSION_DEB = 'v1.61.1'
RCLONE_VERSION_RPM = 'v1.68.2'

def __init__(self,
name: str,
Expand Down Expand Up @@ -3809,7 +3810,12 @@ def get_file_sync_command(base_dir_path, file_names):
'oci os object bulk-upload --no-follow-symlinks --overwrite '
f'--bucket-name {self.name} --namespace-name {self.namespace} '
f'--src-dir "{base_dir_path}" {includes}')
return sync_command

all_commands = oci.goto_oci_cli_venv()
all_commands.append(sync_command)
all_commands.append(oci.leave_oci_cli_venv())

return ' && '.join(all_commands)

def get_dir_sync_command(src_dir_path, dest_dir_name):
if dest_dir_name and not str(dest_dir_name).endswith('/'):
Expand All @@ -3821,7 +3827,12 @@ def get_dir_sync_command(src_dir_path, dest_dir_name):
f'--bucket-name {self.name} --namespace-name {self.namespace} '
f'--object-prefix "{dest_dir_name}" --src-dir "{src_dir_path}" '
f'--exclude ".git/*" ')
return sync_command

all_commands = oci.goto_oci_cli_venv()
all_commands.append(sync_command)
all_commands.append(oci.leave_oci_cli_venv())

return ' && '.join(all_commands)

# Generate message for upload
if len(source_path_list) > 1:
Expand Down Expand Up @@ -3915,13 +3926,13 @@ def mount_command(self, mount_path: str) -> str:
# pylint: disable=line-too-long
install_cmd = (
f'(which dpkg > /dev/null 2>&1 && (which rclone > /dev/null || (cd ~ > /dev/null'
f' && curl -O https://downloads.rclone.org/{self.RCLONE_VERSION}/rclone-{self.RCLONE_VERSION}-linux-amd64.deb'
f' && sudo dpkg -i rclone-{self.RCLONE_VERSION}-linux-amd64.deb'
f' && rm -f rclone-{self.RCLONE_VERSION}-linux-amd64.deb)))'
f' && curl -O https://downloads.rclone.org/{self.RCLONE_VERSION_DEB}/rclone-{self.RCLONE_VERSION_DEB}-linux-amd64.deb'
f' && sudo dpkg -i rclone-{self.RCLONE_VERSION_DEB}-linux-amd64.deb'
f' && rm -f rclone-{self.RCLONE_VERSION_DEB}-linux-amd64.deb)))'
f' || (which rclone > /dev/null || (cd ~ > /dev/null'
f' && curl -O https://downloads.rclone.org/{self.RCLONE_VERSION}/rclone-{self.RCLONE_VERSION}-linux-amd64.rpm'
f' && sudo yum --nogpgcheck install rclone-{self.RCLONE_VERSION}-linux-amd64.rpm -y'
f' && rm -f rclone-{self.RCLONE_VERSION}-linux-amd64.rpm))')
f' && curl -O https://downloads.rclone.org/{self.RCLONE_VERSION_RPM}/rclone-{self.RCLONE_VERSION_RPM}-linux-amd64.rpm'
f' && sudo yum --nogpgcheck install rclone-{self.RCLONE_VERSION_RPM}-linux-amd64.rpm -y'
f' && rm -f rclone-{self.RCLONE_VERSION_RPM}-linux-amd64.rpm))')

# pylint: disable=line-too-long
mount_cmd = (
Expand All @@ -3936,8 +3947,10 @@ def mount_command(self, mount_path: str) -> str:
f' && rclone mount oos_{self.name}:{self.name} {mount_path} --daemon --allow-non-empty'
)

# pylint: disable=line-too-long
version_check_cmd = (
f'rclone --version | grep -q {self.RCLONE_VERSION}')
f'(which dpkg > /dev/null 2>&1 && (rclone --version | grep -q {self.RCLONE_VERSION_DEB}))'
f' || (rclone --version | grep -q {self.RCLONE_VERSION_RPM})')

return mounting_utils.get_mounting_command(mount_path, install_cmd,
mount_cmd, version_check_cmd)
Expand All @@ -3963,6 +3976,13 @@ def _download_file(self, remote_path: str, local_path: str) -> None:
download_command = (f'oci os object get --bucket-name {self.name} '
f'--namespace-name {self.namespace} '
f'--name {remote_path} --file {local_path}')

all_commands = oci.goto_oci_cli_venv()
all_commands.append(download_command)
all_commands.append(oci.leave_oci_cli_venv())

download_command = ' && '.join(all_commands)

try:
with rich_utils.safe_status(
f'[bold cyan]Downloading: {remote_path} -> {local_path}[/]'
Expand Down Expand Up @@ -4012,6 +4032,13 @@ def _delete_oci_bucket(self, bucket_name: str) -> bool:
logger.debug(f'_delete_oci_bucket: {bucket_name}')
remove_command = (f'oci os bucket delete --bucket-name '
f'{bucket_name} --empty --force')

all_commands = oci.goto_oci_cli_venv()
all_commands.append(remove_command)
all_commands.append(oci.leave_oci_cli_venv())

remove_command = ' && '.join(all_commands)

try:
with rich_utils.safe_status(
f'[bold cyan]Deleting OCI bucket {bucket_name}[/]'):
Expand Down
2 changes: 1 addition & 1 deletion sky/setup_files/dependencies.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@
'lambda': local_ray,
'cloudflare': aws_dependencies,
'scp': local_ray,
'oci': ['oci', 'oci-cli'] + local_ray,
'oci': ['oci'] + local_ray,
'kubernetes': ['kubernetes>=20.0.0'],
'remote': remote,
'runpod': ['runpod>=1.5.1'],
Expand Down

0 comments on commit eed2663

Please sign in to comment.