[AWS] Failover Optimization from Quota

sky/backends/cloud_vm_ray_backend.py

@@ -1335,23 +1335,24 @@ def _retry_zones(
         # the instance type in the target region. If not, fail early
         # instead of trying to provision and failing later.
         try:
-            has_non_zero_quota = to_provision.cloud.check_quota_not_zero(
+            need_provision = to_provision.cloud.check_quota_available(
                 to_provision.region, to_provision.instance_type,
                 to_provision.use_spot)
 
         except Exception as e:  # pylint: disable=broad-except
-            has_non_zero_quota = True
-            logger.info(
-                f'Error occurred when trying to check quota. '
-                f'Proceeding assuming quotas are available. Error: {str(e)}')
+            need_provision = True
+            logger.info(f'Error occurred when trying to check quota. '
+                        f'Proceeding assuming quotas are available. Error: '
+                        f'{common_utils.format_exception(e, use_bracket=True)}')
 
-        if not has_non_zero_quota:
+        if not need_provision:
             # if quota is found to be zero, raise exception and skip to
             # the next region
             raise exceptions.ResourcesUnavailableError(
                 f'Found no quota for {to_provision.instance_type} in region '
-                f'{to_provision.region}. To request quotas in this region, '
-                f'visit http://aws.amazon.com/contact-us/ec2-request.')
+                f'{to_provision.region}. To request quotas, check the instruction: '  # pylint: disable=line-too-long
+                f'https://skypilot.readthedocs.io/en/latest/cloud-setup/quota.html.'  # pylint: disable=line-too-long
+            )
 
         for zones in self._yield_zones(to_provision, num_nodes, cluster_name,
                                        prev_cluster_status):

sky/clouds/aws.py

@@ -10,6 +10,9 @@
 import typing
 from typing import Dict, Iterator, List, Optional, Tuple, Any
 
+import boto3
+import botocore
+
 from sky import clouds
 from sky import exceptions
 from sky import sky_logging
@@ -416,13 +419,6 @@ def _make(instance_list):
     @functools.lru_cache(maxsize=1)  # Cache since getting identity is slow.
     def check_credentials(cls) -> Tuple[bool, Optional[str]]:
         """Checks if the user has access credentials to this cloud."""
-        try:
-            # pylint: disable=import-outside-toplevel,unused-import
-            import boto3
-            import botocore
-        except ImportError:
-            raise ImportError('Fail to import dependencies for AWS.'
-                              'Try pip install "skypilot[aws]"') from None
 
         # Checks if the AWS CLI is installed properly
         proc = subprocess.run('aws --version',
@@ -696,12 +692,14 @@ def _get_disk_specs(cls, disk_tier: Optional[str]) -> Dict[str, Any]:
         }
 
     @classmethod
-    def check_quota_not_zero(cls,
-                             region: str,
-                             instance_type: str,
-                             use_spot: bool = False) -> bool:
+    def check_quota_available(cls,
+                              region: str,
+                              instance_type: str,
+                              use_spot: bool = False) -> bool:
         """
-        AWS-specific implmentation of check_quota_not_zero. The function works by
+        Check if the quota is available for the requested instance_type
+
+        AWS-specific implmentation of check_quota_available. The function works by
         matching the instance_type to the corresponding AWS quota code, and then using
         the boto3 Python API to query the region for the specific quota code.
 
@@ -712,14 +710,6 @@ def check_quota_not_zero(cls,
             botocore.exceptions.ClientError: error in Boto3 client request.
         """
 
-        try:
-            # pylint: disable=import-outside-toplevel
-            import boto3
-            import botocore
-        except ImportError:
-            raise ImportError('Fail to import dependencies for AWS.'
-                              'Try pip install "skypilot[aws]"') from None
-
         from sky.clouds.service_catalog import aws_catalog  # pylint: disable=import-outside-toplevel,unused-import
 
         quota_code = aws_catalog.get_quota_code(instance_type, use_spot)

sky/clouds/cloud.py

@@ -470,15 +470,15 @@ def check_disk_tier_enabled(cls, instance_type: str,
 
     @classmethod
     # pylint: disable=unused-argument
-    def check_quota_not_zero(cls,
-                             region: str,
-                             instance_type: str,
-                             use_spot: bool = False) -> bool:
+    def check_quota_available(cls,
+                              region: str,
+                              instance_type: str,
+                              use_spot: bool = False) -> bool:
         """
         Checks to ensure that a particular accelerator has a nonzero quota
         in a region.
 
-        (Currently, check_quota_not_zero is only implemented for AWS.)
+        (Currently, check_quota_available is only implemented for AWS.)
 
         The _retry_zones funtion in cloud_vm_ray_backend goes through different
         candidate regions and attempts to provision the requested instance_type
@@ -509,7 +509,7 @@ def check_quota_not_zero(cls,
         have an adequate quota.
 
         Design choice: We chose a just-in-time approach where
-        check_quota_not_zero is called immediately before a potential
+        check_quota_available is called immediately before a potential
         attempted provision, rather than checking all region quotas
         beforehand, storing them, and using those values on-demand. This is
         because, for example, _retry_zones may only need to go through one or

tests/test_smoke.py

@@ -190,7 +190,7 @@ def get_aws_region_for_quota_failover() -> Optional[str]:
                                                   zone=None)
 
     for region in candidate_regions:
-        if not AWS.check_quota_not_zero(
+        if not AWS.check_quota_available(
                 region=region.name, instance_type="p3.16xlarge", use_spot=True):
             return region.name
 
@@ -2227,7 +2227,7 @@ def test_aws_zero_quota_failover():
 
     test = Test('aws-zero-quota-failover', [
         f'sky launch -y -c {name} --cloud aws --region {region} --gpus V100:8 --use-spot | grep "Found no quota"',
-    ])
+    ], f'sky down -y {name}')
     run_one_test(test)