[SkyServe] Support min_replicas = 0

sky/cli.py

@@ -4386,6 +4386,9 @@ def serve_status(all: bool, endpoint: bool, service_names: List[str]):
       down. This usually indicates resource leakages. If you see such status,
       please login to the cloud console and double-check
 
+    - ``NO_REPLICAS``: The service has no replicas. This usually happens when
+        min_replicas is set to 0.
+
     Each replica can have one of the following statuses:
 
     - ``PENDING``: The maximum number of simultaneous launches has been reached

sky/serve/autoscalers.py

@@ -154,7 +154,7 @@ def _get_desired_num_replicas(self) -> int:
         logger.info(f'Requests per second: {num_requests_per_second}, '
                     f'Current target number of replicas: {target_num_replicas}')
 
-        if not self.bootstrap_done:
+        if (not self.bootstrap_done or self.target_num_replicas == 0):
             self.bootstrap_done = True
             return target_num_replicas
         elif target_num_replicas > self.target_num_replicas:

sky/serve/constants.py

@@ -36,6 +36,8 @@
 # Autoscaler scale decision interval in seconds.
 # We will try to scale up/down every `decision_interval`.
 AUTOSCALER_DEFAULT_DECISION_INTERVAL_SECONDS = 20
+# Autoscaler no replica decision interval in seconds.
+AUTOSCALER_NO_REPLICA_DECISION_INTERVAL_SECONDS = 5
 # Autoscaler default upscale delays in seconds.
 # We will upscale only if the target number of instances
 # is larger than the current launched instances for delay amount of time.

sky/serve/controller.py

@@ -88,7 +88,12 @@ def _run_autoscaler(self):
                              f'{common_utils.format_exception(e)}')
                 with ux_utils.enable_traceback():
                     logger.error(f'  Traceback: {traceback.format_exc()}')
-            time.sleep(constants.AUTOSCALER_DEFAULT_DECISION_INTERVAL_SECONDS)
+            if self._autoscaler.target_num_replicas == 0:
+                time.sleep(
+                    constants.AUTOSCALER_NO_REPLICA_DECISION_INTERVAL_SECONDS)
+            else:
+                time.sleep(
+                    constants.AUTOSCALER_DEFAULT_DECISION_INTERVAL_SECONDS)
 
     def run(self) -> None:
 

sky/serve/serve_state.py

@@ -153,6 +153,9 @@ class ServiceStatus(enum.Enum):
     # Clean up failed
     FAILED_CLEANUP = 'FAILED_CLEANUP'
 
+    # No replica
+    NO_REPLICA = 'NO_REPLICA'
+
     @classmethod
     def failed_statuses(cls) -> List['ServiceStatus']:
         return [cls.CONTROLLER_FAILED, cls.FAILED_CLEANUP]
@@ -175,6 +178,8 @@ def from_replica_statuses(
         if sum(status2num[status]
                for status in ReplicaStatus.failed_statuses()) > 0:
             return cls.FAILED
+        if len(replica_statuses) == 0:
+            return cls.NO_REPLICA
         return cls.REPLICA_INIT
 
 
@@ -186,6 +191,7 @@ def from_replica_statuses(
     ServiceStatus.SHUTTING_DOWN: colorama.Fore.YELLOW,
     ServiceStatus.FAILED: colorama.Fore.RED,
     ServiceStatus.FAILED_CLEANUP: colorama.Fore.RED,
+    ServiceStatus.NO_REPLICA: colorama.Fore.MAGENTA,
 }
 
 

sky/serve/service_spec.py

@@ -33,9 +33,9 @@ def __init__(
         qps_upper_threshold: Optional[float] = None,
         qps_lower_threshold: Optional[float] = None,
     ) -> None:
-        if min_replicas <= 0:
+        if min_replicas < 0:
             with ux_utils.print_exception_no_traceback():
-                raise ValueError('min_replicas must be greater than 0')
+                raise ValueError('min_replicas must be greater or equal to 0')
         if max_replicas is not None and max_replicas < min_replicas:
             with ux_utils.print_exception_no_traceback():
                 raise ValueError(