[Provisioner] Parallelize wait for SSH in provisioner

[Michaelvll]

A user reported that we wait SSH access in provisioner sequentially, which causes inefficiency when the number of nodes increase. We should parallelize it.

skypilot/sky/provision/provisioner.py

Lines 345 to 382 in f2991b1

	def wait_for_ssh(cluster_info: provision_common.ClusterInfo,
	ssh_credentials: Dict[str, str]):
	"""Wait until SSH is ready.
	Raises:
	RuntimeError: If the SSH connection is not ready after timeout.
	"""
	if (cluster_info.has_external_ips() and
	ssh_credentials.get('ssh_proxy_command') is None):
	# If we can access public IPs, then it is more efficient to test SSH
	# connection with raw sockets.
	waiter = _wait_ssh_connection_direct
	else:
	# See https://github.com/skypilot-org/skypilot/pull/1512
	waiter = _wait_ssh_connection_indirect
	ip_list = cluster_info.get_feasible_ips()
	port_list = cluster_info.get_ssh_ports()
	timeout = 60 * 10 # 10-min maximum timeout
	start = time.time()
	# use a queue for SSH querying
	ips = collections.deque(ip_list)
	ssh_ports = collections.deque(port_list)
	while ips:
	ip = ips.popleft()
	ssh_port = ssh_ports.popleft()
	success, stderr = waiter(ip, ssh_port, **ssh_credentials)
	if not success:
	ips.append(ip)
	ssh_ports.append(ssh_port)
	if time.time() - start > timeout:
	with ux_utils.print_exception_no_traceback():
	raise RuntimeError(
	f'Failed to SSH to {ip} after timeout {timeout}s, with '
	f'{stderr}')
	logger.debug('Retrying in 1 second...')
	time.sleep(1)