Always backoff and retry management server (#392)

If we hit connectivity issues, make sure we retry until
the server comes back up.

Co-authored-by: Mark Mandel <[email protected]>

src/xds/ads_client.rs

@@ -132,7 +132,12 @@ impl AdsClient {
         listener_manager_args: ListenerManagerArgs,
         mut shutdown_rx: watch::Receiver<()>,
     ) -> ExecutionResult {
-        let mut backoff = ExponentialBackoff::<SystemClock>::default();
+        let mut backoff = ExponentialBackoff::<SystemClock> {
+            // If we hit connectivity issues, always backoff and retry.
+            max_elapsed_time: None,
+            ..Default::default()
+        };
+
         let log = self.log;
         let metrics = self.metrics;
 
@@ -205,7 +210,7 @@ impl AdsClient {
                         Err(RpcSessionError::Receive(handlers, bk_off, status)) => {
                             resource_handlers = handlers;
                             backoff = bk_off;
-                            error!(log, "Failed to receive from XDS server"; "address" => server_addr, "status" => #?status);
+                            error!(log, "Failed to receive response from XDS server"; "address" => server_addr, "status" => #?status);
                             Self::backoff(
                                 &log,
                                 &mut backoff
@@ -450,9 +455,10 @@ impl AdsClient {
         log: &Logger,
         backoff: &mut ExponentialBackoff<C>,
     ) -> Result<(), ExecutionError> {
-        let delay = backoff
-            .next_backoff()
-            .ok_or(ExecutionError::BackoffLimitExceeded)?;
+        let delay = backoff.next_backoff().ok_or_else(|| {
+            warn!(log, "Backoff limit exceeded");
+            ExecutionError::BackoffLimitExceeded
+        })?;
         info!(log, "Retrying"; "delay" => #?delay);
         tokio::time::sleep(delay).await;
         Ok(())