fix: prevents infinite retries in case of non-transient network retries

Fixes MystenLabs#665
huitseeker · Aug 23, 2022 · 35e23ea · 35e23ea
1 parent e1c068e
commit 35e23ea
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 11 deletions.
diff --git a/network/src/primary.rs b/network/src/primary.rs
@@ -11,7 +11,8 @@ use multiaddr::Multiaddr;
 use rand::{rngs::SmallRng, SeedableRng as _};
 use std::collections::HashMap;
 use tokio::{runtime::Handle, task::JoinHandle};
-use tonic::transport::Channel;
+use tonic::{transport::Channel, Code};
+use tracing::error;
 use types::{
     BincodeEncodedPayload, PrimaryMessage, PrimaryToPrimaryClient, PrimaryToWorkerClient,
     PrimaryWorkerMessage,
@@ -150,9 +151,19 @@ impl ReliableNetwork for PrimaryNetwork {
 
             async move {
                 client.send_message(message).await.map_err(|e| {
-                    // this returns a backoff::Error::Transient
-                    // so that if tonic::Status is returned, we retry
-                    Into::<backoff::Error<eyre::Report>>::into(eyre::Report::from(e))
+                    match e.code() {
+                        Code::FailedPrecondition | Code::InvalidArgument => {
+                            // these errors are not recoverable through retrying, see
+                            // https://github.com/hyperium/tonic/blob/master/tonic/src/status.rs
+                            error!("Irrecoverable network error: {e}");
+                            backoff::Error::permanent(eyre::Report::from(e))
+                        }
+                        _ => {
+                            // this returns a backoff::Error::Transient
+                            // so that if tonic::Status is returned, we retry
+                            Into::<backoff::Error<eyre::Report>>::into(eyre::Report::from(e))
+                        }
+                    }
                 })
             }
         };

diff --git a/network/src/worker.rs b/network/src/worker.rs
@@ -10,7 +10,8 @@ use multiaddr::Multiaddr;
 use rand::{rngs::SmallRng, SeedableRng as _};
 use std::collections::HashMap;
 use tokio::{runtime::Handle, task::JoinHandle};
-use tonic::transport::Channel;
+use tonic::{transport::Channel, Code};
+use tracing::error;
 use types::{
     BincodeEncodedPayload, WorkerMessage, WorkerPrimaryMessage, WorkerToPrimaryClient,
     WorkerToWorkerClient,
@@ -149,9 +150,19 @@ impl ReliableNetwork for WorkerNetwork {
 
             async move {
                 client.send_message(message).await.map_err(|e| {
-                    // this returns a backoff::Error::Transient
-                    // so that if tonic::Status is returned, we retry
-                    Into::<backoff::Error<eyre::Report>>::into(eyre::Report::from(e))
+                    match e.code() {
+                        Code::FailedPrecondition | Code::InvalidArgument => {
+                            // these errors are not recoverable through retrying, see
+                            // https://github.com/hyperium/tonic/blob/master/tonic/src/status.rs
+                            error!("Irrecoverable network error: {e}");
+                            backoff::Error::permanent(eyre::Report::from(e))
+                        }
+                        _ => {
+                            // this returns a backoff::Error::Transient
+                            // so that if tonic::Status is returned, we retry
+                            Into::<backoff::Error<eyre::Report>>::into(eyre::Report::from(e))
+                        }
+                    }
                 })
             }
         };
@@ -237,9 +248,19 @@ impl ReliableNetwork for WorkerToPrimaryNetwork {
 
             async move {
                 client.send_message(message).await.map_err(|e| {
-                    // this returns a backoff::Error::Transient
-                    // so that if tonic::Status is returned, we retry
-                    Into::<backoff::Error<eyre::Report>>::into(eyre::Report::from(e))
+                    match e.code() {
+                        Code::FailedPrecondition | Code::InvalidArgument => {
+                            // these errors are not recoverable through retrying, see
+                            // https://github.com/hyperium/tonic/blob/master/tonic/src/status.rs
+                            error!("Irrecoverable network error: {e}");
+                            backoff::Error::permanent(eyre::Report::from(e))
+                        }
+                        _ => {
+                            // this returns a backoff::Error::Transient
+                            // so that if tonic::Status is returned, we retry
+                            Into::<backoff::Error<eyre::Report>>::into(eyre::Report::from(e))
+                        }
+                    }
                 })
             }
         };