gracefully recover from IncompleteProbeCycle

src/error.rs

@@ -32,6 +32,10 @@ pub enum Error {
     /// steps didn't happen. Bug in the runtime/scheduling
     /// mechanism most likely.
     ///
+    /// Foca tries to resume normal operations after emitting this
+    /// error, but any occurance of it is a sign that something is
+    /// not behaving as expected
+    ///
     /// Must not happen under normal circumstances.
     IncompleteProbeCycle,
 

src/lib.rs

@@ -871,7 +871,15 @@ where
     }
 
     fn probe_random_member(&mut self, mut runtime: impl Runtime<T>) -> Result<()> {
+        debug_assert_eq!(self.connection_state, ConnectionState::Connected);
         if !self.probe.validate() {
+            // Probe has invalid state. We'll reset and submit another timer
+            // so that foca can recover from the issue gracefully
+            self.probe.clear();
+            runtime.submit_after(
+                Timer::ProbeRandomMember(self.timer_token),
+                self.config.probe_period,
+            );
             return Err(Error::IncompleteProbeCycle);
         }
 
@@ -3193,4 +3201,29 @@ mod tests {
             other_foca.handle_data(&broadcast_message, &mut runtime)
         );
     }
+
+    #[test]
+    fn can_recover_from_incomplete_probe_cycle() {
+        // Here we get a foca in the middle of a probe cycle. The correct
+        // sequencing should submit `_send_indirect_probe`
+        let (mut foca, _probed, _send_indirect_probe) = craft_probing_foca(2);
+        let mut runtime = InMemoryRuntime::new();
+        // ... but we'll manually craft a ProbeRandomMember event instead
+        // to trigger the validation failure
+        assert_eq!(
+            Err(Error::IncompleteProbeCycle),
+            foca.handle_timer(Timer::ProbeRandomMember(foca.timer_token()), &mut runtime)
+        );
+
+        // This situation should lead to two things happening:
+        // 1. the probe state should become valid again
+        assert!(foca.probe().validate(), "didn't recover probe state");
+        // 2. should've scheduled a new probe
+        assert!(
+            runtime
+                .find_scheduling(|t| matches!(t, Timer::ProbeRandomMember(_)))
+                .is_some(),
+            "didn't submit a new probe event"
+        );
+    }
 }