Fallback to proc when ebpf timestamps are wrong

probe/endpoint/connection_tracker.go

@@ -88,8 +88,19 @@ func (t *connectionTracker) ReportConnections(rpt *report.Report) {
 	hostNodeID := report.MakeHostNodeID(t.conf.HostID)
 
 	if t.ebpfTracker != nil {
-		t.performEbpfTrack(rpt, hostNodeID)
-		return
+		if t.ebpfTracker.isDead() {
+			// ebpf tracker died, gently fallback to proc scanning
+			if t.conf.WalkProc && t.conf.Scanner == nil {
+				t.conf.Scanner = procspy.NewConnectionScanner(t.conf.ProcessCache)
+			}
+			if t.flowWalker == nil {
+				t.flowWalker = newConntrackFlowWalker(t.conf.UseConntrack, t.conf.ProcRoot, t.conf.BufferSize, "--any-nat")
+			}
+			t.ebpfTracker = nil
+		} else {
+			t.performEbpfTrack(rpt, hostNodeID)
+			return
+		}
 	}
 
 	// seenTuples contains information about connections seen by conntrack and it will be passed to the /proc parser

probe/endpoint/ebpf.go

@@ -25,6 +25,7 @@ type eventTracker interface {
 	walkConnections(f func(ebpfConnection))
 	feedInitialConnections(ci procspy.ConnIter, seenTuples map[string]fourTuple, hostNodeID string)
 	isReadyToHandleConnections() bool
+	isDead() bool
 	stop()
 }
 
@@ -99,7 +100,10 @@ var lastTimestampV4 uint64
 
 func tcpEventCbV4(e tracer.TcpV4) {
 	if lastTimestampV4 > e.Timestamp {
-		log.Errorf("ERROR: late event!\n")
+		// See https://github.com/weaveworks/scope/issues/2334
+		log.Debugf("tcp tracer received event with timestamp %v even though the last timestamp was %v. Stopping the eBPF tracker.", e.Timestamp, lastTimestampV4)
+		ebpfTracker.dead = true
+		ebpfTracker.stop()
 	}
 
 	lastTimestampV4 = e.Timestamp
@@ -197,6 +201,10 @@ func (t *EbpfTracker) isReadyToHandleConnections() bool {
 	return t.readyToHandleConnections
 }
 
+func (t *EbpfTracker) isDead() bool {
+	return t.dead
+}
+
 func (t *EbpfTracker) stop() {
 	// TODO: implement proper stopping logic
 	//