diff --git a/probe/endpoint/connection_tracker.go b/probe/endpoint/connection_tracker.go index 9e753180d5..b0391c5b40 100644 --- a/probe/endpoint/connection_tracker.go +++ b/probe/endpoint/connection_tracker.go @@ -88,8 +88,19 @@ func (t *connectionTracker) ReportConnections(rpt *report.Report) { hostNodeID := report.MakeHostNodeID(t.conf.HostID) if t.ebpfTracker != nil { - t.performEbpfTrack(rpt, hostNodeID) - return + if t.ebpfTracker.isDead() { + // ebpf tracker died, gently fallback to proc scanning + if t.conf.WalkProc && t.conf.Scanner == nil { + t.conf.Scanner = procspy.NewConnectionScanner(t.conf.ProcessCache) + } + if t.flowWalker == nil { + t.flowWalker = newConntrackFlowWalker(t.conf.UseConntrack, t.conf.ProcRoot, t.conf.BufferSize, "--any-nat") + } + t.ebpfTracker = nil + } else { + t.performEbpfTrack(rpt, hostNodeID) + return + } } // seenTuples contains information about connections seen by conntrack and it will be passed to the /proc parser diff --git a/probe/endpoint/ebpf.go b/probe/endpoint/ebpf.go index 07ce10f05a..e68e7d08be 100644 --- a/probe/endpoint/ebpf.go +++ b/probe/endpoint/ebpf.go @@ -25,6 +25,7 @@ type eventTracker interface { walkConnections(f func(ebpfConnection)) feedInitialConnections(ci procspy.ConnIter, seenTuples map[string]fourTuple, hostNodeID string) isReadyToHandleConnections() bool + isDead() bool stop() } @@ -99,7 +100,10 @@ var lastTimestampV4 uint64 func tcpEventCbV4(e tracer.TcpV4) { if lastTimestampV4 > e.Timestamp { - log.Errorf("ERROR: late event!\n") + // See https://github.com/weaveworks/scope/issues/2334 + log.Debugf("tcp tracer received event with timestamp %v even though the last timestamp was %v. Stopping the eBPF tracker.", e.Timestamp, lastTimestampV4) + ebpfTracker.dead = true + ebpfTracker.stop() } lastTimestampV4 = e.Timestamp @@ -197,6 +201,10 @@ func (t *EbpfTracker) isReadyToHandleConnections() bool { return t.readyToHandleConnections } +func (t *EbpfTracker) isDead() bool { + return t.dead +} + func (t *EbpfTracker) stop() { // TODO: implement proper stopping logic //