libev mainloop: dispatch platform manager events via libev mainloop

This eliminates the need for a separate mainloop thread for dispatching events,
thus making sure event handlers cannot run in parallel with timers or
I/O handlers.

src/include/platform/internal/GenericPlatformManagerImpl_POSIX.h

@@ -106,16 +106,26 @@ class GenericPlatformManagerImpl_POSIX : public GenericPlatformManagerImpl<ImplC
 
     inline ImplClass * Impl() { return static_cast<ImplClass *>(this); }
 
-    void ProcessDeviceEvents();
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+    static void _DispatchEventViaScheduleWork(System::Layer * aLayer, void * appState);
+#else
 
     DeviceSafeQueue mChipEventQueue;
     std::atomic<bool> mShouldRunEventLoop;
     static void * EventLoopTaskMain(void * arg);
+#endif
+    void ProcessDeviceEvents();
+    std::atomic<bool> mShouldRunEventLoop;
 };
 
 // Instruct the compiler to instantiate the template only when explicitly told to do so.
 extern template class GenericPlatformManagerImpl_POSIX<PlatformManagerImpl>;
 
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+// with external libev mainloop, this should be implemented externally to terminate the mainloop cleanly
+// (Note that there is a weak default implementation that just calls chipDie() when the external implementation is missing)
+extern void ExitExternalMainLoop();
+#endif
 } // namespace Internal
 } // namespace DeviceLayer
 } // namespace chip

src/include/platform/internal/GenericPlatformManagerImpl_POSIX.ipp

@@ -128,15 +128,35 @@ CHIP_ERROR GenericPlatformManagerImpl_POSIX<ImplClass>::_StartChipTimer(System::
     return CHIP_NO_ERROR;
 }
 
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+template <class ImplClass>
+void GenericPlatformManagerImpl_POSIX<ImplClass>::_DispatchEventViaScheduleWork(System::Layer * aLayer, void * appState)
+{
+    const ChipDeviceEvent * event = static_cast<const ChipDeviceEvent *>(appState);
+    PlatformMgrImpl().DispatchEvent(event);
+    delete event;
+}
+#endif // CHIP_SYSTEM_CONFIG_USE_LIBEV
+
 template <class ImplClass>
 CHIP_ERROR GenericPlatformManagerImpl_POSIX<ImplClass>::_PostEvent(const ChipDeviceEvent * event)
 {
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+    // Schedule dispatching this event via System Layer's ScheduleWork
+    ChipDeviceEvent * eventCopyP = new ChipDeviceEvent;
+    VerifyOrDie(eventCopyP != nullptr);
+    *eventCopyP = *event;
+    SystemLayer().ScheduleWork(&_DispatchEventViaScheduleWork, eventCopyP);
+    return CHIP_NO_ERROR;
+#else
     mChipEventQueue.Push(*event);
 
     SystemLayerSocketsLoop().Signal(); // Trigger wake select on CHIP thread
     return CHIP_NO_ERROR;
+#endif // CHIP_SYSTEM_CONFIG_USE_LIBEV
 }
 
+#if !CHIP_SYSTEM_CONFIG_USE_LIBEV
 template <class ImplClass>
 void GenericPlatformManagerImpl_POSIX<ImplClass>::ProcessDeviceEvents()
 {
@@ -146,6 +166,7 @@ void GenericPlatformManagerImpl_POSIX<ImplClass>::ProcessDeviceEvents()
         Impl()->DispatchEvent(&event);
     }
 }
+#endif // !CHIP_SYSTEM_CONFIG_USE_LIBEV
 
 template <class ImplClass>
 void GenericPlatformManagerImpl_POSIX<ImplClass>::_RunEventLoop()
@@ -197,13 +218,15 @@ void GenericPlatformManagerImpl_POSIX<ImplClass>::_RunEventLoop()
     pthread_cond_signal(&mEventQueueStoppedCond);
 }
 
+#if !CHIP_SYSTEM_CONFIG_USE_LIBEV
 template <class ImplClass>
 void * GenericPlatformManagerImpl_POSIX<ImplClass>::EventLoopTaskMain(void * arg)
 {
     ChipLogDetail(DeviceLayer, "CHIP task running");
     static_cast<GenericPlatformManagerImpl_POSIX<ImplClass> *>(arg)->Impl()->RunEventLoop();
     return nullptr;
 }
+#endif // !CHIP_SYSTEM_CONFIG_USE_LIBEV
 
 template <class ImplClass>
 CHIP_ERROR GenericPlatformManagerImpl_POSIX<ImplClass>::_StartEventLoopTask()
@@ -220,6 +243,10 @@ CHIP_ERROR GenericPlatformManagerImpl_POSIX<ImplClass>::_StartEventLoopTask()
     VerifyOrReturnError(err == 0, CHIP_ERROR_POSIX(err));
 #endif
 
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+    // with libev, we dont need our own mainloop
+    return CHIP_NO_ERROR;
+#else
     //
     // We need to grab the lock here since we have to protect setting
     // mHasValidChipTask, which will be read right away upon creating the
@@ -237,11 +264,26 @@ CHIP_ERROR GenericPlatformManagerImpl_POSIX<ImplClass>::_StartEventLoopTask()
     pthread_mutex_unlock(&mStateLock);
 
     return CHIP_ERROR_POSIX(err);
+#endif // CHIP_SYSTEM_CONFIG_USE_LIBEV
 }
 
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+// fallback implementation
+void __attribute__((weak)) ExitExternalMainLoop()
+{
+    // FIXME: implement better exit
+    VerifyOrDieWithMsg(false, DeviceLayer, "Missing custom ExitExternalMainLoop() implementation for clean shutdown -> just die");
+}
+#endif // CHIP_SYSTEM_CONFIG_USE_LIBEV
+
 template <class ImplClass>
 CHIP_ERROR GenericPlatformManagerImpl_POSIX<ImplClass>::_StopEventLoopTask()
 {
+#if CHIP_SYSTEM_CONFIG_USE_LIBEV
+    // with libev, the mainloop is set up and managed externally
+    ExitExternalMainLoop(); // this callback needs to be implemented.
+    return CHIP_NO_ERROR;
+#else
     int err = 0;
 
     //
@@ -294,6 +336,7 @@ CHIP_ERROR GenericPlatformManagerImpl_POSIX<ImplClass>::_StopEventLoopTask()
 exit:
     mHasValidChipTask = false;
     return CHIP_ERROR_POSIX(err);
+#endif // CHIP_SYSTEM_CONFIG_USE_LIBEV
 }
 
 template <class ImplClass>

src/system/SystemLayerImplSelect.cpp

@@ -162,10 +162,7 @@ CHIP_ERROR LayerImplSelect::StartTimer(Clock::Timeout delay, TimerCompleteCallba
     {
         (void) mTimerList.Add(timer);
         dispatch_source_t timerSource = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, DISPATCH_TIMER_STRICT, dispatchQueue);
-        if (timerSource == nullptr)
-        {
-            chipDie();
-        }
+        VerifyOrDie(timerSource != nullptr);
 
         timer->mTimerSource = timerSource;
         dispatch_source_set_timer(
@@ -181,10 +178,7 @@ CHIP_ERROR LayerImplSelect::StartTimer(Clock::Timeout delay, TimerCompleteCallba
         return CHIP_NO_ERROR;
     }
 #elif CHIP_SYSTEM_CONFIG_USE_LIBEV
-    if (mLibEvLoopP == nullptr)
-    {
-        chipDie();
-    }
+    VerifyOrDie(mLibEvLoopP != nullptr);
     ev_timer_init(&timer->mLibEvTimer, &LayerImplSelect::HandleLibEvTimer, 1, 0);
     timer->mLibEvTimer.data = timer;
     auto t                  = Clock::Milliseconds64(delay).count();
@@ -226,10 +220,7 @@ void LayerImplSelect::CancelTimer(TimerCompleteCallback onComplete, void * appSt
         dispatch_release(timer->mTimerSource);
     }
 #elif CHIP_SYSTEM_CONFIG_USE_LIBEV
-    if (mLibEvLoopP == nullptr)
-    {
-        chipDie();
-    }
+    VerifyOrDie(mLibEvLoopP != nullptr);
     ev_timer_stop(mLibEvLoopP, &timer->mLibEvTimer);
 #endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH/LIBEV
 
@@ -251,12 +242,20 @@ CHIP_ERROR LayerImplSelect::ScheduleWork(TimerCompleteCallback onComplete, void
         return CHIP_NO_ERROR;
     }
 #elif CHIP_SYSTEM_CONFIG_USE_LIBEV
-    // just a timer with no delay
-    return StartTimer(Clock::Timeout(0), onComplete, appState);
+    // schedule as timer with no delay, but do NOT cancel previous timers with same onComplete/appState!
+    TimerList::Node * timer = mTimerPool.Create(*this, SystemClock().GetMonotonicTimestamp(), onComplete, appState);
+    VerifyOrReturnError(timer != nullptr, CHIP_ERROR_NO_MEMORY);
+    VerifyOrDie(mLibEvLoopP != nullptr);
+    ev_timer_init(&timer->mLibEvTimer, &LayerImplSelect::HandleLibEvTimer, 1, 0);
+    timer->mLibEvTimer.data = timer;
+    auto t                  = Clock::Milliseconds64(0).count();
+    ev_timer_set(&timer->mLibEvTimer, static_cast<double>(t) / 1E3, 0.);
+    (void) mTimerList.Add(timer);
+    ev_timer_start(mLibEvLoopP, &timer->mLibEvTimer);
+    return CHIP_NO_ERROR;
 #endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH/LIBEV
 #if !CHIP_SYSTEM_CONFIG_USE_LIBEV
     // Note: dispatch based implementation needs this as fallback, but not LIBEV (and dead code is not allowed with -Werror)
-
     // Ideally we would not use a timer here at all, but if we try to just
     // ScheduleLambda the lambda needs to capture the following:
     // 1) onComplete
@@ -371,10 +370,7 @@ CHIP_ERROR LayerImplSelect::RequestCallbackOnPendingRead(SocketWatchToken token)
         }
     }
 #elif CHIP_SYSTEM_CONFIG_USE_LIBEV
-    if (mLibEvLoopP == nullptr)
-    {
-        chipDie();
-    }
+    VerifyOrDie(mLibEvLoopP != nullptr);
     int evs = (watch->mPendingIO.Has(SocketEventFlags::kRead) ? EV_READ : 0) |
         (watch->mPendingIO.Has(SocketEventFlags::kWrite) ? EV_WRITE : 0);
     if (!ev_is_active(&watch->mIoWatcher))
@@ -434,10 +430,7 @@ CHIP_ERROR LayerImplSelect::RequestCallbackOnPendingWrite(SocketWatchToken token
         }
     }
 #elif CHIP_SYSTEM_CONFIG_USE_LIBEV
-    if (mLibEvLoopP == nullptr)
-    {
-        chipDie();
-    }
+    VerifyOrDie(mLibEvLoopP != nullptr);
     int evs = (watch->mPendingIO.Has(SocketEventFlags::kRead) ? EV_READ : 0) |
         (watch->mPendingIO.Has(SocketEventFlags::kWrite) ? EV_WRITE : 0);
     if (!ev_is_active(&watch->mIoWatcher))
@@ -651,9 +644,9 @@ void LayerImplSelect::HandleTimerComplete(TimerList::Node * timer)
 void LayerImplSelect::HandleLibEvTimer(EV_P_ struct ev_timer * t, int revents)
 {
     TimerList::Node * timer = static_cast<TimerList::Node *>(t->data);
-    VerifyOrDie(timer);
+    VerifyOrDie(timer != nullptr);
     LayerImplSelect * layerP = dynamic_cast<LayerImplSelect *>(timer->mCallback.mSystemLayer);
-    VerifyOrDie(layerP);
+    VerifyOrDie(layerP != nullptr);
     layerP->mTimerList.Remove(timer);
     layerP->mTimerPool.Invoke(timer);
 }

src/system/SystemTimer.h

@@ -100,7 +100,7 @@ class DLL_EXPORT TimerData
 #endif // CHIP_SYSTEM_CONFIG_USE_DISPATCH
 
     // Not defined
-    TimerData(const TimerData &) = delete;
+    TimerData(const TimerData &)             = delete;
     TimerData & operator=(const TimerData &) = delete;
 };