Reroute exceptions in worker threads from shader compilation to main th.

OGRECave · Apr 16, 2024 · b5efa09 · b5efa09
1 parent a5e8d66
commit b5efa09
Show file tree

Hide file tree

Showing 2 changed files with 93 additions and 8 deletions.
diff --git a/OgreMain/include/OgreRenderQueue.h b/OgreMain/include/OgreRenderQueue.h
@@ -90,6 +90,9 @@ namespace Ogre
         Semaphore            mSemaphore;
         std::atomic<bool>    mKeepCompiling;
 
+        bool               mExceptionFound;     // GUARDED_BY( mMutex )
+        std::exception_ptr mThreadedException;  // GUARDED_BY( mMutex )
+
     public:
         ParallelHlmsCompileQueue();
 
@@ -102,13 +105,39 @@ namespace Ogre
 
         inline void pushWarmUpRequest( const Request &&request ) { mRequests.emplace_back( request ); }
 
+        /** Starts worker threads (job queue) so they start accepting work every time pushRequest()
+            gets called and will keep compiling those shaders until stopAndWait() is called.
+
+            The work is done in updateThread() and is in charge of compiling shaders AND generating PSOs.
+        @remarks
+            This function must not be called if RenderSystem::supportsMultithreadedShaderCompliation
+            is false.
+        @param sceneManager
+        */
         void start( SceneManager *sceneManager );
+        /** Signals worker threads we won't be submitting more work, so they should stop once they're
+            done compiling all pending shaders / PSOs.
+
+            Will wait until all shaders are done.
+        @param sceneManager
+        */
         void stopAndWait( SceneManager *sceneManager );
+        /// The actual work done by the job queues.
         void updateThread( size_t threadIdx, HlmsManager *hlmsManager );
 
+        /// Similar to start() and stopAndWait() at the same time: It assumes all work has already been
+        /// gathered in mRequests via pushWarmUpRequest() (instead of gather it as we go)
+        /// and fires all threads to compile the shaders and PSOs in parallel.
+        ///
+        /// It will wait until all threads are done.
         void fireWarmUpParallel( SceneManager *sceneManager );
+
+        /// The actual work done by fireWarmUpParallel().
         void updateWarmUpThread( size_t threadIdx, HlmsManager *hlmsManager,
                                  const HlmsCache *passCaches );
+
+        /// Serial alternative of fireWarmUpParallel() + updateWarmUpThread() for when
+        /// RenderSystem::supportsMultithreadedShaderCompliation is false.
         void warmUpSerial( HlmsManager *hlmsManager, const HlmsCache *passCaches );
     };
 

diff --git a/OgreMain/src/OgreRenderQueue.cpp b/OgreMain/src/OgreRenderQueue.cpp
@@ -999,12 +999,35 @@ namespace Ogre
         mKeepCompiling.store( false, std::memory_order::memory_order_relaxed );
         mSemaphore.increment( static_cast<uint32_t>( sceneManager->getNumWorkerThreads() ) );
         sceneManager->waitForParallelHlmsCompile();
+
+        // No need to use mMutex because we guarantee no write access from other threads
+        // after this point.
+        //
+        // IMPORTANT: After this exception is caught, the Hlms will likely be left in an inconsistent
+        // state. We pretty much can't do anything more unless some heavy reinitialization is done.
+        if( mExceptionFound )
+        {
+            std::exception_ptr threadedException = mThreadedException;
+            mRequests.clear();  // We terminated threads early. (Does it matter?)
+            mThreadedException = nullptr;
+            mExceptionFound = false;
+            std::rethrow_exception( threadedException );
+        }
     }
     //-----------------------------------------------------------------------
     void ParallelHlmsCompileQueue::fireWarmUpParallel( SceneManager *sceneManager )
     {
         sceneManager->_fireWarmUpShadersCompile();
-        OGRE_ASSERT_LOW( mRequests.empty() );
+        OGRE_ASSERT_LOW( mRequests.empty() );  // Should be empty, whether we found an exception or not.
+        // See comments in ParallelHlmsCompileQueue::stopAndWait implementation.
+        // The only difference is that mRequests should be empty by now.
+        if( mExceptionFound )
+        {
+            std::exception_ptr threadedException = mThreadedException;
+            mThreadedException = nullptr;
+            mExceptionFound = false;
+            std::rethrow_exception( threadedException );
+        }
     }
     //-----------------------------------------------------------------------
     void ParallelHlmsCompileQueue::updateWarmUpThread( size_t threadIdx, HlmsManager *hlmsManager,
@@ -1029,9 +1052,23 @@ namespace Ogre
 
             const HlmsDatablock *datablock = request.queuedRenderable.renderable->getDatablock();
             Hlms *hlms = hlmsManager->getHlms( static_cast<HlmsTypes>( datablock->mType ) );
-            hlms->compileStubEntry( passCaches[reinterpret_cast<size_t>( request.passCache )],
-                                    request.reservedStubEntry, request.queuedRenderable,
-                                    request.renderableHash, request.finalHash, threadIdx );
+            try
+            {
+                hlms->compileStubEntry( passCaches[reinterpret_cast<size_t>( request.passCache )],
+                                        request.reservedStubEntry, request.queuedRenderable,
+                                        request.renderableHash, request.finalHash, threadIdx );
+            }
+            catch( Exception & )
+            {
+                ScopedLock lock( mMutex );
+                // We can only report one exception.
+                if( !mExceptionFound )
+                {
+                    mRequests.clear();  // Only way to signal other threads to stop early.
+                    mExceptionFound = true;
+                    mThreadedException = std::current_exception();
+                }
+            }
         }
     }
     //-----------------------------------------------------------------------
@@ -1112,9 +1149,23 @@ namespace Ogre
             {
                 const HlmsDatablock *datablock = request.queuedRenderable.renderable->getDatablock();
                 Hlms *hlms = hlmsManager->getHlms( static_cast<HlmsTypes>( datablock->mType ) );
-                hlms->compileStubEntry( *request.passCache, request.reservedStubEntry,
-                                        request.queuedRenderable, request.renderableHash,
-                                        request.finalHash, threadIdx );
+                try
+                {
+                    hlms->compileStubEntry( *request.passCache, request.reservedStubEntry,
+                                            request.queuedRenderable, request.renderableHash,
+                                            request.finalHash, threadIdx );
+                }
+                catch( Exception & )
+                {
+                    ScopedLock lock( mMutex );
+                    // We can only report one exception.
+                    if( !mExceptionFound )
+                    {
+                        mKeepCompiling.store( false, std::memory_order::memory_order_relaxed );
+                        mExceptionFound = true;
+                        mThreadedException = std::current_exception();
+                    }
+                }
             }
         }
     }
@@ -1213,5 +1264,10 @@ namespace Ogre
     //-----------------------------------------------------------------------
     //-----------------------------------------------------------------------
     //-----------------------------------------------------------------------
-    ParallelHlmsCompileQueue::ParallelHlmsCompileQueue() : mSemaphore( 0u ), mKeepCompiling( false ) {}
+    ParallelHlmsCompileQueue::ParallelHlmsCompileQueue() :
+        mSemaphore( 0u ),
+        mKeepCompiling( false ),
+        mExceptionFound( false )
+    {
+    }
 }  // namespace Ogre