Fix selection buffer crash after resizing window

ogre2/src/Ogre2RayQuery.cc

@@ -123,6 +123,10 @@ RayQueryResult Ogre2RayQuery::ClosestPoint()
 //////////////////////////////////////////////////
 RayQueryResult Ogre2RayQuery::ClosestPointBySelectionBuffer()
 {
+  // update selection buffer dimension in case window is resized
+  this->dataPtr->camera->SelectionBuffer()->SetDimensions(
+    this->dataPtr->camera->ImageWidth(), this->dataPtr->camera->ImageHeight());
+
   RayQueryResult result;
   Ogre::Item *ogreItem = nullptr;
   math::Vector3d point;

ogre2/src/Ogre2SelectionBuffer.cc

@@ -88,6 +88,9 @@ class ignition::rendering::Ogre2SelectionBufferPrivate
   /// into a render target or render texture.
   public: Ogre::CompositorWorkspace *ogreCompositorWorkspace = nullptr;
 
+  /// \brief Name of the compositor workspace definition
+  public: std::string ogreCompWorkspaceDefName;
+
   /// \brief The selection buffer material
   public: Ogre::MaterialPtr selectionMaterial;
 };
@@ -181,10 +184,27 @@ void Ogre2SelectionBuffer::Update()
 /////////////////////////////////////////////////
 void Ogre2SelectionBuffer::DeleteRTTBuffer()
 {
-  if (this->dataPtr->selectionMaterial)
+  // \todo Delete material?
+  // This causes an ogre assertion error about unlinking renderables
+  // when the RTT buffer is created again
+  // if (this->dataPtr->selectionMaterial)
+  // {
+  //   Ogre::MaterialManager::getSingleton().remove(
+  //       this->dataPtr->selectionMaterial->getName());
+  //   this->dataPtr->selectionMaterial.setNull();
+  // }
+
+  if (this->dataPtr->ogreCompositorWorkspace)
   {
-    Ogre::MaterialManager::getSingleton().remove(
-        this->dataPtr->selectionMaterial->getName());
+    // TODO(ahcorde): Remove the workspace. Potential leak here
+    this->dataPtr->ogreCompMgr->removeWorkspace(
+        this->dataPtr->ogreCompositorWorkspace);
+
+    this->dataPtr->ogreCompMgr->removeWorkspaceDefinition(
+        this->dataPtr->ogreCompWorkspaceDefName);
+    this->dataPtr->ogreCompMgr->removeNodeDefinition(
+        this->dataPtr->ogreCompWorkspaceDefName + "/Node");
+    this->dataPtr->ogreCompositorWorkspace = nullptr;
   }
 
   auto engine = Ogre2RenderEngine::Instance();
@@ -222,11 +242,18 @@ void Ogre2SelectionBuffer::CreateRTTBuffer()
   // The SelectionBuffer material is defined in script
   // (selection_buffer.material).
   std::string matSelectionName = "SelectionBuffer";
-  Ogre::MaterialPtr matSelection =
+  std::string matSelectionCloneName =
+      this->dataPtr->camera->getName() + "_" + matSelectionName;
+  this->dataPtr->selectionMaterial =
       Ogre::MaterialManager::getSingleton().getByName(matSelectionName);
-  this->dataPtr->selectionMaterial = matSelection->clone(
-      this->dataPtr->camera->getName() + "_" + matSelectionName);
-  this->dataPtr->selectionMaterial->load();
+  if (this->dataPtr->selectionMaterial.isNull())
+  {
+    Ogre::MaterialPtr matSelection =
+        Ogre::MaterialManager::getSingleton().getByName(matSelectionName);
+    this->dataPtr->selectionMaterial = matSelection->clone(
+        matSelectionCloneName);
+    this->dataPtr->selectionMaterial->load();
+  }
   Ogre::Pass *p = this->dataPtr->selectionMaterial->getTechnique(0)->getPass(0);
   Ogre::GpuProgramParametersSharedPtr psParams =
       p->getFragmentProgramParameters();
@@ -252,13 +279,15 @@ void Ogre2SelectionBuffer::CreateRTTBuffer()
 
   // create compositor workspace for rendering
   // Setup the selection buffer compositor.
-  const Ogre::String workspaceName = "SelectionBufferWorkspace" +
+  this->dataPtr->ogreCompWorkspaceDefName = "SelectionBufferWorkspace" +
       this->dataPtr->camera->getName();
 
+  std::string nodeSpaceDefName =
+      this->dataPtr->ogreCompWorkspaceDefName + "/Node";
+
   Ogre::CompositorNodeDef *nodeDef =
       this->dataPtr->ogreCompMgr->addNodeDefinition(
-      "AutoGen " + Ogre::IdString(workspaceName +
-      "/Node").getReleaseText());
+      nodeSpaceDefName);
   Ogre::TextureDefinitionBase::TextureDefinition *depthTexDef =
       nodeDef->addTextureDefinition("depthTexture");
   depthTexDef->textureType = Ogre::TextureTypes::Type2D;
@@ -346,15 +375,16 @@ void Ogre2SelectionBuffer::CreateRTTBuffer()
   }
 
   Ogre::CompositorWorkspaceDef *workDef =
-      this->dataPtr->ogreCompMgr->addWorkspaceDefinition(workspaceName);
+      this->dataPtr->ogreCompMgr->addWorkspaceDefinition(
+      this->dataPtr->ogreCompWorkspaceDefName);
   workDef->connectExternal(0, nodeDef->getName(), 0);
 
   this->dataPtr->ogreCompositorWorkspace =
       this->dataPtr->ogreCompMgr->addWorkspace(
         this->dataPtr->scene->OgreSceneManager(),
         this->dataPtr->renderTexture,
         this->dataPtr->selectionCamera,
-        workspaceName,
+        this->dataPtr->ogreCompWorkspaceDefName,
         false);
 }
 
@@ -369,14 +399,6 @@ void Ogre2SelectionBuffer::SetDimensions(
   this->dataPtr->height = _height;
 
   this->DeleteRTTBuffer();
-
-  if (this->dataPtr->ogreCompositorWorkspace)
-  {
-    // TODO(ahcorde): Remove the workspace. Potential leak here
-    this->dataPtr->ogreCompMgr->removeWorkspace(
-        this->dataPtr->ogreCompositorWorkspace);
-  }
-
   this->CreateRTTBuffer();
 }
 /////////////////////////////////////////////////
@@ -398,6 +420,14 @@ bool Ogre2SelectionBuffer::ExecuteQuery(const int _x, const int _y,
   if (!this->dataPtr->camera)
     return false;
 
+  // check camera has valid projection matrix
+  // There could be nan values if camera was resized
+  Ogre::Matrix4 projectionMatrix =
+      this->dataPtr->camera->getProjectionMatrix();
+  if (projectionMatrix.getTrans().isNaN() ||
+      projectionMatrix.extractQuaternion().isNaN())
+    return false;
+
    const unsigned int targetWidth = this->dataPtr->width;
    const unsigned int targetHeight = this->dataPtr->height;
 
@@ -426,9 +456,8 @@ bool Ogre2SelectionBuffer::ExecuteQuery(const int _x, const int _y,
   transMatrix[1][3] += y1+y2;
   Ogre::Matrix4 customProjectionMatrix =
       scaleMatrix * transMatrix * this->dataPtr->camera->getProjectionMatrix();
-
-   this->dataPtr->selectionCamera->setCustomProjectionMatrix(true,
-        customProjectionMatrix);
+  this->dataPtr->selectionCamera->setCustomProjectionMatrix(true,
+      customProjectionMatrix);
 
   this->dataPtr->selectionCamera->setPosition(
       this->dataPtr->camera->getDerivedPosition());
@@ -442,7 +471,6 @@ bool Ogre2SelectionBuffer::ExecuteQuery(const int _x, const int _y,
   image.convertFromTexture(this->dataPtr->renderTexture, 0, 0);
 
   Ogre::ColourValue pixel = image.getColourAt(0, 0, 0, 0);
-  // Ogre::ColourValue pixel = image.getColourAt(_x, _y, 0, 0);
   float color = pixel[3];
   uint32_t *rgba = reinterpret_cast<uint32_t *>(&color);
   unsigned int r = *rgba >> 24 & 0xFF;