Switch off 3D rendering on out of memory

src/DynamoCore/Models/DynamoModel.cs

@@ -2109,6 +2109,27 @@ private void DisplayInvalidInputSymbolWarning()
             OnRequestTaskDialog(null, args);
         }
 
+        internal event VoidHandler Preview3DOutage;
+        private void OnPreview3DOutage()
+        {
+            if (Preview3DOutage != null)
+            {
+                Preview3DOutage();
+            }
+        }
+
+        internal void Report3DPreviewOutage(string summary, string description)
+        {
+            OnPreview3DOutage();
+
+            const string imageUri = "/DynamoCoreWpf;component/UI/Images/task_dialog_future_file.png";
+            var args = new TaskDialogEventArgs(
+               new Uri(imageUri, UriKind.Relative),
+               Resources.Preview3DOutageTitle, summary, description);
+
+            OnRequestTaskDialog(null, args);
+        }
+
         /// <summary>
         ///     Remove a workspace from the dynamo model.
         /// </summary>

src/DynamoCore/Models/DynamoModelEvents.cs

@@ -283,7 +283,7 @@ public void OnRequestsCrashPrompt(object sender, CrashPromptArgs args)
 
         internal delegate void TaskDialogHandler(object sender, TaskDialogEventArgs e);
         internal event TaskDialogHandler RequestTaskDialog;
-        internal void OnRequestTaskDialog(object sender, TaskDialogEventArgs args)
+        internal virtual void OnRequestTaskDialog(object sender, TaskDialogEventArgs args)
         {
             if (RequestTaskDialog != null)
                 RequestTaskDialog(sender, args);

src/DynamoCore/Properties/Resources.en-US.resx

@@ -709,4 +709,7 @@ Restart Dynamo to complete the uninstall.</value>
   <data name="FunctionDefinitionOverwrittenMessage" xml:space="preserve">
     <value>Attempting to load customNode {0} loaded by package {1}, but a previous definition named {2} exists with no associated package. The new customNode definition has been loaded, but Dynamo may be in an unstable state, please avoid loading multiple custom nodes with the id.</value>
   </data>
+  <data name="Preview3DOutageTitle" xml:space="preserve">
+    <value>3D preview has been deactivated</value>
+  </data>
 </root>

src/DynamoCore/Properties/Resources.resx

@@ -709,4 +709,7 @@ Restart Dynamo to complete the uninstall.</value>
   <data name="FunctionDefinitionOverwrittenMessage" xml:space="preserve">
     <value>Attempting to load customNode {0} loaded by package {1}, but a previous definition named {2} exists with no associated package. The new customNode definition has been loaded, but Dynamo may be in an unstable state, please avoid loading multiple custom nodes with the id.</value>
   </data>
+  <data name="Preview3DOutageTitle" xml:space="preserve">
+    <value>3D preview has been deactivated</value>
+  </data>
 </root>

src/DynamoCoreWpf/Properties/Resources.en-US.resx

@@ -2239,4 +2239,10 @@ Uninstall the following packages: {0}?</value>
   <data name="NodeTooltipOriginalName" xml:space="preserve">
     <value>Original node name: </value>
   </data>
+  <data name="RenderingMemoryOutageDescription" xml:space="preserve">
+    <value>Please check if the amount of generated geometry is correct. If it is, it might be necessary to disable preview for some nodes before reactivating the background 3D preview.</value>
+  </data>
+  <data name="RenderingMemoryOutageSummary" xml:space="preserve">
+    <value>Dynamo ran out of memory trying to render geometry</value>
+  </data>
 </root>

src/DynamoCoreWpf/Properties/Resources.resx

@@ -2239,4 +2239,10 @@ Uninstall the following packages: {0}?</value>
   <data name="NodeTooltipOriginalName" xml:space="preserve">
     <value>Original node name: </value>
   </data>
+  <data name="RenderingMemoryOutageDescription" xml:space="preserve">
+    <value>Please check if the amount of generated geometry is correct. If it is, it might be necessary to disable preview for some nodes before reactivating the background 3D preview.</value>
+  </data>
+  <data name="RenderingMemoryOutageSummary" xml:space="preserve">
+    <value>Dynamo ran out of memory trying to render geometry</value>
+  </data>
 </root>

src/DynamoCoreWpf/ViewModels/Core/DynamoViewModel.cs

@@ -724,12 +724,14 @@ private void SubscribeModelUiEvents()
         {
             model.RequestBugReport += ReportABug;
             model.RequestDownloadDynamo += DownloadDynamo;
+            model.Preview3DOutage += Disable3DPreview;
         }
 
         private void UnsubscribeModelUiEvents()
         {
             model.RequestBugReport -= ReportABug;
             model.RequestDownloadDynamo -= DownloadDynamo;
+            model.Preview3DOutage -= Disable3DPreview;
         }
 
         private void SubscribeModelCleaningUpEvent()
@@ -868,6 +870,14 @@ internal static void DownloadDynamo()
             Process.Start(new ProcessStartInfo("explorer.exe", Configurations.DynamoDownloadLink));
         }
 
+        private void Disable3DPreview()
+        {
+            foreach (var item in Watch3DViewModels)
+            {
+                item.Active = false;
+            }
+        }
+
         internal bool CanReportABug(object parameter)
         {
             return true;

src/DynamoCoreWpf/ViewModels/Watch3D/HelixWatch3DViewModel.cs

@@ -20,7 +20,9 @@
 using Dynamo.Graph.Nodes.CustomNodes;
 using Dynamo.Graph.Workspaces;
 using Dynamo.Logging;
+using Dynamo.Models;
 using Dynamo.Selection;
+using Dynamo.UI.Prompts;
 using Dynamo.ViewModels;
 using Dynamo.Visualization;
 using Dynamo.Wpf.Properties;
@@ -219,7 +221,7 @@ protected override void OnActiveStateChanged()
         }
 
         public event Action<Model3D> RequestAttachToScene;
-        protected void OnRequestAttachToScene(Model3D model3D)
+        protected virtual void OnRequestAttachToScene(Model3D model3D)
         {
             if (RequestAttachToScene != null)
             {
@@ -829,7 +831,17 @@ public override void GenerateViewGeometryFromRenderPackagesAndRequestUpdate(Rend
             var meshPackages = packages.Cast<HelixRenderPackage>().Where(rp => rp.MeshVertexCount % 3 == 0);
 
             RemoveGeometryForUpdatedPackages(meshPackages);
-            AggregateRenderPackages(meshPackages);
+            try
+            {
+                AggregateRenderPackages(meshPackages);
+            }
+            catch (OutOfMemoryException)
+            {
+                // This can happen when the amount of packages to render is too large
+                string summary = Resources.RenderingMemoryOutageSummary;
+                var description = Resources.RenderingMemoryOutageDescription;
+                (dynamoModel as DynamoModel).Report3DPreviewOutage(summary, description);
+            }
 #if DEBUG
             renderTimer.Stop();
             Debug.WriteLine(string.Format("RENDER: {0} ellapsed for compiling assets for rendering.", renderTimer.Elapsed));

src/VisualizationTests/HelixWatch3DViewModelTests.cs

@@ -86,28 +86,21 @@ protected override void StartDynamo(TestSessionConfiguration testConfig)
                 }
             }
 
-            Model = DynamoModel.Start(
-                new DynamoModel.DefaultStartConfiguration()
-                {
-                    StartInTestMode = true,
-                    PathResolver = pathResolver,
-                    GeometryFactoryPath = preloader.GeometryFactoryPath,
-                    UpdateManager = this.UpdateManager,
-                    ProcessMode = TaskProcessMode.Synchronous
-                });
+            Model = CreateModel(new DynamoModel.DefaultStartConfiguration()
+            {
+                StartInTestMode = true,
+                PathResolver = pathResolver,
+                GeometryFactoryPath = preloader.GeometryFactoryPath,
+                UpdateManager = this.UpdateManager,
+                ProcessMode = TaskProcessMode.Synchronous
+            });
 
             Model.EvaluationCompleted += Model_EvaluationCompleted;
 
-            ViewModel = DynamoViewModel.Start(
-                new DynamoViewModel.StartConfiguration()
-                {
-                    DynamoModel = Model,
-                    Watch3DViewModel = 
-                        HelixWatch3DViewModel.TryCreateHelixWatch3DViewModel(
-                            null,
-                            new Watch3DViewModelStartupParams(Model), 
-                            Model.Logger)
-                });
+            var vmConfig = CreateViewModelStartConfiguration();
+            vmConfig.DynamoModel = Model;
+
+            ViewModel = DynamoViewModel.Start(vmConfig);
 
             //create the view
             View = new DynamoView(ViewModel);
@@ -116,6 +109,29 @@ protected override void StartDynamo(TestSessionConfiguration testConfig)
             SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
         }
 
+        /// <summary>
+        /// Derived test classes can override this method to provide a customized Dynamo model.
+        /// </summary>
+        protected virtual DynamoModel CreateModel(DynamoModel.IStartConfiguration configuration)
+        {
+            return DynamoModel.Start(configuration);
+        }
+
+        /// <summary>
+        /// Derived test classes can override this method to provide a customized view model configuration.
+        /// </summary>
+        protected virtual DynamoViewModel.StartConfiguration CreateViewModelStartConfiguration()
+        {
+            return new DynamoViewModel.StartConfiguration()
+            {
+                DynamoModel = Model,
+                Watch3DViewModel = HelixWatch3DViewModel.TryCreateHelixWatch3DViewModel(
+                    null,
+                    new Watch3DViewModelStartupParams(Model),
+                    Model.Logger)
+            };
+        }
+
         private async void Model_EvaluationCompleted(object sender, EvaluationCompletedEventArgs e)
         {
             DispatcherUtil.DoEvents();

src/VisualizationTests/Preview3DMemoryOutageTest.cs

@@ -0,0 +1,74 @@
+using Dynamo.Models;
+using Dynamo.ViewModels;
+using Dynamo.Wpf.ViewModels.Watch3D;
+using DynamoCoreWpfTests.Utility;
+using HelixToolkit.Wpf.SharpDX;
+using Moq;
+using Moq.Protected;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace WpfVisualizationTests
+{
+    [TestFixture]
+    public class Preview3DMemoryOutageTest : VisualizationTest
+    {
+        private Mock<DynamoModel> modelMock;
+
+        /// <summary>
+        /// Creates a mock DynamoModel that does not show a task dialog but instead sets its call
+        /// as a verifiable call.
+        /// </summary>
+        /// <param name="configuration">Default DynamoModel configuration</param>
+        /// <returns>Mock DynamoModel</returns>
+        protected override DynamoModel CreateModel(DynamoModel.IStartConfiguration configuration)
+        {
+            modelMock = new Mock<DynamoModel>(configuration)
+            {
+                CallBase = true
+            };
+            modelMock.Setup(m => m.OnRequestTaskDialog(It.IsAny<object>(), It.IsAny<TaskDialogEventArgs>()))
+                .Callback(() => { }) // Prevent dialog from blocking the test
+                .Verifiable();
+
+            return modelMock.Object;
+        }
+
+        /// <summary>
+        /// Sets up a mock HelixWatch3DViewModel which will throw OutOfMemoryException when rendering.
+        /// </summary>
+        /// <returns>DynamoViewModel configuration referencing the mock 3D preview</returns>
+        protected override DynamoViewModel.StartConfiguration CreateViewModelStartConfiguration()
+        {
+            var watch3DMock = new Mock<HelixWatch3DViewModel>(null, new Watch3DViewModelStartupParams(Model))
+            {
+                CallBase = true
+            };
+            watch3DMock.Protected().Setup("OnRequestAttachToScene", ItExpr.IsAny<Model3D>()).Throws<OutOfMemoryException>();
+            return new DynamoViewModel.StartConfiguration()
+            {
+                Watch3DViewModel = watch3DMock.Object
+            };
+        }
+
+        /// <summary>
+        /// Opens any file that produces geometry and checks that the 3D preview is disabled after the error
+        /// and also that a dialog is shown.
+        /// </summary>
+        [Test]
+        public void HandlesRenderMemoryOutageGracefully()
+        {
+            Assert.Greater(ViewModel.Watch3DViewModels.Count(), 0);
+            Assert.True(ViewModel.Watch3DViewModels.All(w => w.Active));
+
+            OpenVisualizationTest("ASM_cuboid.dyn");
+
+            Assert.True(ViewModel.Watch3DViewModels.All(w => !w.Active));
+            modelMock.Verify();
+        }
+    }
+}

src/VisualizationTests/WpfVisualizationTests.csproj

@@ -59,6 +59,9 @@
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\..\extern\prism\Microsoft.Practices.Prism.dll</HintPath>
     </Reference>
+    <Reference Include="Moq, Version=4.2.1507.118, Culture=neutral, PublicKeyToken=69f491c39445e920, processorArchitecture=MSIL">
+      <HintPath>..\packages\Moq.4.2.1507.0118\lib\net40\Moq.dll</HintPath>
+    </Reference>
     <Reference Include="nunit.framework, Version=2.6.3.13283, Culture=neutral, PublicKeyToken=96d09a1eb7f44a77, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\..\extern\NUnit\nunit.framework.dll</HintPath>
@@ -115,6 +118,7 @@
     <Compile Include="HelixRenderPackageTests.cs" />
     <Compile Include="HelixWatch3DViewModelTests.cs" />
     <Compile Include="HelixWatch3DViewModelUnitTests.cs" />
+    <Compile Include="Preview3DMemoryOutageTest.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Watch3DViewModelTests.cs" />
   </ItemGroup>

src/VisualizationTests/packages.config

@@ -4,6 +4,7 @@
   <package id="HelixToolkit" version="2015.1.629" targetFramework="net45" />
   <package id="HelixToolkit.Wpf" version="2015.1.629" targetFramework="net45" />
   <package id="HelixToolkit.Wpf.SharpDX" version="2015.1.629" targetFramework="net45" />
+  <package id="Moq" version="4.2.1507.0118" targetFramework="net47" />
   <package id="SharpDX" version="2.6.3" targetFramework="net45" />
   <package id="SharpDX.D3DCompiler" version="2.6.3" targetFramework="net45" />
   <package id="SharpDX.Direct2D1" version="2.6.3" targetFramework="net45" />