Handle memory outage on render (cherry pick)

src/DynamoCore/Models/DynamoModel.cs

@@ -2109,6 +2109,27 @@ private void DisplayInvalidInputSymbolWarning()
             OnRequestTaskDialog(null, args);
         }
 
+        internal event VoidHandler Preview3DOutage;
+        private void OnPreview3DOutage()
+        {
+            if (Preview3DOutage != null)
+            {
+                Preview3DOutage();
+            }
+        }
+
+        internal void Report3DPreviewOutage(string summary, string description)
+        {
+            OnPreview3DOutage();
+
+            const string imageUri = "/DynamoCoreWpf;component/UI/Images/task_dialog_future_file.png";
+            var args = new TaskDialogEventArgs(
+               new Uri(imageUri, UriKind.Relative),
+               Resources.Preview3DOutageTitle, summary, description);
+
+            OnRequestTaskDialog(null, args);
+        }
+
         /// <summary>
         ///     Remove a workspace from the dynamo model.
         /// </summary>

src/DynamoCore/Models/DynamoModelEvents.cs

@@ -283,7 +283,7 @@ public void OnRequestsCrashPrompt(object sender, CrashPromptArgs args)
 
         internal delegate void TaskDialogHandler(object sender, TaskDialogEventArgs e);
         internal event TaskDialogHandler RequestTaskDialog;
-        internal void OnRequestTaskDialog(object sender, TaskDialogEventArgs args)
+        internal virtual void OnRequestTaskDialog(object sender, TaskDialogEventArgs args)
         {
             if (RequestTaskDialog != null)
                 RequestTaskDialog(sender, args);

src/DynamoCore/Properties/Resources.en-US.resx

@@ -709,4 +709,7 @@ Restart Dynamo to complete the uninstall.</value>
   <data name="FunctionDefinitionOverwrittenMessage" xml:space="preserve">
     <value>Attempting to load customNode {0} loaded by package {1}, but a previous definition named {2} exists with no associated package. The new customNode definition has been loaded, but Dynamo may be in an unstable state, please avoid loading multiple custom nodes with the id.</value>
   </data>
+  <data name="Preview3DOutageTitle" xml:space="preserve">
+    <value>3D preview has been deactivated</value>
+  </data>
 </root>

src/DynamoCore/Properties/Resources.resx

@@ -709,4 +709,7 @@ Restart Dynamo to complete the uninstall.</value>
   <data name="FunctionDefinitionOverwrittenMessage" xml:space="preserve">
     <value>Attempting to load customNode {0} loaded by package {1}, but a previous definition named {2} exists with no associated package. The new customNode definition has been loaded, but Dynamo may be in an unstable state, please avoid loading multiple custom nodes with the id.</value>
   </data>
+  <data name="Preview3DOutageTitle" xml:space="preserve">
+    <value>3D preview has been deactivated</value>
+  </data>
 </root>

src/DynamoCoreWpf/Properties/Resources.en-US.resx

@@ -2242,4 +2242,10 @@ Uninstall the following packages: {0}?</value>
   <data name="NodeTooltipOriginalName" xml:space="preserve">
     <value>Original node name: </value>
   </data>
+  <data name="RenderingMemoryOutageDescription" xml:space="preserve">
+    <value>Please check if you intended to render this amount of geometry, and consider turning off the preview of other nodes within your graph, lowering the amount of Geometry you wish to render, or turning down the render precision.</value>
+  </data>
+  <data name="RenderingMemoryOutageSummary" xml:space="preserve">
+    <value>Dynamo has run out of memory trying to render your geometry. The geometry preview has been disabled.</value>
+  </data>
 </root>

src/DynamoCoreWpf/Properties/Resources.resx

@@ -2242,4 +2242,10 @@ Uninstall the following packages: {0}?</value>
   <data name="NodeTooltipOriginalName" xml:space="preserve">
     <value>Original node name: </value>
   </data>
+  <data name="RenderingMemoryOutageDescription" xml:space="preserve">
+    <value>Please check if you intended to render this amount of geometry, and consider turning off the preview of other nodes within your graph, lowering the amount of Geometry you wish to render, or turning down the render precision.</value>
+  </data>
+  <data name="RenderingMemoryOutageSummary" xml:space="preserve">
+    <value>Dynamo has run out of memory trying to render your geometry. The geometry preview has been disabled.</value>
+  </data>
 </root>

src/DynamoCoreWpf/ViewModels/Core/DynamoViewModel.cs

@@ -724,12 +724,14 @@ private void SubscribeModelUiEvents()
         {
             model.RequestBugReport += ReportABug;
             model.RequestDownloadDynamo += DownloadDynamo;
+            model.Preview3DOutage += Disable3DPreview;
         }
 
         private void UnsubscribeModelUiEvents()
         {
             model.RequestBugReport -= ReportABug;
             model.RequestDownloadDynamo -= DownloadDynamo;
+            model.Preview3DOutage -= Disable3DPreview;
         }
 
         private void SubscribeModelCleaningUpEvent()
@@ -868,6 +870,14 @@ internal static void DownloadDynamo()
             Process.Start(new ProcessStartInfo("explorer.exe", Configurations.DynamoDownloadLink));
         }
 
+        private void Disable3DPreview()
+        {
+            foreach (var item in Watch3DViewModels)
+            {
+                item.Active = false;
+            }
+        }
+
         internal bool CanReportABug(object parameter)
         {
             return true;

src/DynamoCoreWpf/ViewModels/Watch3D/HelixWatch3DViewModel.cs

@@ -21,7 +21,9 @@
 using Dynamo.Graph.Nodes.CustomNodes;
 using Dynamo.Graph.Workspaces;
 using Dynamo.Logging;
+using Dynamo.Models;
 using Dynamo.Selection;
+using Dynamo.UI.Prompts;
 using Dynamo.Utilities;
 using Dynamo.ViewModels;
 using Dynamo.Visualization;
@@ -859,7 +861,17 @@ public override void GenerateViewGeometryFromRenderPackagesAndRequestUpdate(Rend
             var meshPackages = packages.Cast<HelixRenderPackage>().Where(rp => rp.MeshVertexCount % 3 == 0);
 
             RemoveGeometryForUpdatedPackages(meshPackages);
-            AggregateRenderPackages(meshPackages);
+            try
+            {
+                AggregateRenderPackages(meshPackages);
+            }
+            catch (OutOfMemoryException)
+            {
+                // This can happen when the amount of packages to render is too large
+                string summary = Resources.RenderingMemoryOutageSummary;
+                var description = Resources.RenderingMemoryOutageDescription;
+                (dynamoModel as DynamoModel).Report3DPreviewOutage(summary, description);
+            }
 #if DEBUG
             renderTimer.Stop();
             Debug.WriteLine(string.Format("RENDER: {0} ellapsed for compiling assets for rendering.", renderTimer.Elapsed));
@@ -1635,7 +1647,7 @@ private void ToggleTreeViewItemHighlighting(string path, bool isSelected)
         /// attaches them to the visual scene.
         /// </summary>
         /// <param name="packages">An <see cref="IEnumerable"/> of <see cref="HelixRenderPackage"/>.</param>
-        private void AggregateRenderPackages(IEnumerable<HelixRenderPackage> packages)
+        protected virtual void AggregateRenderPackages(IEnumerable<HelixRenderPackage> packages)
         {
             IEnumerable<string> customNodeIdents = null;
             if (InCustomNode())

src/VisualizationTests/HelixWatch3DViewModelTests.cs

@@ -86,28 +86,21 @@ protected override void StartDynamo(TestSessionConfiguration testConfig)
                 }
             }
 
-            Model = DynamoModel.Start(
-                new DynamoModel.DefaultStartConfiguration()
-                {
-                    StartInTestMode = true,
-                    PathResolver = pathResolver,
-                    GeometryFactoryPath = preloader.GeometryFactoryPath,
-                    UpdateManager = this.UpdateManager,
-                    ProcessMode = TaskProcessMode.Synchronous
-                });
+            Model = CreateModel(new DynamoModel.DefaultStartConfiguration()
+            {
+                StartInTestMode = true,
+                PathResolver = pathResolver,
+                GeometryFactoryPath = preloader.GeometryFactoryPath,
+                UpdateManager = this.UpdateManager,
+                ProcessMode = TaskProcessMode.Synchronous
+            });
 
             Model.EvaluationCompleted += Model_EvaluationCompleted;
 
-            ViewModel = DynamoViewModel.Start(
-                new DynamoViewModel.StartConfiguration()
-                {
-                    DynamoModel = Model,
-                    Watch3DViewModel = 
-                        HelixWatch3DViewModel.TryCreateHelixWatch3DViewModel(
-                            null,
-                            new Watch3DViewModelStartupParams(Model), 
-                            Model.Logger)
-                });
+            var vmConfig = CreateViewModelStartConfiguration();
+            vmConfig.DynamoModel = Model;
+
+            ViewModel = DynamoViewModel.Start(vmConfig);
 
             //create the view
             View = new DynamoView(ViewModel);
@@ -116,6 +109,29 @@ protected override void StartDynamo(TestSessionConfiguration testConfig)
             SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
         }
 
+        /// <summary>
+        /// Derived test classes can override this method to provide a customized Dynamo model.
+        /// </summary>
+        protected virtual DynamoModel CreateModel(DynamoModel.IStartConfiguration configuration)
+        {
+            return DynamoModel.Start(configuration);
+        }
+
+        /// <summary>
+        /// Derived test classes can override this method to provide a customized view model configuration.
+        /// </summary>
+        protected virtual DynamoViewModel.StartConfiguration CreateViewModelStartConfiguration()
+        {
+            return new DynamoViewModel.StartConfiguration()
+            {
+                DynamoModel = Model,
+                Watch3DViewModel = HelixWatch3DViewModel.TryCreateHelixWatch3DViewModel(
+                    null,
+                    new Watch3DViewModelStartupParams(Model),
+                    Model.Logger)
+            };
+        }
+
         private async void Model_EvaluationCompleted(object sender, EvaluationCompletedEventArgs e)
         {
             DispatcherUtil.DoEvents();

src/VisualizationTests/Preview3DMemoryOutageTest.cs

@@ -0,0 +1,75 @@
+using Dynamo.Models;
+using Dynamo.ViewModels;
+using Dynamo.Wpf.Rendering;
+using Dynamo.Wpf.ViewModels.Watch3D;
+using DynamoCoreWpfTests.Utility;
+using HelixToolkit.Wpf.SharpDX;
+using Moq;
+using Moq.Protected;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace WpfVisualizationTests
+{
+    [TestFixture]
+    public class Preview3DMemoryOutageTest : VisualizationTest
+    {
+        private Mock<DynamoModel> modelMock;
+
+        /// <summary>
+        /// Creates a mock DynamoModel that does not show a task dialog but instead sets its call
+        /// as a verifiable call.
+        /// </summary>
+        /// <param name="configuration">Default DynamoModel configuration</param>
+        /// <returns>Mock DynamoModel</returns>
+        protected override DynamoModel CreateModel(DynamoModel.IStartConfiguration configuration)
+        {
+            modelMock = new Mock<DynamoModel>(configuration)
+            {
+                CallBase = true
+            };
+            modelMock.Setup(m => m.OnRequestTaskDialog(It.IsAny<object>(), It.IsAny<TaskDialogEventArgs>()))
+                .Callback(() => { }) // Prevent dialog from blocking the test
+                .Verifiable();
+
+            return modelMock.Object;
+        }
+
+        /// <summary>
+        /// Sets up a mock HelixWatch3DViewModel which will throw OutOfMemoryException when rendering.
+        /// </summary>
+        /// <returns>DynamoViewModel configuration referencing the mock 3D preview</returns>
+        protected override DynamoViewModel.StartConfiguration CreateViewModelStartConfiguration()
+        {
+            var watch3DMock = new Mock<HelixWatch3DViewModel>(null, new Watch3DViewModelStartupParams(Model))
+            {
+                CallBase = true
+            };
+            watch3DMock.Protected().Setup("AggregateRenderPackages", ItExpr.IsAny<IEnumerable<HelixRenderPackage>>()).Throws<OutOfMemoryException>();
+            return new DynamoViewModel.StartConfiguration()
+            {
+                Watch3DViewModel = watch3DMock.Object
+            };
+        }
+
+        /// <summary>
+        /// Opens any file that produces geometry and checks that the 3D preview is disabled after the error
+        /// and also that a dialog is shown.
+        /// </summary>
+        [Test]
+        public void HandlesRenderMemoryOutageGracefully()
+        {
+            Assert.Greater(ViewModel.Watch3DViewModels.Count(), 0);
+            Assert.True(ViewModel.Watch3DViewModels.All(w => w.Active));
+
+            OpenVisualizationTest("ASM_cuboid.dyn");
+
+            Assert.True(ViewModel.Watch3DViewModels.All(w => !w.Active));
+            modelMock.Verify();
+        }
+    }
+}

src/VisualizationTests/WpfVisualizationTests.csproj

@@ -53,6 +53,9 @@
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\..\extern\prism\Microsoft.Practices.Prism.dll</HintPath>
     </Reference>
+    <Reference Include="Moq, Version=4.2.1507.118, Culture=neutral, PublicKeyToken=69f491c39445e920, processorArchitecture=MSIL">
+      <HintPath>..\packages\Moq.4.2.1507.0118\lib\net40\Moq.dll</HintPath>
+    </Reference>
     <Reference Include="nunit.framework, Version=2.6.3.13283, Culture=neutral, PublicKeyToken=96d09a1eb7f44a77, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\..\extern\NUnit\nunit.framework.dll</HintPath>
@@ -105,6 +108,7 @@
     <Compile Include="HelixWatch3dViewModelPrimitiveTests.cs" />
     <Compile Include="HelixWatch3DViewModelTests.cs" />
     <Compile Include="HelixWatch3DViewModelUnitTests.cs" />
+    <Compile Include="Preview3DMemoryOutageTest.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Watch3DViewModelTests.cs" />
   </ItemGroup>

src/VisualizationTests/packages.config

@@ -4,6 +4,7 @@
   <package id="HelixToolkit" version="2.9.0" targetFramework="net47" />
   <package id="HelixToolkit.Wpf" version="2.9.0" targetFramework="net47" />
   <package id="HelixToolkit.Wpf.SharpDX" version="2.9.0" targetFramework="net47" />
+  <package id="Moq" version="4.2.1507.0118" targetFramework="net47" />
   <package id="SharpDX" version="4.2.0" targetFramework="net47" />
   <package id="SharpDX.D3DCompiler" version="4.2.0" targetFramework="net47" />
   <package id="SharpDX.Direct2D1" version="4.2.0" targetFramework="net47" />