From 424177f436cd453eda35d7f53c174ad32345044c Mon Sep 17 00:00:00 2001 From: Loren Kuich Date: Wed, 18 Dec 2019 02:15:39 -0800 Subject: [PATCH] Fixed memory leak, added CumulativeConfidence function, fixed prediction result sorting --- Assets/Coach-ML/Coach.cs | 62 +++++++++++++++++++++++---------- Assets/Scenes/SampleScene.unity | 18 +++++----- 2 files changed, 52 insertions(+), 28 deletions(-) diff --git a/Assets/Coach-ML/Coach.cs b/Assets/Coach-ML/Coach.cs index b51e98f..d7b3224 100644 --- a/Assets/Coach-ML/Coach.cs +++ b/Assets/Coach-ML/Coach.cs @@ -7,7 +7,6 @@ using Barracuda; using UnityEngine.Networking; using System.Net; -using UnityEngine.Serialization; namespace Coach { @@ -100,25 +99,27 @@ public static class ImageUtil private static Texture2D Scale(this Texture2D tex, int width, int height, FilterMode mode = FilterMode.Trilinear) { Rect texR = new Rect(0, 0, width, height); - GpuScale(tex, width, height, mode); + RenderTexture rt = GpuScale(tex, width, height, mode); // Update new texture tex.Resize(width, height); tex.ReadPixels(texR, 0, 0, true); tex.Apply(true); + RenderTexture.ReleaseTemporary(rt); + return tex; } // Internal unility that renders the source texture into the RTT - the scaling method itself. - private static void GpuScale(Texture2D src, int width, int height, FilterMode fmode) + private static RenderTexture GpuScale(Texture2D src, int width, int height, FilterMode fmode) { //We need the source texture in VRAM because we render with it src.filterMode = fmode; src.Apply(true); //Using RTT for best quality and performance. Thanks, Unity 5 - RenderTexture rtt = new RenderTexture(width, height, 32); + var rtt = RenderTexture.GetTemporary(width, height, 32); //Set the RTT in order to render to it Graphics.SetRenderTarget(rtt); @@ -129,6 +130,8 @@ private static void GpuScale(Texture2D src, int width, int height, FilterMode fm //Then clear & draw the texture to fill the entire RTT. GL.Clear(true, true, new Color(0, 0, 0, 0)); Graphics.DrawTexture(new Rect(0, 0, 1, 1), src); + + return rtt; } private static Tensor ToTensor(this Texture2D tex, ImageDims dims) @@ -180,30 +183,29 @@ public class CoachResult /// //Unsorted prediction results /// - public List Results { get; private set; } + public LabelProbability[] Results { get; private set; } /// //Sorted prediction results, descending in Confidence /// - // public List SortedResults { get; private set; } + public LabelProbability[] SortedResults { get; private set; } public CoachResult(string[] labels, Tensor output) { - Debug.LogWarning(output); - Results = new List(); + Results = new LabelProbability[labels.Length]; for (var i = 0; i < labels.Length; i++) { string label = labels[i]; float probability = output[i]; - Results.Add(new LabelProbability() + Results[i] = new LabelProbability() { Label = label, Confidence = probability - }); + }; } - // SortedResults = Results.OrderByDescending(r => r.Confidence).ToList(); + SortedResults = Results.OrderByDescending(r => r.Confidence).ToArray(); output.Dispose(); } @@ -213,7 +215,7 @@ public CoachResult(string[] labels, Tensor output) /// public LabelProbability Best() { - return Results.FirstOrDefault(); + return SortedResults.FirstOrDefault(); } /// @@ -221,7 +223,19 @@ public LabelProbability Best() /// public LabelProbability Worst() { - return Results.LastOrDefault(); + return SortedResults.LastOrDefault(); + } + } + + public struct CumulativeConfidenceResult + { + public float Threshhold; + public CoachResult LastResult; + public float CumulativeConfidence; + + public bool IsPassedThreshold() + { + return CumulativeConfidence >= Threshhold; } } @@ -314,6 +328,18 @@ public CoachResult Predict(byte[] image, string inputName = "input", string outp return GetModelResult(imageTensor, inputName, outputName); } + public void CumulativeConfidence(Texture2D image, float threshhold, ref CumulativeConfidenceResult result) + { + var prediction = Predict(image); + result.LastResult = prediction; + result.Threshhold = threshhold; + + if (result.LastResult.Best().Label != prediction.Best().Label) + result.CumulativeConfidence = 0; + else if (result.CumulativeConfidence <= threshhold) + result.CumulativeConfidence += prediction.Best().Confidence; + } + private CoachResult GetModelResult(Tensor imageTensor, string inputName = "input", string outputName = "output") { var inputs = new Dictionary(); @@ -338,10 +364,7 @@ public void CleanUp() [Serializable] public class StatusDef { - [FormerlySerializedAs("short")] public string _short; - - [FormerlySerializedAs("long")] public string _long; } @@ -349,7 +372,7 @@ public class StatusDef public class ModelDef { public string name; - public StatusDef status; + // public StatusDef status; public int version; public string module; public string[] labels; @@ -456,7 +479,10 @@ public async Task CacheModel(string modelName, string path = ".", bool skipMatch if (!IsAuthenticated()) throw new Exception("User is not authenticated"); - ModelDef model = this.Profile.models.Single(m => m.name == modelName); + ModelDef model = this.Profile.models.SingleOrDefault(m => m.name == modelName); + if (model == null) + throw new Exception($"{modelName} is an invalid model"); + int version = model.version; string modelDir = Path.Combine(path, modelName); diff --git a/Assets/Scenes/SampleScene.unity b/Assets/Scenes/SampleScene.unity index 1443d58..b5cfaca 100644 --- a/Assets/Scenes/SampleScene.unity +++ b/Assets/Scenes/SampleScene.unity @@ -94,8 +94,9 @@ LightmapSettings: m_PVRFilteringAtrousPositionSigmaDirect: 0.5 m_PVRFilteringAtrousPositionSigmaIndirect: 2 m_PVRFilteringAtrousPositionSigmaAO: 1 - m_ShowResolutionOverlay: 1 m_ExportTrainingData: 0 + m_TrainingDataDestination: TrainingData + m_LightProbeSampleCountMultiplier: 4 m_LightingDataAsset: {fileID: 0} m_UseShadowmask: 1 --- !u!196 &4 @@ -170,7 +171,7 @@ MonoBehaviour: m_GameObject: {fileID: 211247794} m_Enabled: 1 m_EditorHideFlags: 0 - m_Script: {fileID: 1301386320, guid: f70555f144d8491a825f0804e09c671c, type: 3} + m_Script: {fileID: 11500000, guid: dc42784cf147c0c48a680349fa168899, type: 3} m_Name: m_EditorClassIdentifier: m_IgnoreReversedGraphics: 1 @@ -187,7 +188,7 @@ MonoBehaviour: m_GameObject: {fileID: 211247794} m_Enabled: 1 m_EditorHideFlags: 0 - m_Script: {fileID: 1980459831, guid: f70555f144d8491a825f0804e09c671c, type: 3} + m_Script: {fileID: 11500000, guid: 0cd44c1031e13a943bb63640046fad76, type: 3} m_Name: m_EditorClassIdentifier: m_UiScaleMode: 1 @@ -230,10 +231,9 @@ MonoBehaviour: m_GameObject: {fileID: 211247794} m_Enabled: 1 m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 0c0359307f108214fa7f019386a8b150, type: 3} + m_Script: {fileID: 11500000, guid: ac400c37f901663459a68cbfc8c1dfc0, type: 3} m_Name: m_EditorClassIdentifier: - Image: {fileID: 211247801} --- !u!222 &211247800 CanvasRenderer: m_ObjectHideFlags: 0 @@ -251,7 +251,7 @@ MonoBehaviour: m_GameObject: {fileID: 211247794} m_Enabled: 1 m_EditorHideFlags: 0 - m_Script: {fileID: -98529514, guid: f70555f144d8491a825f0804e09c671c, type: 3} + m_Script: {fileID: 11500000, guid: 1344c3c82d62a2a41a3576d8abb8e3ea, type: 3} m_Name: m_EditorClassIdentifier: m_Material: {fileID: 0} @@ -260,8 +260,6 @@ MonoBehaviour: m_OnCullStateChanged: m_PersistentCalls: m_Calls: [] - m_TypeName: UnityEngine.UI.MaskableGraphic+CullStateChangedEvent, UnityEngine.UI, - Version=1.0.0.0, Culture=neutral, PublicKeyToken=null m_Texture: {fileID: 2800000, guid: 835e52329d78a3d44b7a1df76348098b, type: 3} m_UVRect: serializedVersion: 2 @@ -380,7 +378,7 @@ MonoBehaviour: m_GameObject: {fileID: 632981135} m_Enabled: 1 m_EditorHideFlags: 0 - m_Script: {fileID: 1077351063, guid: f70555f144d8491a825f0804e09c671c, type: 3} + m_Script: {fileID: 11500000, guid: 4f231c4fb786f3946a6b90b886c48677, type: 3} m_Name: m_EditorClassIdentifier: m_HorizontalAxis: Horizontal @@ -399,7 +397,7 @@ MonoBehaviour: m_GameObject: {fileID: 632981135} m_Enabled: 1 m_EditorHideFlags: 0 - m_Script: {fileID: -619905303, guid: f70555f144d8491a825f0804e09c671c, type: 3} + m_Script: {fileID: 11500000, guid: 76c392e42b5098c458856cdf6ecaaaa1, type: 3} m_Name: m_EditorClassIdentifier: m_FirstSelected: {fileID: 0}