Feature/reviewer improv 2

CHANGELOG.md

@@ -11,8 +11,15 @@ Each entry in the changelog should use the following template:
 ```
 
 ## [Unreleased]
+### Added
+
+- IsIdentifiableReviewer shows progress when loading large files (with cancellation support)
+- IsIdentifiableReviewer groups outstanding failures by column
+- [#616](https://github.com/SMI/SmiServices/pull/616) by `rkm`. Check for clobbered files during package build
+
+### Fixed
 
--   [#616](https://github.com/SMI/SmiServices/pull/616) by `rkm`. Check for clobbered files during package build
+- Fixed rules being flagged as 'Identical' when classifying different input columns in IsIdentifiableReviewer
 
 ## [1.15.1] 2021-02-17
 

src/applications/IsIdentifiableReviewer/MainWindow.cs

@@ -4,6 +4,8 @@
 using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
 using IsIdentifiableReviewer.Out;
 using IsIdentifiableReviewer.Views;
 using Microservices.IsIdentifiable.Reporting;
@@ -386,16 +388,43 @@ private void OpenReport(string path, Action<Exception> exceptionHandler)
             if ( path == null)
                 return;
 
-            try
-            {
-                CurrentReport = new ReportReader(new FileInfo(path));
-                SetupToShow(CurrentReport.Failures.FirstOrDefault());
-                Next();
-            }
-            catch (Exception e)
-            {
-                exceptionHandler(e);
-            }
+            var cts = new CancellationTokenSource();
+
+            var btn = new Button("Cancel");
+            Action cancelFunc = ()=>{cts.Cancel();};
+            Action closeFunc = ()=>{Application.RequestStop();};
+            btn.Clicked += cancelFunc;
+
+            var dlg = new Dialog("Opening",MainWindow.DlgWidth,5,btn);
+            var rows = new Label($"Loaded: 0 rows"){
+                Width = Dim.Fill() };
+            dlg.Add(rows);
+
+            Task.Run(()=>{
+                    try
+                    {
+                        CurrentReport = new ReportReader(new FileInfo(path),(s)=>
+                        rows.Text = $"Loaded: {s:N0} rows",cts.Token);
+                        SetupToShow(CurrentReport.Failures.FirstOrDefault());
+                        Next();
+                    }
+                    catch (Exception e)
+                    {
+                        exceptionHandler(e);
+                    } 
+
+                }
+            ).ContinueWith((t)=>{
+
+                    btn.Clicked -= cancelFunc;
+                    btn.Text = "Done";
+                    btn.Clicked += closeFunc;
+                    dlg.SetNeedsDisplay();
+
+                    cts.Dispose();
+            });
+
+            Application.Run(dlg);
         }
 
         public void ShowMessage(string title, string body)

src/applications/IsIdentifiableReviewer/ReportReader.cs

@@ -3,6 +3,7 @@
 using System.IO;
 using System.Linq;
 using System.Text;
+using System.Threading;
 using Microservices.IsIdentifiable.Reporting;
 using Microservices.IsIdentifiable.Reporting.Reports;
 
@@ -22,6 +23,11 @@ public ReportReader(FileInfo csvFile)
             var report = new FailureStoreReport("", 0);
             Failures = report.Deserialize(csvFile).ToArray();
         }
+        public ReportReader(FileInfo csvFile,Action<int> loadedRows, CancellationToken token)
+        {
+            var report = new FailureStoreReport("", 0);
+            Failures = report.Deserialize(csvFile,loadedRows,token).ToArray();
+        }
 
         public bool Next()
         {

src/applications/IsIdentifiableReviewer/Views/FailureGroupingNode.cs

@@ -0,0 +1,20 @@
+using System.Collections.Generic;
+using System.Linq;
+using Terminal.Gui;
+
+namespace IsIdentifiableReviewer.Views
+{
+    internal class FailureGroupingNode : TreeNodeWithCount
+    {
+        public string Group {get;}
+        public OutstandingFailureNode[] Failures {get;}
+
+        public FailureGroupingNode(string group, OutstandingFailureNode[] failures):base(group)
+        {
+            this.Group = group;
+            this.Failures = failures;
+
+            Children = failures.OrderByDescending(f=>f.NumberOfTimesReported).Cast<ITreeNode>().ToList();
+        }
+    }
+}

src/applications/IsIdentifiableReviewer/Views/RulesView.cs

@@ -6,6 +6,8 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
 using Terminal.Gui;
 
 namespace IsIdentifiableReviewer.Views
@@ -193,11 +195,53 @@ private void EvaluateRuleCoverage()
 
             _treeView.AddObjects(new []{ colliding,ignore,update,outstanding});
 
+            var cts = new CancellationTokenSource();
+
+            var btn = new Button("Cancel");
+            Action cancelFunc = ()=>{cts.Cancel();};
+            Action closeFunc = ()=>{Application.RequestStop();};
+            btn.Clicked += cancelFunc;
+
+            var dlg = new Dialog("Evaluating",MainWindow.DlgWidth,6,btn);
+
+            var stage = new Label("Evaluating Failures"){Width = Dim.Fill(), X = 0,Y = 0};
+            var progress = new ProgressBar(){Height= 2,Width = Dim.Fill(), X=0,Y = 1};
+            var textProgress = new Label("0/0"){TextAlignment = TextAlignment.Right ,Width = Dim.Fill(), X=0,Y = 2};
+
+            dlg.Add(stage);
+            dlg.Add(progress);
+            dlg.Add(textProgress);
+
+            Task.Run(()=>{
+                EvaluateRuleCoverageAsync(stage,progress,textProgress,cts.Token,colliding,ignore,update,outstanding);
+                },cts.Token).ContinueWith((t)=>{
+
+                    btn.Clicked -= cancelFunc;
+                    btn.Text = "Done";
+                    btn.Clicked += closeFunc;
+                    dlg.SetNeedsDisplay();
+
+                    cts.Dispose();
+            });;
+
+            Application.Run(dlg);
+        }
+
+        private void EvaluateRuleCoverageAsync(Label stage,ProgressBar progress, Label textProgress, CancellationToken token,TreeNodeWithCount colliding,TreeNodeWithCount ignore,TreeNodeWithCount update,TreeNodeWithCount outstanding)
+        {
             Dictionary<IsIdentifiableRule,int> rulesUsed = new Dictionary<IsIdentifiableRule, int>();
             Dictionary<string,OutstandingFailureNode> outstandingFailures = new Dictionary<string, OutstandingFailureNode>();
 
+            int done = 0;
+            var max = CurrentReport.Failures.Count();
+
             foreach(Failure f in CurrentReport.Failures)
             {
+                done++;
+                token.ThrowIfCancellationRequested();
+                if(done % 1000 == 0)
+                    SetProgress(progress,textProgress,done,max);
+
                 var ignoreRule = Ignorer.Rules.FirstOrDefault(r=>r.Apply(f.ProblemField,f.ProblemValue, out _) != RuleAction.None);
                 var updateRule = Updater.Rules.FirstOrDefault(r=>r.Apply(f.ProblemField,f.ProblemValue, out _) != RuleAction.None);
 
@@ -230,14 +274,59 @@ private void EvaluateRuleCoverage()
                         outstandingFailures[f.ProblemValue].NumberOfTimesReported++;
                 }
             }
+
+            SetProgress(progress,textProgress,done,max);
+
+            var ignoreRulesUsed = rulesUsed.Where(r=>r.Key.Action == RuleAction.Ignore).ToList();
+            stage.Text = "Evaluating Ignore Rules Used";
+            max = ignoreRulesUsed.Count();
+            done = 0;
+
+            foreach(var used in ignoreRulesUsed.OrderByDescending(kvp => kvp.Value))
+            {
+                done++;
+                token.ThrowIfCancellationRequested();
+                if(done % 1000 == 0)
+                    SetProgress(progress,textProgress,done,max);
 
-            foreach(var used in rulesUsed.Where(r=>r.Key.Action == RuleAction.Ignore).OrderByDescending(kvp=>kvp.Value))
                 ignore.Children.Add(new RuleUsageNode(Ignorer,used.Key,used.Value));
+            }
+
+            SetProgress(progress,textProgress,done,max);
+
+
+            stage.Text = "Evaluating Update Rules Used";
+            var updateRulesUsed = rulesUsed.Where(r=>r.Key.Action == RuleAction.Report).ToList();
+            max = updateRulesUsed.Count();
+            done = 0;
+
+            foreach(var used in updateRulesUsed.OrderByDescending(kvp=>kvp.Value)){
+                done++;
+
+                token.ThrowIfCancellationRequested();
+                if(done % 1000 == 0)
+                    SetProgress(progress,textProgress,done,max);
+
+                update.Children.Add(new RuleUsageNode(Updater,used.Key,used.Value)); 
+            }
 
-            foreach(var used in rulesUsed.Where(r=>r.Key.Action == RuleAction.Report).OrderByDescending(kvp=>kvp.Value))
-                update.Children.Add(new RuleUsageNode(Updater,used.Key,used.Value));
+            SetProgress(progress,textProgress,done,max);
+
+            stage.Text = "Evaluating Outstanding Failures";
 
-            outstanding.Children = outstandingFailures.Select(kvp=>kvp.Value).OrderByDescending(v=>v.NumberOfTimesReported).Cast<ITreeNode>().ToList();
+            outstanding.Children = 
+                outstandingFailures.Select(f=>f.Value).GroupBy(f=>f.Failure.ProblemField)               
+                    .Select(g=>new FailureGroupingNode(g.Key,g.ToArray()))
+                    .OrderByDescending(v=>v.Failures.Sum(f=>f.NumberOfTimesReported))
+                    .Cast<ITreeNode>()
+                    .ToList();
+        }
+
+        private void SetProgress(ProgressBar pb, Label tp, int done, int max)
+        {
+            if(max != 0)
+                pb.Fraction = done/(float)max;
+            tp.Text = $"{done:N0}/{max:N0}";
         }
 
         private void AddDuplicatesToTree(List<IsIdentifiableRule> allRules)
@@ -256,7 +345,11 @@ public IEnumerable<DuplicateRulesNode> GetDuplicates(IList<IsIdentifiableRule> r
             {
                 var duplicateRules = dup.ToArray();
 
-                if(duplicateRules.Length > 1)
+                if(
+                    // Multiple rules with same pattern
+                    duplicateRules.Length > 1 &&
+                    // targeting the same column
+                    duplicateRules.Select(r=>r.IfColumn).Distinct().Count() == 1)
                 {
                     yield return new DuplicateRulesNode(dup.Key,duplicateRules);
                 }

src/applications/IsIdentifiableReviewer/Views/TreeNodeWithCount.cs

@@ -4,12 +4,16 @@ namespace IsIdentifiableReviewer.Views
 {
     internal class TreeNodeWithCount : TreeNode
     {
-        public TreeNodeWithCount(string text):base(text)
+        public string Heading { get; }
+
+        public TreeNodeWithCount(string heading)
         {
+            Heading = heading;
         }
+
         public override string ToString()
         {
-            return base.ToString() + $" ({Children.Count:N0})";
+            return Heading + $" ({Children.Count:N0})";
         }
     }
 }