[DotLiquid] Add TraceInfo for HL7 v2 conversion

src/Microsoft.Health.Fhir.Liquid.Converter.Tool/ConverterLogicHandler.cs

@@ -9,6 +9,7 @@
 using System.Linq;
 using Microsoft.Extensions.Logging;
 using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2;
+using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models;
 using Microsoft.Health.Fhir.Liquid.Converter.Models;
 using Microsoft.Health.Fhir.Liquid.Converter.Tool.Models;
 using Newtonsoft.Json;
@@ -56,9 +57,10 @@ private static void ConvertSingleFile(ConverterOptions options)
                 var dataType = GetDataTypes(options.TemplateDirectory);
                 var dataProcessor = CreateDataProcessor(dataType);
                 var templateProvider = CreateTemplateProvider(dataType, options.TemplateDirectory);
-                var resultString = dataProcessor.Convert(options.InputDataContent, options.RootTemplate, templateProvider);
-                var result = new ConverterResult(ProcessStatus.OK, resultString);
-                WriteOutputFile(options.OutputDataFile, JsonConvert.SerializeObject(result, Formatting.Indented));
+                var traceInfo = CreateTraceInfo(dataType, options.IsTraceInfo);
+                var resultString = dataProcessor.Convert(options.InputDataContent, options.RootTemplate, templateProvider, traceInfo);
+                var result = new ConverterResult(ProcessStatus.OK, resultString, traceInfo);
+                WriteOutputFile(options.OutputDataFile, JsonConvert.SerializeObject(result, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }));
                 Logger.LogInformation("Process completed");
             }
             catch (Exception ex)
@@ -83,10 +85,12 @@ private static void ConvertBatchFiles(ConverterOptions options)
                 {
                     try
                     {
-                        var result = dataProcessor.Convert(File.ReadAllText(file), options.RootTemplate, templateProvider);
+                        var traceInfo = CreateTraceInfo(dataType, options.IsTraceInfo);
+                        var resultString = dataProcessor.Convert(File.ReadAllText(file), options.RootTemplate, templateProvider, traceInfo);
+                        var result = new ConverterResult(ProcessStatus.OK, resultString, traceInfo);
                         var outputFileDirectory = Path.Join(options.OutputDataFolder, Path.GetRelativePath(options.InputDataFolder, Path.GetDirectoryName(file)));
                         var outputFilePath = Path.Join(outputFileDirectory, Path.GetFileNameWithoutExtension(file) + ".json");
-                        WriteOutputFile(outputFilePath, result);
+                        WriteOutputFile(outputFilePath, JsonConvert.SerializeObject(result, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }));
                         succeededCount++;
                     }
                     catch (Exception ex)
@@ -151,6 +155,11 @@ private static ITemplateProvider CreateTemplateProvider(DataType dataType, strin
             throw new NotImplementedException($"The conversion from data type {dataType} to FHIR is not supported");
         }
 
+        private static TraceInfo CreateTraceInfo(DataType dataType, bool isTraceInfo)
+        {
+            return isTraceInfo ? (dataType == DataType.Hl7v2 ? new Hl7v2TraceInfo() : new TraceInfo()) : null;
+        }
+
         private static List<string> GetInputFiles(DataType dataType, string inputDataFolder)
         {
             if (dataType == DataType.Hl7v2)

src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/ConverterOptions.cs

@@ -27,5 +27,8 @@ public class ConverterOptions
 
         [Option('o', "OutputDataFolder", Required = false, HelpText = "Output data folder")]
         public string OutputDataFolder { get; set; }
+
+        [Option('t', "IsTraceInfo", Required = false, HelpText = "Provide trace information in the output")]
+        public bool IsTraceInfo { get; set; }
     }
 }

src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/ConverterResult.cs

@@ -3,20 +3,24 @@
 // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
 // -------------------------------------------------------------------------------------------------
 
+using Microsoft.Health.Fhir.Liquid.Converter.Models;
 using Newtonsoft.Json.Linq;
 
 namespace Microsoft.Health.Fhir.Liquid.Converter.Tool.Models
 {
     public class ConverterResult
     {
-        public ConverterResult(ProcessStatus status, string fhirResource)
+        public ConverterResult(ProcessStatus status, string fhirResource, TraceInfo traceInfo)
         {
             Status = status;
             FhirResource = new JRaw(fhirResource);
+            TraceInfo = traceInfo;
         }
 
         public ProcessStatus Status { get; set; }
 
         public JRaw FhirResource { get; set; }
+
+        public TraceInfo TraceInfo { get; set; }
     }
 }

src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs

@@ -55,7 +55,7 @@ public void GivenValidEvaluateTemplateContent_WhenParseAndRender_CorrectResultSh
                 timeout: 0,
                 formatProvider: CultureInfo.InvariantCulture);
 
-            Assert.Empty(template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context }));
+            Assert.Empty(template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture)));
         }
 
         [Theory]
@@ -81,7 +81,7 @@ public void GivenInvalidSnippet_WhenRender_ExceptionsShouldBeThrown()
                 maxIterations: 0,
                 timeout: 0,
                 formatProvider: CultureInfo.InvariantCulture);
-            Assert.Throws<FileSystemException>(() => template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context }));
+            Assert.Throws<FileSystemException>(() => template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture)));
 
             // Valid template file system but no such template
             template = Template.Parse(@"{% evaluate bundleId using 'ID/Foo' Data: hl7v2Data -%}");
@@ -93,7 +93,7 @@ public void GivenInvalidSnippet_WhenRender_ExceptionsShouldBeThrown()
                 maxIterations: 0,
                 timeout: 0,
                 formatProvider: CultureInfo.InvariantCulture);
-            Assert.Throws<Exceptions.RenderException>(() => template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context }));
+            Assert.Throws<Exceptions.RenderException>(() => template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture)));
         }
     }
 }

src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Filters/FiltersRenderingTests.cs

@@ -42,7 +42,7 @@ public void FiltersRenderingTest()
             var context = new Context(CultureInfo.InvariantCulture);
             context.AddFilters(typeof(Filters));
 
-            var actual = template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context });
+            var actual = template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture));
             Assert.Equal(Expected, actual);
         }
     }

src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Hl7v2DataParserTests.cs

@@ -66,7 +66,7 @@ public void GivenValidHl7v2Message_WhenParse_CorrectHl7v2DataShouldBeReturned()
             // Hl7v2Field tests
             var field = (Hl7v2Field)segment[5];
             Assert.Equal("(130) 724-0433^PRN^PH^^^431^2780404~(330) 274-8214^ORN^PH^^^330^2748214", field["Value"]);
-            Assert.Equal(2, ((List<Hl7v2Field>)field["Repeats"]).Count);
+            Assert.Equal(2, ((SafeList<Hl7v2Field>)field["Repeats"]).Count);
             Assert.Equal("(130) 724-0433", ((Hl7v2Component)field[1]).Value);
             Assert.Throws<RenderException>(() => (Hl7v2Component)field[null]);
             Assert.Throws<RenderException>(() => (Hl7v2Component)field[1.2]);

src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Models/Hl7v2TraceInfoTests.cs

@@ -0,0 +1,70 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Collections.Generic;
+using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2;
+using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models;
+using Xunit;
+
+namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.Hl7v2.Models
+{
+    public class Hl7v2TraceInfoTests
+    {
+        [Fact]
+        public void GivenHl7v2Data_WhenCreate_CorrectHl7v2TraceInfoShouldBeReturned()
+        {
+            // Null Hl7v2Data
+            var traceInfo = Hl7v2TraceInfo.CreateTraceInfo(null);
+            Assert.Empty(traceInfo.UnusedSegments);
+
+            // Empty Hl7v2Data
+            var data = new Hl7v2Data();
+            traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data);
+            Assert.Empty(traceInfo.UnusedSegments);
+
+            // Null data
+            data = new Hl7v2Data()
+            {
+                Meta = null,
+                Data = null,
+            };
+            traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data);
+            Assert.Empty(traceInfo.UnusedSegments);
+
+            // Null segment
+            data = new Hl7v2Data()
+            {
+                Meta = new List<string>() { null },
+                Data = new List<Hl7v2Segment>() { null },
+            };
+            traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data);
+            Assert.Empty(traceInfo.UnusedSegments);
+
+            // Valid Hl7v2Data before render
+            var content = @"MSH|^~\&|AccMgr|1|||20050110045504||ADT^A01|599102|P|2.3||| 
+PID|1||10006579^^^1^MR^1||DUCK^DONALD^D||19241010|M||1|111 DUCK ST^^FOWL^CA^999990000^^M|1|8885551212|8885551212|1|2||40007716^^^AccMgr^VN^1|123121234|||||||||||NO ";
+            var parser = new Hl7v2DataParser();
+            data = parser.Parse(content);
+            traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data);
+            Assert.Equal(2, traceInfo.UnusedSegments.Count);
+            Assert.Equal(15, traceInfo.UnusedSegments[1].Field.Count);
+
+            // Valid Hl7v2Data after render
+            var processor = new Hl7v2Processor();
+            var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory);
+            _ = processor.Convert(content, "ADT_A01", templateProvider, traceInfo);
+            Assert.Equal(2, traceInfo.UnusedSegments.Count);
+
+            var unusedPid = traceInfo.UnusedSegments[1];
+            Assert.Equal("PID", unusedPid.Type);
+            Assert.Equal(1, unusedPid.Line);
+            Assert.Equal(3, unusedPid.Field.Count);
+            Assert.Equal(1, unusedPid.Field[0].Index);
+            Assert.Single(unusedPid.Field[0].Component);
+            Assert.Equal(1, unusedPid.Field[0].Index);
+            Assert.Equal("1", unusedPid.Field[0].Component[0].Value);
+        }
+    }
+}

src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/SafeList.cs

@@ -3,13 +3,14 @@
 // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
 // -------------------------------------------------------------------------------------------------
 
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using DotLiquid;
 
 namespace Microsoft.Health.Fhir.Liquid.Converter.DotLiquids
 {
-    public class SafeList<T> : IIndexable, ILiquidizable
+    public class SafeList<T> : IEnumerable, IIndexable, ILiquidizable
         where T : class
     {
         private IList<T> _internalList;
@@ -54,11 +55,16 @@ public virtual bool ContainsKey(object key)
             return key is int index && index >= 0 && index < _internalList.Count;
         }
 
-        public void Add(T item)
+        public virtual void Add(T item)
         {
             _internalList.Add(item);
         }
 
+        public virtual IEnumerator GetEnumerator()
+        {
+            return _internalList.GetEnumerator();
+        }
+
         public virtual object ToLiquid()
         {
             return this;

src/Microsoft.Health.Fhir.Liquid.Converter/Filters/GeneralFilters.cs

@@ -26,7 +26,7 @@ public static string GetProperty(Context context, string originalCode, string ma
                 return null;
             }
 
-            var map = ((CodeSystemMapping)context["CodeSystemMapping"])?.Mapping;
+            var map = (context["CodeSystemMapping"] as CodeSystemMapping)?.Mapping;
             return map != null &&
                 map.ContainsKey(mapping) &&
                 map[mapping].ContainsKey(originalCode) &&

src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2Processor.cs

@@ -10,6 +10,7 @@
 using System.Threading;
 using DotLiquid;
 using Microsoft.Health.Fhir.Liquid.Converter.Exceptions;
+using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models;
 using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.OutputProcessor;
 using Microsoft.Health.Fhir.Liquid.Converter.Models;
 using Newtonsoft.Json;
@@ -44,9 +45,15 @@ public string Convert(string data, string rootTemplate, ITemplateProvider templa
                 throw new RenderException(FhirConverterErrorCode.TemplateNotFound, string.Format(Resources.TemplateNotFound, rootTemplate));
             }
 
-            var context = CreateContext(templateProvider, data);
+            var hl7v2Data = _dataParser.Parse(data);
+            var context = CreateContext(templateProvider, hl7v2Data);
             var rawResult = RenderTemplates(template, context);
             var result = PostProcessor.Process(rawResult);
+            if (traceInfo is Hl7v2TraceInfo hl7V2TraceInfo)
+            {
+                hl7V2TraceInfo.UnusedSegments = Hl7v2TraceInfo.CreateTraceInfo(hl7v2Data).UnusedSegments;
+            }
+
             return result.ToString(Formatting.Indented);
         }
 
@@ -56,10 +63,9 @@ public string Convert(string data, string rootTemplate, ITemplateProvider templa
             return Convert(data, rootTemplate, templateProvider, traceInfo);
         }
 
-        private Context CreateContext(ITemplateProvider templateProvider, string data)
+        private Context CreateContext(ITemplateProvider templateProvider, Hl7v2Data hl7v2Data)
         {
             // Load data and templates
-            var hl7v2Data = _dataParser.Parse(data);
             var timeout = _settings != null ? _settings.TimeOut : 0;
             var context = new Context(
                 environments: new List<Hash>() { Hash.FromAnonymousObject(new { hl7v2Data }) },
@@ -88,7 +94,7 @@ private string RenderTemplates(Template template, Context context)
             try
             {
                 template.MakeThreadSafe();
-                return template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context });
+                return template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture));
             }
             catch (TimeoutException ex)
             {

src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Component.cs

@@ -16,10 +16,13 @@ public class Hl7v2Component : Drop
     {
         public Hl7v2Component(string value, IEnumerable<string> subcomponents)
         {
+            IsAccessed = false;
             Value = value;
             Subcomponents = new SafeList<string>(subcomponents);
         }
 
+        public bool IsAccessed { get; set; }
+
         public string Value { get; set; }
 
         public SafeList<string> Subcomponents { get; set; }
@@ -33,6 +36,7 @@ public override object this[object index]
                     throw new RenderException(FhirConverterErrorCode.PropertyNotFound, string.Format(Resources.PropertyNotFound, index, this.GetType().Name));
                 }
 
+                IsAccessed = true;
                 var indexString = index.ToString();
                 if (string.Equals(indexString, "Value", StringComparison.InvariantCultureIgnoreCase))
                 {

src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Field.cs

@@ -18,14 +18,14 @@ public Hl7v2Field(string value, IEnumerable<Hl7v2Component> components)
         {
             Value = value;
             Components = new SafeList<Hl7v2Component>(components);
-            Repeats = new List<Hl7v2Field>();
+            Repeats = new SafeList<Hl7v2Field>();
         }
 
         public string Value { get; set; }
 
         public SafeList<Hl7v2Component> Components { get; set; }
 
-        public List<Hl7v2Field> Repeats { get; set; }
+        public SafeList<Hl7v2Field> Repeats { get; set; }
 
         public override object this[object index]
         {
@@ -39,6 +39,7 @@ public override object this[object index]
                 var indexString = index.ToString();
                 if (string.Equals(indexString, "Value", StringComparison.InvariantCultureIgnoreCase))
                 {
+                    SetAccessForAllComponents();
                     return Value;
                 }
                 else if (string.Equals(indexString, "Components", StringComparison.InvariantCultureIgnoreCase))
@@ -47,6 +48,7 @@ public override object this[object index]
                 }
                 else if (string.Equals(indexString, "Repeats", StringComparison.InvariantCultureIgnoreCase))
                 {
+                    SetAccessForAllComponents();
                     return Repeats;
                 }
                 else if (int.TryParse(indexString, out int result))
@@ -59,5 +61,16 @@ public override object this[object index]
                 }
             }
         }
+
+        private void SetAccessForAllComponents()
+        {
+            foreach (var component in Components)
+            {
+                if (component is Hl7v2Component hl7V2Component)
+                {
+                    hl7V2Component.IsAccessed = true;
+                }
+            }
+        }
     }
 }

src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2Segment.cs

@@ -36,6 +36,7 @@ public override object this[object index]
                 var indexString = index.ToString();
                 if (string.Equals(indexString, "Value", StringComparison.InvariantCultureIgnoreCase))
                 {
+                    SetAccessForAllComponents();
                     return Value;
                 }
                 else if (string.Equals(indexString, "Fields", StringComparison.InvariantCultureIgnoreCase))
@@ -52,5 +53,22 @@ public override object this[object index]
                 }
             }
         }
+
+        private void SetAccessForAllComponents()
+        {
+            foreach (var field in Fields)
+            {
+                if (field is Hl7v2Field hl7v2Field)
+                {
+                    foreach (var component in hl7v2Field.Components)
+                    {
+                        if (component is Hl7v2Component hl7V2Component)
+                        {
+                            hl7V2Component.IsAccessed = true;
+                        }
+                    }
+                }
+            }
+        }
     }
 }