[DotLiquid] Add TraceInfo for HL7 v2 conversion

Fhir.Liquid.Converter.sln

@@ -16,6 +16,9 @@ EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.TemplateManagement.FunctionalTests", "src\Microsoft.Health.Fhir.TemplateManagement.FunctionalTests\Microsoft.Health.Fhir.TemplateManagement.FunctionalTests.csproj", "{530D0D28-B6AC-4B2F-8B6A-DD77CAB537E6}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{FB441B10-F38D-40E7-B2B4-D8424DA1D595}"
+	ProjectSection(SolutionItems) = preProject
+		.editorconfig = .editorconfig
+	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.TemplateManagement.UnitTests", "src\Microsoft.Health.Fhir.TemplateManagement.UnitTests\Microsoft.Health.Fhir.TemplateManagement.UnitTests.csproj", "{C4049E7B-7977-4691-BE86-4BD7E3E8C401}"
 EndProject

src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/FunctionalTests.cs

@@ -11,6 +11,7 @@
 using DotLiquid;
 using Microsoft.Health.Fhir.Liquid.Converter.Exceptions;
 using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2;
+using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Xunit;
@@ -54,7 +55,8 @@ public void GivenHl7v2Message_WhenConverting_ExpectedFhirResourceShouldBeReturne
 
             var inputContent = File.ReadAllText(inputFile);
             var expectedContent = File.ReadAllText(expectedFile);
-            var actualContent = hl7v2Processor.Convert(inputContent, rootTemplate, new Hl7v2TemplateProvider(templateDirectory));
+            var traceInfo = new Hl7v2TraceInfo();
+            var actualContent = hl7v2Processor.Convert(inputContent, rootTemplate, new Hl7v2TemplateProvider(templateDirectory), traceInfo);
 
             // Remove ID
             var regex = new Regex(@"(?<=(""urn:uuid:|""|/))([A-Za-z0-9\-]{36})(?="")");
@@ -66,6 +68,7 @@ public void GivenHl7v2Message_WhenConverting_ExpectedFhirResourceShouldBeReturne
             var expectedObject = serializer.Deserialize<JObject>(new JsonTextReader(new StringReader(expectedContent)));
             var actualObject = serializer.Deserialize<JObject>(new JsonTextReader(new StringReader(actualContent)));
             Assert.True(JToken.DeepEquals(expectedObject, actualObject));
+            Assert.True(traceInfo.UnusedSegments.Count > 0);
         }
 
         [Fact]

src/Microsoft.Health.Fhir.Liquid.Converter.Tool/ConverterLogicHandler.cs

@@ -7,8 +7,8 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using Microsoft.Extensions.Logging;
 using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2;
+using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models;
 using Microsoft.Health.Fhir.Liquid.Converter.Models;
 using Microsoft.Health.Fhir.Liquid.Converter.Tool.Models;
 using Newtonsoft.Json;
@@ -18,89 +18,48 @@ namespace Microsoft.Health.Fhir.Liquid.Converter.Tool
     internal static class ConverterLogicHandler
     {
         private const string MetadataFileName = "metadata.json";
-        private static readonly ILogger Logger;
-
-        static ConverterLogicHandler()
-        {
-            using var loggerFactory = LoggerFactory.Create(builder =>
-            {
-                builder.AddFilter("Microsoft.Health.Fhir.Liquid.Converter", LogLevel.Trace)
-                       .AddConsole();
-            });
-            FhirConverterLogging.LoggerFactory = loggerFactory;
-            Logger = FhirConverterLogging.CreateLogger(typeof(ConverterLogicHandler));
-        }
 
         internal static void Convert(ConverterOptions options)
         {
             if (!IsValidOptions(options))
             {
-                Logger.LogError("Invalid command-line options.");
-                return;
+                throw new InputParameterException("Invalid command-line options.");
             }
 
+            var dataType = GetDataTypes(options.TemplateDirectory);
+            var dataProcessor = CreateDataProcessor(dataType);
+            var templateProvider = CreateTemplateProvider(dataType, options.TemplateDirectory);
+
             if (!string.IsNullOrEmpty(options.InputDataContent))
             {
-                ConvertSingleFile(options);
+                ConvertSingleFile(dataProcessor, templateProvider, dataType, options.RootTemplate, options.InputDataContent, options.OutputDataFile, options.IsTraceInfo);
             }
             else
             {
-                ConvertBatchFiles(options);
+                ConvertBatchFiles(dataProcessor, templateProvider, dataType, options.RootTemplate, options.InputDataFolder, options.OutputDataFolder, options.IsTraceInfo);
             }
+
+            Console.WriteLine($"Conversion completed!");
         }
 
-        private static void ConvertSingleFile(ConverterOptions options)
+        private static void ConvertSingleFile(IFhirConverter dataProcessor, ITemplateProvider templateProvider, DataType dataType, string rootTemplate, string inputContent, string outputFile, bool isTraceInfo)
         {
-            try
-            {
-                var dataType = GetDataTypes(options.TemplateDirectory);
-                var dataProcessor = CreateDataProcessor(dataType);
-                var templateProvider = CreateTemplateProvider(dataType, options.TemplateDirectory);
-                var resultString = dataProcessor.Convert(options.InputDataContent, options.RootTemplate, templateProvider);
-                var result = new ConverterResult(ProcessStatus.OK, resultString);
-                WriteOutputFile(options.OutputDataFile, JsonConvert.SerializeObject(result, Formatting.Indented));
-                Logger.LogInformation("Process completed");
-            }
-            catch (Exception ex)
-            {
-                var error = new ConverterError(ex, options.TemplateDirectory);
-                WriteOutputFile(options.OutputDataFile, JsonConvert.SerializeObject(error, Formatting.Indented));
-                Logger.LogError($"Error occurred when converting input data: {error.ErrorMessage}");
-            }
+            var traceInfo = CreateTraceInfo(dataType, isTraceInfo);
+            var resultString = dataProcessor.Convert(inputContent, rootTemplate, templateProvider, traceInfo);
+            var result = new ConverterResult(ProcessStatus.OK, resultString, traceInfo);
+            SaveConverterResult(outputFile, result);
         }
 
-        private static void ConvertBatchFiles(ConverterOptions options)
+        private static void ConvertBatchFiles(IFhirConverter dataProcessor, ITemplateProvider templateProvider, DataType dataType, string rootTemplate, string inputFolder, string outputFolder, bool isTraceInfo)
         {
-            try
+            var files = GetInputFiles(dataType, inputFolder);
+            foreach (var file in files)
             {
-                int succeededCount = 0;
-                int failedCount = 0;
-                var dataType = GetDataTypes(options.TemplateDirectory);
-                var dataProcessor = CreateDataProcessor(dataType);
-                var templateProvider = CreateTemplateProvider(dataType, options.TemplateDirectory);
-                var files = GetInputFiles(dataType, options.InputDataFolder);
-                foreach (var file in files)
-                {
-                    try
-                    {
-                        var result = dataProcessor.Convert(File.ReadAllText(file), options.RootTemplate, templateProvider);
-                        var outputFileDirectory = Path.Join(options.OutputDataFolder, Path.GetRelativePath(options.InputDataFolder, Path.GetDirectoryName(file)));
-                        var outputFilePath = Path.Join(outputFileDirectory, Path.GetFileNameWithoutExtension(file) + ".json");
-                        WriteOutputFile(outputFilePath, result);
-                        succeededCount++;
-                    }
-                    catch (Exception ex)
-                    {
-                        Logger.LogError($"Error occurred when converting file {file}: {ex.Message}");
-                        failedCount++;
-                    }
-                }
-
-                Logger.LogInformation($"Process completed with {succeededCount} files succeeded and {failedCount} files failed");
-            }
-            catch (Exception ex)
-            {
-                Logger.LogError(ex.Message);
+                Console.WriteLine($"Processing {Path.GetFullPath(file)}");
+                var fileContent = File.ReadAllText(file);
+                var outputFileDirectory = Path.Join(outputFolder, Path.GetRelativePath(inputFolder, Path.GetDirectoryName(file)));
+                var outputFilePath = Path.Join(outputFileDirectory, Path.GetFileNameWithoutExtension(file) + ".json");
+                ConvertSingleFile(dataProcessor, templateProvider, dataType, rootTemplate, fileContent, outputFilePath, isTraceInfo);
             }
         }
 
@@ -151,6 +110,11 @@ private static ITemplateProvider CreateTemplateProvider(DataType dataType, strin
             throw new NotImplementedException($"The conversion from data type {dataType} to FHIR is not supported");
         }
 
+        private static TraceInfo CreateTraceInfo(DataType dataType, bool isTraceInfo)
+        {
+            return isTraceInfo ? (dataType == DataType.Hl7v2 ? new Hl7v2TraceInfo() : new TraceInfo()) : null;
+        }
+
         private static List<string> GetInputFiles(DataType dataType, string inputDataFolder)
         {
             if (dataType == DataType.Hl7v2)
@@ -161,10 +125,12 @@ private static List<string> GetInputFiles(DataType dataType, string inputDataFol
             return new List<string>();
         }
 
-        private static void WriteOutputFile(string outputFilePath, string content)
+        private static void SaveConverterResult(string outputFilePath, ConverterResult result)
         {
             var outputFileDirectory = Path.GetDirectoryName(outputFilePath);
             Directory.CreateDirectory(outputFileDirectory);
+
+            var content = JsonConvert.SerializeObject(result, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore });
             File.WriteAllText(outputFilePath, content);
         }
 

src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/ConverterOptions.cs

@@ -27,5 +27,8 @@ public class ConverterOptions
 
         [Option('o', "OutputDataFolder", Required = false, HelpText = "Output data folder")]
         public string OutputDataFolder { get; set; }
+
+        [Option('t', "IsTraceInfo", Required = false, HelpText = "Provide trace information in the output")]
+        public bool IsTraceInfo { get; set; }
     }
 }

src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/ConverterResult.cs

@@ -3,20 +3,24 @@
 // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
 // -------------------------------------------------------------------------------------------------
 
+using Microsoft.Health.Fhir.Liquid.Converter.Models;
 using Newtonsoft.Json.Linq;
 
 namespace Microsoft.Health.Fhir.Liquid.Converter.Tool.Models
 {
     public class ConverterResult
     {
-        public ConverterResult(ProcessStatus status, string fhirResource)
+        public ConverterResult(ProcessStatus status, string fhirResource, TraceInfo traceInfo)
         {
             Status = status;
             FhirResource = new JRaw(fhirResource);
+            TraceInfo = traceInfo;
         }
 
         public ProcessStatus Status { get; set; }
 
         public JRaw FhirResource { get; set; }
+
+        public TraceInfo TraceInfo { get; set; }
     }
 }

src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Program.cs

@@ -37,4 +37,4 @@ private static void HandleOptionsParseError(ParserResult<object> parseResult)
             throw new InputParameterException(usageText);
         }
     }
-}
+}

src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs

@@ -55,7 +55,7 @@ public void GivenValidEvaluateTemplateContent_WhenParseAndRender_CorrectResultSh
                 timeout: 0,
                 formatProvider: CultureInfo.InvariantCulture);
 
-            Assert.Empty(template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context }));
+            Assert.Empty(template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture)));
         }
 
         [Theory]
@@ -81,7 +81,7 @@ public void GivenInvalidSnippet_WhenRender_ExceptionsShouldBeThrown()
                 maxIterations: 0,
                 timeout: 0,
                 formatProvider: CultureInfo.InvariantCulture);
-            Assert.Throws<FileSystemException>(() => template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context }));
+            Assert.Throws<FileSystemException>(() => template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture)));
 
             // Valid template file system but no such template
             template = Template.Parse(@"{% evaluate bundleId using 'ID/Foo' Data: hl7v2Data -%}");
@@ -93,7 +93,7 @@ public void GivenInvalidSnippet_WhenRender_ExceptionsShouldBeThrown()
                 maxIterations: 0,
                 timeout: 0,
                 formatProvider: CultureInfo.InvariantCulture);
-            Assert.Throws<Exceptions.RenderException>(() => template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context }));
+            Assert.Throws<Exceptions.RenderException>(() => template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture)));
         }
     }
 }

src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Extensions/StringExtensionsTests.cs

@@ -0,0 +1,41 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Collections.Generic;
+using Microsoft.Health.Fhir.Liquid.Converter.Extensions;
+using Xunit;
+
+namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.Extensions
+{
+    public class StringExtensionsTests
+    {
+        public static IEnumerable<object[]> GetIndexOfNthOccurrenceData()
+        {
+            // Null or empty string or char
+            yield return new object[] { null, null, 1, -1 };
+            yield return new object[] { string.Empty, null, 1, -1 };
+            yield return new object[] { "abc|abc", null, 1, -1 };
+
+            // Nth occurrence smaller than one
+            yield return new object[] { "abc|abc", '|', 0, -1 };
+            yield return new object[] { "abc|abc", '|', -1, -1 };
+
+            // Nth occurrence hit
+            yield return new object[] { "abc|abc", '|', 1, 3 };
+            yield return new object[] { "abc|abc|abc", '|', 2, 7 };
+
+            // Nth occurrence not hit
+            yield return new object[] { "abc|abc", '|', 3, -1 };
+            yield return new object[] { "abc|abc", '^', 1, -1 };
+        }
+
+        [Theory]
+        [MemberData(nameof(GetIndexOfNthOccurrenceData))]
+        public void IndexOfNthOccurrenceTests(string s, char c, int n, int expected)
+        {
+            Assert.Equal(expected, s.IndexOfNthOccurrence(c, n));
+        }
+    }
+}

src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Filters/FiltersRenderingTests.cs

@@ -42,7 +42,7 @@ public void FiltersRenderingTest()
             var context = new Context(CultureInfo.InvariantCulture);
             context.AddFilters(typeof(Filters));
 
-            var actual = template.Render(new RenderParameters(CultureInfo.InvariantCulture) { Context = context });
+            var actual = template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture));
             Assert.Equal(Expected, actual);
         }
     }

src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Hl7v2DataParserTests.cs

@@ -66,7 +66,7 @@ public void GivenValidHl7v2Message_WhenParse_CorrectHl7v2DataShouldBeReturned()
             // Hl7v2Field tests
             var field = (Hl7v2Field)segment[5];
             Assert.Equal("(130) 724-0433^PRN^PH^^^431^2780404~(330) 274-8214^ORN^PH^^^330^2748214", field["Value"]);
-            Assert.Equal(2, ((List<Hl7v2Field>)field["Repeats"]).Count);
+            Assert.Equal(2, ((SafeList<Hl7v2Field>)field["Repeats"]).Count);
             Assert.Equal("(130) 724-0433", ((Hl7v2Component)field[1]).Value);
             Assert.Throws<RenderException>(() => (Hl7v2Component)field[null]);
             Assert.Throws<RenderException>(() => (Hl7v2Component)field[1.2]);

src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Models/Hl7v2TraceInfoTests.cs

@@ -0,0 +1,68 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Collections.Generic;
+using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2;
+using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models;
+using Xunit;
+
+namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.Hl7v2.Models
+{
+    public class Hl7v2TraceInfoTests
+    {
+        [Fact]
+        public void GivenHl7v2Data_WhenCreate_CorrectHl7v2TraceInfoShouldBeReturned()
+        {
+            // Null Hl7v2Data
+            var traceInfo = Hl7v2TraceInfo.CreateTraceInfo(null);
+            Assert.Empty(traceInfo.UnusedSegments);
+
+            // Empty Hl7v2Data
+            var data = new Hl7v2Data();
+            traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data);
+            Assert.Empty(traceInfo.UnusedSegments);
+
+            // Null data
+            data = new Hl7v2Data()
+            {
+                Meta = null,
+                Data = null,
+            };
+            traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data);
+            Assert.Empty(traceInfo.UnusedSegments);
+
+            // Null segment
+            data = new Hl7v2Data()
+            {
+                Meta = new List<string>() { null },
+                Data = new List<Hl7v2Segment>() { null },
+            };
+            traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data);
+            Assert.Empty(traceInfo.UnusedSegments);
+
+            // Valid Hl7v2Data before render
+            var content = @"MSH|^~\&|AccMgr|1|||20050110045504||ADT^A01|599102|P|2.3||| 
+PID|1||10006579^^^1^MR^1||DUCK^DONALD^D||19241010|M||1|111 DUCK ST^^FOWL^CA^999990000^^M|1|8885551212|8885551212|1|2||40007716^^^AccMgr^VN^1|123121234|||||||||||NO ";
+            var parser = new Hl7v2DataParser();
+            data = parser.Parse(content);
+            traceInfo = Hl7v2TraceInfo.CreateTraceInfo(data);
+            Assert.Equal(2, traceInfo.UnusedSegments.Count);
+            Assert.Equal(27, traceInfo.UnusedSegments[1].Components.Count);
+
+            // Valid Hl7v2Data after render
+            var processor = new Hl7v2Processor();
+            var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory);
+            _ = processor.Convert(content, "ADT_A01", templateProvider, traceInfo);
+            Assert.Equal(2, traceInfo.UnusedSegments.Count);
+
+            var unusedPid = traceInfo.UnusedSegments[1];
+            Assert.Equal("PID", unusedPid.Type);
+            Assert.Equal(1, unusedPid.Line);
+            Assert.Equal(6, unusedPid.Components.Count);
+            Assert.Equal(118, unusedPid.Components[2].Offset);
+            Assert.Equal("40007716", unusedPid.Components[2].Value);
+        }
+    }
+}