From 09dc067d37fc940aace8f767019921034bbe3289 Mon Sep 17 00:00:00 2001 From: Dustin Burson Date: Sat, 6 Jun 2020 18:43:02 -0700 Subject: [PATCH 01/13] Add correlation id support to measurements/groups & ObservationGroups/Factory --- .../CorrelationMeasurementObservationGroup.cs | 46 +++++++ ...ationMeasurementObservationGroupFactory.cs | 26 ++++ .../Data/IMeasurement.cs | 2 + .../Data/IMeasurementGroup.cs | 2 + .../Data/IObservationGroup.cs | 2 + .../Data/Measurement.cs | 3 +- .../Data/MeasurementGroup.cs | 2 + .../MeasurementObservationGroupFactory.cs | 114 +++--------------- .../Data/SortedObservationGroup.cs | 48 ++++++++ .../TimePeriodMeasurementObservationGroup.cs | 31 +++++ ...eriodMeasurementObservationGroupFactory.cs | 90 ++++++++++++++ .../Service/FhirImportService.cs | 6 +- .../Template/ObservationPeriodInterval.cs | 5 + ...easurementObservationGroupFactoryTests.cs} | 10 +- .../Service/FhirImportServiceTests.cs | 8 +- 15 files changed, 280 insertions(+), 115 deletions(-) create mode 100644 src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroup.cs create mode 100644 src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroupFactory.cs create mode 100644 src/lib/Microsoft.Health.Fhir.Ingest/Data/SortedObservationGroup.cs create mode 100644 src/lib/Microsoft.Health.Fhir.Ingest/Data/TimePeriodMeasurementObservationGroup.cs create mode 100644 src/lib/Microsoft.Health.Fhir.Ingest/Data/TimePeriodMeasurementObservationGroupFactory.cs rename test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/{MeasurementObservationGroupFactoryTests.cs => TimePeriodMeasurementObservationGroupFactoryTests.cs} (95%) diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroup.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroup.cs new file mode 100644 index 00000000..31ca9400 --- /dev/null +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroup.cs @@ -0,0 +1,46 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using EnsureThat; + +namespace Microsoft.Health.Fhir.Ingest.Data +{ + public class CorrelationMeasurementObservationGroup : SortedObservationGroup + { + private readonly string _correlationId; + private DateTime _start = DateTime.MaxValue; + private DateTime _end = DateTime.MinValue; + + public CorrelationMeasurementObservationGroup(string correlationId) + { + _correlationId = EnsureArg.IsNotNullOrWhiteSpace(correlationId, nameof(correlationId)); + } + + public override (DateTime Start, DateTime End) Boundary => (_start, _end); + + public override void AddMeasurement(IMeasurement measurement) + { + EnsureArg.IsNotNull(measurement, nameof(measurement)); + + base.AddMeasurement(measurement); + + if (measurement.OccurrenceTimeUtc < _start) + { + _start = measurement.OccurrenceTimeUtc; + } + + if (measurement.OccurrenceTimeUtc > _end) + { + _end = measurement.OccurrenceTimeUtc; + } + } + + public override string GetIdSegment() + { + return _correlationId; + } + } +} \ No newline at end of file diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroupFactory.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroupFactory.cs new file mode 100644 index 00000000..1a2079ab --- /dev/null +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroupFactory.cs @@ -0,0 +1,26 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using EnsureThat; + +namespace Microsoft.Health.Fhir.Ingest.Data +{ + public class CorrelationMeasurementObservationGroupFactory : IObservationGroupFactory + { + public IEnumerable Build(IMeasurementGroup input) + { + EnsureArg.IsNotNull(input, nameof(input)); + + var observationGroup = new CorrelationMeasurementObservationGroup(input.CorrelationId); + foreach (var m in input.Data) + { + observationGroup.AddMeasurement(m); + } + + yield return observationGroup; + } + } +} \ No newline at end of file diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/IMeasurement.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/IMeasurement.cs index cf1c8d33..0ffb2e8f 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Data/IMeasurement.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/IMeasurement.cs @@ -22,6 +22,8 @@ public interface IMeasurement string EncounterId { get; } + string CorrelationId { get; } + IEnumerable Properties { get; } } } diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/IMeasurementGroup.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/IMeasurementGroup.cs index 50d7561e..4ee6c412 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Data/IMeasurementGroup.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/IMeasurementGroup.cs @@ -17,6 +17,8 @@ public interface IMeasurementGroup string EncounterId { get; } + string CorrelationId { get; } + IEnumerable Data { get; } } } diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/IObservationGroup.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/IObservationGroup.cs index 661dac3c..ddf826d0 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Data/IObservationGroup.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/IObservationGroup.cs @@ -17,5 +17,7 @@ public interface IObservationGroup void AddMeasurement(IMeasurement measurement); IDictionary> GetValues(); + + string GetIdSegment(); } } diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/Measurement.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/Measurement.cs index 9ff4e29a..135c7449 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Data/Measurement.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/Measurement.cs @@ -27,9 +27,10 @@ public Measurement() public string EncounterId { get; set; } + public string CorrelationId { get; set; } + #pragma warning disable CA2227 public IList Properties { get; set; } - #pragma warning restore CA2227 IEnumerable IMeasurement.Properties => Properties; diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/MeasurementGroup.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/MeasurementGroup.cs index 78a4de2e..3c41cb62 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Data/MeasurementGroup.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/MeasurementGroup.cs @@ -16,6 +16,8 @@ public class MeasurementGroup : IMeasurementGroup public string EncounterId { get; set; } + public string CorrelationId { get; set; } + public DateTime WindowTime { get; set; } public string MeasureType { get; set; } diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/MeasurementObservationGroupFactory.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/MeasurementObservationGroupFactory.cs index 115133ca..81cef5c8 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Data/MeasurementObservationGroupFactory.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/MeasurementObservationGroupFactory.cs @@ -5,118 +5,34 @@ using System; using System.Collections.Generic; -using System.Linq; -using EnsureThat; using Microsoft.Health.Fhir.Ingest.Template; namespace Microsoft.Health.Fhir.Ingest.Data { public class MeasurementObservationGroupFactory : IObservationGroupFactory { - private static readonly IDictionary> BoundaryFunctions = new Dictionary> - { - { ObservationPeriodInterval.Single, GetSingleBoundary }, - { ObservationPeriodInterval.Hourly, GetHourlyBoundary }, - { ObservationPeriodInterval.Daily, GetDailyBoundary }, - }; - - private readonly Func _boundaryFunction; + private readonly IObservationGroupFactory _internalFactory; public MeasurementObservationGroupFactory(ObservationPeriodInterval period) { - _boundaryFunction = BoundaryFunctions[period]; - } - - public IEnumerable Build(IMeasurementGroup input) - { - EnsureArg.IsNotNull(input, nameof(input)); - - var lookup = new Dictionary(); - foreach (var m in input.Data) + switch (period) { - var boundary = GetBoundaryKey(m); - if (lookup.TryGetValue(boundary.Start, out var grp)) - { - grp.AddMeasurement(m); - } - else - { - var newGrp = CreateObservationGroup(input, boundary); - newGrp.AddMeasurement(m); - lookup.Add(boundary.Start, newGrp); - } + case ObservationPeriodInterval.CorrelationId: + _internalFactory = new CorrelationMeasurementObservationGroupFactory(); + break; + case ObservationPeriodInterval.Single: + case ObservationPeriodInterval.Hourly: + case ObservationPeriodInterval.Daily: + _internalFactory = new TimePeriodMeasurementObservationGroupFactory(period); + break; + default: + throw new NotSupportedException($"ObservationPeriodInterval {period} is not supported."); } - - return lookup.Values; - } - - public virtual (DateTime Start, DateTime End) GetBoundaryKey(IMeasurement measurement) - { - EnsureArg.IsNotNull(measurement); - - return _boundaryFunction(measurement.OccurrenceTimeUtc.ToUniversalTime()); - } - - public virtual IObservationGroup CreateObservationGroup(IMeasurementGroup group, (DateTime Start, DateTime End) boundary) - { - EnsureArg.IsNotNull(group, nameof(group)); - - return new MeasurementObservationGroup - { - Name = group.MeasureType, - Boundary = boundary, - }; - } - - private static (DateTime Start, DateTime End) GetSingleBoundary(DateTime utcDateTime) - { - return (utcDateTime, utcDateTime); } - private static (DateTime Start, DateTime End) GetHourlyBoundary(DateTime utcDateTime) - { - var start = utcDateTime.Date.AddHours(utcDateTime.Hour); - var end = start.AddHours(1).AddTicks(-1); - - return (start, end); - } - - private static (DateTime Start, DateTime End) GetDailyBoundary(DateTime utcDateTime) - { - var start = utcDateTime.Date; - var end = start.AddDays(1).AddTicks(-1); - - return (start, end); - } - - private class MeasurementObservationGroup : IObservationGroup, IComparer<(DateTime Time, string Value)> + public IEnumerable Build(IMeasurementGroup input) { - private readonly Dictionary> _propertyTimeValues = new Dictionary>(); - - public string Name { get; set; } - - public (DateTime Start, DateTime End) Boundary { get; set; } - - public void AddMeasurement(IMeasurement measurement) - { - foreach (var mp in measurement.Properties) - { - if (!_propertyTimeValues.TryGetValue(mp.Name, out SortedSet<(DateTime Time, string Value)> values)) - { - values = new SortedSet<(DateTime Time, string Value)>(this); - _propertyTimeValues.Add(mp.Name, values); - } - - values.Add((measurement.OccurrenceTimeUtc, mp.Value)); - } - } - - public int Compare((DateTime Time, string Value) x, (DateTime Time, string Value) y) - { - return DateTime.Compare(x.Time, y.Time); - } - - public IDictionary> GetValues() => _propertyTimeValues.ToDictionary(m => m.Key, m => (IEnumerable<(DateTime Time, string Value)>)m.Value); + return _internalFactory.Build(input); } } -} \ No newline at end of file +} diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/SortedObservationGroup.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/SortedObservationGroup.cs new file mode 100644 index 00000000..e70aa5b7 --- /dev/null +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/SortedObservationGroup.cs @@ -0,0 +1,48 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using EnsureThat; + +namespace Microsoft.Health.Fhir.Ingest.Data +{ + public abstract class SortedObservationGroup : IObservationGroup, IComparer<(DateTime Time, string Value)> + { + private readonly Dictionary> _propertyTimeValues = new Dictionary>(); + + public virtual string Name { get; set; } + + public abstract (DateTime Start, DateTime End) Boundary { get; } + + protected IDictionary> PropertyTimeValues => _propertyTimeValues; + + public virtual void AddMeasurement(IMeasurement measurement) + { + EnsureArg.IsNotNull(measurement); + + foreach (var mp in measurement.Properties) + { + if (!_propertyTimeValues.TryGetValue(mp.Name, out SortedSet<(DateTime Time, string Value)> values)) + { + values = new SortedSet<(DateTime Time, string Value)>(this); + _propertyTimeValues.Add(mp.Name, values); + } + + values.Add((measurement.OccurrenceTimeUtc, mp.Value)); + } + } + + public abstract string GetIdSegment(); + + public virtual IDictionary> GetValues() => _propertyTimeValues.ToDictionary(m => m.Key, m => (IEnumerable<(DateTime Time, string Value)>)m.Value); + + public int Compare((DateTime Time, string Value) x, (DateTime Time, string Value) y) + { + return DateTime.Compare(x.Time, y.Time); + } + } +} \ No newline at end of file diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/TimePeriodMeasurementObservationGroup.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/TimePeriodMeasurementObservationGroup.cs new file mode 100644 index 00000000..7b252fdb --- /dev/null +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/TimePeriodMeasurementObservationGroup.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Globalization; + +namespace Microsoft.Health.Fhir.Ingest.Data +{ + public class TimePeriodMeasurementObservationGroup : SortedObservationGroup + { + public const string DateFormat = "yyyyMMddHHmmssZ"; + private readonly (DateTime Start, DateTime End) _bondary; + + public TimePeriodMeasurementObservationGroup((DateTime Start, DateTime End) boundary) + { + _bondary = boundary; + } + + public override (DateTime Start, DateTime End) Boundary => _bondary; + + public override string GetIdSegment() + { + var startToken = Boundary.Start.ToString(DateFormat, CultureInfo.InvariantCulture.DateTimeFormat); + var endToken = Boundary.End.ToString(DateFormat, CultureInfo.InvariantCulture.DateTimeFormat); + + return $"{startToken}.{endToken}"; + } + } +} \ No newline at end of file diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/TimePeriodMeasurementObservationGroupFactory.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/TimePeriodMeasurementObservationGroupFactory.cs new file mode 100644 index 00000000..80155863 --- /dev/null +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/TimePeriodMeasurementObservationGroupFactory.cs @@ -0,0 +1,90 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using EnsureThat; +using Microsoft.Health.Fhir.Ingest.Template; + +namespace Microsoft.Health.Fhir.Ingest.Data +{ + public class TimePeriodMeasurementObservationGroupFactory : IObservationGroupFactory + { + private static readonly IDictionary> BoundaryFunctions = new Dictionary> + { + { ObservationPeriodInterval.Single, GetSingleBoundary }, + { ObservationPeriodInterval.Hourly, GetHourlyBoundary }, + { ObservationPeriodInterval.Daily, GetDailyBoundary }, + }; + + private readonly Func _boundaryFunction; + + public TimePeriodMeasurementObservationGroupFactory(ObservationPeriodInterval period) + { + _boundaryFunction = BoundaryFunctions[period]; + } + + public IEnumerable Build(IMeasurementGroup input) + { + EnsureArg.IsNotNull(input, nameof(input)); + + var lookup = new Dictionary(); + foreach (var m in input.Data) + { + var boundary = GetBoundaryKey(m); + if (lookup.TryGetValue(boundary.Start, out var grp)) + { + grp.AddMeasurement(m); + } + else + { + var newGrp = CreateObservationGroup(input, boundary); + newGrp.AddMeasurement(m); + lookup.Add(boundary.Start, newGrp); + } + } + + return lookup.Values; + } + + public virtual (DateTime Start, DateTime End) GetBoundaryKey(IMeasurement measurement) + { + EnsureArg.IsNotNull(measurement); + + return _boundaryFunction(measurement.OccurrenceTimeUtc.ToUniversalTime()); + } + + protected virtual IObservationGroup CreateObservationGroup(IMeasurementGroup group, (DateTime Start, DateTime End) boundary) + { + EnsureArg.IsNotNull(group, nameof(group)); + + return new TimePeriodMeasurementObservationGroup(boundary) + { + Name = group.MeasureType, + }; + } + + private static (DateTime Start, DateTime End) GetSingleBoundary(DateTime utcDateTime) + { + return (utcDateTime, utcDateTime); + } + + private static (DateTime Start, DateTime End) GetHourlyBoundary(DateTime utcDateTime) + { + var start = utcDateTime.Date.AddHours(utcDateTime.Hour); + var end = start.AddHours(1).AddTicks(-1); + + return (start, end); + } + + private static (DateTime Start, DateTime End) GetDailyBoundary(DateTime utcDateTime) + { + var start = utcDateTime.Date; + var end = start.AddDays(1).AddTicks(-1); + + return (start, end); + } + } +} \ No newline at end of file diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Service/FhirImportService.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Service/FhirImportService.cs index 45262658..0f17dd8a 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Service/FhirImportService.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Service/FhirImportService.cs @@ -4,7 +4,6 @@ // ------------------------------------------------------------------------------------------------- using System; -using System.Globalization; using System.Threading.Tasks; using EnsureThat; using Microsoft.Health.Fhir.Ingest.Data; @@ -14,7 +13,6 @@ namespace Microsoft.Health.Fhir.Ingest.Service { public abstract class FhirImportService : IFhirImportService> { - public const string DateFormat = "yyyyMMddHHmmssZ"; public const string ServiceSystem = @"https://azure.microsoft.com/en-us/services/iomt-fhir-connector/"; protected static (string Identifer, string System) GenerateObservationId(IObservationGroup observationGroup, string deviceId, string patientId) @@ -23,9 +21,7 @@ protected static (string Identifer, string System) GenerateObservationId(IObserv EnsureArg.IsNotNullOrWhiteSpace(deviceId, nameof(deviceId)); EnsureArg.IsNotNullOrWhiteSpace(patientId, nameof(patientId)); - var startToken = observationGroup.Boundary.Start.ToString(DateFormat, CultureInfo.InvariantCulture.DateTimeFormat); - var endToken = observationGroup.Boundary.End.ToString(DateFormat, CultureInfo.InvariantCulture.DateTimeFormat); - var value = $"{patientId}.{deviceId}.{observationGroup.Name}.{startToken}.{endToken}"; + var value = $"{patientId}.{deviceId}.{observationGroup.Name}.{observationGroup.GetIdSegment()}"; return (value, ServiceSystem); } diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Template/ObservationPeriodInterval.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Template/ObservationPeriodInterval.cs index 72539100..090660f2 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Template/ObservationPeriodInterval.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Template/ObservationPeriodInterval.cs @@ -7,6 +7,11 @@ namespace Microsoft.Health.Fhir.Ingest.Template { public enum ObservationPeriodInterval { + /// + /// Do not group according to time, group according to a correlation id supplied in the measurement group. + /// + CorrelationId = -1, + /// /// Do not group measurements. Each measurement will be mapped to one observation. /// diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/MeasurementObservationGroupFactoryTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/TimePeriodMeasurementObservationGroupFactoryTests.cs similarity index 95% rename from test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/MeasurementObservationGroupFactoryTests.cs rename to test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/TimePeriodMeasurementObservationGroupFactoryTests.cs index 36903dd9..b37f51c1 100644 --- a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/MeasurementObservationGroupFactoryTests.cs +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/TimePeriodMeasurementObservationGroupFactoryTests.cs @@ -13,7 +13,7 @@ namespace Microsoft.Health.Fhir.Ingest.Data { - public class MeasurementObservationGroupFactoryTests + public class TimePeriodMeasurementObservationGroupFactoryTests { [Fact] public void GivenSingleBoundaryWithSingleMeasurementSingleValue_WhenBuild_SingleObservationGroupWithSingleValueReturned_Test() @@ -36,7 +36,7 @@ public void GivenSingleBoundaryWithSingleMeasurementSingleValue_WhenBuild_Single .Mock(mg => mg.MeasureType.Returns("a")) .Mock(mg => mg.Data.Returns(measurement)); - var factory = new MeasurementObservationGroupFactory(ObservationPeriodInterval.Single); + var factory = new TimePeriodMeasurementObservationGroupFactory(ObservationPeriodInterval.Single); var result = factory.Build(measureGroup)?.ToArray(); Assert.NotNull(result); @@ -106,7 +106,7 @@ public void GivenSingleBoundaryWithMultipleMeasurementMultipleValue_WhenBuild_Mu .Mock(mg => mg.MeasureType.Returns("a")) .Mock(mg => mg.Data.Returns(measurement)); - var factory = new MeasurementObservationGroupFactory(ObservationPeriodInterval.Single); + var factory = new TimePeriodMeasurementObservationGroupFactory(ObservationPeriodInterval.Single); var result = factory.Build(measureGroup)?.ToArray(); Assert.NotNull(result); @@ -246,7 +246,7 @@ public void GivenHourlyBoundaryWithMultipleMeasurementMultipleValue_WhenBuild_Mu .Mock(mg => mg.MeasureType.Returns("a")) .Mock(mg => mg.Data.Returns(measurement)); - var factory = new MeasurementObservationGroupFactory(ObservationPeriodInterval.Hourly); + var factory = new TimePeriodMeasurementObservationGroupFactory(ObservationPeriodInterval.Hourly); var result = factory.Build(measureGroup)?.ToArray(); Assert.NotNull(result); @@ -370,7 +370,7 @@ public void GivenDailyBoundaryWithMultipleMeasurementMultipleValue_WhenBuild_Mul .Mock(mg => mg.MeasureType.Returns("a")) .Mock(mg => mg.Data.Returns(measurement)); - var factory = new MeasurementObservationGroupFactory(ObservationPeriodInterval.Daily); + var factory = new TimePeriodMeasurementObservationGroupFactory(ObservationPeriodInterval.Daily); var result = factory.Build(measureGroup)?.ToArray(); Assert.NotNull(result); diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/FhirImportServiceTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/FhirImportServiceTests.cs index 8905923a..45dad464 100644 --- a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/FhirImportServiceTests.cs +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/FhirImportServiceTests.cs @@ -18,14 +18,12 @@ public class FhirImportServiceTests [Fact] public void GivenObservationGroupDeviceIdAndPatientId_WhenGenerateObservationId_CorrectIdReturned_Test() { - var startDate = new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var endDate = startDate.AddHours(1).AddTicks(-1); var observationGroup = Substitute.For() - .Mock(m => m.Boundary.Returns((startDate, endDate))) - .Mock(m => m.Name.Returns("heartrate")); + .Mock(m => m.Name.Returns("heartrate")) + .Mock(m => m.GetIdSegment().ReturnsForAnyArgs("segment")); var result = TestFhirImportService.TestGenerateObservationId(observationGroup, "deviceId", "patientId"); - Assert.Equal("patientId.deviceId.heartrate.20190101000000Z.20190101005959Z", result.Identifer); + Assert.Equal("patientId.deviceId.heartrate.segment", result.Identifer); Assert.Equal(FhirImportService.ServiceSystem, result.System); } From 5096d629240a0205cdfb6b8e0087866758d471fc Mon Sep 17 00:00:00 2001 From: Dustin Burson Date: Wed, 10 Jun 2020 16:41:18 -0700 Subject: [PATCH 02/13] * Add correlation id to SA queries in collection query project and SA query property of ARM templates * Add ability to extract correlation id to JsonPathContentTemplate * Add tests for extraction logic --- deploy/templates/consumption-azuredeploy.json | 2 +- deploy/templates/default-azuredeploy.json | 2 +- .../managed-identity-azuredeploy.json | 2 +- deploy/templates/premium-azuredeploy.json | 2 +- .../Template/JsonPathContentTemplate.cs | 29 ++++++++-- src/query/CollectionQuery/CollectionQuery.sln | 25 ++++++++ src/query/CollectionQuery/Script.asaql | 4 +- .../JsonContentTemplateFactoryTests.cs | 2 + .../Template/JsonPathContentTemplateTests.cs | 58 +++++++++++++++++++ ...nPathContentTemplateValidWithOptional.json | 1 + 10 files changed, 118 insertions(+), 9 deletions(-) create mode 100644 src/query/CollectionQuery/CollectionQuery.sln diff --git a/deploy/templates/consumption-azuredeploy.json b/deploy/templates/consumption-azuredeploy.json index bc12d329..2f8eadf1 100644 --- a/deploy/templates/consumption-azuredeploy.json +++ b/deploy/templates/consumption-azuredeploy.json @@ -251,7 +251,7 @@ "name": "Transformation", "properties": { "streamingUnits": "[parameters('StreamingUnits')]", - "query": "[concat('SELECT \r\n DeviceId [DeviceId], \r\n PatientId [PatientId],\r\n EncounterId [EncounterId],\r\n collect() [Data],\r\n System.Timestamp [WindowTime],\r\n Type [MeasureType],\r\n count(*) [Count]\r\nINTO\r\n [FhirImportOutput]\r\nFROM\r\n [NormalizedData] PARTITION BY PartitionId TIMESTAMP BY OccurrenceTimeUtc\r\nGROUP BY PartitionId, \r\n DeviceId, \r\n PatientId, \r\n EncounterId, \r\n Type, \r\n TUMBLINGWINDOW(', parameters('JobWindowUnit'), ', ', parameters('JobWindowMagnitude'), ')')]" + "query": "[concat('SELECT \r\n DeviceId [DeviceId], \r\n PatientId [PatientId],\r\n EncounterId [EncounterId],\r\n CorrelationId [CorrelationId],\r\n collect() [Data],\r\n System.Timestamp [WindowTime],\r\n Type [MeasureType],\r\n count(*) [Count]\r\nINTO\r\n [FhirImportOutput]\r\nFROM\r\n [NormalizedData] PARTITION BY PartitionId TIMESTAMP BY OccurrenceTimeUtc\r\nGROUP BY PartitionId, \r\n DeviceId, \r\n PatientId, \r\n EncounterId, \r\n CorrelationId, \r\n Type, \r\n TUMBLINGWINDOW(', parameters('JobWindowUnit'), ', ', parameters('JobWindowMagnitude'), ')')]" } }, "functions": [ diff --git a/deploy/templates/default-azuredeploy.json b/deploy/templates/default-azuredeploy.json index 221c1510..bc518258 100644 --- a/deploy/templates/default-azuredeploy.json +++ b/deploy/templates/default-azuredeploy.json @@ -272,7 +272,7 @@ "name": "Transformation", "properties": { "streamingUnits": "[parameters('StreamingUnits')]", - "query": "[concat('SELECT \r\n DeviceId [DeviceId], \r\n PatientId [PatientId],\r\n EncounterId [EncounterId],\r\n collect() [Data],\r\n System.Timestamp [WindowTime],\r\n Type [MeasureType],\r\n count(*) [Count]\r\nINTO\r\n [FhirImportOutput]\r\nFROM\r\n [NormalizedData] PARTITION BY PartitionId TIMESTAMP BY OccurrenceTimeUtc\r\nGROUP BY PartitionId, \r\n DeviceId, \r\n PatientId, \r\n EncounterId, \r\n Type, \r\n TUMBLINGWINDOW(', parameters('JobWindowUnit'), ', ', parameters('JobWindowMagnitude'), ')')]" + "query": "[concat('SELECT \r\n DeviceId [DeviceId], \r\n PatientId [PatientId],\r\n EncounterId [EncounterId],\r\n CorrelationId [CorrelationId],\r\n collect() [Data],\r\n System.Timestamp [WindowTime],\r\n Type [MeasureType],\r\n count(*) [Count]\r\nINTO\r\n [FhirImportOutput]\r\nFROM\r\n [NormalizedData] PARTITION BY PartitionId TIMESTAMP BY OccurrenceTimeUtc\r\nGROUP BY PartitionId, \r\n DeviceId, \r\n PatientId, \r\n EncounterId, \r\n CorrelationId, \r\n Type, \r\n TUMBLINGWINDOW(', parameters('JobWindowUnit'), ', ', parameters('JobWindowMagnitude'), ')')]" } }, "functions": [ diff --git a/deploy/templates/managed-identity-azuredeploy.json b/deploy/templates/managed-identity-azuredeploy.json index 0ef2a44a..daa7bb68 100644 --- a/deploy/templates/managed-identity-azuredeploy.json +++ b/deploy/templates/managed-identity-azuredeploy.json @@ -248,7 +248,7 @@ "name": "Transformation", "properties": { "streamingUnits": "[parameters('StreamingUnits')]", - "query": "[concat('SELECT \r\n DeviceId [DeviceId], \r\n PatientId [PatientId],\r\n EncounterId [EncounterId],\r\n collect() [Data],\r\n System.Timestamp [WindowTime],\r\n Type [MeasureType],\r\n count(*) [Count]\r\nINTO\r\n [FhirImportOutput]\r\nFROM\r\n [NormalizedData] PARTITION BY PartitionId TIMESTAMP BY OccurrenceTimeUtc\r\nGROUP BY PartitionId, \r\n DeviceId, \r\n PatientId, \r\n EncounterId, \r\n Type, \r\n TUMBLINGWINDOW(', parameters('JobWindowUnit'), ', ', parameters('JobWindowMagnitude'), ')')]" + "query": "[concat('SELECT \r\n DeviceId [DeviceId], \r\n PatientId [PatientId],\r\n EncounterId [EncounterId],\r\n CorrelationId [CorrelationId],\r\n collect() [Data],\r\n System.Timestamp [WindowTime],\r\n Type [MeasureType],\r\n count(*) [Count]\r\nINTO\r\n [FhirImportOutput]\r\nFROM\r\n [NormalizedData] PARTITION BY PartitionId TIMESTAMP BY OccurrenceTimeUtc\r\nGROUP BY PartitionId, \r\n DeviceId, \r\n PatientId, \r\n EncounterId, \r\n CorrelationId, \r\n Type, \r\n TUMBLINGWINDOW(', parameters('JobWindowUnit'), ', ', parameters('JobWindowMagnitude'), ')')]" } }, "functions": [ diff --git a/deploy/templates/premium-azuredeploy.json b/deploy/templates/premium-azuredeploy.json index d499e695..cea24fb6 100644 --- a/deploy/templates/premium-azuredeploy.json +++ b/deploy/templates/premium-azuredeploy.json @@ -251,7 +251,7 @@ "name": "Transformation", "properties": { "streamingUnits": "[parameters('StreamingUnits')]", - "query": "[concat('SELECT \r\n DeviceId [DeviceId], \r\n PatientId [PatientId],\r\n EncounterId [EncounterId],\r\n collect() [Data],\r\n System.Timestamp [WindowTime],\r\n Type [MeasureType],\r\n count(*) [Count]\r\nINTO\r\n [FhirImportOutput]\r\nFROM\r\n [NormalizedData] PARTITION BY PartitionId TIMESTAMP BY OccurrenceTimeUtc\r\nGROUP BY PartitionId, \r\n DeviceId, \r\n PatientId, \r\n EncounterId, \r\n Type, \r\n TUMBLINGWINDOW(', parameters('JobWindowUnit'), ', ', parameters('JobWindowMagnitude'), ')')]" + "query": "[concat('SELECT \r\n DeviceId [DeviceId], \r\n PatientId [PatientId],\r\n EncounterId [EncounterId],\r\n CorrelationId [CorrelationId],\r\n collect() [Data],\r\n System.Timestamp [WindowTime],\r\n Type [MeasureType],\r\n count(*) [Count]\r\nINTO\r\n [FhirImportOutput]\r\nFROM\r\n [NormalizedData] PARTITION BY PartitionId TIMESTAMP BY OccurrenceTimeUtc\r\nGROUP BY PartitionId, \r\n DeviceId, \r\n PatientId, \r\n EncounterId, \r\n CorrelationId, \r\n Type, \r\n TUMBLINGWINDOW(', parameters('JobWindowUnit'), ', ', parameters('JobWindowMagnitude'), ')')]" } }, "functions": [ diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Template/JsonPathContentTemplate.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Template/JsonPathContentTemplate.cs index 50867db5..90ab36d3 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Template/JsonPathContentTemplate.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Template/JsonPathContentTemplate.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Runtime.CompilerServices; using EnsureThat; using Microsoft.Health.Fhir.Ingest.Data; @@ -26,6 +27,8 @@ public class JsonPathContentTemplate : IContentTemplate public virtual string TimestampExpression { get; set; } + public virtual string CorrelationIdExpression { get; set; } + #pragma warning disable CA2227 public virtual IList Values { get; set; } #pragma warning restore CA2227 @@ -43,10 +46,11 @@ public virtual IEnumerable GetMeasurements(JToken token) protected static T EvalExpression(JToken token, params string[] expressions) { EnsureArg.IsNotNull(token, nameof(token)); + EnsureArg.IsNotNull(expressions, nameof(expressions)); if (expressions == null) { - return default(T); + return default; } foreach (var expression in expressions) @@ -66,7 +70,14 @@ protected static T EvalExpression(JToken token, params string[] expressions) return evaluatedToken.Value(); } - return default(T); + return default; + } + + protected static bool IsExpressionDefined(params string[] expressions) + { + EnsureArg.IsNotNull(expressions, nameof(expressions)); + + return expressions.Any(ex => !string.IsNullOrWhiteSpace(ex)); } protected virtual DateTime? GetTimestamp(JToken token) => EvalExpression(token, TimestampExpression); @@ -77,6 +88,8 @@ protected static T EvalExpression(JToken token, params string[] expressions) protected virtual string GetEncounterId(JToken token) => EvalExpression(token, EncounterIdExpression); + protected virtual string GetCorrelationId(JToken token) => EvalExpression(token, CorrelationIdExpression); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private IEnumerable MatchTypeTokens(JToken token) { @@ -87,12 +100,19 @@ private Measurement CreateMeasurementFromToken(JToken token) { // Current assumption is that the expressions should match a single element and will error otherwise. - var deviceId = GetDeviceId(token); + string deviceId = GetDeviceId(token); EnsureArg.IsNotNull(deviceId, nameof(deviceId)); - var timestamp = GetTimestamp(token); + DateTime? timestamp = GetTimestamp(token); EnsureArg.IsNotNull(timestamp, nameof(timestamp)); + string correlationId = null; + if (IsExpressionDefined(CorrelationIdExpression)) + { + correlationId = GetCorrelationId(token); + EnsureArg.IsNotNull(correlationId, nameof(correlationId)); + } + var measurement = new Measurement { DeviceId = deviceId, @@ -100,6 +120,7 @@ private Measurement CreateMeasurementFromToken(JToken token) Type = TypeName, PatientId = GetPatientId(token), EncounterId = GetEncounterId(token), + CorrelationId = correlationId, }; if (Values != null) diff --git a/src/query/CollectionQuery/CollectionQuery.sln b/src/query/CollectionQuery/CollectionQuery.sln new file mode 100644 index 00000000..48d58f01 --- /dev/null +++ b/src/query/CollectionQuery/CollectionQuery.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30204.135 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{E6C563F2-BDC2-4A24-ACBF-01715157DCBB}") = "CollectionQuery", "CollectionQuery.asaproj", "{423A55D5-A156-4261-A2D8-C79E9ABF4E27}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {423A55D5-A156-4261-A2D8-C79E9ABF4E27}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {423A55D5-A156-4261-A2D8-C79E9ABF4E27}.Debug|Any CPU.Build.0 = Debug|Any CPU + {423A55D5-A156-4261-A2D8-C79E9ABF4E27}.Release|Any CPU.ActiveCfg = Release|Any CPU + {423A55D5-A156-4261-A2D8-C79E9ABF4E27}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {0EC2916C-C7DD-49E2-B135-098A7DC47C03} + EndGlobalSection +EndGlobal diff --git a/src/query/CollectionQuery/Script.asaql b/src/query/CollectionQuery/Script.asaql index 56f571f3..4758070b 100644 --- a/src/query/CollectionQuery/Script.asaql +++ b/src/query/CollectionQuery/Script.asaql @@ -2,6 +2,7 @@ DeviceId [DeviceId], PatientId [PatientId], EncounterId [EncounterId], + CorrelationId [CorrelationId], collect() [Data], System.Timestamp [WindowTime], Type [MeasureType], @@ -13,6 +14,7 @@ FROM GROUP BY PartitionId, DeviceId, PatientId, - EncounterId, + EncounterId, + CorrelationId, Type, TUMBLINGWINDOW(MINUTE, 5) \ No newline at end of file diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Template/JsonContentTemplateFactoryTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Template/JsonContentTemplateFactoryTests.cs index 36bae75e..f5f36758 100644 --- a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Template/JsonContentTemplateFactoryTests.cs +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Template/JsonContentTemplateFactoryTests.cs @@ -30,6 +30,7 @@ public void GivenValidTemplateJson_WhenFactoryCreate_ThenTemplateCreated_Test(st Assert.Equal("$.device", jsonPathTemplate.DeviceIdExpression); Assert.Equal("$.date", jsonPathTemplate.TimestampExpression); Assert.Null(jsonPathTemplate.PatientIdExpression); + Assert.Null(jsonPathTemplate.CorrelationIdExpression); Assert.Collection(jsonPathTemplate.Values, v => { Assert.True(v.Required); @@ -57,6 +58,7 @@ public void GivenValidTemplateJsonWithOptionalExpressions_WhenFactoryCreate_Then Assert.Equal("$.device", jsonPathTemplate.DeviceIdExpression); Assert.Equal("$.date", jsonPathTemplate.TimestampExpression); Assert.Equal("$.patient", jsonPathTemplate.PatientIdExpression); + Assert.Equal("$.sessionid", jsonPathTemplate.CorrelationIdExpression); Assert.Collection(jsonPathTemplate.Values, v => { Assert.True(v.Required); diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Template/JsonPathContentTemplateTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Template/JsonPathContentTemplateTests.cs index 25fa8628..c8a2af7b 100644 --- a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Template/JsonPathContentTemplateTests.cs +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Template/JsonPathContentTemplateTests.cs @@ -90,6 +90,19 @@ public class JsonPathContentTemplateTests }, }; + private static readonly IContentTemplate CorrelationIdTemplate = new JsonPathContentTemplate + { + TypeName = "heartrate", + TypeMatchExpression = "$..[?(@heartrate)]", + DeviceIdExpression = "$.device", + TimestampExpression = "$.date", + CorrelationIdExpression = "$.session", + Values = new List + { + new JsonPathValueExpression { ValueName = "hr", ValueExpression = "$.heartrate", Required = false }, + }, + }; + [Fact] public void GivenMultiValueTemplateAndValidTokenWithMissingValue_WhenGetMeasurements_ThenSingleMeasurementReturned_Test() { @@ -106,6 +119,7 @@ public void GivenMultiValueTemplateAndValidTokenWithMissingValue_WhenGetMeasurem Assert.Equal("abc", m.DeviceId); Assert.Null(m.PatientId); Assert.Null(m.EncounterId); + Assert.Null(m.CorrelationId); Assert.Collection(m.Properties, p => { Assert.Equal("hr", p.Name); @@ -130,6 +144,7 @@ public void GivenMultiValueTemplateAndValidTokenWithAllValues_WhenGetMeasurement Assert.Equal("abc", m.DeviceId); Assert.Null(m.PatientId); Assert.Null(m.EncounterId); + Assert.Null(m.CorrelationId); Assert.Collection( m.Properties, p => @@ -169,6 +184,7 @@ public void GivenMultiValueTemplateAndValidTokenArrayWithAllValues_WhenGetMeasur Assert.Equal("abc", m.DeviceId); Assert.Null(m.PatientId); Assert.Null(m.EncounterId); + Assert.Null(m.CorrelationId); Assert.Collection( m.Properties, p => @@ -249,6 +265,7 @@ public void GivenMultiValueRequiredTemplateAndValidTokenWithAllValues_WhenGetMea Assert.Equal("abc", m.DeviceId); Assert.Null(m.PatientId); Assert.Null(m.EncounterId); + Assert.Null(m.CorrelationId); Assert.Collection( m.Properties, p => @@ -280,6 +297,7 @@ public void GivenSingleValueTemplateAndValidToken_WhenGetMeasurements_ThenSingle Assert.Equal("abc", m.DeviceId); Assert.Null(m.PatientId); Assert.Null(m.EncounterId); + Assert.Null(m.CorrelationId); Assert.Collection(m.Properties, p => { Assert.Equal("hr", p.Name); @@ -304,6 +322,7 @@ public void GivenSingleValueOptionalContentTemplateAndValidToken_WhenGetMeasurem Assert.Equal("abc", m.DeviceId); Assert.Equal("123", m.PatientId); Assert.Equal("789", m.EncounterId); + Assert.Null(m.CorrelationId); Assert.Collection(m.Properties, p => { Assert.Equal("hr", p.Name); @@ -328,6 +347,7 @@ public void GivenSingleValueCompoundAndTemplateAndValidToken_WhenGetMeasurements Assert.Equal("abc", m.DeviceId); Assert.Null(m.PatientId); Assert.Null(m.EncounterId); + Assert.Null(m.CorrelationId); Assert.Collection(m.Properties, p => { Assert.Equal("hr", p.Name); @@ -344,6 +364,7 @@ public void GivenSingleValueCompoundAndTemplateAndValidToken_WhenGetMeasurements Assert.Equal("abc", m.DeviceId); Assert.Null(m.PatientId); Assert.Null(m.EncounterId); + Assert.Null(m.CorrelationId); Assert.Collection(m.Properties, p => { Assert.Equal("hr", p.Name); @@ -380,6 +401,7 @@ public void GivenPropertyWithSpace_WhenGetMeasurements_ThenSingleMeasurementRetu Assert.Equal("data", m.DeviceId); Assert.Null(m.PatientId); Assert.Null(m.EncounterId); + Assert.Null(m.CorrelationId); Assert.Collection(m.Properties, p => { Assert.Equal("prop", p.Name); @@ -412,6 +434,42 @@ public void GivenTemplateAndInvalidToken_WhenGetMeasurements_ThenEmptyIEnumerabl Assert.Empty(result); } + [Fact] + public void GivenTemplateWithCorrelationIdAndIdPresent_WhenGetMeasurements_ThenCorrelationIdReturn_Test() + { + var time = DateTime.UtcNow; + var session = Guid.NewGuid().ToString(); + var token = JToken.FromObject(new { heartrate = "60", device = "abc", date = time, session }); + + var result = CorrelationIdTemplate.GetMeasurements(token).ToArray(); + + Assert.NotNull(result); + Assert.Collection(result, m => + { + Assert.Equal("heartrate", m.Type); + Assert.Equal(time, m.OccurrenceTimeUtc); + Assert.Equal("abc", m.DeviceId); + Assert.Null(m.PatientId); + Assert.Null(m.EncounterId); + Assert.Equal(session, m.CorrelationId); + Assert.Collection(m.Properties, p => + { + Assert.Equal("hr", p.Name); + Assert.Equal("60", p.Value); + }); + }); + } + + [Fact] + public void GivenTemplateWithCorrelationIdAndIdMissing_WhenGetMeasurements_ThenArgumentNullExceptionThrown_Test() + { + var time = DateTime.UtcNow; + var token = JToken.FromObject(new { heartrate = "60", device = "abc", date = time }); + + var ex = Assert.Throws(() => CorrelationIdTemplate.GetMeasurements(token).ToArray()); + Assert.Contains("correlationId", ex.Message); + } + public class JsonWidget { [JsonProperty(PropertyName = "My Property")] diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/TestInput/data_JsonPathContentTemplateValidWithOptional.json b/test/Microsoft.Health.Fhir.Ingest.UnitTests/TestInput/data_JsonPathContentTemplateValidWithOptional.json index 7975827f..2a216e40 100644 --- a/test/Microsoft.Health.Fhir.Ingest.UnitTests/TestInput/data_JsonPathContentTemplateValidWithOptional.json +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/TestInput/data_JsonPathContentTemplateValidWithOptional.json @@ -6,6 +6,7 @@ "deviceIdExpression": "$.device", "timestampExpression": "$.date", "patientIdExpression": "$.patient", + "correlationIdExpression": "$.sessionid", "values": [ { "required": "true", From 7b20386f9c3579f7de659180568c288f38948790 Mon Sep 17 00:00:00 2001 From: Dustin Burson Date: Wed, 10 Jun 2020 20:01:05 -0700 Subject: [PATCH 03/13] * Update exception in CorrelationObservationGroup to throw new CorrelationIdNotDefinedException with id not set * Add CorrelationIdNotDefinedException to telemetry processor as a known non-recoverable error. * Add unit tests for new observation groups and observation group factories. --- .../Data/CorrelationIdNotDefinedException.cs | 26 +++ .../CorrelationMeasurementObservationGroup.cs | 7 +- .../Telemetry/ExceptionTelemetryProcessor.cs | 4 +- ...MeasurementObservationGroupFactoryTests.cs | 102 +++++++++++ ...elationMeasurementObservationGroupTests.cs | 167 ++++++++++++++++++ ...MeasurementObservationGroupFactoryTests.cs | 78 ++++++++ ...ePeriodMeasurementObservationGroupTests.cs | 150 ++++++++++++++++ 7 files changed, 532 insertions(+), 2 deletions(-) create mode 100644 src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationIdNotDefinedException.cs create mode 100644 test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupFactoryTests.cs create mode 100644 test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupTests.cs create mode 100644 test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/MeasurementObservationGroupFactoryTests.cs create mode 100644 test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/TimePeriodMeasurementObservationGroupTests.cs diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationIdNotDefinedException.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationIdNotDefinedException.cs new file mode 100644 index 00000000..15e4e220 --- /dev/null +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationIdNotDefinedException.cs @@ -0,0 +1,26 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Fhir.Ingest.Data +{ + public class CorrelationIdNotDefinedException : Exception + { + public CorrelationIdNotDefinedException(string message) + : base(message) + { + } + + public CorrelationIdNotDefinedException(string message, Exception innerException) + : base(message, innerException) + { + } + + public CorrelationIdNotDefinedException() + { + } + } +} diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroup.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroup.cs index 31ca9400..91c9481f 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroup.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroup.cs @@ -16,7 +16,12 @@ public class CorrelationMeasurementObservationGroup : SortedObservationGroup public CorrelationMeasurementObservationGroup(string correlationId) { - _correlationId = EnsureArg.IsNotNullOrWhiteSpace(correlationId, nameof(correlationId)); + if (string.IsNullOrWhiteSpace(correlationId)) + { + throw new CorrelationIdNotDefinedException(); + } + + _correlationId = correlationId; } public override (DateTime Start, DateTime End) Boundary => (_start, _end); diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ExceptionTelemetryProcessor.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ExceptionTelemetryProcessor.cs index cd48d257..b815d616 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ExceptionTelemetryProcessor.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ExceptionTelemetryProcessor.cs @@ -9,6 +9,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Health.Common.Telemetry; using Microsoft.Health.Extensions.Fhir; +using Microsoft.Health.Fhir.Ingest.Data; using Microsoft.Health.Fhir.Ingest.Service; using Microsoft.Health.Fhir.Ingest.Template; @@ -25,7 +26,8 @@ public ExceptionTelemetryProcessor() typeof(NotSupportedException), typeof(FhirResourceNotFoundException), typeof(MultipleResourceFoundException<>), - typeof(TemplateNotFoundException)) + typeof(TemplateNotFoundException), + typeof(CorrelationIdNotDefinedException)) { } diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupFactoryTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupFactoryTests.cs new file mode 100644 index 00000000..00f46039 --- /dev/null +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupFactoryTests.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Health.Tests.Common; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.Ingest.Data +{ + public class CorrelationMeasurementObservationGroupFactoryTests + { + [Fact] + public void GivenMultipleMeasurementsWithCorrelationId_WhenBuild_AllMeasurementsReturnInSingleObservationGroupSortedByDate_Test() + { + var seedDate = new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + var measurement = new IMeasurement[] + { + new Measurement + { + OccurrenceTimeUtc = seedDate, + Properties = new List + { + new MeasurementProperty { Name = "a", Value = "1" }, + }, + }, + new Measurement + { + OccurrenceTimeUtc = seedDate.AddDays(1), + Properties = new List + { + new MeasurementProperty { Name = "a", Value = "2" }, + }, + }, + new Measurement + { + OccurrenceTimeUtc = seedDate.AddHours(1), + Properties = new List + { + new MeasurementProperty { Name = "a", Value = "3" }, + }, + }, + }; + + var measureGroup = Substitute.For() + .Mock(mg => mg.MeasureType.Returns("a")) + .Mock(mg => mg.CorrelationId.Returns("id")) + .Mock(mg => mg.Data.Returns(measurement)); + + var factory = new CorrelationMeasurementObservationGroupFactory(); + + var result = factory.Build(measureGroup)?.ToArray(); + Assert.NotNull(result); + Assert.Single(result); + + Assert.Collection( + result, + og => + { + Assert.Equal(seedDate, og.Boundary.Start); + Assert.Equal(measurement[1].OccurrenceTimeUtc, og.Boundary.End); + + var properties = og.GetValues().ToArray(); + Assert.Single(properties); + Assert.Collection( + properties, + p => + { + Assert.Equal("a", p.Key); + Assert.Collection( + p.Value, + v => + { + Assert.Equal(seedDate, v.Time); + Assert.Equal("1", v.Value); + }, + v => + { + Assert.Equal(measurement[2].OccurrenceTimeUtc, v.Time); + Assert.Equal("3", v.Value); + }, + v => + { + Assert.Equal(measurement[1].OccurrenceTimeUtc, v.Time); + Assert.Equal("2", v.Value); + }); + }); + }); + } + + [Fact] + public void GivenMeasurementGroupWithoutCorrelationId_WhenBuild_ThenCorrelationIdNotDefinedExceptionThrown_Test() + { + var measureGroup = Substitute.For() + .Mock(mg => mg.CorrelationId.Returns((string)null)); + + var factory = new CorrelationMeasurementObservationGroupFactory(); + + Assert.Throws(() => factory.Build(measureGroup)?.ToArray()); + } + } +} diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupTests.cs new file mode 100644 index 00000000..7f404eb2 --- /dev/null +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupTests.cs @@ -0,0 +1,167 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Fhir.Ingest.Data +{ + public class CorrelationMeasurementObservationGroupTests + { + [Fact] + public void GivenSingleMeasurement_WhenGetBoundary_ThenBoundarySet_Test() + { + var start = DateTime.UtcNow.AddDays(-1); + + var og = new CorrelationMeasurementObservationGroup("test"); + + og.AddMeasurement(new Measurement { OccurrenceTimeUtc = start }); + + Assert.Equal(start, og.Boundary.Start); + Assert.Equal(start, og.Boundary.End); + } + + [Fact] + public void GivenMultipleMeasurement_WhenGetBoundary_ThenBoundaryMatchesMinAndMax_Test() + { + var start = DateTime.UtcNow.AddDays(-1); + var end = DateTime.UtcNow; + + var og = new CorrelationMeasurementObservationGroup("test"); + + og.AddMeasurement(new Measurement { OccurrenceTimeUtc = end }); + og.AddMeasurement(new Measurement { OccurrenceTimeUtc = start.AddHours(2) }); + og.AddMeasurement(new Measurement { OccurrenceTimeUtc = start }); + og.AddMeasurement(new Measurement { OccurrenceTimeUtc = start.AddHours(1) }); + + Assert.Equal(start, og.Boundary.Start); + Assert.Equal(end, og.Boundary.End); + } + + [Fact] + public void GivenValidObservationGroup_WhenGetIdSegment_CorrectValueReturned_Test() + { + var id = Guid.NewGuid().ToString(); + var og = new CorrelationMeasurementObservationGroup(id); + + var segment = og.GetIdSegment(); + Assert.Equal(id, segment); + } + + [Fact] + public void GivenDifferentNamedValues_WhenAddMeasurement_ThenGetValuesReturnsSorted_Test() + { + var startDate = new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var endDate = startDate.AddDays(1); + + var measurement = new IMeasurement[] + { + new Measurement + { + OccurrenceTimeUtc = startDate, + Properties = new List + { + new MeasurementProperty { Name = "a", Value = "1" }, + }, + }, + new Measurement + { + OccurrenceTimeUtc = endDate, + Properties = new List + { + new MeasurementProperty { Name = "a", Value = "2" }, + }, + }, + new Measurement + { + OccurrenceTimeUtc = startDate.AddHours(1), + Properties = new List + { + new MeasurementProperty { Name = "a", Value = "3" }, + }, + }, + new Measurement + { + OccurrenceTimeUtc = endDate, + Properties = new List + { + new MeasurementProperty { Name = "b", Value = "1" }, + }, + }, + new Measurement + { + OccurrenceTimeUtc = startDate, + Properties = new List + { + new MeasurementProperty { Name = "b", Value = "2" }, + }, + }, + new Measurement + { + OccurrenceTimeUtc = startDate, + Properties = new List + { + new MeasurementProperty { Name = "c", Value = "3" }, + }, + }, + }; + + var og = new CorrelationMeasurementObservationGroup("test"); + + foreach (var m in measurement) + { + og.AddMeasurement(m); + } + + var values = og.GetValues(); + + var aValues = values["a"]; + + Assert.Collection( + aValues, + v => + { + Assert.Equal(startDate, v.Time); + Assert.Equal("1", v.Value); + }, + v => + { + Assert.Equal(measurement[2].OccurrenceTimeUtc, v.Time); + Assert.Equal("3", v.Value); + }, + v => + { + Assert.Equal(endDate, v.Time); + Assert.Equal("2", v.Value); + }); + + var bValues = values["b"]; + + Assert.Collection( + bValues, + v => + { + Assert.Equal(startDate, v.Time); + Assert.Equal("2", v.Value); + }, + v => + { + Assert.Equal(endDate, v.Time); + Assert.Equal("1", v.Value); + }); + + var cValues = values["c"]; + + Assert.Collection( + cValues, + v => + { + Assert.Equal(startDate, v.Time); + Assert.Equal("3", v.Value); + }); + } + } +} diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/MeasurementObservationGroupFactoryTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/MeasurementObservationGroupFactoryTests.cs new file mode 100644 index 00000000..ba7c8eef --- /dev/null +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/MeasurementObservationGroupFactoryTests.cs @@ -0,0 +1,78 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Health.Fhir.Ingest.Template; +using Microsoft.Health.Tests.Common; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.Ingest.Data +{ + public class MeasurementObservationGroupFactoryTests + { + [Fact] + public void GivenPeriodIntervalCorrelationId_WhenBuild_ThenCorrelationMeasurementObservationGroupsReturned_Test() + { + var factory = new MeasurementObservationGroupFactory(ObservationPeriodInterval.CorrelationId); + + var seedDate = new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + var measurement = new IMeasurement[] + { + new Measurement + { + OccurrenceTimeUtc = seedDate, + Properties = new List + { + new MeasurementProperty { Name = "a", Value = "1" }, + }, + }, + }; + + var measureGroup = Substitute.For() + .Mock(mg => mg.MeasureType.Returns("a")) + .Mock(mg => mg.Data.Returns(measurement)) + .Mock(mg => mg.CorrelationId.Returns("id")); + + var observationGroup = factory.Build(measureGroup).First(); + + Assert.IsType(observationGroup); + } + + [Theory] + [InlineData(ObservationPeriodInterval.Daily)] + [InlineData(ObservationPeriodInterval.Hourly)] + [InlineData(ObservationPeriodInterval.Single)] + public void GivenOtherPeriodInterval_WhenBuild_ThenTimePeriodMeasurementObservationGroupsReturned_Test(ObservationPeriodInterval periodInterval) + { + var factory = new MeasurementObservationGroupFactory(periodInterval); + + var seedDate = new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + var measurement = new IMeasurement[] + { + new Measurement + { + OccurrenceTimeUtc = seedDate, + Properties = new List + { + new MeasurementProperty { Name = "a", Value = "1" }, + }, + }, + }; + + var measureGroup = Substitute.For() + .Mock(mg => mg.MeasureType.Returns("a")) + .Mock(mg => mg.Data.Returns(measurement)); + + var observationGroup = factory.Build(measureGroup).First(); + + Assert.IsType(observationGroup); + } + } +} diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/TimePeriodMeasurementObservationGroupTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/TimePeriodMeasurementObservationGroupTests.cs new file mode 100644 index 00000000..871f8cf7 --- /dev/null +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/TimePeriodMeasurementObservationGroupTests.cs @@ -0,0 +1,150 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Xunit; + +namespace Microsoft.Health.Fhir.Ingest.Data +{ + public class TimePeriodMeasurementObservationGroupTests + { + [Fact] + public void GivenBoundary_WhenCtor_ThenBoundarySet_Test() + { + var start = DateTime.UtcNow.AddDays(-1); + var end = DateTime.UtcNow; + + var og = new TimePeriodMeasurementObservationGroup((start, end)); + + Assert.Equal(start, og.Boundary.Start); + Assert.Equal(end, og.Boundary.End); + } + + [Fact] + public void GivenValidObservationGroup_WhenGetIdSegment_CorrectValueReturned_Test() + { + var start = new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var end = start.AddHours(1).AddTicks(-1); + var og = new TimePeriodMeasurementObservationGroup((start, end)); + + var segment = og.GetIdSegment(); + Assert.Equal("20190101000000Z.20190101005959Z", segment); + } + + [Fact] + public void GivenDifferentNamedValues_WhenAddMeasurement_ThenGetValuesReturnsSorted_Test() + { + var startDate = new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var endDate = startDate.AddDays(1); + + var measurement = new IMeasurement[] + { + new Measurement + { + OccurrenceTimeUtc = startDate, + Properties = new List + { + new MeasurementProperty { Name = "a", Value = "1" }, + }, + }, + new Measurement + { + OccurrenceTimeUtc = endDate, + Properties = new List + { + new MeasurementProperty { Name = "a", Value = "2" }, + }, + }, + new Measurement + { + OccurrenceTimeUtc = startDate.AddHours(1), + Properties = new List + { + new MeasurementProperty { Name = "a", Value = "3" }, + }, + }, + new Measurement + { + OccurrenceTimeUtc = endDate, + Properties = new List + { + new MeasurementProperty { Name = "b", Value = "1" }, + }, + }, + new Measurement + { + OccurrenceTimeUtc = startDate, + Properties = new List + { + new MeasurementProperty { Name = "b", Value = "2" }, + }, + }, + new Measurement + { + OccurrenceTimeUtc = startDate, + Properties = new List + { + new MeasurementProperty { Name = "c", Value = "3" }, + }, + }, + }; + + var og = new TimePeriodMeasurementObservationGroup((startDate, endDate)); + + foreach (var m in measurement) + { + og.AddMeasurement(m); + } + + var values = og.GetValues(); + + var aValues = values["a"]; + + Assert.Collection( + aValues, + v => + { + Assert.Equal(startDate, v.Time); + Assert.Equal("1", v.Value); + }, + v => + { + Assert.Equal(measurement[2].OccurrenceTimeUtc, v.Time); + Assert.Equal("3", v.Value); + }, + v => + { + Assert.Equal(endDate, v.Time); + Assert.Equal("2", v.Value); + }); + + var bValues = values["b"]; + + Assert.Collection( + bValues, + v => + { + Assert.Equal(startDate, v.Time); + Assert.Equal("2", v.Value); + }, + v => + { + Assert.Equal(endDate, v.Time); + Assert.Equal("1", v.Value); + }); + + var cValues = values["c"]; + + Assert.Collection( + cValues, + v => + { + Assert.Equal(startDate, v.Time); + Assert.Equal("3", v.Value); + }); + } + } +} From 602e42fd3ef88862fa28c35678f361c93cd134bc Mon Sep 17 00:00:00 2001 From: Dustin Burson Date: Wed, 10 Jun 2020 21:58:07 -0700 Subject: [PATCH 04/13] Add missing file header --- .../CorrelationMeasurementObservationGroupFactoryTests.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupFactoryTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupFactoryTests.cs index 00f46039..3ebb165d 100644 --- a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupFactoryTests.cs +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupFactoryTests.cs @@ -1,4 +1,9 @@ -using System; +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; using System.Collections.Generic; using System.Linq; using Microsoft.Health.Tests.Common; From bfbe3f73a4d874714c2eeeefdcaad53109fd7f7d Mon Sep 17 00:00:00 2001 From: Dustin Burson Date: Thu, 11 Jun 2020 11:10:12 -0700 Subject: [PATCH 05/13] Add unit test for FhirTemplate to ensure default value for PeriodInterval hasn't changed. This is to ensure no backwards breaking changes. --- .../Template/FhirTemplateTests.cs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 test/Microsoft.Health.Fhir.Ingest.UnitTests/Template/FhirTemplateTests.cs diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Template/FhirTemplateTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Template/FhirTemplateTests.cs new file mode 100644 index 00000000..653c800e --- /dev/null +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Template/FhirTemplateTests.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Fhir.Ingest.Template +{ + public class FhirTemplateTests + { + [Fact] + public void GivenDefaultFhirTemplate_WhenGetPeriodInterval_ThenValueIsSingle_Test() + { + var template = new TestFhirTemplate(); + + Assert.Equal(ObservationPeriodInterval.Single, template.PeriodInterval); + } + + private class TestFhirTemplate : FhirTemplate + { + } + } +} From 7b7249b3f31c6c1b2e64a8e87e2d914f86f43375 Mon Sep 17 00:00:00 2001 From: Dustin Burson Date: Thu, 11 Jun 2020 17:25:13 -0700 Subject: [PATCH 06/13] FIx CorrelationMeasurementObservationGroupFactory not setting ObservationGroup name --- .../Data/CorrelationMeasurementObservationGroupFactory.cs | 6 +++++- .../Service/R4FhirImportService.cs | 2 +- .../CorrelationMeasurementObservationGroupFactoryTests.cs | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroupFactory.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroupFactory.cs index 1a2079ab..595ee7a8 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroupFactory.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroupFactory.cs @@ -14,7 +14,11 @@ public IEnumerable Build(IMeasurementGroup input) { EnsureArg.IsNotNull(input, nameof(input)); - var observationGroup = new CorrelationMeasurementObservationGroup(input.CorrelationId); + var observationGroup = new CorrelationMeasurementObservationGroup(input.CorrelationId) + { + Name = input.MeasureType, + }; + foreach (var m in input.Data) { observationGroup.AddMeasurement(m); diff --git a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Service/R4FhirImportService.cs b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Service/R4FhirImportService.cs index 6c3ed106..c3cfb608 100644 --- a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Service/R4FhirImportService.cs +++ b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Service/R4FhirImportService.cs @@ -103,7 +103,7 @@ protected static Model.Identifier GenerateObservationIdentifier(IObservationGrou EnsureArg.IsNotNull(grp, nameof(grp)); EnsureArg.IsNotNull(ids, nameof(ids)); - var identity = FhirImportService.GenerateObservationId(grp, ids[ResourceType.Device], ids[ResourceType.Patient]); + var identity = GenerateObservationId(grp, ids[ResourceType.Device], ids[ResourceType.Patient]); return new Model.Identifier { System = identity.System, diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupFactoryTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupFactoryTests.cs index 3ebb165d..93c29ada 100644 --- a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupFactoryTests.cs +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/CorrelationMeasurementObservationGroupFactoryTests.cs @@ -65,8 +65,11 @@ public void GivenMultipleMeasurementsWithCorrelationId_WhenBuild_AllMeasurements Assert.Equal(seedDate, og.Boundary.Start); Assert.Equal(measurement[1].OccurrenceTimeUtc, og.Boundary.End); + Assert.Equal("a", og.Name); + var properties = og.GetValues().ToArray(); Assert.Single(properties); + Assert.Collection( properties, p => From e312af80e56bdc4b8a237d9f9dfb7c41a0889202 Mon Sep 17 00:00:00 2001 From: Dustin Burson Date: Thu, 11 Jun 2020 18:57:27 -0700 Subject: [PATCH 07/13] Add additional validations --- .../Data/CorrelationMeasurementObservationGroupFactory.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroupFactory.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroupFactory.cs index 595ee7a8..84e39514 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroupFactory.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/CorrelationMeasurementObservationGroupFactory.cs @@ -13,6 +13,8 @@ public class CorrelationMeasurementObservationGroupFactory : IObservationGroupFa public IEnumerable Build(IMeasurementGroup input) { EnsureArg.IsNotNull(input, nameof(input)); + EnsureArg.IsNotNull(input.MeasureType, nameof(input.MeasureType)); + EnsureArg.IsNotNull(input.Data, nameof(input.Data)); var observationGroup = new CorrelationMeasurementObservationGroup(input.CorrelationId) { From de9cd1d25d0436ccd73afddbd66a9dbaf202425b Mon Sep 17 00:00:00 2001 From: Dustin Burson Date: Mon, 15 Jun 2020 12:32:15 -0700 Subject: [PATCH 08/13] Refactor value processors to accept new interface IObservationData that includes data to be created and information about the existing observation. --- .../Template/IObservationData.cs | 19 ++ .../Template/ObservationData.cs | 19 ++ .../Service/R4FhirImportService.cs | 8 +- .../CodeValueFhirTemplateProcessor.cs | 41 ++- .../CodeableConceptFhirValueProcessor.cs | 9 +- .../Template/QuantityFhirValueProcessor.cs | 10 +- .../Template/R4FhirValueProcessor.cs | 4 +- .../Template/SampledDataFhirValueProcessor.cs | 22 +- .../Template/StringFhirValueProcessor.cs | 12 +- .../CodeValueFhirTemplateProcessorTests.cs | 254 +++++++++++------- .../CodeableConceptFhirValueProcessorTests.cs | 15 +- .../QuantityFhirValueProcessorTests.cs | 17 +- .../SampledDataFhirValueProcessorTests.cs | 40 ++- .../Template/StringFhirValueProcessorTests.cs | 17 +- 14 files changed, 331 insertions(+), 156 deletions(-) create mode 100644 src/lib/Microsoft.Health.Fhir.Ingest/Template/IObservationData.cs create mode 100644 src/lib/Microsoft.Health.Fhir.Ingest/Template/ObservationData.cs diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Template/IObservationData.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Template/IObservationData.cs new file mode 100644 index 00000000..daa06e95 --- /dev/null +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Template/IObservationData.cs @@ -0,0 +1,19 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; + +namespace Microsoft.Health.Fhir.Ingest.Template +{ + public interface IObservationData + { + (DateTime start, DateTime end) ObservationPeriod { get; } + + (DateTime start, DateTime end) DataPeriod { get; } + + IEnumerable<(DateTime, string)> Data { get; } + } +} diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Template/ObservationData.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Template/ObservationData.cs new file mode 100644 index 00000000..98381886 --- /dev/null +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Template/ObservationData.cs @@ -0,0 +1,19 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; + +namespace Microsoft.Health.Fhir.Ingest.Template +{ + public class ObservationData : IObservationData + { + public (DateTime start, DateTime end) ObservationPeriod { get; set; } + + public (DateTime start, DateTime end) DataPeriod { get; set; } + + public IEnumerable<(DateTime, string)> Data { get; set; } + } +} diff --git a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Service/R4FhirImportService.cs b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Service/R4FhirImportService.cs index c3cfb608..4eee55a9 100644 --- a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Service/R4FhirImportService.cs +++ b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Service/R4FhirImportService.cs @@ -23,18 +23,18 @@ public class R4FhirImportService : { private readonly IFhirClient _client; private readonly IFhirTemplateProcessor, Model.Observation> _fhirTemplateProcessor; - private readonly IResourceIdentityService _resourceIdentityService; private readonly IMemoryCache _observationCache; public R4FhirImportService(IResourceIdentityService resourceIdentityService, IFhirClient fhirClient, IFhirTemplateProcessor, Model.Observation> fhirTemplateProcessor, IMemoryCache observationCache) { _fhirTemplateProcessor = EnsureArg.IsNotNull(fhirTemplateProcessor, nameof(fhirTemplateProcessor)); _client = EnsureArg.IsNotNull(fhirClient, nameof(fhirClient)); - _resourceIdentityService = EnsureArg.IsNotNull(resourceIdentityService, nameof(resourceIdentityService)); _observationCache = EnsureArg.IsNotNull(observationCache, nameof(observationCache)); + + ResourceIdentityService = EnsureArg.IsNotNull(resourceIdentityService, nameof(resourceIdentityService)); } - protected IResourceIdentityService ResourceIdentityService => _resourceIdentityService; + protected IResourceIdentityService ResourceIdentityService { get; } public override async Task ProcessAsync(ILookupTemplate config, IMeasurementGroup data, Func> errorConsumer = null) { @@ -63,7 +63,7 @@ public virtual async Task SaveObservationAsync(ILookupTemplate(newObservation).ConfigureAwait(false); + result = await _client.CreateAsync(newObservation).ConfigureAwait(false); } else { diff --git a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeValueFhirTemplateProcessor.cs b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeValueFhirTemplateProcessor.cs index ecef6606..14af04c1 100644 --- a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeValueFhirTemplateProcessor.cs +++ b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeValueFhirTemplateProcessor.cs @@ -16,14 +16,14 @@ namespace Microsoft.Health.Fhir.Ingest.Template { public class CodeValueFhirTemplateProcessor : FhirTemplateProcessor { - private readonly IFhirValueProcessor<(DateTime start, DateTime end, IEnumerable<(DateTime, string)> values), Element> _valueProcessor; + private readonly IFhirValueProcessor _valueProcessor; public CodeValueFhirTemplateProcessor() : this(new R4FhirValueProcessor()) { } - public CodeValueFhirTemplateProcessor(IFhirValueProcessor<(DateTime start, DateTime end, IEnumerable<(DateTime, string)> values), Element> valueProcessor) + public CodeValueFhirTemplateProcessor(IFhirValueProcessor valueProcessor) { EnsureArg.IsNotNull(valueProcessor); _valueProcessor = valueProcessor; @@ -55,7 +55,7 @@ protected override Observation CreateObseravtionImpl(CodeValueFhirTemplate templ if (!string.IsNullOrWhiteSpace(template?.Value?.ValueName) && values.TryGetValue(template?.Value?.ValueName, out var obValues)) { - observation.Value = _valueProcessor.CreateValue(template.Value, (grp.Boundary.Start, grp.Boundary.End, obValues)); + observation.Value = _valueProcessor.CreateValue(template.Value, CreateMergeData(grp.Boundary, grp.Boundary, obValues)); } if (template?.Components?.Count > 0) @@ -70,7 +70,7 @@ protected override Observation CreateObseravtionImpl(CodeValueFhirTemplate templ new Observation.ComponentComponent { Code = ResolveCode(component.Value.ValueName, component.Codes), - Value = _valueProcessor.CreateValue(component.Value, (grp.Boundary.Start, grp.Boundary.End, compValues)), + Value = _valueProcessor.CreateValue(component.Value, CreateMergeData(grp.Boundary, grp.Boundary, compValues)), }); } } @@ -93,10 +93,11 @@ protected override Observation MergeObservationImpl(CodeValueFhirTemplate templa } var values = grp.GetValues(); + var observatiobPeriod = GetObservationPeriod(existingObservation); if (!string.IsNullOrWhiteSpace(template?.Value?.ValueName) && values.TryGetValue(template?.Value?.ValueName, out var obValues)) { - existingObservation.Value = _valueProcessor.MergeValue(template.Value, (grp.Boundary.Start, grp.Boundary.End, obValues), existingObservation.Value); + existingObservation.Value = _valueProcessor.MergeValue(template.Value, CreateMergeData(grp.Boundary, observatiobPeriod, obValues), existingObservation.Value); } if (template?.Components?.Count > 0) @@ -120,12 +121,12 @@ protected override Observation MergeObservationImpl(CodeValueFhirTemplate templa new Observation.ComponentComponent { Code = ResolveCode(component.Value.ValueName, component.Codes), - Value = _valueProcessor.CreateValue(component.Value, (grp.Boundary.Start, grp.Boundary.End, compValues)), + Value = _valueProcessor.CreateValue(component.Value, CreateMergeData(grp.Boundary, observatiobPeriod, compValues)), }); } else { - foundComponent.Value = _valueProcessor.MergeValue(component.Value, (grp.Boundary.Start, grp.Boundary.End, compValues), foundComponent.Value); + foundComponent.Value = _valueProcessor.MergeValue(component.Value, CreateMergeData(grp.Boundary, observatiobPeriod, compValues), foundComponent.Value); } } } @@ -163,5 +164,31 @@ protected static List ResolveCategory(IList data) + { + return new ObservationData + { + DataPeriod = dataPeriod, + ObservationPeriod = observationPeriod, + Data = EnsureArg.IsNotNull(data, nameof(data)), + }; + } + + protected static (DateTime start, DateTime end) GetObservationPeriod(Observation observation) + { + EnsureArg.IsNotNull(observation, nameof(observation)); + var effective = EnsureArg.IsNotNull(observation.Effective, nameof(observation.Effective)); + + switch (effective) + { + case FhirDateTime dt: + return (dt.ToDateTimeOffset(TimeSpan.Zero).UtcDateTime, dt.ToDateTimeOffset(TimeSpan.Zero).UtcDateTime); + case Period p: + return (p.StartElement.ToDateTimeOffset(TimeSpan.Zero).UtcDateTime, p.EndElement.ToDateTimeOffset(TimeSpan.Zero).UtcDateTime); + default: + throw new NotSupportedException($"Observation effective type of {effective.GetType()} is not supported."); + } + } } } diff --git a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeableConceptFhirValueProcessor.cs b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeableConceptFhirValueProcessor.cs index 701c7c4d..e3f72b7c 100644 --- a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeableConceptFhirValueProcessor.cs +++ b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeableConceptFhirValueProcessor.cs @@ -4,18 +4,17 @@ // ------------------------------------------------------------------------------------------------- using System; -using System.Collections.Generic; using System.Linq; using EnsureThat; using Hl7.Fhir.Model; namespace Microsoft.Health.Fhir.Ingest.Template { - public class CodeableConceptFhirValueProcessor : FhirValueProcessor values), Element> + public class CodeableConceptFhirValueProcessor : FhirValueProcessor { - protected override Element CreateValueImpl(CodeableConceptFhirValueType template, (DateTime start, DateTime end, IEnumerable<(DateTime, string)> values) inValue) + protected override Element CreateValueImpl(CodeableConceptFhirValueType template, IObservationData inValue) { - // Values for codeable concepts currently have no meaning. The existance of the measurement means the code applies. + // Values for codeable concepts currently have no meaning. The existence of the measurement means the code applies. EnsureArg.IsNotNull(template, nameof(template)); @@ -32,7 +31,7 @@ protected override Element CreateValueImpl(CodeableConceptFhirValueType template }; } - protected override Element MergeValueImpl(CodeableConceptFhirValueType template, (DateTime start, DateTime end, IEnumerable<(DateTime, string)> values) inValue, Element existingValue) + protected override Element MergeValueImpl(CodeableConceptFhirValueType template, IObservationData inValue, Element existingValue) { if (!(existingValue is CodeableConcept)) { diff --git a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/QuantityFhirValueProcessor.cs b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/QuantityFhirValueProcessor.cs index 325d0268..1e88c720 100644 --- a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/QuantityFhirValueProcessor.cs +++ b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/QuantityFhirValueProcessor.cs @@ -12,22 +12,24 @@ namespace Microsoft.Health.Fhir.Ingest.Template { - public class QuantityFhirValueProcessor : FhirValueProcessor values), Element> + public class QuantityFhirValueProcessor : FhirValueProcessor { - protected override Element CreateValueImpl(QuantityFhirValueType template, (DateTime start, DateTime end, IEnumerable<(DateTime, string)> values) inValue) + protected override Element CreateValueImpl(QuantityFhirValueType template, IObservationData inValue) { EnsureArg.IsNotNull(template, nameof(template)); + EnsureArg.IsNotNull(inValue, nameof(inValue)); + IEnumerable<(DateTime, string)> values = EnsureArg.IsNotNull(inValue.Data, nameof(IObservationData.Data)); return new Quantity { - Value = decimal.Parse(inValue.values.Single().Item2, CultureInfo.InvariantCulture), + Value = decimal.Parse(values.Single().Item2, CultureInfo.InvariantCulture), Unit = template.Unit, System = template.System, Code = template.Code, }; } - protected override Element MergeValueImpl(QuantityFhirValueType template, (DateTime start, DateTime end, IEnumerable<(DateTime, string)> values) inValue, Element existingValue) + protected override Element MergeValueImpl(QuantityFhirValueType template, IObservationData inValue, Element existingValue) { if (!(existingValue is Quantity)) { diff --git a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/R4FhirValueProcessor.cs b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/R4FhirValueProcessor.cs index 62887600..c6f619ab 100644 --- a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/R4FhirValueProcessor.cs +++ b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/R4FhirValueProcessor.cs @@ -3,13 +3,11 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; -using System.Collections.Generic; using Hl7.Fhir.Model; namespace Microsoft.Health.Fhir.Ingest.Template { - public class R4FhirValueProcessor : CollectionFhirValueProcessor<(DateTime start, DateTime end, IEnumerable<(DateTime, string)> values), Element> + public class R4FhirValueProcessor : CollectionFhirValueProcessor { public R4FhirValueProcessor() : base( diff --git a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/SampledDataFhirValueProcessor.cs b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/SampledDataFhirValueProcessor.cs index 0d782d38..f1125b0f 100644 --- a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/SampledDataFhirValueProcessor.cs +++ b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/SampledDataFhirValueProcessor.cs @@ -12,7 +12,7 @@ namespace Microsoft.Health.Fhir.Ingest.Template { - public class SampledDataFhirValueProcessor : FhirValueProcessor values), Element> + public class SampledDataFhirValueProcessor : FhirValueProcessor { private readonly SampledDataProcessor _sampledDataProcessor; @@ -21,22 +21,30 @@ public SampledDataFhirValueProcessor(SampledDataProcessor sampledDataProcessor = _sampledDataProcessor = sampledDataProcessor ?? SampledDataProcessor.Instance; } - protected override Element CreateValueImpl(SampledDataFhirValueType template, (DateTime start, DateTime end, IEnumerable<(DateTime, string)> values) inValue) + protected override Element CreateValueImpl(SampledDataFhirValueType template, IObservationData inValue) { EnsureArg.IsNotNull(template, nameof(template)); + EnsureArg.IsNotNull(inValue, nameof(inValue)); + IEnumerable<(DateTime, string)> values = EnsureArg.IsNotNull(inValue.Data, nameof(IObservationData.Data)); + DateTime dataStart = inValue.DataPeriod.start; + DateTime dataEnd = inValue.DataPeriod.end; return new SampledData { Origin = new SimpleQuantity { Value = 0, Unit = template.Unit }, Period = template.DefaultPeriod, Dimensions = 1, - Data = _sampledDataProcessor.BuildSampledData(inValue.values.ToArray(), inValue.start, inValue.end, template.DefaultPeriod), + Data = _sampledDataProcessor.BuildSampledData(values.ToArray(), dataStart, dataEnd, template.DefaultPeriod), }; } - protected override Element MergeValueImpl(SampledDataFhirValueType template, (DateTime start, DateTime end, IEnumerable<(DateTime, string)> values) inValue, Element existingValue) + protected override Element MergeValueImpl(SampledDataFhirValueType template, IObservationData inValue, Element existingValue) { EnsureArg.IsNotNull(template, nameof(template)); + EnsureArg.IsNotNull(inValue, nameof(inValue)); + IEnumerable<(DateTime, string)> values = EnsureArg.IsNotNull(inValue.Data, nameof(IObservationData.Data)); + DateTime dataStart = inValue.DataPeriod.start; + DateTime dataEnd = inValue.DataPeriod.end; if (!(existingValue is SampledData sampledData)) { @@ -48,9 +56,9 @@ protected override Element MergeValueImpl(SampledDataFhirValueType template, (Da throw new NotSupportedException($"Existing {typeof(SampledData)} value has more than 1 dimension."); } - var existingTimeValues = _sampledDataProcessor.SampledDataToTimeValues(sampledData.Data, inValue.start, template.DefaultPeriod); - var mergedTimeValues = _sampledDataProcessor.MergeData(existingTimeValues, inValue.values.ToArray()); - sampledData.Data = _sampledDataProcessor.BuildSampledData(mergedTimeValues, inValue.start, inValue.end, template.DefaultPeriod); + var existingTimeValues = _sampledDataProcessor.SampledDataToTimeValues(sampledData.Data, dataStart, template.DefaultPeriod); + var mergedTimeValues = _sampledDataProcessor.MergeData(existingTimeValues, values.ToArray()); + sampledData.Data = _sampledDataProcessor.BuildSampledData(mergedTimeValues, dataStart, dataEnd, template.DefaultPeriod); return existingValue; } diff --git a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/StringFhirValueProcessor.cs b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/StringFhirValueProcessor.cs index bf9c09e0..0cd0492f 100644 --- a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/StringFhirValueProcessor.cs +++ b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/StringFhirValueProcessor.cs @@ -11,16 +11,18 @@ namespace Microsoft.Health.Fhir.Ingest.Template { - public class StringFhirValueProcessor : FhirValueProcessor values), Element> + public class StringFhirValueProcessor : FhirValueProcessor { - protected override Element CreateValueImpl(StringFhirValueType template, (DateTime start, DateTime end, IEnumerable<(DateTime, string)> values) inValue) + protected override Element CreateValueImpl(StringFhirValueType template, IObservationData inValue) { - EnsureArg.IsNotNull(template, nameof(template)); + EnsureArg.IsNotNull(template, nameof(template)); + EnsureArg.IsNotNull(inValue, nameof(inValue)); + IEnumerable<(DateTime, string)> values = EnsureArg.IsNotNull(inValue.Data, nameof(IObservationData.Data)); - return new FhirString(inValue.values.Single().Item2); + return new FhirString(values.Single().Item2); } - protected override Element MergeValueImpl(StringFhirValueType template, (DateTime start, DateTime end, IEnumerable<(DateTime, string)> values) inValue, Element existingValue) + protected override Element MergeValueImpl(StringFhirValueType template, IObservationData inValue, Element existingValue) { if (!(existingValue is FhirString)) { diff --git a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs index 67d7e64d..ba62b051 100644 --- a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs +++ b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Ingest.Data; @@ -27,7 +28,7 @@ public void GivenDefaultCtor_WhenCtor_ThenProcessorCreated_Test() [Fact] public void GivenTemplate_WhenCreateObservationGroups_ThenPeriodIntervalCorrectlyUsed_Test() { - var valueProcessor = Substitute.For values), Element>>(); + var valueProcessor = Substitute.For>(); var template = Substitute.For().Mock(m => m.PeriodInterval.Returns(ObservationPeriodInterval.Single)); var measurementGroup = new MeasurementGroup { @@ -48,7 +49,7 @@ public void GivenTemplate_WhenCreateObservationGroups_ThenPeriodIntervalCorrectl [Fact] public void GivenEmptyTemplate_WhenCreateObservation_ThenShellObservationReturned_Test() { - var valueProcessor = Substitute.For values), Element>>(); + var valueProcessor = Substitute.For>(); var template = Substitute.For(); @@ -92,7 +93,7 @@ public void GivenEmptyTemplate_WhenCreateObservation_ThenShellObservationReturne public void GivenTemplateWithValue_WhenCreateObservation_ThenObservationReturned_Test() { var element = Substitute.For(); - var valueProcessor = Substitute.For values), Element>>() + var valueProcessor = Substitute.For>() .Mock(m => m.CreateValue(null, default).ReturnsForAnyArgs(element)); var valueType = Substitute.For() @@ -148,17 +149,19 @@ public void GivenTemplateWithValue_WhenCreateObservation_ThenObservationReturned valueProcessor.Received(1) .CreateValue( valueType, - Arg.Is<(DateTime start, DateTime end, IEnumerable<(DateTime, string)> values)>( - v => v.start == boundary.start - && v.end == boundary.end - && v.values.First().Item2 == "v1")); + Arg.Is( + v => v.DataPeriod.start == boundary.start + && v.DataPeriod.end == boundary.end + && v.ObservationPeriod.start == boundary.start + && v.ObservationPeriod.end == boundary.end + && v.Data.First().Item2 == "v1")); } [Fact] public void GivenTemplateWithComponent_WhenCreateObservation_ThenObservationReturned_Test() { var element = Substitute.For(); - var valueProcessor = Substitute.For values), Element>>() + var valueProcessor = Substitute.For>() .Mock(m => m.CreateValue(null, default).ReturnsForAnyArgs(element)); var valueType = Substitute.For() @@ -246,18 +249,18 @@ public void GivenTemplateWithComponent_WhenCreateObservation_ThenObservationRetu valueProcessor.Received(1) .CreateValue( valueType, - Arg.Is<(DateTime start, DateTime end, IEnumerable<(DateTime, string)> values)>( - v => v.start == boundary.start - && v.end == boundary.end - && v.values.First().Item2 == "v2")); + Arg.Is( + v => v.DataPeriod.start == boundary.start + && v.DataPeriod.end == boundary.end + && v.ObservationPeriod.start == boundary.start + && v.ObservationPeriod.end == boundary.end + && v.Data.First().Item2 == "v2")); } [Fact] public void GivenEmptyTemplate_WhenMergObservation_ThenObservationReturned_Test() { - var oldObservation = new Observation(); - - var valueProcessor = Substitute.For values), Element>>(); + var valueProcessor = Substitute.For>(); var template = Substitute.For(); @@ -268,6 +271,15 @@ public void GivenEmptyTemplate_WhenMergObservation_ThenObservationReturned_Test( (DateTime start, DateTime end) boundary = (new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc), new DateTime(2019, 1, 2, 0, 0, 0, DateTimeKind.Utc)); + var oldObservation = new Observation + { + Effective = new Period + { + Start = boundary.start.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), + End = boundary.end.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), + }, + }; + var observationGroup = Substitute.For() .Mock(m => m.Boundary.Returns(boundary)) .Mock(m => m.Name.Returns("code")) @@ -285,13 +297,9 @@ public void GivenEmptyTemplate_WhenMergObservation_ThenObservationReturned_Test( public void GivenTemplateWithValue_WhenMergObservation_ThenObservationReturned_Test() { Element oldValue = new Quantity(); - var oldObservation = new Observation - { - Value = oldValue, - }; var element = Substitute.For(); - var valueProcessor = Substitute.For values), Element>>() + var valueProcessor = Substitute.For>() .Mock(m => m.MergeValue(default, default, default).ReturnsForAnyArgs(element)); var valueType = Substitute.For() @@ -316,6 +324,16 @@ public void GivenTemplateWithValue_WhenMergObservation_ThenObservationReturned_T .Mock(m => m.Name.Returns("code")) .Mock(m => m.GetValues().Returns(values)); + var oldObservation = new Observation + { + Effective = new Period + { + Start = boundary.start.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), + End = boundary.end.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), + }, + Value = oldValue, + }; + var processor = new CodeValueFhirTemplateProcessor(valueProcessor); var newObservation = processor.MergeObservation(template, observationGroup, oldObservation); @@ -326,10 +344,12 @@ public void GivenTemplateWithValue_WhenMergObservation_ThenObservationReturned_T valueProcessor.Received(1) .MergeValue( valueType, - Arg.Is<(DateTime start, DateTime end, IEnumerable<(DateTime, string)> values)>( - v => v.start == boundary.start - && v.end == boundary.end - && v.values.First().Item2 == "v1"), + Arg.Is( + v => v.DataPeriod.start == boundary.start + && v.DataPeriod.end == boundary.end + && v.ObservationPeriod.start == boundary.start + && v.ObservationPeriod.end == boundary.end + && v.Data.First().Item2 == "v1"), oldValue); } @@ -337,31 +357,9 @@ public void GivenTemplateWithValue_WhenMergObservation_ThenObservationReturned_T public void GivenTemplateWithComponent_WhenMergObservation_ThenObservationReturned_Test() { Element oldValue = new Quantity(); - var oldObservation = new Observation - { - Component = new List - { - new Observation.ComponentComponent - { - Code = new CodeableConcept - { - Coding = new List - { - new Coding - { - Display = "p2", - System = FhirImportService.ServiceSystem, - Code = "p2", - }, - }, - }, - Value = oldValue, - }, - }, - }; var element = Substitute.For(); - var valueProcessor = Substitute.For values), Element>>() + var valueProcessor = Substitute.For>() .Mock(m => m.MergeValue(default, default, default).ReturnsForAnyArgs(element)) .Mock(m => m.CreateValue(null, default).ReturnsForAnyArgs(element)); @@ -410,6 +408,34 @@ public void GivenTemplateWithComponent_WhenMergObservation_ThenObservationReturn .Mock(m => m.Name.Returns("code")) .Mock(m => m.GetValues().Returns(values)); + var oldObservation = new Observation + { + Effective = new Period + { + Start = boundary.start.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), + End = boundary.end.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), + }, + Component = new List + { + new Observation.ComponentComponent + { + Code = new CodeableConcept + { + Coding = new List + { + new Coding + { + Display = "p2", + System = FhirImportService.ServiceSystem, + Code = "p2", + }, + }, + }, + Value = oldValue, + }, + }, + }; + var processor = new CodeValueFhirTemplateProcessor(valueProcessor); var newObservation = processor.MergeObservation(template, observationGroup, oldObservation); @@ -446,29 +472,32 @@ public void GivenTemplateWithComponent_WhenMergObservation_ThenObservationReturn valueProcessor.Received(1) .MergeValue( valueType1, - Arg.Is<(DateTime start, DateTime end, IEnumerable<(DateTime, string)> values)>( - v => v.start == boundary.start - && v.end == boundary.end - && v.values.First().Item2 == "v2"), + Arg.Is( + v => v.DataPeriod.start == boundary.start + && v.DataPeriod.end == boundary.end + && v.ObservationPeriod.start == boundary.start + && v.ObservationPeriod.end == boundary.end + && v.Data.First().Item2 == "v2"), oldValue); valueProcessor.Received(1) .CreateValue( valueType2, - Arg.Is<(DateTime start, DateTime end, IEnumerable<(DateTime, string)> values)>( - v => v.start == boundary.start - && v.end == boundary.end - && v.values.First().Item2 == "v3")); + Arg.Is( + v => v.DataPeriod.start == boundary.start + && v.DataPeriod.end == boundary.end + && v.ObservationPeriod.start == boundary.start + && v.ObservationPeriod.end == boundary.end + && v.Data.First().Item2 == "v3")); } [Fact] public void GivenTemplateWithComponentAndObservationWithOutComponent_WhenMergObservation_ThenObservationWithComponentAddedReturned_Test() { Element oldValue = new Quantity(); - var oldObservation = new Observation(); var element = Substitute.For(); - var valueProcessor = Substitute.For values), Element>>() + var valueProcessor = Substitute.For>() .Mock(m => m.CreateValue(null, default).ReturnsForAnyArgs(element)); var valueType1 = Substitute.For() @@ -505,6 +534,15 @@ public void GivenTemplateWithComponentAndObservationWithOutComponent_WhenMergObs .Mock(m => m.Name.Returns("code")) .Mock(m => m.GetValues().Returns(values)); + var oldObservation = new Observation + { + Effective = new Period + { + Start = boundary.start.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), + End = boundary.end.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), + }, + }; + var processor = new CodeValueFhirTemplateProcessor(valueProcessor); var newObservation = processor.MergeObservation(template, observationGroup, oldObservation); @@ -537,16 +575,18 @@ public void GivenTemplateWithComponentAndObservationWithOutComponent_WhenMergObs valueProcessor.Received(1) .CreateValue( valueType1, - Arg.Is<(DateTime start, DateTime end, IEnumerable<(DateTime, string)> values)>( - v => v.start == boundary.start - && v.end == boundary.end - && v.values.First().Item2 == "v2")); + Arg.Is( + v => v.DataPeriod.start == boundary.start + && v.DataPeriod.end == boundary.end + && v.ObservationPeriod.start == boundary.start + && v.ObservationPeriod.end == boundary.end + && v.Data.First().Item2 == "v2")); } [Fact] public void GivenTemplateWithCategory_WhenCreateObservation_ThenCategoryReturned() { - var valueProcessor = Substitute.For values), Element>>(); + var valueProcessor = Substitute.For>(); var template = Substitute.For() .Mock(m => m.Category.Returns( new List @@ -632,26 +672,7 @@ public void GivenTemplateWithCategory_WhenCreateObservation_ThenCategoryReturned [Fact] public void GivenTemplateWithCategory_WhenMergeObservationWithCategory_ThenCategoryReplaced() { - var oldObservation = new Observation() - { - Category = new List - { - new CodeableConcept - { - Coding = new List - { - new Coding - { - Display = "old category display", - System = "old category system", - Code = "old category code", - }, - }, - Text = "old category text", - }, - }, - }; - var valueProcessor = Substitute.For values), Element>>(); + var valueProcessor = Substitute.For>(); var template = Substitute.For() .Mock(m => m.Category.Returns( new List @@ -681,6 +702,31 @@ public void GivenTemplateWithCategory_WhenMergeObservationWithCategory_ThenCateg .Mock(m => m.Name.Returns("code")) .Mock(m => m.GetValues().Returns(values)); + var oldObservation = new Observation() + { + Effective = new Period + { + Start = boundary.start.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), + End = boundary.end.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), + }, + Category = new List + { + new CodeableConcept + { + Coding = new List + { + new Coding + { + Display = "old category display", + System = "old category system", + Code = "old category code", + }, + }, + Text = "old category text", + }, + }, + }; + var newObservation = processor.MergeObservation(template, observationGroup, oldObservation); Assert.Collection( @@ -702,8 +748,33 @@ public void GivenTemplateWithCategory_WhenMergeObservationWithCategory_ThenCateg [Fact] public void GivenTemplateWithoutCategory_WhenMergeObservationWithCategory_ThenCategoryRemoved() { + var valueProcessor = Substitute.For>(); + var template = Substitute.For() + .Mock(m => m.Category.Returns( + new List { })); + + var processor = new CodeValueFhirTemplateProcessor(valueProcessor); + + var values = new Dictionary> + { + { "p1", new[] { (DateTime.UtcNow, "v1") } }, + { "p2", new[] { (DateTime.UtcNow, "v2") } }, + }; + + (DateTime start, DateTime end) boundary = (new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc), new DateTime(2019, 1, 2, 0, 0, 0, DateTimeKind.Utc)); + + var observationGroup = Substitute.For() + .Mock(m => m.Boundary.Returns(boundary)) + .Mock(m => m.Name.Returns("code")) + .Mock(m => m.GetValues().Returns(values)); + var oldObservation = new Observation() { + Effective = new Period + { + Start = boundary.start.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), + End = boundary.end.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), + }, Category = new List { new CodeableConcept @@ -721,25 +792,6 @@ public void GivenTemplateWithoutCategory_WhenMergeObservationWithCategory_ThenCa }, }, }; - var valueProcessor = Substitute.For values), Element>>(); - var template = Substitute.For() - .Mock(m => m.Category.Returns( - new List { })); - - var processor = new CodeValueFhirTemplateProcessor(valueProcessor); - - var values = new Dictionary> - { - { "p1", new[] { (DateTime.UtcNow, "v1") } }, - { "p2", new[] { (DateTime.UtcNow, "v2") } }, - }; - - (DateTime start, DateTime end) boundary = (new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc), new DateTime(2019, 1, 2, 0, 0, 0, DateTimeKind.Utc)); - - var observationGroup = Substitute.For() - .Mock(m => m.Boundary.Returns(boundary)) - .Mock(m => m.Name.Returns("code")) - .Mock(m => m.GetValues().Returns(values)); var newObservation = processor.MergeObservation(template, observationGroup, oldObservation); diff --git a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeableConceptFhirValueProcessorTests.cs b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeableConceptFhirValueProcessorTests.cs index 1587dbda..fd371dee 100644 --- a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeableConceptFhirValueProcessorTests.cs +++ b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeableConceptFhirValueProcessorTests.cs @@ -6,6 +6,8 @@ using System; using System.Collections.Generic; using Hl7.Fhir.Model; +using Microsoft.Health.Tests.Common; +using NSubstitute; using Xunit; namespace Microsoft.Health.Fhir.Ingest.Template @@ -26,7 +28,10 @@ public void GivenValidTemplate_WhenCreateValue_ThenSampledDataProperlyConfigured }, }; - var data = (DateTime.Now, DateTime.UtcNow, new (DateTime, string)[] { (DateTime.UtcNow, "value") }); + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns((DateTime.UtcNow, DateTime.UtcNow))) + .Mock(m => m.Data.Returns(new (DateTime, string)[] { (DateTime.UtcNow, "value") })); + var result = processor.CreateValue(template, data) as CodeableConcept; Assert.NotNull(result); Assert.Equal("myText", result.Text); @@ -51,9 +56,8 @@ public void GivenInvalidElementType_WhenMergeValue_ThenNotSupportedExceptionThro { var processor = new CodeableConceptFhirValueProcessor(); var template = new CodeableConceptFhirValueType(); - var data = (DateTime.Now, DateTime.UtcNow, new (DateTime, string)[] { (DateTime.UtcNow, "value") }); - Assert.Throws(() => processor.MergeValue(template, data, new FhirDateTime())); + Assert.Throws(() => processor.MergeValue(template, default, new FhirDateTime())); } [Fact] @@ -79,7 +83,10 @@ public void GivenValidTemplate_WhenMergeValue_ThenMergeValueReturned_Test() }, }; - var data = (DateTime.Now, DateTime.UtcNow, new (DateTime, string)[] { (DateTime.UtcNow, "value") }); + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns((DateTime.UtcNow, DateTime.UtcNow))) + .Mock(m => m.Data.Returns(new (DateTime, string)[] { (DateTime.UtcNow, "value") })); + var result = processor.MergeValue(template, data, oldValue) as CodeableConcept; Assert.NotNull(result); Assert.Equal("myText", result.Text); diff --git a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/QuantityFhirValueProcessorTests.cs b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/QuantityFhirValueProcessorTests.cs index e9f0ea1a..4876856f 100644 --- a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/QuantityFhirValueProcessorTests.cs +++ b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/QuantityFhirValueProcessorTests.cs @@ -5,6 +5,8 @@ using System; using Hl7.Fhir.Model; +using Microsoft.Health.Tests.Common; +using NSubstitute; using Xunit; namespace Microsoft.Health.Fhir.Ingest.Template @@ -22,7 +24,10 @@ public void GivenValidTemplate_WhenCreateValue_ThenSampledDataProperlyConfigured Code = "myCode", }; - var data = (DateTime.Now, DateTime.UtcNow, new (DateTime, string)[] { (DateTime.UtcNow, "22.4") }); + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns((DateTime.UtcNow, DateTime.UtcNow))) + .Mock(m => m.Data.Returns(new (DateTime, string)[] { (DateTime.UtcNow, "22.4") })); + var result = processor.CreateValue(template, data) as Quantity; Assert.NotNull(result); Assert.Equal("myUnit", result.Unit); @@ -36,7 +41,10 @@ public void GivenInvalidElementType_WhenMergeValue_ThenNotSupportedExceptionThro { var processor = new QuantityFhirValueProcessor(); var template = new QuantityFhirValueType(); - var data = (DateTime.Now, DateTime.UtcNow, new (DateTime, string)[] { (DateTime.UtcNow, "value") }); + + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns((DateTime.UtcNow, DateTime.UtcNow))) + .Mock(m => m.Data.Returns(new (DateTime, string)[] { (DateTime.UtcNow, "value") })); Assert.Throws(() => processor.MergeValue(template, data, new FhirDateTime())); } @@ -54,7 +62,10 @@ public void GivenValidTemplate_WhenMergeValue_ThenMergeValueReturned_Test() var oldQuantity = new Quantity { Value = 1, System = "s", Code = "c", Unit = "u" }; - var data = (DateTime.Now, DateTime.UtcNow, new (DateTime, string)[] { (DateTime.UtcNow, "22.4") }); + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns((DateTime.UtcNow, DateTime.UtcNow))) + .Mock(m => m.Data.Returns(new (DateTime, string)[] { (DateTime.UtcNow, "22.4") })); + var result = processor.MergeValue(template, data, oldQuantity) as Quantity; Assert.NotNull(result); Assert.Equal("myUnit", result.Unit); diff --git a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/SampledDataFhirValueProcessorTests.cs b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/SampledDataFhirValueProcessorTests.cs index e7218589..456678e3 100644 --- a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/SampledDataFhirValueProcessorTests.cs +++ b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/SampledDataFhirValueProcessorTests.cs @@ -27,7 +27,12 @@ public void GivenValidTemplate_WhenCreateValue_ThenSampledDataProperlyConfigured DefaultPeriod = 10, Unit = "myUnits", }; - var data = (new DateTime(2019, 1, 1), new DateTime(2019, 1, 3), new (DateTime, string)[] { (new DateTime(2019, 1, 2), "value") }); + + var values = new (DateTime, string)[] { (new DateTime(2019, 1, 2), "value") }; + + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns((new DateTime(2019, 1, 1), new DateTime(2019, 1, 3)))) + .Mock(m => m.Data.Returns(values)); var result = processor.CreateValue(template, data) as SampledData; Assert.NotNull(result); @@ -38,9 +43,9 @@ public void GivenValidTemplate_WhenCreateValue_ThenSampledDataProperlyConfigured sdp.Received(1).BuildSampledData( Arg.Is<(DateTime, string)[]>( - v => v.Length == 1 && v.All(i => i.Item1 == data.Item3[0].Item1 && i.Item2 == data.Item3[0].Item2)), - data.Item1, - data.Item2, + v => v.Length == 1 && v.All(i => i.Item1 == values[0].Item1 && i.Item2 == values[0].Item2)), + data.DataPeriod.start, + data.DataPeriod.end, template.DefaultPeriod); } @@ -51,7 +56,12 @@ public void GivenInvalidElementType_WhenMergeValue_ThenNotSupportedExceptionThro var processor = new SampledDataFhirValueProcessor(sdp); var template = new SampledDataFhirValueType(); - var data = (new DateTime(2019, 1, 1), new DateTime(2019, 1, 3), new (DateTime, string)[] { (new DateTime(2019, 1, 2), "value") }); + + var values = new (DateTime, string)[] { (new DateTime(2019, 1, 2), "value") }; + + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns((new DateTime(2019, 1, 1), new DateTime(2019, 1, 3)))) + .Mock(m => m.Data.Returns(values)); Assert.Throws(() => processor.MergeValue(template, data, new FhirDateTime())); } @@ -70,18 +80,23 @@ public void GivenValidTemplate_WhenMergeValue_ThenMergeValueReturned_Test() var processor = new SampledDataFhirValueProcessor(sdp); var template = new SampledDataFhirValueType { DefaultPeriod = 100 }; var existingSampledData = new SampledData { Data = "data" }; - var data = (new DateTime(2019, 1, 1), new DateTime(2019, 1, 3), new (DateTime, string)[] { (new DateTime(2019, 1, 2), "value") }); + + var values = new (DateTime, string)[] { (new DateTime(2019, 1, 2), "value") }; + + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns((new DateTime(2019, 1, 1), new DateTime(2019, 1, 3)))) + .Mock(m => m.Data.Returns(values)); var result = processor.MergeValue(template, data, existingSampledData) as SampledData; Assert.NotNull(result); Assert.Equal("merged", result.Data); - sdp.Received(1).SampledDataToTimeValues("data", data.Item1, 100); + sdp.Received(1).SampledDataToTimeValues("data", data.DataPeriod.start, 100); sdp.Received(1).MergeData( existingValues, Arg.Is<(DateTime, string)[]>( - v => v.Length == 1 && v.All(i => i.Item1 == data.Item3[0].Item1 && i.Item2 == data.Item3[0].Item2))); - sdp.Received(1).BuildSampledData(mergeData, data.Item1, data.Item2, 100); + v => v.Length == 1 && v.All(i => i.Item1 == values[0].Item1 && i.Item2 == values[0].Item2))); + sdp.Received(1).BuildSampledData(mergeData, data.DataPeriod.start, data.DataPeriod.end, 100); } [Fact] @@ -91,7 +106,12 @@ public void GivenSampledDataWithGreaterThanOneDimension_WhenMergeValue_ThenNotSu var processor = new SampledDataFhirValueProcessor(sdp); var template = new SampledDataFhirValueType(); - var data = (new DateTime(2019, 1, 1), new DateTime(2019, 1, 3), new (DateTime, string)[] { (new DateTime(2019, 1, 2), "value") }); + + var values = new (DateTime, string)[] { (new DateTime(2019, 1, 2), "value") }; + + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns((new DateTime(2019, 1, 1), new DateTime(2019, 1, 3)))) + .Mock(m => m.Data.Returns(values)); Assert.Throws(() => processor.MergeValue(template, data, new SampledData { Dimensions = 2 })); } diff --git a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/StringFhirValueProcessorTests.cs b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/StringFhirValueProcessorTests.cs index 2fa3c780..1dd5e2ce 100644 --- a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/StringFhirValueProcessorTests.cs +++ b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/StringFhirValueProcessorTests.cs @@ -5,6 +5,8 @@ using System; using Hl7.Fhir.Model; +using Microsoft.Health.Tests.Common; +using NSubstitute; using Xunit; namespace Microsoft.Health.Fhir.Ingest.Template @@ -17,7 +19,10 @@ public void GivenValidTemplate_WhenCreateValue_ThenStringProperlyConfigured_Test var processor = new StringFhirValueProcessor(); var template = new StringFhirValueType() { }; - var data = (DateTime.Now, DateTime.UtcNow, new (DateTime, string)[] { (DateTime.UtcNow, "my string value") }); + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns((DateTime.UtcNow, DateTime.UtcNow))) + .Mock(m => m.Data.Returns(new (DateTime, string)[] { (DateTime.UtcNow, "my string value") })); + var result = processor.CreateValue(template, data) as FhirString; Assert.NotNull(result); Assert.Equal("my string value", result.Value); @@ -28,7 +33,10 @@ public void GivenInvalidElementType_WhenMergeValue_ThenNotSupportedExceptionThro { var processor = new StringFhirValueProcessor(); var template = new StringFhirValueType(); - var data = (DateTime.Now, DateTime.UtcNow, new (DateTime, string)[] { (DateTime.UtcNow, "a string") }); + + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns((DateTime.UtcNow, DateTime.UtcNow))) + .Mock(m => m.Data.Returns(new (DateTime, string)[] { (DateTime.UtcNow, "a string") })); Assert.Throws(() => processor.MergeValue(template, data, new FhirDateTime())); } @@ -41,7 +49,10 @@ public void GivenValidTemplate_WhenMergeValue_ThenMergeValueReturned_Test() FhirString oldString = new FhirString("old string"); - var data = (DateTime.Now, DateTime.UtcNow, new (DateTime, string)[] { (DateTime.UtcNow, "new string") }); + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns((DateTime.UtcNow, DateTime.UtcNow))) + .Mock(m => m.Data.Returns(new (DateTime, string)[] { (DateTime.UtcNow, "new string") })); + var result = processor.MergeValue(template, data, oldString) as FhirString; Assert.NotNull(result); Assert.Equal("new string", result.Value); From 2cf1364987ef8b6c29cecb3df95f1b738441e89d Mon Sep 17 00:00:00 2001 From: Dustin Burson Date: Wed, 17 Jun 2020 13:16:57 -0700 Subject: [PATCH 09/13] * Add DateTime extensions for common FHIR operations on DateTime values. * Add ability to update the period of observation if the merged values our outside the current period. * Add unit tests for observation period update scenario. --- .../DateTimeExtensions.cs | 38 +++++ .../CodeValueFhirTemplateProcessor.cs | 37 +++-- .../CodeValueFhirTemplateProcessorTests.cs | 147 ++++++++++++++---- 3 files changed, 183 insertions(+), 39 deletions(-) create mode 100644 src/lib/Microsoft.Health.Extensions.Fhir.R4/DateTimeExtensions.cs diff --git a/src/lib/Microsoft.Health.Extensions.Fhir.R4/DateTimeExtensions.cs b/src/lib/Microsoft.Health.Extensions.Fhir.R4/DateTimeExtensions.cs new file mode 100644 index 00000000..2e7a3363 --- /dev/null +++ b/src/lib/Microsoft.Health.Extensions.Fhir.R4/DateTimeExtensions.cs @@ -0,0 +1,38 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Globalization; +using EnsureThat; +using Hl7.Fhir.Model; + +namespace Microsoft.Health.Extensions.Fhir +{ + public static class DateTimeExtensions + { + public static DateTime ToUtcDateTime(this FhirDateTime fhirDateTime) + { + return EnsureArg.IsNotNull(fhirDateTime, nameof(fhirDateTime)) + .ToDateTimeOffset(TimeSpan.Zero) + .UtcDateTime; + } + + public static (DateTime start, DateTime end) ToUtcDateTimePeriod(this Period period) + { + EnsureArg.IsNotNull(period, nameof(period)); + + return (period.StartElement.ToUtcDateTime(), period.EndElement.ToUtcDateTime()); + } + + public static Period ToPeriod(this (DateTime start, DateTime end) boundary) + { + return new Period + { + Start = boundary.start.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), + End = boundary.end.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), + }; + } + } +} diff --git a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeValueFhirTemplateProcessor.cs b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeValueFhirTemplateProcessor.cs index 14af04c1..614eaa62 100644 --- a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeValueFhirTemplateProcessor.cs +++ b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeValueFhirTemplateProcessor.cs @@ -9,6 +9,7 @@ using System.Linq; using EnsureThat; using Hl7.Fhir.Model; +using Microsoft.Health.Extensions.Fhir; using Microsoft.Health.Fhir.Ingest.Data; using Microsoft.Health.Fhir.Ingest.Service; @@ -39,11 +40,7 @@ protected override Observation CreateObseravtionImpl(CodeValueFhirTemplate templ Status = ObservationStatus.Final, Code = ResolveCode(grp.Name, template.Codes), Issued = DateTimeOffset.UtcNow, - Effective = new Period - { - Start = grp.Boundary.Start.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), - End = grp.Boundary.End.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), - }, + Effective = grp.Boundary.ToPeriod(), }; if (template?.Category?.Count > 0) @@ -93,13 +90,31 @@ protected override Observation MergeObservationImpl(CodeValueFhirTemplate templa } var values = grp.GetValues(); - var observatiobPeriod = GetObservationPeriod(existingObservation); + + (DateTime start, DateTime end) observationPeriod = GetObservationPeriod(existingObservation); + + // Update observation effective period if merge values exist outside the current period + if (grp.Boundary.Start < observationPeriod.start) + { + observationPeriod.start = grp.Boundary.Start; + } + + if (grp.Boundary.End > observationPeriod.end) + { + observationPeriod.end = grp.Boundary.End; + } + + existingObservation.Effective = observationPeriod.ToPeriod(); + + // Update observation value if (!string.IsNullOrWhiteSpace(template?.Value?.ValueName) && values.TryGetValue(template?.Value?.ValueName, out var obValues)) { - existingObservation.Value = _valueProcessor.MergeValue(template.Value, CreateMergeData(grp.Boundary, observatiobPeriod, obValues), existingObservation.Value); + existingObservation.Value = _valueProcessor.MergeValue(template.Value, CreateMergeData(grp.Boundary, observationPeriod, obValues), existingObservation.Value); } + // Update observation component values + if (template?.Components?.Count > 0) { if (existingObservation.Component == null) @@ -121,12 +136,12 @@ protected override Observation MergeObservationImpl(CodeValueFhirTemplate templa new Observation.ComponentComponent { Code = ResolveCode(component.Value.ValueName, component.Codes), - Value = _valueProcessor.CreateValue(component.Value, CreateMergeData(grp.Boundary, observatiobPeriod, compValues)), + Value = _valueProcessor.CreateValue(component.Value, CreateMergeData(grp.Boundary, observationPeriod, compValues)), }); } else { - foundComponent.Value = _valueProcessor.MergeValue(component.Value, CreateMergeData(grp.Boundary, observatiobPeriod, compValues), foundComponent.Value); + foundComponent.Value = _valueProcessor.MergeValue(component.Value, CreateMergeData(grp.Boundary, observationPeriod, compValues), foundComponent.Value); } } } @@ -183,9 +198,9 @@ protected static (DateTime start, DateTime end) GetObservationPeriod(Observation switch (effective) { case FhirDateTime dt: - return (dt.ToDateTimeOffset(TimeSpan.Zero).UtcDateTime, dt.ToDateTimeOffset(TimeSpan.Zero).UtcDateTime); + return (dt.ToUtcDateTime(), dt.ToUtcDateTime()); case Period p: - return (p.StartElement.ToDateTimeOffset(TimeSpan.Zero).UtcDateTime, p.EndElement.ToDateTimeOffset(TimeSpan.Zero).UtcDateTime); + return p.ToUtcDateTimePeriod(); default: throw new NotSupportedException($"Observation effective type of {effective.GetType()} is not supported."); } diff --git a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs index ba62b051..601ce1a2 100644 --- a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs +++ b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs @@ -8,6 +8,7 @@ using System.Globalization; using System.Linq; using Hl7.Fhir.Model; +using Microsoft.Health.Extensions.Fhir; using Microsoft.Health.Fhir.Ingest.Data; using Microsoft.Health.Fhir.Ingest.Service; using Microsoft.Health.Tests.Common; @@ -326,11 +327,7 @@ public void GivenTemplateWithValue_WhenMergObservation_ThenObservationReturned_T var oldObservation = new Observation { - Effective = new Period - { - Start = boundary.start.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), - End = boundary.end.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), - }, + Effective = boundary.ToPeriod(), Value = oldValue, }; @@ -339,7 +336,6 @@ public void GivenTemplateWithValue_WhenMergObservation_ThenObservationReturned_T var newObservation = processor.MergeObservation(template, observationGroup, oldObservation); Assert.Equal(element, newObservation.Value); - // oldObservation.Value Assert.Equal(ObservationStatus.Amended, newObservation.Status); valueProcessor.Received(1) .MergeValue( @@ -410,11 +406,7 @@ public void GivenTemplateWithComponent_WhenMergObservation_ThenObservationReturn var oldObservation = new Observation { - Effective = new Period - { - Start = boundary.start.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), - End = boundary.end.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), - }, + Effective = boundary.ToPeriod(), Component = new List { new Observation.ComponentComponent @@ -536,11 +528,7 @@ public void GivenTemplateWithComponentAndObservationWithOutComponent_WhenMergObs var oldObservation = new Observation { - Effective = new Period - { - Start = boundary.start.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), - End = boundary.end.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), - }, + Effective = boundary.ToPeriod(), }; var processor = new CodeValueFhirTemplateProcessor(valueProcessor); @@ -584,7 +572,7 @@ public void GivenTemplateWithComponentAndObservationWithOutComponent_WhenMergObs } [Fact] - public void GivenTemplateWithCategory_WhenCreateObservation_ThenCategoryReturned() + public void GivenTemplateWithCategory_WhenCreateObservation_ThenCategoryReturned_Test() { var valueProcessor = Substitute.For>(); var template = Substitute.For() @@ -670,7 +658,7 @@ public void GivenTemplateWithCategory_WhenCreateObservation_ThenCategoryReturned } [Fact] - public void GivenTemplateWithCategory_WhenMergeObservationWithCategory_ThenCategoryReplaced() + public void GivenTemplateWithCategory_WhenMergeObservationWithCategory_ThenCategoryReplaced_Test() { var valueProcessor = Substitute.For>(); var template = Substitute.For() @@ -704,11 +692,7 @@ public void GivenTemplateWithCategory_WhenMergeObservationWithCategory_ThenCateg var oldObservation = new Observation() { - Effective = new Period - { - Start = boundary.start.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), - End = boundary.end.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), - }, + Effective = boundary.ToPeriod(), Category = new List { new CodeableConcept @@ -770,11 +754,7 @@ public void GivenTemplateWithoutCategory_WhenMergeObservationWithCategory_ThenCa var oldObservation = new Observation() { - Effective = new Period - { - Start = boundary.start.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), - End = boundary.end.ToString("o", CultureInfo.InvariantCulture.DateTimeFormat), - }, + Effective = boundary.ToPeriod(), Category = new List { new CodeableConcept @@ -797,5 +777,116 @@ public void GivenTemplateWithoutCategory_WhenMergeObservationWithCategory_ThenCa Assert.False(newObservation.Category.Any()); } + + [Fact] + public void GivenExistingObservation_WhenMergeObservationWithDataOutsideEffectivePeriod_ThenPeriodUpdated_Test() + { + Element oldValue = new Quantity(); + + var element = Substitute.For(); + var valueProcessor = Substitute.For>() + .Mock(m => m.MergeValue(default, default, default).ReturnsForAnyArgs(element)); + + var valueType = Substitute.For() + .Mock(m => m.ValueName.Returns("p1")); + var template = Substitute.For() + .Mock(m => m.Value.Returns(valueType)); + + var values = new Dictionary> + { + { "p1", new[] { (DateTime.UtcNow, "v1") } }, + }; + + DateTime observationStart = new DateTime(2019, 1, 2, 0, 0, 0, DateTimeKind.Utc); + DateTime observationEnd = new DateTime(2019, 1, 3, 0, 0, 0, DateTimeKind.Utc); + (DateTime start, DateTime end) boundary = (new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc), new DateTime(2019, 1, 4, 0, 0, 0, DateTimeKind.Utc)); + + var observationGroup = Substitute.For() + .Mock(m => m.Boundary.Returns(boundary)) + .Mock(m => m.GetValues().Returns(values)); + + var oldObservation = new Observation + { + Effective = (observationStart, observationEnd).ToPeriod(), + Value = oldValue, + }; + + var processor = new CodeValueFhirTemplateProcessor(valueProcessor); + + var newObservation = processor.MergeObservation(template, observationGroup, oldObservation); + + var (start, end) = Assert.IsType(newObservation.Effective) + .ToUtcDateTimePeriod(); + Assert.Equal(boundary.start, start); + Assert.Equal(boundary.end, end); + + Assert.Equal(ObservationStatus.Amended, newObservation.Status); + valueProcessor.Received(1) + .MergeValue( + valueType, + Arg.Is( + v => v.DataPeriod.start == boundary.start + && v.DataPeriod.end == boundary.end + && v.ObservationPeriod.start == boundary.start + && v.ObservationPeriod.end == boundary.end + && v.Data.First().Item2 == "v1"), + oldValue); + } + + [Fact] + public void GivenExistingObservation_WhenMergeObservationWithDataInsideEffectivePeriod_ThenPeriodNotUpdated_Test() + { + Element oldValue = new Quantity(); + + var element = Substitute.For(); + var valueProcessor = Substitute.For>() + .Mock(m => m.MergeValue(default, default, default).ReturnsForAnyArgs(element)); + + var valueType = Substitute.For() + .Mock(m => m.ValueName.Returns("p1")); + var template = Substitute.For() + .Mock(m => m.Value.Returns(valueType)); + + var values = new Dictionary> + { + { "p1", new[] { (DateTime.UtcNow, "v1") } }, + }; + + DateTime observationStart = new DateTime(2018, 12, 31, 0, 0, 0, DateTimeKind.Utc); + DateTime observationEnd = new DateTime(2019, 1, 3, 0, 0, 0, DateTimeKind.Utc); + (DateTime start, DateTime end) boundary = (new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Utc), new DateTime(2019, 1, 2, 0, 0, 0, DateTimeKind.Utc)); + + var observationGroup = Substitute.For() + .Mock(m => m.Boundary.Returns(boundary)) + .Mock(m => m.GetValues().Returns(values)); + + var oldObservation = new Observation + { + Effective = (observationStart, observationEnd).ToPeriod(), + Value = oldValue, + }; + + var processor = new CodeValueFhirTemplateProcessor(valueProcessor); + + var newObservation = processor.MergeObservation(template, observationGroup, oldObservation); + + var (start, end) = Assert.IsType(newObservation.Effective) + .ToUtcDateTimePeriod(); + Assert.Equal(observationStart, start); + Assert.Equal(observationEnd, end); + + Assert.Equal(ObservationStatus.Amended, newObservation.Status); + valueProcessor.Received(1) + .MergeValue( + valueType, + Arg.Is( + v => v.DataPeriod.start == boundary.start + && v.DataPeriod.end == boundary.end + && v.ObservationPeriod.start == observationStart + && v.ObservationPeriod.end == observationEnd + && v.Data.First().Item2 == "v1"), + oldValue); + } + } } From 8776d0fbb2e360c3c0dd99937a049ece87b8f2c3 Mon Sep 17 00:00:00 2001 From: Dustin Burson Date: Wed, 17 Jun 2020 13:54:29 -0700 Subject: [PATCH 10/13] * Fixed CodeValueFhirTemplateProcessor using new period instead of old period when merging data. Old period will now be used and tests updated. * Updated SampledDataFhirValueProcessor to use values from existing observation when merging sample data. Tests updated. --- .../CodeValueFhirTemplateProcessor.cs | 30 ++-- .../Template/SampledDataFhirValueProcessor.cs | 12 +- .../CodeValueFhirTemplateProcessorTests.cs | 4 +- .../SampledDataFhirValueProcessorTests.cs | 160 +++++++++++++++++- 4 files changed, 179 insertions(+), 27 deletions(-) diff --git a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeValueFhirTemplateProcessor.cs b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeValueFhirTemplateProcessor.cs index 614eaa62..5e5d2903 100644 --- a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeValueFhirTemplateProcessor.cs +++ b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/CodeValueFhirTemplateProcessor.cs @@ -5,7 +5,6 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.Linq; using EnsureThat; using Hl7.Fhir.Model; @@ -90,31 +89,15 @@ protected override Observation MergeObservationImpl(CodeValueFhirTemplate templa } var values = grp.GetValues(); - (DateTime start, DateTime end) observationPeriod = GetObservationPeriod(existingObservation); - // Update observation effective period if merge values exist outside the current period - if (grp.Boundary.Start < observationPeriod.start) - { - observationPeriod.start = grp.Boundary.Start; - } - - if (grp.Boundary.End > observationPeriod.end) - { - observationPeriod.end = grp.Boundary.End; - } - - existingObservation.Effective = observationPeriod.ToPeriod(); - // Update observation value - if (!string.IsNullOrWhiteSpace(template?.Value?.ValueName) && values.TryGetValue(template?.Value?.ValueName, out var obValues)) { existingObservation.Value = _valueProcessor.MergeValue(template.Value, CreateMergeData(grp.Boundary, observationPeriod, obValues), existingObservation.Value); } // Update observation component values - if (template?.Components?.Count > 0) { if (existingObservation.Component == null) @@ -147,6 +130,19 @@ protected override Observation MergeObservationImpl(CodeValueFhirTemplate templa } } + // Update observation effective period if merge values exist outside the current period. + if (grp.Boundary.Start < observationPeriod.start) + { + observationPeriod.start = grp.Boundary.Start; + } + + if (grp.Boundary.End > observationPeriod.end) + { + observationPeriod.end = grp.Boundary.End; + } + + existingObservation.Effective = observationPeriod.ToPeriod(); + return existingObservation; } diff --git a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/SampledDataFhirValueProcessor.cs b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/SampledDataFhirValueProcessor.cs index f1125b0f..1a0aba06 100644 --- a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/SampledDataFhirValueProcessor.cs +++ b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/SampledDataFhirValueProcessor.cs @@ -43,8 +43,6 @@ protected override Element MergeValueImpl(SampledDataFhirValueType template, IOb EnsureArg.IsNotNull(template, nameof(template)); EnsureArg.IsNotNull(inValue, nameof(inValue)); IEnumerable<(DateTime, string)> values = EnsureArg.IsNotNull(inValue.Data, nameof(IObservationData.Data)); - DateTime dataStart = inValue.DataPeriod.start; - DateTime dataEnd = inValue.DataPeriod.end; if (!(existingValue is SampledData sampledData)) { @@ -56,9 +54,15 @@ protected override Element MergeValueImpl(SampledDataFhirValueType template, IOb throw new NotSupportedException($"Existing {typeof(SampledData)} value has more than 1 dimension."); } - var existingTimeValues = _sampledDataProcessor.SampledDataToTimeValues(sampledData.Data, dataStart, template.DefaultPeriod); + (DateTime dataStart, DateTime dataEnd) = inValue.DataPeriod; + (DateTime observationStart, DateTime observationEnd) = inValue.ObservationPeriod; + + DateTime mergeStart = dataStart < observationStart ? dataStart : observationStart; + DateTime mergeEnd = dataEnd > observationEnd ? dataEnd : observationEnd; + + var existingTimeValues = _sampledDataProcessor.SampledDataToTimeValues(sampledData.Data, observationStart, template.DefaultPeriod); var mergedTimeValues = _sampledDataProcessor.MergeData(existingTimeValues, values.ToArray()); - sampledData.Data = _sampledDataProcessor.BuildSampledData(mergedTimeValues, dataStart, dataEnd, template.DefaultPeriod); + sampledData.Data = _sampledDataProcessor.BuildSampledData(mergedTimeValues, mergeStart, mergeEnd, template.DefaultPeriod); return existingValue; } diff --git a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs index 601ce1a2..27b3e7b5 100644 --- a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs +++ b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs @@ -827,8 +827,8 @@ public void GivenExistingObservation_WhenMergeObservationWithDataOutsideEffectiv Arg.Is( v => v.DataPeriod.start == boundary.start && v.DataPeriod.end == boundary.end - && v.ObservationPeriod.start == boundary.start - && v.ObservationPeriod.end == boundary.end + && v.ObservationPeriod.start == observationStart + && v.ObservationPeriod.end == observationEnd && v.Data.First().Item2 == "v1"), oldValue); } diff --git a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/SampledDataFhirValueProcessorTests.cs b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/SampledDataFhirValueProcessorTests.cs index 456678e3..8931dd04 100644 --- a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/SampledDataFhirValueProcessorTests.cs +++ b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/SampledDataFhirValueProcessorTests.cs @@ -29,9 +29,11 @@ public void GivenValidTemplate_WhenCreateValue_ThenSampledDataProperlyConfigured }; var values = new (DateTime, string)[] { (new DateTime(2019, 1, 2), "value") }; + var dataPeriod = (new DateTime(2019, 1, 1), new DateTime(2019, 1, 3)); var data = Substitute.For() - .Mock(m => m.DataPeriod.Returns((new DateTime(2019, 1, 1), new DateTime(2019, 1, 3)))) + .Mock(m => m.DataPeriod.Returns(dataPeriod)) + .Mock(m => m.ObservationPeriod.Returns(dataPeriod)) .Mock(m => m.Data.Returns(values)); var result = processor.CreateValue(template, data) as SampledData; @@ -58,9 +60,11 @@ public void GivenInvalidElementType_WhenMergeValue_ThenNotSupportedExceptionThro var template = new SampledDataFhirValueType(); var values = new (DateTime, string)[] { (new DateTime(2019, 1, 2), "value") }; + var dataPeriod = (new DateTime(2019, 1, 1), new DateTime(2019, 1, 3)); var data = Substitute.For() - .Mock(m => m.DataPeriod.Returns((new DateTime(2019, 1, 1), new DateTime(2019, 1, 3)))) + .Mock(m => m.DataPeriod.Returns(dataPeriod)) + .Mock(m => m.ObservationPeriod.Returns(dataPeriod)) .Mock(m => m.Data.Returns(values)); Assert.Throws(() => processor.MergeValue(template, data, new FhirDateTime())); @@ -82,9 +86,11 @@ public void GivenValidTemplate_WhenMergeValue_ThenMergeValueReturned_Test() var existingSampledData = new SampledData { Data = "data" }; var values = new (DateTime, string)[] { (new DateTime(2019, 1, 2), "value") }; + var dataPeriod = (new DateTime(2019, 1, 1), new DateTime(2019, 1, 3)); var data = Substitute.For() - .Mock(m => m.DataPeriod.Returns((new DateTime(2019, 1, 1), new DateTime(2019, 1, 3)))) + .Mock(m => m.DataPeriod.Returns(dataPeriod)) + .Mock(m => m.ObservationPeriod.Returns(dataPeriod)) .Mock(m => m.Data.Returns(values)); var result = processor.MergeValue(template, data, existingSampledData) as SampledData; @@ -108,12 +114,158 @@ public void GivenSampledDataWithGreaterThanOneDimension_WhenMergeValue_ThenNotSu var template = new SampledDataFhirValueType(); var values = new (DateTime, string)[] { (new DateTime(2019, 1, 2), "value") }; + var dataPeriod = (new DateTime(2019, 1, 1), new DateTime(2019, 1, 3)); var data = Substitute.For() - .Mock(m => m.DataPeriod.Returns((new DateTime(2019, 1, 1), new DateTime(2019, 1, 3)))) + .Mock(m => m.DataPeriod.Returns(dataPeriod)) + .Mock(m => m.ObservationPeriod.Returns(dataPeriod)) .Mock(m => m.Data.Returns(values)); Assert.Throws(() => processor.MergeValue(template, data, new SampledData { Dimensions = 2 })); } + + [Fact] + public void GivenObservationWithSmallerEffectivePeriodThanNewData_WhenMergeValue_ThenCorrectDatesUsedWhenBuilding_Test() + { + var existingValues = new (DateTime Time, string Value)[] { }; + var mergeData = new (DateTime Time, string Value)[] { }; + + var sdp = Substitute.For() + .Mock(m => m.SampledDataToTimeValues(default, default, default).ReturnsForAnyArgs(existingValues)) + .Mock(m => m.MergeData(default, default).ReturnsForAnyArgs(mergeData)) + .Mock(m => m.BuildSampledData(default, default, default, default).ReturnsForAnyArgs("merged")); + + var processor = new SampledDataFhirValueProcessor(sdp); + var template = new SampledDataFhirValueType { DefaultPeriod = 100 }; + var existingSampledData = new SampledData { Data = "data" }; + + var values = new (DateTime, string)[] { (new DateTime(2019, 1, 1), "value") }; + var dataPeriod = (new DateTime(2019, 1, 1), new DateTime(2019, 1, 4)); + var observationPeriod = (new DateTime(2019, 1, 2), new DateTime(2019, 1, 3)); + + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns(dataPeriod)) + .Mock(m => m.ObservationPeriod.Returns(observationPeriod)) + .Mock(m => m.Data.Returns(values)); + + var result = processor.MergeValue(template, data, existingSampledData) as SampledData; + Assert.NotNull(result); + Assert.Equal("merged", result.Data); + + sdp.Received(1).SampledDataToTimeValues("data", data.ObservationPeriod.start, 100); + sdp.Received(1).MergeData( + existingValues, + Arg.Is<(DateTime, string)[]>( + v => v.Length == 1 && v.All(i => i.Item1 == values[0].Item1 && i.Item2 == values[0].Item2))); + sdp.Received(1).BuildSampledData(mergeData, data.DataPeriod.start, data.DataPeriod.end, 100); + } + + [Fact] + public void GivenObservationWithGreaterEffectivePeriodThanNewData_WhenMergeValue_ThenCorrectDatesUsedWhenBuilding_Test() + { + var existingValues = new (DateTime Time, string Value)[] { }; + var mergeData = new (DateTime Time, string Value)[] { }; + + var sdp = Substitute.For() + .Mock(m => m.SampledDataToTimeValues(default, default, default).ReturnsForAnyArgs(existingValues)) + .Mock(m => m.MergeData(default, default).ReturnsForAnyArgs(mergeData)) + .Mock(m => m.BuildSampledData(default, default, default, default).ReturnsForAnyArgs("merged")); + + var processor = new SampledDataFhirValueProcessor(sdp); + var template = new SampledDataFhirValueType { DefaultPeriod = 100 }; + var existingSampledData = new SampledData { Data = "data" }; + + var values = new (DateTime, string)[] { (new DateTime(2019, 1, 1), "value") }; + var dataPeriod = (new DateTime(2019, 1, 1), new DateTime(2019, 1, 4)); + var observationPeriod = (new DateTime(2018, 12, 31), new DateTime(2019, 1, 5)); + + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns(dataPeriod)) + .Mock(m => m.ObservationPeriod.Returns(observationPeriod)) + .Mock(m => m.Data.Returns(values)); + + var result = processor.MergeValue(template, data, existingSampledData) as SampledData; + Assert.NotNull(result); + Assert.Equal("merged", result.Data); + + sdp.Received(1).SampledDataToTimeValues("data", data.ObservationPeriod.start, 100); + sdp.Received(1).MergeData( + existingValues, + Arg.Is<(DateTime, string)[]>( + v => v.Length == 1 && v.All(i => i.Item1 == values[0].Item1 && i.Item2 == values[0].Item2))); + sdp.Received(1).BuildSampledData(mergeData, data.ObservationPeriod.start, data.ObservationPeriod.end, 100); + } + + [Fact] + public void GivenObservationWithPriorEffectivePeriodThanNewData_WhenMergeValue_ThenCorrectDatesUsedWhenBuilding_Test() + { + var existingValues = new (DateTime Time, string Value)[] { }; + var mergeData = new (DateTime Time, string Value)[] { }; + + var sdp = Substitute.For() + .Mock(m => m.SampledDataToTimeValues(default, default, default).ReturnsForAnyArgs(existingValues)) + .Mock(m => m.MergeData(default, default).ReturnsForAnyArgs(mergeData)) + .Mock(m => m.BuildSampledData(default, default, default, default).ReturnsForAnyArgs("merged")); + + var processor = new SampledDataFhirValueProcessor(sdp); + var template = new SampledDataFhirValueType { DefaultPeriod = 100 }; + var existingSampledData = new SampledData { Data = "data" }; + + var values = new (DateTime, string)[] { (new DateTime(2019, 1, 2), "value") }; + var dataPeriod = (new DateTime(2019, 1, 1), new DateTime(2019, 1, 4)); + var observationPeriod = (new DateTime(2018, 12, 31), new DateTime(2019, 1, 1)); + + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns(dataPeriod)) + .Mock(m => m.ObservationPeriod.Returns(observationPeriod)) + .Mock(m => m.Data.Returns(values)); + + var result = processor.MergeValue(template, data, existingSampledData) as SampledData; + Assert.NotNull(result); + Assert.Equal("merged", result.Data); + + sdp.Received(1).SampledDataToTimeValues("data", data.ObservationPeriod.start, 100); + sdp.Received(1).MergeData( + existingValues, + Arg.Is<(DateTime, string)[]>( + v => v.Length == 1 && v.All(i => i.Item1 == values[0].Item1 && i.Item2 == values[0].Item2))); + sdp.Received(1).BuildSampledData(mergeData, data.ObservationPeriod.start, data.DataPeriod.end, 100); + } + + [Fact] + public void GivenNewDataBeforeEffectivePeriodThanNewData_WhenMergeValue_ThenCorrectDatesUsedWhenBuilding_Test() + { + var existingValues = new (DateTime Time, string Value)[] { }; + var mergeData = new (DateTime Time, string Value)[] { }; + + var sdp = Substitute.For() + .Mock(m => m.SampledDataToTimeValues(default, default, default).ReturnsForAnyArgs(existingValues)) + .Mock(m => m.MergeData(default, default).ReturnsForAnyArgs(mergeData)) + .Mock(m => m.BuildSampledData(default, default, default, default).ReturnsForAnyArgs("merged")); + + var processor = new SampledDataFhirValueProcessor(sdp); + var template = new SampledDataFhirValueType { DefaultPeriod = 100 }; + var existingSampledData = new SampledData { Data = "data" }; + + var values = new (DateTime, string)[] { (new DateTime(2019, 1, 2), "value") }; + var dataPeriod = (new DateTime(2019, 1, 1), new DateTime(2019, 1, 4)); + var observationPeriod = (new DateTime(2019, 1, 5), new DateTime(2019, 1, 6)); + + var data = Substitute.For() + .Mock(m => m.DataPeriod.Returns(dataPeriod)) + .Mock(m => m.ObservationPeriod.Returns(observationPeriod)) + .Mock(m => m.Data.Returns(values)); + + var result = processor.MergeValue(template, data, existingSampledData) as SampledData; + Assert.NotNull(result); + Assert.Equal("merged", result.Data); + + sdp.Received(1).SampledDataToTimeValues("data", data.ObservationPeriod.start, 100); + sdp.Received(1).MergeData( + existingValues, + Arg.Is<(DateTime, string)[]>( + v => v.Length == 1 && v.All(i => i.Item1 == values[0].Item1 && i.Item2 == values[0].Item2))); + sdp.Received(1).BuildSampledData(mergeData, data.DataPeriod.start, data.ObservationPeriod.end, 100); + } } } From f0767a62d131ab8c883f804a05d07d2c2665bbc2 Mon Sep 17 00:00:00 2001 From: Dustin Burson Date: Wed, 17 Jun 2020 16:52:05 -0700 Subject: [PATCH 11/13] * Update SampledDataFhirValueProcessor to use observation period instead of data period during create to future proof. Right now they would always be the same. * Fix issue in SampledDataProcessor were the last value was omitted from the stream if it was on the end boundary. Never hit before because the hourly and daily time periods would end on the time boundary - 1 ms. --- .../Data/SampledDataProcessor.cs | 2 +- .../Template/SampledDataFhirValueProcessor.cs | 6 +-- .../Data/SampledDataProcessorTests.cs | 47 ++++++++++++++++--- .../SampledDataFhirValueProcessorTests.cs | 8 ++-- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Data/SampledDataProcessor.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Data/SampledDataProcessor.cs index 21b10cce..cba61755 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Data/SampledDataProcessor.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Data/SampledDataProcessor.cs @@ -25,7 +25,7 @@ public virtual string BuildSampledData((DateTime Time, string Value)[] values, D var i = 0; var sb = new StringBuilder(); - while (currentDateTime < endBoundary) + while (currentDateTime <= endBoundary) { string value = null; diff --git a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/SampledDataFhirValueProcessor.cs b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/SampledDataFhirValueProcessor.cs index 1a0aba06..e886e291 100644 --- a/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/SampledDataFhirValueProcessor.cs +++ b/src/lib/Microsoft.Health.Fhir.R4.Ingest/Template/SampledDataFhirValueProcessor.cs @@ -26,15 +26,15 @@ protected override Element CreateValueImpl(SampledDataFhirValueType template, IO EnsureArg.IsNotNull(template, nameof(template)); EnsureArg.IsNotNull(inValue, nameof(inValue)); IEnumerable<(DateTime, string)> values = EnsureArg.IsNotNull(inValue.Data, nameof(IObservationData.Data)); - DateTime dataStart = inValue.DataPeriod.start; - DateTime dataEnd = inValue.DataPeriod.end; + + (DateTime observationStart, DateTime observationEnd) = inValue.ObservationPeriod; return new SampledData { Origin = new SimpleQuantity { Value = 0, Unit = template.Unit }, Period = template.DefaultPeriod, Dimensions = 1, - Data = _sampledDataProcessor.BuildSampledData(values.ToArray(), dataStart, dataEnd, template.DefaultPeriod), + Data = _sampledDataProcessor.BuildSampledData(values.ToArray(), observationStart, observationEnd, template.DefaultPeriod), }; } diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/SampledDataProcessorTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/SampledDataProcessorTests.cs index 4b4fded7..c32597d5 100644 --- a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/SampledDataProcessorTests.cs +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Data/SampledDataProcessorTests.cs @@ -19,7 +19,7 @@ public void GivenDataGoingPastEndBoundary_WhenBuildSampledData_ThenExcessDiscard .Select(i => (seedDateTime.AddMinutes(i), i.ToString())) .ToArray(); - var result = SampledDataProcessor.Instance.BuildSampledData(values, seedDateTime, seedDateTime.AddMinutes(10), (decimal)TimeSpan.FromMinutes(1).TotalMilliseconds); + var result = SampledDataProcessor.Instance.BuildSampledData(values, seedDateTime, seedDateTime.AddMinutes(10).AddMilliseconds(-1), (decimal)TimeSpan.FromMinutes(1).TotalMilliseconds); Assert.NotNull(result); Assert.Equal("0 1 2 3 4 5 6 7 8 9", result); } @@ -27,17 +27,19 @@ public void GivenDataGoingPastEndBoundary_WhenBuildSampledData_ThenExcessDiscard [Theory] [InlineData(10)] [InlineData(100)] - public void GivenDataStartingOnBoundary_WhenBuildSampledData_ThenSampledDataPopulated_Test(int totalSamples) + public void GivenDataStartingOnBoundaryAndEndingOnBoundary_WhenBuildSampledData_ThenSampledDataPopulated_Test(int totalSamples) { var period = TimeSpan.FromSeconds(1); // Normalize endBoundary to seconds being the most significant value var endBoundary = NormalizeToSecond(DateTime.UtcNow); - var startBoundary = endBoundary.AddSeconds(-1 * totalSamples); + var startBoundary = endBoundary.AddSeconds(-1 * (totalSamples - 1)); var values = Enumerable.Range(1, totalSamples) .Select(i => (startBoundary.AddSeconds(i - 1), i.ToString())) .ToArray(); + Assert.Equal(endBoundary, values.Last().Item1); + var result = SampledDataProcessor.Instance.BuildSampledData(values, startBoundary, endBoundary, (decimal)period.TotalMilliseconds); Assert.NotNull(result); @@ -53,13 +55,46 @@ public void GivenDataStartingOnBoundary_WhenBuildSampledData_ThenSampledDataPopu [Theory] [InlineData(10)] [InlineData(100)] - public void GivenDataStartingOnBoundaryAndMissingValues_WhenBuildSampledData_ThenSampledDataPopulatedWithEForMissingValues_Test(int totalSamples) + public void GivenDataStartingOnBoundaryAndEndingBeforeBoundary_WhenBuildSampledData_ThenSampledDataPopulatedWithERemaining_Test(int totalSamples) { var period = TimeSpan.FromSeconds(1); // Normalize endBoundary to seconds being the most significant value var endBoundary = NormalizeToSecond(DateTime.UtcNow); var startBoundary = endBoundary.AddSeconds(-1 * totalSamples); + var values = Enumerable.Range(1, totalSamples) + .Select(i => (startBoundary.AddSeconds(i - 1), i.ToString())) + .ToArray(); + + var result = SampledDataProcessor.Instance.BuildSampledData(values, startBoundary, endBoundary, (decimal)period.TotalMilliseconds); + Assert.NotNull(result); + + var resultValues = result.Split(" "); + Assert.Equal(totalSamples + 1, resultValues.Length); + + for (int i = 0; i < resultValues.Length; i++) + { + if (i < values.Length) + { + Assert.Equal(values[i].Item2, resultValues[i]); + } + else + { + Assert.Equal("E", resultValues[i]); + } + } + } + + [Theory] + [InlineData(10)] + [InlineData(100)] + public void GivenDataStartingOnBoundaryAndMissingValues_WhenBuildSampledData_ThenSampledDataPopulatedWithEForMissingValues_Test(int totalSamples) + { + var period = TimeSpan.FromSeconds(1); + + // Normalize endBoundary to seconds being the most significant value + var endBoundary = NormalizeToSecond(DateTime.UtcNow); + var startBoundary = endBoundary.AddSeconds(-1 * (totalSamples - 1)); var values = Enumerable.Range(1, totalSamples / 2) .Select(i => (startBoundary.AddSeconds((i - 1) * 2), i.ToString())) .ToArray(); @@ -92,7 +127,7 @@ public void GivenDataStartingOffBoundary_WhenBuildSampledData_ThenSampledDataPop // Normalize endBoundary to seconds being the most significant value var endBoundary = NormalizeToSecond(DateTime.UtcNow); - var startBoundary = endBoundary.AddSeconds(-1 * totalSamples); + var startBoundary = endBoundary.AddSeconds(-1 * (totalSamples - 1)); var values = Enumerable.Range(1, totalSamples) .Select(i => (startBoundary.AddSeconds(i - 1).AddMilliseconds(5), i.ToString())) .ToArray(); @@ -116,7 +151,7 @@ public void GivenDataCollisions_WhenBuildSampledData_ThenLastValueUsed_Test() // Normalize endBoundary to seconds being the most significant value var endBoundary = NormalizeToSecond(DateTime.UtcNow); - var startBoundary = endBoundary.AddSeconds(-1 * 10); + var startBoundary = endBoundary.AddSeconds(-1 * 9); var values = Enumerable.Range(1, 20) .Select(i => (startBoundary.AddMilliseconds((i * 500) - 499), i.ToString())) .ToArray(); diff --git a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/SampledDataFhirValueProcessorTests.cs b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/SampledDataFhirValueProcessorTests.cs index 8931dd04..c8f0a171 100644 --- a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/SampledDataFhirValueProcessorTests.cs +++ b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/SampledDataFhirValueProcessorTests.cs @@ -46,8 +46,8 @@ public void GivenValidTemplate_WhenCreateValue_ThenSampledDataProperlyConfigured sdp.Received(1).BuildSampledData( Arg.Is<(DateTime, string)[]>( v => v.Length == 1 && v.All(i => i.Item1 == values[0].Item1 && i.Item2 == values[0].Item2)), - data.DataPeriod.start, - data.DataPeriod.end, + data.ObservationPeriod.start, + data.ObservationPeriod.end, template.DefaultPeriod); } @@ -97,12 +97,12 @@ public void GivenValidTemplate_WhenMergeValue_ThenMergeValueReturned_Test() Assert.NotNull(result); Assert.Equal("merged", result.Data); - sdp.Received(1).SampledDataToTimeValues("data", data.DataPeriod.start, 100); + sdp.Received(1).SampledDataToTimeValues("data", data.ObservationPeriod.start, 100); sdp.Received(1).MergeData( existingValues, Arg.Is<(DateTime, string)[]>( v => v.Length == 1 && v.All(i => i.Item1 == values[0].Item1 && i.Item2 == values[0].Item2))); - sdp.Received(1).BuildSampledData(mergeData, data.DataPeriod.start, data.DataPeriod.end, 100); + sdp.Received(1).BuildSampledData(mergeData, data.ObservationPeriod.start, data.ObservationPeriod.end, 100); } [Fact] From a444e1bcc1cb03a537a42b8bc601b8b53df86785 Mon Sep 17 00:00:00 2001 From: Dustin Burson Date: Wed, 17 Jun 2020 17:01:54 -0700 Subject: [PATCH 12/13] Fix code style error --- .../Template/CodeValueFhirTemplateProcessorTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs index 27b3e7b5..ef2d9e90 100644 --- a/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs +++ b/test/Microsoft.Health.Fhir.R4.Ingest.UnitTests/Template/CodeValueFhirTemplateProcessorTests.cs @@ -887,6 +887,5 @@ public void GivenExistingObservation_WhenMergeObservationWithDataInsideEffective && v.Data.First().Item2 == "v1"), oldValue); } - } } From c8c93efacb2115d9ce9856316064b9bf7217cbc7 Mon Sep 17 00:00:00 2001 From: Dustin Burson Date: Thu, 18 Jun 2020 12:36:09 -0700 Subject: [PATCH 13/13] Update documentation to include the new correlation id feature. --- docs/Configuration.md | 91 +++++++++++++++++++++----- src/query/CollectionQuery/asaproj.json | 42 ++++++------ 2 files changed, 96 insertions(+), 37 deletions(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index ba907a99..7930b02c 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -1,10 +1,12 @@ # Configuration + This article details how to configure your instance of the IoMT FHIR Connector for Azure. -The IoMT FHIR Connector for Azure requires two JSON configuration files. The first, device content, is responsible for mapping the payloads sent to the Event Hub end point and extracting types, device identifiers, measurement date time, and the measurement value(s). The second template controls the FHIR mapping. The FHIR mapping allows configuration of the length of the observation period, FHIR data type used to store the values, and code(s). The two configuration files should be uploaded to the storage container "template" created under the blob storage account provisioned during the [ARM template deployment](ARMInstallation.md). The device content mapping file should be call `devicecontent.json` and the FHIR mapping file should be called `fhirmapping.json`. Full examples can be found in the repository under [/sample/templates](../sample/templates). Configuration files are loaded from blob per compute execution. Once updated they should take effect immediately. +The IoMT FHIR Connector for Azure requires two JSON configuration files. The first, device content, is responsible for mapping the payloads sent to the Event Hub end point and extracting types, device identifiers, measurement date time, and the measurement value(s). The second template controls the FHIR mapping. The FHIR mapping allows configuration of the length of the observation period, FHIR data type used to store the values, and code(s). The two configuration files should be uploaded to the storage container "template" created under the blob storage account provisioned during the [ARM template deployment](ARMInstallation.md). The device content mapping file should be call `devicecontent.json` and the FHIR mapping file should be called `fhirmapping.json`. Full examples can be found in the repository under [/sample/templates](../sample/templates). Configuration files are loaded from blob per compute execution. Once updated they should take effect immediately. # Device Content Mapping -The IoMT FHIR Connector for Azure provides mapping functionality to extract device content into a common format for further evaluation. Each event hub message received is evaluated against all templates. This allows a single inbound message to be projected to multiple outbound messages and subsequently mapped to different observations in FHIR. The result is a normalized data object representing the value or values parsed by the templates. The normalized data model has a few required properties that must be found and extracted: + +The IoMT FHIR Connector for Azure provides mapping functionality to extract device content into a common format for further evaluation. Each event hub message received is evaluated against all templates. This allows a single inbound message to be projected to multiple outbound messages and subsequently mapped to different observations in FHIR. The result is a normalized data object representing the value or values parsed by the templates. The normalized data model has a few required properties that must be found and extracted: | Property | Description | | - | - | @@ -43,29 +45,35 @@ Below is a conceptual example of what happens during normalization. ``` ## Mapping with JSON Path + The two device content template types supported today rely on JSON Path to both match the desired template and extract values. More information on JSON Path can be found [here](https://goessner.net/articles/JsonPath/). Both template types use the [JSON .NET implementation](https://www.newtonsoft.com/json/help/html/QueryJsonSelectTokenJsonPath.htm) for resolving JSON Path expressions. Additional examples can be found in the [unit tests](../test/Microsoft.Health.Fhir.Ingest.UnitTests/Template/JsonPathContentTemplateTests.cs). ### **JsonPathContentTemplate** + The JsonPathContentTemplate allows matching on and extracting values from an EventHub message using JSON Path. | Property | Description |
Example
-| --- | --- | --- +| --- | --- | --- |**TypeName**|The type to associate with measurements that match the template.|`heartrate` |**TypeMatchExpression**|The JSON Path expression that is evaluated against the EventData payload. If a matching JToken is found the template is considered a match. All subsequent expressions are evaluated against the extracted JToken matched here.|`$..[?(@heartRate)]` |**TimestampExpression**|The JSON Path expression to extract the timestamp value for the measurement's OccurenceTimeUtc.|`$.endDate` |**DeviceIdExpression**|The JSON Path expression to extract the device identifier.|`$.deviceId` |**PatientIdExpression**|*Optional*: The JSON Path expression to extract the patient identifier.|`$.patientId` |**EncounterIdExpression**|*Optional*: The JSON Path expression to extract the encounter identifier.|`$.encounterId` +|**CorrelationIdExpression**|*Optional*: The JSON Path expression to extract the correlation identifier. If extracted this value can be used to group values into a single observation in the FHIR mapping template.|`$.correlationId` |**Values[].ValueName**|The name to associate with the value extracted by the subsequent expression. Used to bind the desired value/component in the FHIR mapping template. |`hr` |**Values[].ValueExpression**|The JSON Path expression to extract the desired value.|`$.heartRate` |**Values[].Required**|Will require the value to be present in the payload. If not found a measurement will not be generated and an InvalidOperationException will be thrown.|`true` #### Examples + --- + **Heart Rate** *Message* + ```json { "Body": { @@ -77,7 +85,9 @@ The JsonPathContentTemplate allows matching on and extracting values from an Eve "SystemProperties": {} } ``` + *Template* + ```json { "templateType": "JsonPathContent", @@ -96,10 +106,13 @@ The JsonPathContentTemplate allows matching on and extracting values from an Eve } } ``` + --- + **Blood Pressure** *Message* + ```json { "Body": { @@ -112,7 +125,9 @@ The JsonPathContentTemplate allows matching on and extracting values from an Eve "SystemProperties": {} } ``` + *Template* + ```json { "typeName": "bloodpressure", @@ -133,9 +148,13 @@ The JsonPathContentTemplate allows matching on and extracting values from an Eve ] } ``` + +--- + **Project Multiple Measurements from Single Message** *Message* + ```json { "Body": { @@ -148,7 +167,9 @@ The JsonPathContentTemplate allows matching on and extracting values from an Eve "SystemProperties": {} } ``` + *Template 1* + ```json { "templateType": "JsonPathContent", @@ -167,7 +188,9 @@ The JsonPathContentTemplate allows matching on and extracting values from an Eve } } ``` + *Template 2* + ```json { "templateType": "JsonPathContent", @@ -186,9 +209,13 @@ The JsonPathContentTemplate allows matching on and extracting values from an Eve } } ``` + +--- + **Project Multiple Measurements from Array in Message** *Message* + ```json { "Body": [ @@ -212,7 +239,9 @@ The JsonPathContentTemplate allows matching on and extracting values from an Eve "SystemProperties": {} } ``` + *Template* + ```json { "templateType": "JsonPathContent", @@ -231,21 +260,26 @@ The JsonPathContentTemplate allows matching on and extracting values from an Eve } } ``` + ### **IotJsonPathContentTemplate** + The IotJsonPathContentTemplate is similar to the JsonPathContentTemplate except the DeviceIdExpression and TimestampExpression are not required. The assumption when using this template is the messages being evaluated were sent using the [Azure IoT Hub Device SDKs](https://docs.microsoft.com/en-us/azure/iot-hub/iot-hub-devguide-sdks#azure-iot-hub-device-sdks). When using these SDKs the device identity (assuming the device id from Iot Hub/Central is registered as an identifer for a device resource on the destination FHIR server) is known as well as the timestamp of the message. If you are using Azure IoT Hub Device SDKs but are using custom properties in the message body for the device identity or measurement timestamp you can still use the JsonPathContentTemplate. -*Note: When using the IotJsonPathContentTemplate the TypeMatchExpression should resolve to the entire message as a JToken. Please see the examples below.* +*Note: When using the IotJsonPathContentTemplate the TypeMatchExpression should resolve to the entire message as a JToken. Please see the examples below.* + #### Examples + --- + **Heart Rate** *Message* ```json { "Body": { - "heartRate": "78" + "heartRate": "78" }, "Properties": { "iothub-creation-time-utc" : "2019-02-01T22:46:01.8750000Z" @@ -275,6 +309,7 @@ The assumption when using this template is the messages being evaluated were sen } ``` --- + **Blood Pressure** *Message* @@ -313,15 +348,19 @@ The assumption when using this template is the messages being evaluated were sen ``` # FHIR Mapping + Once the device content is extracted into [Measurement](../src/lib/Microsoft.Health.Fhir.Ingest/Data/Measurement.cs) definitions the data is collected and grouped according to a window of time (set during deployment), device id, and type. The output of this grouping is sent to be converted into a FHIR resource (observation currently). Here the FHIR mapping controls how the data is mapped into a FHIR observation. Should an observation be created for a point in time or over a period of an hour? What codes should be added to the observation? Should be value be represented as SampledData or a Quantity? These are all options the FHIR mapping configuration controls. +The FHIR mapping also controls how the measurements are grouped into an observation. This is controlled by the ```PeriodInterval``` setting in the template documentation below. The simplest option is ```Instance``` which means each measurement will map to a single observation. This is option appropriate if your data collection is infrequent (a couple times a day or less). If you are collecting data with high frequency (every second or ever minute) then using a period of ```Hour``` or ```Day``` is recommended. When set data that arrived will be mapped to the correct hourly and daily observation. This should be used with a value type SampledData which will capture measurements according to the period you configure. The last option is grouping by ```CorrelationId``` which will group all measurements that share the same device, type, and correlation id (defined and extracted during device mapping). This should also use the SampledData value type so all values with in the observation period can be represented. + ## CodeValueFhirTemplate -The CodeValueFhirTemplate is currently the only template supported in FHIR mapping at this time. It allows you defined codes, the effective period, and value of the observation. Multiple value types are supported: SampledData, CodeableConcept, and Quantity. In addition to these configurable values the identifier for the observation, along with linking to the proper device and patient are handled automatically. An additional code used by IoMT FHIR Connector for Azure is also added. -| Property | Description +The CodeValueFhirTemplate is currently the only template supported in FHIR mapping at this time. It allows you defined codes, the effective period, and value of the observation. Multiple value types are supported: SampledData, CodeableConcept, String, and Quantity. In addition to these configurable values the identifier for the observation, along with linking to the proper device and patient are handled automatically. An additional code used by IoMT FHIR Connector for Azure is also added. + +| Property | Description | --- | --- |**TypeName**| The type of measurement this template should bind to. There should be at least one DeviceContent template that outputs this type. -|**PeriodInterval**|The period of time the observation created should represent. Supported values are 0 (an instance), 60 (an hour), 1440 (a day). +|**PeriodInterval**|How measurements should be grouped into observations. Supported values are 0 (an instance), 60 (an hour), 1440 (a day), and -1 (correlation id). |**Category**|Any number of [CodeableConcepts](http://hl7.org/fhir/datatypes-definitions.html#codeableconcept) to classify the type of observation created. |**Codes**|One or more [Codings](http://hl7.org/fhir/datatypes-definitions.html#coding) to apply to the observation created. |**Codes[].Code**|The code for the [Coding](http://hl7.org/fhir/datatypes-definitions.html#coding). @@ -333,6 +372,7 @@ The CodeValueFhirTemplate is currently the only template supported in FHIR mappi |**Components[].Value**|The value to extract and represent in the component. See [Value Type Templates](#valuetypes) for more information. ## Value Type Templates + Each Value type defines at least the following properties: | Property | Description | --- | --- @@ -342,40 +382,46 @@ Each Value type defines at least the following properties: Below are the currently supported value type templates. In the future further templates may be added. ### SampledData + Represents the [SampledData](http://hl7.org/fhir/datatypes.html#SampledData) FHIR data type. Measurements are written to value stream starting with start of the observations and incrementing forward using the period defined. If no value is present an `E` will be written into the data stream. If the period is such that two more values occupy the same position in the data stream the latest value is used. The same logic is applied when an observation using the SampledData is updated. -| Property | Description +| Property | Description | --- | --- |**DefaultPeriod**|The default period in milliseconds to use. |**Unit**|The unit to set on the origin of the SampledData. ### Quantity + Represents the [Quantity](http://hl7.org/fhir/datatypes.html#Quantity) FHIR data type. If more than one value is present in the grouping only the first value is used. If new value arrives that maps to the same observation it will overwrite the old value. -| Property | Description -| --- | --- +| Property | Description +| --- | --- |**Unit**| Unit representation. |**Code**| Coded form of the unit. |**System**| System that defines the coded unit form. ### String + Represent the [string](https://www.hl7.org/fhir/datatypes.html#string) FHIR data type. If more than one value is present in the grouping only the first value is used. If new value arrives that maps to the same observation it will overwrite the old value. No additional properties are defined. ### CodeableConcept + Represents the [CodeableConcept](http://hl7.org/fhir/datatypes.html#CodeableConcept) FHIR data type. The actual value isn't used. -| Property | Description -| --- | --- -|**Text**|Plain text representation. +| Property | Description +| --- | --- +|**Text**|Plain text representation. |**Codes**|One or more [Codings](http://hl7.org/fhir/datatypes-definitions.html#coding) to apply to the observation created. |**Codes[].Code**|The code for the [Coding](http://hl7.org/fhir/datatypes-definitions.html#coding). |**Codes[].System**|The system for the [Coding](http://hl7.org/fhir/datatypes-definitions.html#coding). |**Codes[].Display**|The display for the [Coding](http://hl7.org/fhir/datatypes-definitions.html#coding). ## Examples + **Heart Rate - Sampled Data** + ```json { "templateType": "CodeValueFhir", @@ -398,8 +444,11 @@ Represents the [CodeableConcept](http://hl7.org/fhir/datatypes.html#CodeableConc } } ``` + --- + **Steps - Sampled Data** + ```json { "templateType": "CodeValueFhir", @@ -410,7 +459,7 @@ Represents the [CodeableConcept](http://hl7.org/fhir/datatypes.html#CodeableConc "system": "http://loinc.org", "display": "Number of steps" } - ], + ], "periodInterval": 60, "typeName": "stepsCount", "value": { @@ -422,8 +471,11 @@ Represents the [CodeableConcept](http://hl7.org/fhir/datatypes.html#CodeableConc } } ``` + --- + **Blood Pressure - Sampled Data** + ```json { "templateType": "CodeValueFhir", @@ -472,8 +524,11 @@ Represents the [CodeableConcept](http://hl7.org/fhir/datatypes.html#CodeableConc } } ``` + --- + **Blood Pressure - Quantity** + ```json { "templateType": "CodeValueFhir", @@ -520,8 +575,11 @@ Represents the [CodeableConcept](http://hl7.org/fhir/datatypes.html#CodeableConc } } ``` + --- + **Device Removed - Codeable Concept** + ```json { "templateType": "CodeValueFhir", @@ -550,5 +608,6 @@ Represents the [CodeableConcept](http://hl7.org/fhir/datatypes.html#CodeableConc } } ``` + --- -FHIR® is the registered trademark of HL7 and is used with the permission of HL7. \ No newline at end of file +FHIR® is the registered trademark of HL7 and is used with the permission of HL7. \ No newline at end of file diff --git a/src/query/CollectionQuery/asaproj.json b/src/query/CollectionQuery/asaproj.json index 5c359428..c01cc576 100644 --- a/src/query/CollectionQuery/asaproj.json +++ b/src/query/CollectionQuery/asaproj.json @@ -1,22 +1,22 @@ -{ - "name": "CollectionQuery", - "startFile": "Script.asaql", - "configurations": [ - { - "filePath": "Inputs\\NormalizedDeviceInput.json", - "subType": "Input" - }, - { - "filePath": "Outputs\\CollectedBlobOutput.json", - "subType": "Output" - }, - { - "filePath": "Outputs\\FhirImportOutput.json", - "subType": "Output" - }, - { - "filePath": "JobConfig.json", - "subType": "JobConfig" - } - ] +{ + "name": "CollectionQuery", + "startFile": "Script.asaql", + "configurations": [ + { + "filePath": "Inputs\\NormalizedDeviceInput.json", + "subType": "Input" + }, + { + "filePath": "Outputs\\CollectedBlobOutput.json", + "subType": "Output" + }, + { + "filePath": "Outputs\\FhirImportOutput.json", + "subType": "Output" + }, + { + "filePath": "JobConfig.json", + "subType": "JobConfig" + } + ] } \ No newline at end of file