From 3b400d4a8684e90dbc709f0965a11eeb60f5f8eb Mon Sep 17 00:00:00 2001 From: William Yochum Date: Fri, 28 Aug 2020 15:50:54 -0700 Subject: [PATCH 01/14] method to prefix metric name --- src/lib/Microsoft.Health.Common/Telemetry/Metric.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/lib/Microsoft.Health.Common/Telemetry/Metric.cs b/src/lib/Microsoft.Health.Common/Telemetry/Metric.cs index 9bd0fe60..63e505da 100644 --- a/src/lib/Microsoft.Health.Common/Telemetry/Metric.cs +++ b/src/lib/Microsoft.Health.Common/Telemetry/Metric.cs @@ -15,8 +15,16 @@ public Metric(string name, IDictionary dimensions) Dimensions = dimensions; } - public string Name { get; } + public string Name { get; set; } public IDictionary Dimensions { get; } + + public void AddPrefixToName(string prefix) + { + if (!Name.StartsWith(prefix, System.StringComparison.CurrentCulture)) + { + Name = $"{prefix}{Name}"; + } + } } } From 923fff07904cd5c5307fe9a6f561dc18c5fa94fd Mon Sep 17 00:00:00 2001 From: Will Yochum Date: Fri, 20 Nov 2020 17:38:40 -0800 Subject: [PATCH 02/14] console app stream analytics replacement --- ...icrosoft.Health.Fhir.Ingest.Console.csproj | 8 + .../Program.cs | 12 ++ Microsoft.Health.Fhir.Ingest.sln | 20 +- .../MeasurementCollectionToFhir/Processor.cs | 60 ++++++ .../ProcessorStartup.cs | 73 +++++++ ...icrosoft.Health.Fhir.Ingest.Console.csproj | 38 ++++ src/console/Normalize/Processor.cs | 94 +++++++++ src/console/Normalize/ProcessorStartup.cs | 36 ++++ src/console/Program.cs | 134 +++++++++++++ src/console/appsettings.json | 20 ++ src/console/devicecontent.json | 115 +++++++++++ src/console/fhirmapping.json | 176 +++++++++++++++++ .../EventCheckpointing/ICheckpointClient.cs | 21 ++ .../StorageCheckpointClient.cs | 183 ++++++++++++++++++ .../EventConsumers/EventPrinter.cs | 31 +++ .../EventConsumers/IEventConsumer.cs | 17 ++ .../Service/EventBatchingOptions.cs | 16 ++ .../Service/EventBatchingService.cs | 139 +++++++++++++ .../Service/EventConsumerService.cs | 37 ++++ .../Service/IEventConsumerService.cs | 18 ++ .../Service/Infrastructure/EventQueue.cs | 92 +++++++++ .../Infrastructure/EventQueueWindow.cs | 34 ++++ .../EventHubProcessor/EventHubOptions.cs | 22 +++ .../EventHubProcessor/EventProcessor.cs | 113 +++++++++++ .../Microsoft.Health.Events.csproj | 37 ++++ .../Microsoft.Health.Events.sln | 27 +++ .../Model/Checkpoint.cs | 18 ++ .../Microsoft.Health.Events/Model/Event.cs | 47 +++++ .../Model/MaximumWaitEvent.cs | 17 ++ .../Storage/StorageOptions.cs | 18 ++ .../Microsoft.Health.Fhir.Ingest.csproj | 1 + .../Service/MeasurementFhirImportService.cs | 51 ++++- 32 files changed, 1720 insertions(+), 5 deletions(-) create mode 100644 Microsoft.Health.Fhir.Ingest.Console/Microsoft.Health.Fhir.Ingest.Console.csproj create mode 100644 Microsoft.Health.Fhir.Ingest.Console/Program.cs create mode 100644 src/console/MeasurementCollectionToFhir/Processor.cs create mode 100644 src/console/MeasurementCollectionToFhir/ProcessorStartup.cs create mode 100644 src/console/Microsoft.Health.Fhir.Ingest.Console.csproj create mode 100644 src/console/Normalize/Processor.cs create mode 100644 src/console/Normalize/ProcessorStartup.cs create mode 100644 src/console/Program.cs create mode 100644 src/console/appsettings.json create mode 100644 src/console/devicecontent.json create mode 100644 src/console/fhirmapping.json create mode 100644 src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs create mode 100644 src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs create mode 100644 src/lib/Microsoft.Health.Events/EventConsumers/EventPrinter.cs create mode 100644 src/lib/Microsoft.Health.Events/EventConsumers/IEventConsumer.cs create mode 100644 src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingOptions.cs create mode 100644 src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs create mode 100644 src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs create mode 100644 src/lib/Microsoft.Health.Events/EventConsumers/Service/IEventConsumerService.cs create mode 100644 src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueue.cs create mode 100644 src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueueWindow.cs create mode 100644 src/lib/Microsoft.Health.Events/EventHubProcessor/EventHubOptions.cs create mode 100644 src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs create mode 100644 src/lib/Microsoft.Health.Events/Microsoft.Health.Events.csproj create mode 100644 src/lib/Microsoft.Health.Events/Microsoft.Health.Events.sln create mode 100644 src/lib/Microsoft.Health.Events/Model/Checkpoint.cs create mode 100644 src/lib/Microsoft.Health.Events/Model/Event.cs create mode 100644 src/lib/Microsoft.Health.Events/Model/MaximumWaitEvent.cs create mode 100644 src/lib/Microsoft.Health.Events/Storage/StorageOptions.cs diff --git a/Microsoft.Health.Fhir.Ingest.Console/Microsoft.Health.Fhir.Ingest.Console.csproj b/Microsoft.Health.Fhir.Ingest.Console/Microsoft.Health.Fhir.Ingest.Console.csproj new file mode 100644 index 00000000..c73e0d16 --- /dev/null +++ b/Microsoft.Health.Fhir.Ingest.Console/Microsoft.Health.Fhir.Ingest.Console.csproj @@ -0,0 +1,8 @@ + + + + Exe + netcoreapp3.1 + + + diff --git a/Microsoft.Health.Fhir.Ingest.Console/Program.cs b/Microsoft.Health.Fhir.Ingest.Console/Program.cs new file mode 100644 index 00000000..5e450f23 --- /dev/null +++ b/Microsoft.Health.Fhir.Ingest.Console/Program.cs @@ -0,0 +1,12 @@ +using System; + +namespace Microsoft.Health.Fhir.Ingest.Console +{ + class Program + { + static void Main(string[] args) + { + Console.WriteLine("Hello World!"); + } + } +} diff --git a/Microsoft.Health.Fhir.Ingest.sln b/Microsoft.Health.Fhir.Ingest.sln index 18b41eeb..59fb0fbd 100644 --- a/Microsoft.Health.Fhir.Ingest.sln +++ b/Microsoft.Health.Fhir.Ingest.sln @@ -73,9 +73,15 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "healthkitOnFhir", "healthki EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.Ingest.Template", "src\lib\Microsoft.Health.Fhir.Ingest.Template\Microsoft.Health.Fhir.Ingest.Template.csproj", "{85D653A7-0E63-4751-8904-728156807A14}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Ingest.Schema", "src\lib\Microsoft.Health.Fhir.Ingest.Schema\Microsoft.Health.Fhir.Ingest.Schema.csproj", "{A85AB6BC-698C-460F-81D4-9D1D2BD14B71}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.Ingest.Schema", "src\lib\Microsoft.Health.Fhir.Ingest.Schema\Microsoft.Health.Fhir.Ingest.Schema.csproj", "{A85AB6BC-698C-460F-81D4-9D1D2BD14B71}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Ingest.Template.UnitTests", "test\Microsoft.Health.Fhir.Ingest.Template.UnitTests\Microsoft.Health.Fhir.Ingest.Template.UnitTests.csproj", "{EE072537-807D-4FE2-BFEB-424B64DCD7F9}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.Ingest.Template.UnitTests", "test\Microsoft.Health.Fhir.Ingest.Template.UnitTests\Microsoft.Health.Fhir.Ingest.Template.UnitTests.csproj", "{EE072537-807D-4FE2-BFEB-424B64DCD7F9}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Events", "src\lib\Microsoft.Health.Events\Microsoft.Health.Events.csproj", "{22275DE3-859D-40F0-9547-7711568164C0}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "console", "console", "{1EF3584A-C437-4B45-8BF8-1597D5A8DBC7}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.Ingest.Console", "src\console\Microsoft.Health.Fhir.Ingest.Console.csproj", "{927BC214-ABD9-4A1B-9F7C-75973513D141}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -147,6 +153,14 @@ Global {EE072537-807D-4FE2-BFEB-424B64DCD7F9}.Debug|Any CPU.Build.0 = Debug|Any CPU {EE072537-807D-4FE2-BFEB-424B64DCD7F9}.Release|Any CPU.ActiveCfg = Release|Any CPU {EE072537-807D-4FE2-BFEB-424B64DCD7F9}.Release|Any CPU.Build.0 = Release|Any CPU + {22275DE3-859D-40F0-9547-7711568164C0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {22275DE3-859D-40F0-9547-7711568164C0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {22275DE3-859D-40F0-9547-7711568164C0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {22275DE3-859D-40F0-9547-7711568164C0}.Release|Any CPU.Build.0 = Release|Any CPU + {927BC214-ABD9-4A1B-9F7C-75973513D141}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {927BC214-ABD9-4A1B-9F7C-75973513D141}.Debug|Any CPU.Build.0 = Debug|Any CPU + {927BC214-ABD9-4A1B-9F7C-75973513D141}.Release|Any CPU.ActiveCfg = Release|Any CPU + {927BC214-ABD9-4A1B-9F7C-75973513D141}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -172,6 +186,8 @@ Global {85D653A7-0E63-4751-8904-728156807A14} = {513D67B4-80E1-476D-955F-E7E7C79D144A} {A85AB6BC-698C-460F-81D4-9D1D2BD14B71} = {513D67B4-80E1-476D-955F-E7E7C79D144A} {EE072537-807D-4FE2-BFEB-424B64DCD7F9} = {FAF8B402-892E-4EA2-B4CF-69B0C70BA762} + {22275DE3-859D-40F0-9547-7711568164C0} = {513D67B4-80E1-476D-955F-E7E7C79D144A} + {927BC214-ABD9-4A1B-9F7C-75973513D141} = {1EF3584A-C437-4B45-8BF8-1597D5A8DBC7} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A358924D-F948-4AE8-8CD0-A0F56225CE0C} diff --git a/src/console/MeasurementCollectionToFhir/Processor.cs b/src/console/MeasurementCollectionToFhir/Processor.cs new file mode 100644 index 00000000..c5c062af --- /dev/null +++ b/src/console/MeasurementCollectionToFhir/Processor.cs @@ -0,0 +1,60 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs; +using Microsoft.Health.Events.EventConsumers; +using Microsoft.Health.Events.Model; +using Microsoft.Health.Fhir.Ingest.Host; +using Microsoft.Health.Fhir.Ingest.Service; +using Microsoft.Health.Fhir.Ingest.Telemetry; + +namespace Microsoft.Health.Fhir.Ingest.Console.MeasurementCollectionToFhir +{ + public class Processor : IEventConsumer + { + private MeasurementFhirImportService _measurementImportService; + private string _templateDefinition; + private ITelemetryLogger _logger; + + public Processor( + [Blob("template/%Template:FhirMapping%", FileAccess.Read)] string templateDefinition, + [MeasurementFhirImport] MeasurementFhirImportService measurementImportService) + { + _templateDefinition = templateDefinition; + _measurementImportService = measurementImportService; + + // todo: inject logger + var config = new TelemetryConfiguration(); + var telemetryClient = new TelemetryClient(config); + _logger = new IomtTelemetryLogger(telemetryClient); + } + + public async Task ConsumeAsync(IEnumerable events) + { + EnsureArg.IsNotNull(events); + + try + { + // todo: get template from blob container + string template = File.ReadAllText("./fhirmapping.json"); + _templateDefinition = template; + + await _measurementImportService.ProcessEventsAsync(events, _templateDefinition, _logger).ConfigureAwait(false); + return new AcceptedResult(); + } + catch + { + throw; + } + } + } +} diff --git a/src/console/MeasurementCollectionToFhir/ProcessorStartup.cs b/src/console/MeasurementCollectionToFhir/ProcessorStartup.cs new file mode 100644 index 00000000..00f69fe3 --- /dev/null +++ b/src/console/MeasurementCollectionToFhir/ProcessorStartup.cs @@ -0,0 +1,73 @@ +using EnsureThat; +using Hl7.Fhir.Model; +using Hl7.Fhir.Rest; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.Common; +using Microsoft.Health.Extensions.Fhir; +using Microsoft.Health.Extensions.Fhir.Config; +using Microsoft.Health.Fhir.Ingest.Config; +using Microsoft.Health.Fhir.Ingest.Host; +using Microsoft.Health.Fhir.Ingest.Service; +using Microsoft.Health.Fhir.Ingest.Template; +using System; +using System.Linq; + +namespace Microsoft.Health.Fhir.Ingest.Console.MeasurementCollectionToFhir +{ + public class ProcessorStartup + { + public ProcessorStartup(IConfiguration configuration) + { + Configuration = configuration; + } + + public IConfiguration Configuration { get; } + + public void ConfigureServices(IServiceCollection services) + { + Configuration.GetSection("FhirService") + .GetChildren() + .ToList() + .ForEach(env => Environment.SetEnvironmentVariable(env.Path, env.Value)); + + services.Configure(Configuration.GetSection("ResourceIdentity")); + services.Configure(Configuration.GetSection("FhirClient")); + + services.TryAddSingleton, FhirClientFactory>(); + services.TryAddSingleton(sp => sp.GetRequiredService>().Create()); + services.TryAddSingleton, Observation>, R4FhirLookupTemplateProcessor>(); + services.TryAddSingleton(ResolveResourceIdentityService); + services.TryAddSingleton(sp => new MemoryCache(Options.Create(new MemoryCacheOptions { SizeLimit = 5000 }))); + services.TryAddSingleton(); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(ResolveMeasurementImportProvider); + } + + private MeasurementFhirImportProvider ResolveMeasurementImportProvider(IServiceProvider serviceProvider) + { + EnsureArg.IsNotNull(serviceProvider, nameof(serviceProvider)); + + IOptions options = Options.Create(new MeasurementFhirImportOptions()); + var logger = new LoggerFactory(); + var measurementImportService = new MeasurementFhirImportProvider(Configuration, options, logger, serviceProvider); + + return measurementImportService; + } + + private static IResourceIdentityService ResolveResourceIdentityService(IServiceProvider serviceProvider) + { + EnsureArg.IsNotNull(serviceProvider, nameof(serviceProvider)); + + var fhirClient = serviceProvider.GetRequiredService(); + var resourceIdentityOptions = serviceProvider.GetRequiredService>(); + return ResourceIdentityServiceFactory.Instance.Create(resourceIdentityOptions.Value, fhirClient); + } + } +} diff --git a/src/console/Microsoft.Health.Fhir.Ingest.Console.csproj b/src/console/Microsoft.Health.Fhir.Ingest.Console.csproj new file mode 100644 index 00000000..eed0da10 --- /dev/null +++ b/src/console/Microsoft.Health.Fhir.Ingest.Console.csproj @@ -0,0 +1,38 @@ + + + + Exe + netcoreapp3.1 + + + + + + + + + + + + + + + + + + Always + + + + + + Always + + + + + + Always + + + diff --git a/src/console/Normalize/Processor.cs b/src/console/Normalize/Processor.cs new file mode 100644 index 00000000..4bc0e51f --- /dev/null +++ b/src/console/Normalize/Processor.cs @@ -0,0 +1,94 @@ +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.EventHubs; +using Microsoft.Azure.WebJobs; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; +using Microsoft.Health.Events.EventConsumers; +using Microsoft.Health.Events.Model; +using Microsoft.Health.Fhir.Ingest.Config; +using Microsoft.Health.Fhir.Ingest.Data; +using Microsoft.Health.Fhir.Ingest.Service; +using Microsoft.Health.Fhir.Ingest.Telemetry; +using Microsoft.Health.Fhir.Ingest.Template; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using static Microsoft.Azure.EventHubs.EventData; + +namespace Microsoft.Health.Fhir.Ingest.Console.Normalize +{ + public class Processor : IEventConsumer + { + private string _templateDefinitions; + private ITelemetryLogger _logger; + private IConfiguration _env; + private IOptions _options; + + public Processor( + [Blob("template/%Template:DeviceContent%", FileAccess.Read)] string templateDefinitions, + IConfiguration configuration, + IOptions options) + { + _templateDefinitions = templateDefinitions; + + var config = new TelemetryConfiguration(); + var telemetryClient = new TelemetryClient(config); + _logger = new IomtTelemetryLogger(telemetryClient); + _env = configuration; + _options = options; + } + + public async Task ConsumeAsync(IEnumerable events) + { + // todo: get template from blob container + string readText = File.ReadAllText("./devicecontent.json"); + _templateDefinitions = readText; + + var templateContext = CollectionContentTemplateFactory.Default.Create(_templateDefinitions); + templateContext.EnsureValid(); + var template = templateContext.Template; + + _logger.LogMetric( + IomtMetrics.DeviceEvent(), + events.Count()); + + IEnumerable eventHubEvents = events + .Select(x => { + var eventData = new EventData(x.Body.ToArray()); + eventData.SystemProperties = new SystemPropertiesCollection( + x.SequenceNumber, + x.EnqueuedTime.UtcDateTime, + x.Offset.ToString(), + x.PartitionId); + + foreach (KeyValuePair entry in x.SystemProperties) + { + eventData.SystemProperties.TryAdd(entry.Key, entry.Value); + } + + return eventData; + }) + .ToList(); + + var dataNormalizationService = new MeasurementEventNormalizationService(_logger, template); + + var connectionString = _env.GetSection("OutputEventHub").Value; + var eventHubName = connectionString.Substring(connectionString.LastIndexOf('=') + 1); + + var collector = CreateCollector(eventHubName, connectionString, _options); + + await dataNormalizationService.ProcessAsync(eventHubEvents, collector).ConfigureAwait(false); + + return new AcceptedResult(); + } + + private IAsyncCollector CreateCollector(string eventHubName, string connectionString, IOptions options) + { + var client = options.Value.GetEventHubClient(eventHubName, connectionString); + return new MeasurementToEventAsyncCollector(new EventHubService(client)); + } + } +} diff --git a/src/console/Normalize/ProcessorStartup.cs b/src/console/Normalize/ProcessorStartup.cs new file mode 100644 index 00000000..1ff56b4d --- /dev/null +++ b/src/console/Normalize/ProcessorStartup.cs @@ -0,0 +1,36 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using EnsureThat; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Ingest.Config; + +namespace Microsoft.Health.Fhir.Ingest.Console.Normalize +{ + public class ProcessorStartup + { + public ProcessorStartup(IConfiguration configuration) + { + Configuration = configuration; + } + + public IConfiguration Configuration { get; } + + public void ConfigureServices(IServiceCollection services) + { + var outputEventHubConnection = Configuration.GetSection("OutputEventHub").Value; + var outputEventHubName = outputEventHubConnection.Substring(outputEventHubConnection.LastIndexOf('=') + 1); + + EnsureArg.IsNotNullOrEmpty(outputEventHubConnection); + EnsureArg.IsNotNullOrEmpty(outputEventHubName); + + services.Configure(options => + { + options.AddSender(outputEventHubName, outputEventHubConnection); + }); + } + } +} diff --git a/src/console/Program.cs b/src/console/Program.cs new file mode 100644 index 00000000..7e231d09 --- /dev/null +++ b/src/console/Program.cs @@ -0,0 +1,134 @@ +using Azure.Messaging.EventHubs; +using Azure.Messaging.EventHubs.Consumer; +using Azure.Storage.Blobs; +using EnsureThat; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Microsoft.Health.Events.EventCheckpointing; +using Microsoft.Health.Events.EventConsumers; +using Microsoft.Health.Events.EventConsumers.Service; +using Microsoft.Health.Events.EventHubProcessor; +using Microsoft.Health.Events.Storage; +using Microsoft.Health.Fhir.Ingest.Config; +using Microsoft.Health.Fhir.Ingest.Service; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Health.Fhir.Ingest.Console +{ + public class Program + { + public static async Task Main() + { + var config = GetEnvironmentConfig(); + + var eventHub = config.GetSection("Console:EventHub").Value; + var eventHubOptions = GetEventHubInfo(config, eventHub); + + EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubConnectionString); + EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubName); + + var eventBatchingOptions = new EventBatchingOptions(); + config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions); + + var storageOptions = new StorageOptions(); + config.GetSection(StorageOptions.Settings).Bind(storageOptions); + + var serviceProvider = GetRequiredServiceProvider(config, eventHub); + var eventConsumers = GetEventConsumers(config, eventHub, serviceProvider); + + var eventConsumerService = new EventConsumerService(eventConsumers); + var checkpointClient = new StorageCheckpointClient(storageOptions); + + var ct = new CancellationToken(); + + string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName; + BlobContainerClient storageClient = new BlobContainerClient(storageOptions.BlobStorageConnectionString, storageOptions.BlobContainerName); + + var eventProcessorClientOptions = new EventProcessorClientOptions(); + eventProcessorClientOptions.MaximumWaitTime = TimeSpan.FromSeconds(60); + EventProcessorClient client = new EventProcessorClient(storageClient, consumerGroup, eventHubOptions.EventHubConnectionString, eventHubOptions.EventHubName, eventProcessorClientOptions); + + var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient); + var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient); + await eventHubReader.RunAsync(client, ct); + } + + public static IConfiguration GetEnvironmentConfig() + { + IConfiguration config = new ConfigurationBuilder() + .AddJsonFile("appsettings.json", true, true) + .Build(); + + return config; + } + + public static ServiceProvider GetRequiredServiceProvider(IConfiguration config, string eventHub) + { + if (eventHub == "devicedata") + { + var serviceCollection = new ServiceCollection(); + Normalize.ProcessorStartup startup = new Normalize.ProcessorStartup(config); + startup.ConfigureServices(serviceCollection); + var serviceProvider = serviceCollection.BuildServiceProvider(); + return serviceProvider; + } + else if (eventHub == "normalizeddata") + { + var serviceCollection = new ServiceCollection(); + MeasurementCollectionToFhir.ProcessorStartup startup = new MeasurementCollectionToFhir.ProcessorStartup(config); + startup.ConfigureServices(serviceCollection); + var serviceProvider = serviceCollection.BuildServiceProvider(); + return serviceProvider; + } + else + { + throw new Exception("No valid event hub type was found"); + } + } + + public static EventHubOptions GetEventHubInfo(IConfiguration config, string eventHub) + { + var connectionString = eventHub == "devicedata" + ? config.GetSection("InputEventHub").Value + : config.GetSection("OutputEventHub").Value; + + var eventHubName = connectionString.Substring(connectionString.LastIndexOf('=') + 1); + return new EventHubOptions(connectionString, eventHubName); + } + + public static List GetEventConsumers(IConfiguration config, string inputEventHub, ServiceProvider sp) + { + var eventConsumers = new List(); + if (inputEventHub == "devicedata") + { + var template = config.GetSection("Template:DeviceContent").Value; + var deviceDataNormalization = new Normalize.Processor(template, config, sp.GetRequiredService>()); + eventConsumers.Add(deviceDataNormalization); + } + + else if (inputEventHub == "normalizeddata") + { + var template = config.GetSection("Template:FhirMapping").Value; + var measurementImportService = ResolveMeasurementService(sp); + var measurementToFhirConsumer = new MeasurementCollectionToFhir.Processor(template, measurementImportService); + eventConsumers.Add(measurementToFhirConsumer); + } + + if (config.GetSection("Console:Debug")?.Value == "true") + { + eventConsumers.Add(new EventPrinter()); + } + + return eventConsumers; + } + + public static MeasurementFhirImportService ResolveMeasurementService(IServiceProvider services) + { + return services.GetRequiredService(); + } + } +} diff --git a/src/console/appsettings.json b/src/console/appsettings.json new file mode 100644 index 00000000..d01a0efe --- /dev/null +++ b/src/console/appsettings.json @@ -0,0 +1,20 @@ +{ + "EventBatching:FlushTimespan": 300, + "EventBatching:MaxEvents": 500, + "Storage:BlobStorageConnectionString": "", + "Storage:BlobContainerName": "", + "Storage:BlobPrefix": "testing", + "APPINSIGHTS_INSTRUMENTATIONKEY": "", + "Console:EventHub": "normalizeddata", + "FhirService:Authority": "", + "FhirService:ClientId": "", + "FhirService:ClientSecret": "", + "FhirService:Resource": "", + "FhirService:Url": "", + "InputEventHub": "", + "OutputEventHub": "", + "ResourceIdentity:ResourceIdentityServiceType": "Create", + "ResourceIdentity:DefaultDeviceIdentifierSystem": "", + "Template:DeviceContent": "devicecontent.json", + "Template:FhirMapping": "fhirmapping.json" +} \ No newline at end of file diff --git a/src/console/devicecontent.json b/src/console/devicecontent.json new file mode 100644 index 00000000..99e7696b --- /dev/null +++ b/src/console/devicecontent.json @@ -0,0 +1,115 @@ +{ + "templateType": "CollectionContent", + "template": [ + { + "templateType": "IotJsonPathContent", + "template": { + "typeName": "heartrate", + "typeMatchExpression": "$..[?(@Body.HeartRate)]", + "patientIdExpression": "$.SystemProperties.iothub-connection-device-id", + "values": [ + { + "required": "true", + "valueExpression": "$.Body.HeartRate", + "valueName": "hr" + } + ] + } + }, + { + "templateType": "IotJsonPathContent", + "template": { + "typeName": "respiratoryrate", + "typeMatchExpression": "$..[?(@Body.RespiratoryRate)]", + "patientIdExpression": "$.SystemProperties.iothub-connection-device-id", + "values": [ + { + "required": "true", + "valueExpression": "$.Body.RespiratoryRate", + "valueName": "respiratoryrate" + } + ] + } + }, + { + "templateType": "IotJsonPathContent", + "template": { + "typeName": "hrv", + "typeMatchExpression": "$..[?(@Body.HeartRateVariability)]", + "patientIdExpression": "$.SystemProperties.iothub-connection-device-id", + "values": [ + { + "required": "true", + "valueExpression": "$.Body.HeartRateVariability", + "valueName": "hrv" + } + ] + } + }, + { + "templateType": "IotJsonPathContent", + "template": { + "typeName": "bodytemperature", + "typeMatchExpression": "$..[?(@Body.BodyTemperature)]", + "patientIdExpression": "$.SystemProperties.iothub-connection-device-id", + "values": [ + { + "required": "true", + "valueExpression": "$.Body.BodyTemperature", + "valueName": "temp" + } + ] + } + }, + { + "templateType": "IotJsonPathContent", + "template": { + "typeName": "bp", + "typeMatchExpression": "$..[?(@Body.Systolic && @Body.Diastolic)]", + "patientIdExpression": "$.SystemProperties.iothub-connection-device-id", + "values": [ + { + "required": "true", + "valueExpression": "$.Body.Systolic", + "valueName": "systolic" + }, + { + "required": "true", + "valueExpression": "$.Body.Diastolic", + "valueName": "diastolic" + } + ] + } + }, + { + "templateType": "IotJsonPathContent", + "template": { + "typeName": "rangeofmotion", + "typeMatchExpression": "$..[?(@Body.RangeOfMotion)]", + "patientIdExpression": "$.SystemProperties.iothub-connection-device-id", + "values": [ + { + "required": "true", + "valueExpression": "$.Body.RangeOfMotion", + "valueName": "rangeofmotion" + } + ] + } + }, + { + "templateType": "IotJsonPathContent", + "template": { + "typeName": "kneebend", + "typeMatchExpression": "$..[?(@Body.KneeBend)]", + "patientIdExpression": "$.SystemProperties.iothub-connection-device-id", + "values": [ + { + "required": "true", + "valueExpression": "$.Body.KneeBend", + "valueName": "kneebend" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/src/console/fhirmapping.json b/src/console/fhirmapping.json new file mode 100644 index 00000000..b93574cf --- /dev/null +++ b/src/console/fhirmapping.json @@ -0,0 +1,176 @@ +{ + "templateType": "CollectionFhir", + "template": [ + { + "templateType": "CodeValueFhir", + "template": { + "codes": [ + { + "code": "8867-4", + "system": "http://loinc.org", + "display": "Heart rate" + } + ], + "periodInterval": 60, + "typeName": "heartrate", + "value": { + "defaultPeriod": 30000, + "unit": "count/min", + "valueName": "hr", + "valueType": "SampledData" + } + } + }, + { + "templateType": "CodeValueFhir", + "template": { + "codes": [ + { + "code": "9279-1", + "system": "http://loinc.org", + "display": "Respiratory rate" + } + ], + "periodInterval": 60, + "typeName": "respiratoryrate", + "value": { + "defaultPeriod": 30000, + "unit": "breaths/minute", + "valueName": "respiratoryrate", + "valueType": "SampledData" + } + } + }, + { + "templateType": "CodeValueFhir", + "template": { + "codes": [ + { + "code": "80404-7", + "system": "http://loinc.org", + "display": "Heart rate variability" + } + ], + "periodInterval": 60, + "typeName": "hrv", + "value": { + "defaultPeriod": 30000, + "unit": "", + "valueName": "hrv", + "valueType": "SampledData" + } + } + }, + { + "templateType": "CodeValueFhir", + "template": { + "codes": [ + { + "code": "8310-5", + "system": "http://loinc.org", + "display": "Body temperature" + } + ], + "periodInterval": 60, + "typeName": "bodytemperature", + "value": { + "defaultPeriod": 30000, + "unit": "", + "valueName": "temp", + "valueType": "SampledData" + } + } + }, + { + "templateType": "CodeValueFhir", + "template": { + "codes": [ + { + "code": "85354-9", + "display": "Blood pressure panel", + "system": "http://loinc.org" + } + ], + "periodInterval": 60, + "typeName": "bloodpressure", + "components": [ + { + "codes": [ + { + "code": "8867-4", + "display": "Diastolic blood pressure", + "system": "http://loinc.org" + } + ], + "value": { + "defaultPeriod": 30000, + "unit": "mmHg", + "valueName": "diastolic", + "valueType": "SampledData" + } + }, + { + "codes": [ + { + "code": "8480-6", + "display": "Systolic blood pressure", + "system": "http://loinc.org" + }, + { + "code": "271649006", + "display": "Systolic blood pressure", + "system": "http://snomed.info/sct" + } + ], + "value": { + "defaultPeriod": 30000, + "unit": "mmHg", + "valueName": "systolic", + "valueType": "SampledData" + } + } + ] + } + }, + { + "templateType": "CodeValueFhir", + "template": { + "codes": [ + { + "code": "rangeOfMotion", + "system": "https://www.mydevice.com/v1", + "display": "Range Of Motion" + } + ], + "periodInterval": 60, + "typeName": "rangeofmotion", + "value": { + "defaultPeriod": 30000, + "unit": "", + "valueName": "rangeofmotion", + "valueType": "SampledData" + } + } + }, + { + "templateType": "CodeValueFhir", + "template": { + "codes": [ + { + "code": "kneeBend", + "system": "https://www.mydevice.com/v1", + "display": "Knee Bend" + } + ], + "periodInterval": 60, + "typeName": "kneebend", + "value": { + "defaultPeriod": 30000, + "unit": "", + "valueName": "kneebend", + "valueType": "SampledData" + } + } + } + ] +} \ No newline at end of file diff --git a/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs b/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs new file mode 100644 index 00000000..41453db5 --- /dev/null +++ b/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs @@ -0,0 +1,21 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Health.Events.Model; + +namespace Microsoft.Health.Events.EventCheckpointing +{ + public interface ICheckpointClient + { + Task SetCheckpointAsync(Event eventArg); + + Task PublishCheckpointsAsync(CancellationToken cancellationToken); + + Task> ListCheckpointsAsync(); + } +} diff --git a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs new file mode 100644 index 00000000..6749f4bb --- /dev/null +++ b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs @@ -0,0 +1,183 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Timers; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using EnsureThat; +using Microsoft.Health.Events.Model; +using Microsoft.Health.Events.Storage; + +namespace Microsoft.Health.Events.EventCheckpointing +{ + public class StorageCheckpointClient : ICheckpointClient + { + private Dictionary _checkpoints; + private BlobContainerClient _storageClient; + private static System.Timers.Timer _publisherTimer; + private int _publishTimerInterval = 10000; + private bool _canPublish = false; + + public StorageCheckpointClient(StorageOptions options) + { + EnsureArg.IsNotNull(options); + EnsureArg.IsNotNullOrWhiteSpace(options.BlobPrefix); + EnsureArg.IsNotNullOrWhiteSpace(options.BlobStorageConnectionString); + EnsureArg.IsNotNullOrWhiteSpace(options.BlobContainerName); + + BlobPrefix = options.BlobPrefix; + + _checkpoints = new Dictionary(); + _storageClient = new BlobContainerClient(options.BlobStorageConnectionString, options.BlobContainerName); + + SetPublisherTimer(); + } + + public string BlobPrefix { get; } + + public async Task UpdateCheckpointAsync(Checkpoint checkpoint) + { + EnsureArg.IsNotNull(checkpoint); + EnsureArg.IsNotNullOrWhiteSpace(checkpoint.Id); + + var blobName = $"{BlobPrefix}/checkpoint/{checkpoint.Id}"; + var blobClient = _storageClient.GetBlobClient(blobName); + + var metadata = new Dictionary() + { + { "LastProcessed", checkpoint.LastProcessed.DateTime.ToString("MM/dd/yyyy hh:mm:ss.fff tt") }, + }; + + try + { + try + { + await blobClient.SetMetadataAsync(metadata); + } + catch (RequestFailedException ex) when ((ex.ErrorCode == BlobErrorCode.BlobNotFound) || (ex.ErrorCode == BlobErrorCode.ContainerNotFound)) + { + using (var blobContent = new MemoryStream(Array.Empty())) + { + await blobClient.UploadAsync(blobContent, metadata: metadata).ConfigureAwait(false); + } + } + } + catch (RequestFailedException ex) when (ex.ErrorCode == BlobErrorCode.ContainerNotFound) + { + // todo: log + throw; + } + catch + { + // todo: log + throw; + } + finally + { + // todo: log + } + } + + public Task> ListCheckpointsAsync() + { + var prefix = $"{BlobPrefix}/checkpoint"; + + Task> GetCheckpointsAsync() + { + var checkpoints = new List(); + + foreach (BlobItem blob in _storageClient.GetBlobs(traits: BlobTraits.Metadata, states: BlobStates.All, prefix: prefix, cancellationToken: CancellationToken.None)) + { + var partitionId = blob.Name.Split('/').Last(); + DateTimeOffset lastEventTimestamp = DateTime.MinValue; + + if (blob.Metadata.TryGetValue("LastProcessed", out var str)) + { + DateTimeOffset.TryParse(str, null, DateTimeStyles.AssumeUniversal, out lastEventTimestamp); + } + + checkpoints.Add(new Checkpoint + { + Prefix = BlobPrefix, + Id = partitionId, + LastProcessed = lastEventTimestamp, + }); + } + + return Task.FromResult(checkpoints); + } + + try + { + // todo: consider retries + return GetCheckpointsAsync(); + } + catch (RequestFailedException ex) when (ex.ErrorCode == BlobErrorCode.ContainerNotFound) + { + // todo: log errors + throw; + } + catch + { + // todo: log errors + throw; + } + finally + { + // todo: log complete + } + } + + public Task SetCheckpointAsync(Event eventArgs) + { + EnsureArg.IsNotNull(eventArgs); + EnsureArg.IsNotNullOrWhiteSpace(eventArgs.PartitionId); + + _canPublish = true; + var checkpoint = new Checkpoint(); + checkpoint.LastProcessed = eventArgs.EnqueuedTime; + checkpoint.Id = eventArgs.PartitionId; + checkpoint.Prefix = BlobPrefix; + + _checkpoints[eventArgs.PartitionId] = checkpoint; + + return Task.CompletedTask; + } + + public async Task PublishCheckpointsAsync(CancellationToken ct) + { + foreach (KeyValuePair checkpoint in _checkpoints) + { + await UpdateCheckpointAsync(checkpoint.Value); + } + } + + private void SetPublisherTimer() + { + _publisherTimer = new System.Timers.Timer(_publishTimerInterval); + _publisherTimer.Elapsed += OnTimedEvent; + _publisherTimer.AutoReset = true; + _publisherTimer.Enabled = true; + } + + private async void OnTimedEvent(object source, ElapsedEventArgs e) + { + if (_canPublish) + { + // await PublishCheckpointsAsync(CancellationToken.None); + await Task.Delay(1000); + _canPublish = false; + } + } + } +} diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/EventPrinter.cs b/src/lib/Microsoft.Health.Events/EventConsumers/EventPrinter.cs new file mode 100644 index 00000000..6aee5935 --- /dev/null +++ b/src/lib/Microsoft.Health.Events/EventConsumers/EventPrinter.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Health.Events.Model; + +namespace Microsoft.Health.Events.EventConsumers +{ + public class EventPrinter : IEventConsumer + { + public async Task ConsumeAsync(IEnumerable events) + { + EnsureArg.IsNotNull(events); + foreach (Event evt in events) + { + string message = Encoding.UTF8.GetString(evt.Body.ToArray()); + var enqueuedTime = evt.EnqueuedTime.UtcDateTime; + Console.WriteLine($"Enqueued Time: {enqueuedTime} Event Message: \"{message}\""); + } + + return await Task.FromResult(new AcceptedResult()); + } + } +} diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/IEventConsumer.cs b/src/lib/Microsoft.Health.Events/EventConsumers/IEventConsumer.cs new file mode 100644 index 00000000..a088a37b --- /dev/null +++ b/src/lib/Microsoft.Health.Events/EventConsumers/IEventConsumer.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Health.Events.Model; + +namespace Microsoft.Health.Events.EventConsumers +{ + public interface IEventConsumer + { + Task ConsumeAsync(IEnumerable events); + } +} diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingOptions.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingOptions.cs new file mode 100644 index 00000000..20f8f40c --- /dev/null +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingOptions.cs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Events.EventConsumers.Service +{ + public class EventBatchingOptions + { + public const string Settings = "EventBatching"; + + public int FlushTimespan { get; set; } + + public int MaxEvents { get; set; } + } +} diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs new file mode 100644 index 00000000..d704e4f8 --- /dev/null +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs @@ -0,0 +1,139 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Health.Events.EventCheckpointing; +using Microsoft.Health.Events.EventConsumers.Service.Infrastructure; +using Microsoft.Health.Events.Model; + +namespace Microsoft.Health.Events.EventConsumers.Service +{ + public class EventBatchingService : IEventConsumerService + { + private ConcurrentDictionary _eventQueues; + private int _maxEvents = 100; + private TimeSpan _flushTimespan = TimeSpan.FromSeconds(300); + private IEventConsumerService _eventConsumerService; + private ICheckpointClient _checkpointClient; + private const int _timeBuffer = -5; + + public EventBatchingService(IEventConsumerService eventConsumerService, EventBatchingOptions options, ICheckpointClient checkpointClient) + { + EnsureArg.IsNotNull(options); + _eventQueues = new ConcurrentDictionary(); + _eventConsumerService = eventConsumerService; + _maxEvents = options.MaxEvents; + _flushTimespan = TimeSpan.FromSeconds(options.FlushTimespan); + _checkpointClient = checkpointClient; + } + + public EventQueue GetQueue(string queueId) + { + EnsureArg.IsNotNullOrWhiteSpace(queueId); + + if (!_eventQueues.ContainsKey(queueId)) + { + throw new Exception($"Queue with identifier {queueId} does not exist"); + } + + return _eventQueues[queueId]; + } + + private bool EventQueueExists(string queueId) + { + return _eventQueues.ContainsKey(queueId); + } + + private EventQueue CreateQueueIfMissing(string queueId, DateTime initTime, TimeSpan flushTimespan) + { + return _eventQueues.GetOrAdd(queueId, new EventQueue(queueId, initTime, flushTimespan)); + } + + public Task ConsumeEvent(Event eventArg) + { + EnsureArg.IsNotNull(eventArg); + + var queueId = eventArg.PartitionId; + var eventEnqueuedTime = eventArg.EnqueuedTime.UtcDateTime; + + if (eventArg is MaximumWaitEvent) + { + if (EventQueueExists(queueId)) + { + var windowThresholdTime = GetQueue(queueId).GetQueueWindow(); + ThresholdWaitReached(queueId, windowThresholdTime); + } + } + else + { + var queue = CreateQueueIfMissing(queueId, eventEnqueuedTime, _flushTimespan); + + queue.Enqueue(eventArg); + + var windowThresholdTime = queue.GetQueueWindow(); + if (eventEnqueuedTime > windowThresholdTime) + { + ThresholdTimeReached(queueId, eventArg, windowThresholdTime); + return Task.CompletedTask; + } + + if (queue.GetQueueCount() >= _maxEvents) + { + ThresholdCountReached(queueId); + } + } + + return Task.CompletedTask; + } + + // todo: fix -"Collection was modified; enumeration operation may not execute." + private async void ThresholdCountReached(string queueId) + { + Console.WriteLine($"The threshold count {_maxEvents} was reached."); + var events = await GetQueue(queueId).Flush(_maxEvents); + await _eventConsumerService.ConsumeEvents(events); + UpdateCheckpoint(events); + } + + private async void ThresholdTimeReached(string queueId, Event eventArg, DateTime windowEnd) + { + Console.WriteLine($"The threshold time {_eventQueues[queueId].GetQueueWindow()} was reached."); + var queue = GetQueue(queueId); + var events = await queue.Flush(windowEnd); + await _eventConsumerService.ConsumeEvents(events); + queue.IncrementQueueWindow(eventArg.EnqueuedTime.UtcDateTime); + UpdateCheckpoint(events); + } + + private async void ThresholdWaitReached(string queueId, DateTime windowEnd) + { + if (windowEnd < DateTime.UtcNow.AddSeconds(_timeBuffer)) + { + Console.WriteLine($"Threshold wait reached. Flushing {_eventQueues[queueId].GetQueueCount()} events up to: {windowEnd}"); + var events = await GetQueue(queueId).Flush(windowEnd); + await _eventConsumerService.ConsumeEvents(events); + UpdateCheckpoint(events); + } + } + + private async void UpdateCheckpoint(List events) + { + if (events.Count > 0) + { + var eventCheckpoint = events[events.Count - 1]; + await _checkpointClient.SetCheckpointAsync(eventCheckpoint); + } + } + + public Task ConsumeEvents(IEnumerable events) + { + throw new NotImplementedException(); + } + } +} diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs new file mode 100644 index 00000000..fef4477b --- /dev/null +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs @@ -0,0 +1,37 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Health.Events.EventConsumers; +using Microsoft.Health.Events.Model; + +namespace Microsoft.Health.Events.EventConsumers.Service +{ + public class EventConsumerService : IEventConsumerService + { + private readonly IEnumerable eventConsumers; + + public EventConsumerService(IEnumerable eventConsumers) + { + this.eventConsumers = eventConsumers; + } + + public Task ConsumeEvent(Event eventArg) + { + throw new System.NotImplementedException(); + } + + public Task ConsumeEvents(IEnumerable events) + { + foreach (IEventConsumer eventConsumer in eventConsumers) + { + eventConsumer.ConsumeAsync(events); + } + + return Task.CompletedTask; + } + } +} diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/IEventConsumerService.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/IEventConsumerService.cs new file mode 100644 index 00000000..51f5f81a --- /dev/null +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/IEventConsumerService.cs @@ -0,0 +1,18 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Health.Events.Model; + +namespace Microsoft.Health.Events.EventConsumers.Service +{ + public interface IEventConsumerService + { + Task ConsumeEvents(IEnumerable events); + + Task ConsumeEvent(Event eventArg); + } +} diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueue.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueue.cs new file mode 100644 index 00000000..b4ad3c22 --- /dev/null +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueue.cs @@ -0,0 +1,92 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Health.Events.Model; + +namespace Microsoft.Health.Events.EventConsumers.Service.Infrastructure +{ + public class EventQueue + { + private string _queueId; + private ConcurrentQueue _queue; + private EventQueueWindow _queueWindow; + + public EventQueue(string queueId, DateTime initDateTime, TimeSpan flushTimespan) + { + _queueId = queueId; + _queue = new ConcurrentQueue(); + _queueWindow = new EventQueueWindow(initDateTime, flushTimespan); + } + + public void IncrementQueueWindow(DateTime dateTime) + { + _queueWindow.IncrementWindow(dateTime); + } + + public DateTime GetQueueWindow() + { + return _queueWindow.GetWindowEnd(); + } + + public int GetQueueCount() + { + return _queue.Count; + } + + public void Enqueue(Event eventArg) + { + _queue.Enqueue(eventArg); + } + + // flush a fixed number of events + public Task> Flush(int numEvents) + { + Console.WriteLine($"Flushing {numEvents} events"); + + var count = 0; + var events = new List(); + + while (count < numEvents) + { + if (_queue.TryDequeue(out var dequeuedEvent)) + { + events.Add(dequeuedEvent); + count++; + } + } + + Console.WriteLine($"Current window {GetQueueWindow()}"); + return Task.FromResult(events); + } + + // flush up to a date time + public Task> Flush(DateTime dateTime) + { + Console.WriteLine($"Attempt to flush queue up to {dateTime}"); + + var events = new List(); + while (_queue.TryPeek(out var eventData)) + { + var enqueuedUtc = eventData.EnqueuedTime.UtcDateTime; + if (enqueuedUtc <= dateTime) + { + _queue.TryDequeue(out var dequeuedEvent); + events.Add(dequeuedEvent); + } + else + { + break; + } + } + + Console.WriteLine($"Flushed {events.Count} events up to {dateTime}"); + return Task.FromResult(events); + } + } +} \ No newline at end of file diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueueWindow.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueueWindow.cs new file mode 100644 index 00000000..c9d4126e --- /dev/null +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueueWindow.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Events.EventConsumers.Service.Infrastructure +{ + public class EventQueueWindow + { + private DateTime _windowEnd = DateTime.MinValue; + private TimeSpan _flushTimespan; + + public EventQueueWindow(DateTime initDateTime, TimeSpan flushTimespan) + { + _windowEnd = initDateTime.Add(flushTimespan); + _flushTimespan = flushTimespan; + } + + public void IncrementWindow(DateTime currentEnqueudTime) + { + while (currentEnqueudTime >= _windowEnd) + { + _windowEnd = _windowEnd.Add(_flushTimespan); + } + } + + public DateTime GetWindowEnd() + { + return _windowEnd; + } + } +} diff --git a/src/lib/Microsoft.Health.Events/EventHubProcessor/EventHubOptions.cs b/src/lib/Microsoft.Health.Events/EventHubProcessor/EventHubOptions.cs new file mode 100644 index 00000000..74e787a2 --- /dev/null +++ b/src/lib/Microsoft.Health.Events/EventHubProcessor/EventHubOptions.cs @@ -0,0 +1,22 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Events.EventHubProcessor +{ + public class EventHubOptions + { + public const string Settings = "EventHub"; + + public EventHubOptions(string connectionString, string name) + { + EventHubConnectionString = connectionString; + EventHubName = name; + } + + public string EventHubConnectionString { get; set; } + + public string EventHubName { get; set; } + } +} diff --git a/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs b/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs new file mode 100644 index 00000000..b89bdca5 --- /dev/null +++ b/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs @@ -0,0 +1,113 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Threading; +using System.Threading.Tasks; +using Azure.Messaging.EventHubs; +using Azure.Messaging.EventHubs.Consumer; +using Azure.Messaging.EventHubs.Processor; +using EnsureThat; +using Microsoft.Health.Events.EventCheckpointing; +using Microsoft.Health.Events.EventConsumers.Service; +using Microsoft.Health.Events.Model; + +namespace Microsoft.Health.Events.EventHubProcessor +{ + public class EventProcessor + { + private IEventConsumerService _eventConsumerService; + private StorageCheckpointClient _checkpointClient; + + public EventProcessor(IEventConsumerService eventConsumerService, StorageCheckpointClient checkpointClient) + { + _eventConsumerService = eventConsumerService; + _checkpointClient = checkpointClient; + } + + public async Task RunAsync(EventProcessorClient processor, CancellationToken ct) + { + EnsureArg.IsNotNull(processor); + + Task ProcessEventHandler(ProcessEventArgs eventArgs) + { + try + { + if (eventArgs.HasEvent) + { + var evt = new Event( + eventArgs.Partition.PartitionId, + eventArgs.Data.Body, + eventArgs.Data.Offset, + eventArgs.Data.SequenceNumber, + eventArgs.Data.EnqueuedTime.UtcDateTime, + eventArgs.Data.SystemProperties); + + _eventConsumerService.ConsumeEvent(evt); + } + else + { + var evt = new MaximumWaitEvent(eventArgs.Partition.PartitionId, DateTime.UtcNow); + _eventConsumerService.ConsumeEvent(evt); + } + } + catch + { + throw; + } + + return Task.CompletedTask; + } + + Task ProcessErrorHandler(ProcessErrorEventArgs eventArgs) + { + // todo: add an error processor + Console.WriteLine(eventArgs.Exception.Message); + return Task.CompletedTask; + } + + async Task ProcessInitializingHandler(PartitionInitializingEventArgs initArgs) + { + Console.WriteLine($"Initializing partition {initArgs.PartitionId}"); + + var partitionId = initArgs.PartitionId; + + // todo: only get checkpoint for partition instead of listing them all + var checkpoints = await _checkpointClient.ListCheckpointsAsync(); + foreach (var checkpoint in checkpoints) + { + if (checkpoint.Id == partitionId) + { + initArgs.DefaultStartingPosition = EventPosition.FromEnqueuedTime(checkpoint.LastProcessed); + Console.WriteLine($"Starting to read partition {partitionId} from checkpoint {checkpoint.LastProcessed}"); + break; + } + } + } + + processor.ProcessEventAsync += ProcessEventHandler; + processor.ProcessErrorAsync += ProcessErrorHandler; + processor.PartitionInitializingAsync += ProcessInitializingHandler; + + try + { + Console.WriteLine($"Starting event hub processor at {DateTime.UtcNow}"); + await processor.StartProcessingAsync(); + + while (!ct.IsCancellationRequested) + { + } + + await processor.StopProcessingAsync(); + } + finally + { + processor.ProcessEventAsync -= ProcessEventHandler; + processor.ProcessErrorAsync -= ProcessErrorHandler; + processor.PartitionInitializingAsync -= ProcessInitializingHandler; + } + } + } +} diff --git a/src/lib/Microsoft.Health.Events/Microsoft.Health.Events.csproj b/src/lib/Microsoft.Health.Events/Microsoft.Health.Events.csproj new file mode 100644 index 00000000..d9196a5c --- /dev/null +++ b/src/lib/Microsoft.Health.Events/Microsoft.Health.Events.csproj @@ -0,0 +1,37 @@ + + + netcoreapp3.1 + ..\..\..\CustomAnalysisRules.ruleset + true + 7.3 + + + true + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/src/lib/Microsoft.Health.Events/Microsoft.Health.Events.sln b/src/lib/Microsoft.Health.Events/Microsoft.Health.Events.sln new file mode 100644 index 00000000..dc46272d --- /dev/null +++ b/src/lib/Microsoft.Health.Events/Microsoft.Health.Events.sln @@ -0,0 +1,27 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30523.141 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Events", "Microsoft.Health.Events.csproj", "{62ABBEA7-F031-4BCC-A4D4-2CC535BBFEE7}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{2D9C34D3-4743-45D9-B208-D87283BF783A}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {62ABBEA7-F031-4BCC-A4D4-2CC535BBFEE7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {62ABBEA7-F031-4BCC-A4D4-2CC535BBFEE7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {62ABBEA7-F031-4BCC-A4D4-2CC535BBFEE7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {62ABBEA7-F031-4BCC-A4D4-2CC535BBFEE7}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {B4435849-B7E6-4A6E-95AD-672F6F3E188A} + EndGlobalSection +EndGlobal diff --git a/src/lib/Microsoft.Health.Events/Model/Checkpoint.cs b/src/lib/Microsoft.Health.Events/Model/Checkpoint.cs new file mode 100644 index 00000000..d0e15dcd --- /dev/null +++ b/src/lib/Microsoft.Health.Events/Model/Checkpoint.cs @@ -0,0 +1,18 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Events.Model +{ + public class Checkpoint + { + public string Prefix { get; set; } + + public string Id { get; set; } + + public DateTimeOffset LastProcessed { get; set; } + } +} diff --git a/src/lib/Microsoft.Health.Events/Model/Event.cs b/src/lib/Microsoft.Health.Events/Model/Event.cs new file mode 100644 index 00000000..1abb96ef --- /dev/null +++ b/src/lib/Microsoft.Health.Events/Model/Event.cs @@ -0,0 +1,47 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; + +namespace Microsoft.Health.Events.Model +{ + public class Event + { + public Event(string partitionId, DateTime dateTime) + { + EnqueuedTime = dateTime; + PartitionId = partitionId; + } + + public Event( + string partitionId, + ReadOnlyMemory body, + long sequenceNumber, + long offset, + DateTimeOffset enqueuedTime, + IReadOnlyDictionary systemProperties) + { + PartitionId = partitionId; + Body = body; + SequenceNumber = sequenceNumber; + Offset = offset; + EnqueuedTime = enqueuedTime; + SystemProperties = new Dictionary(systemProperties); + } + + public string PartitionId { get; } + + public ReadOnlyMemory Body { get; } + + public long SequenceNumber { get; } + + public long Offset { get; } + + public DateTimeOffset EnqueuedTime { get; } + + public Dictionary SystemProperties { get; } + } +} diff --git a/src/lib/Microsoft.Health.Events/Model/MaximumWaitEvent.cs b/src/lib/Microsoft.Health.Events/Model/MaximumWaitEvent.cs new file mode 100644 index 00000000..ee56b69b --- /dev/null +++ b/src/lib/Microsoft.Health.Events/Model/MaximumWaitEvent.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Events.Model +{ + public class MaximumWaitEvent : Event + { + public MaximumWaitEvent(string partitionId, DateTime dateTime) + : base(partitionId, dateTime) + { + } + } +} diff --git a/src/lib/Microsoft.Health.Events/Storage/StorageOptions.cs b/src/lib/Microsoft.Health.Events/Storage/StorageOptions.cs new file mode 100644 index 00000000..01e44aac --- /dev/null +++ b/src/lib/Microsoft.Health.Events/Storage/StorageOptions.cs @@ -0,0 +1,18 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Events.Storage +{ + public class StorageOptions + { + public const string Settings = "Storage"; + + public string BlobStorageConnectionString { get; set; } + + public string BlobContainerName { get; set; } + + public string BlobPrefix { get; set; } + } +} diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Microsoft.Health.Fhir.Ingest.csproj b/src/lib/Microsoft.Health.Fhir.Ingest/Microsoft.Health.Fhir.Ingest.csproj index be734a7e..a50366a6 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Microsoft.Health.Fhir.Ingest.csproj +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Microsoft.Health.Fhir.Ingest.csproj @@ -46,6 +46,7 @@ + diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs index 6a38f369..377cc853 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs @@ -11,6 +11,7 @@ using EnsureThat; using Microsoft.Health.Common.Service; using Microsoft.Health.Common.Telemetry; +using Microsoft.Health.Events.Model; using Microsoft.Health.Fhir.Ingest.Config; using Microsoft.Health.Fhir.Ingest.Data; using Microsoft.Health.Fhir.Ingest.Telemetry; @@ -31,15 +32,34 @@ public MeasurementFhirImportService(FhirImportService fhirImportService, Measure } public async Task ProcessStreamAsync(Stream data, string templateDefinition, ITelemetryLogger log) + { + var template = BuildTemplate(templateDefinition, log); + var measurementGroups = await ParseAsync(data, log).ConfigureAwait(false); + + await ProcessMeasurementGroups(measurementGroups, template, log).ConfigureAwait(false); + } + + public async Task ProcessEventsAsync(IEnumerable events, string templateDefinition, ITelemetryLogger log) + { + var template = BuildTemplate(templateDefinition, log); + var measurementGroups = ParseEventData(events); + + await ProcessMeasurementGroups(measurementGroups, template, log).ConfigureAwait(false); + } + + private ILookupTemplate BuildTemplate(string templateDefinition, ITelemetryLogger log) { EnsureArg.IsNotNull(templateDefinition, nameof(templateDefinition)); EnsureArg.IsNotNull(log, nameof(log)); + var templateContext = Options.TemplateFactory.Create(templateDefinition); templateContext.EnsureValid(); - var template = templateContext.Template; - var measurementGroups = await ParseAsync(data, log).ConfigureAwait(false); + return templateContext.Template; + } + private async Task ProcessMeasurementGroups(IEnumerable measurementGroups, ILookupTemplate template, ITelemetryLogger log) + { // Group work by device to avoid race conditions when resource creation is enabled. var workItems = measurementGroups.GroupBy(grp => grp.DeviceId) .Select(grp => new Func( @@ -87,6 +107,31 @@ private static async Task> ParseAsync(Stream data return measurementGroups; } + private static IEnumerable ParseEventData(IEnumerable data) + { + // Deserialize events into measurements and then group according to the device, type, and other factors + var body = data.First().Body.ToArray(); + var text = System.Text.Encoding.Default.GetString(body); + var measurement = JsonConvert.DeserializeObject(text); + + return data.Select(e => JsonConvert.DeserializeObject(System.Text.Encoding.Default.GetString(e.Body.ToArray()))) + .GroupBy(m => $"{m.DeviceId}-{m.Type}-{m.PatientId}-{m.EncounterId}-{m.CorrelationId}") + .Select(g => + { + var measurements = g.ToList(); + return new MeasurementGroup + { + Data = measurements, + MeasureType = measurements[0].Type, + CorrelationId = measurements[0].CorrelationId, + DeviceId = measurements[0].DeviceId, + EncounterId = measurements[0].EncounterId, + PatientId = measurements[0].PatientId, + }; + }) + .ToArray(); + } + private static async Task CalculateMetricsAsync(IList measurements, ITelemetryLogger log) { await Task.Run(() => @@ -120,4 +165,4 @@ await Task.Run(() => }).ConfigureAwait(false); } } -} +} \ No newline at end of file From b5aab811465d30db9a3fd81d726ed6baf3475b20 Mon Sep 17 00:00:00 2001 From: Will Yochum Date: Fri, 20 Nov 2020 17:44:11 -0800 Subject: [PATCH 03/14] remove test app files --- .../Microsoft.Health.Fhir.Ingest.Console.csproj | 8 -------- Microsoft.Health.Fhir.Ingest.Console/Program.cs | 12 ------------ 2 files changed, 20 deletions(-) delete mode 100644 Microsoft.Health.Fhir.Ingest.Console/Microsoft.Health.Fhir.Ingest.Console.csproj delete mode 100644 Microsoft.Health.Fhir.Ingest.Console/Program.cs diff --git a/Microsoft.Health.Fhir.Ingest.Console/Microsoft.Health.Fhir.Ingest.Console.csproj b/Microsoft.Health.Fhir.Ingest.Console/Microsoft.Health.Fhir.Ingest.Console.csproj deleted file mode 100644 index c73e0d16..00000000 --- a/Microsoft.Health.Fhir.Ingest.Console/Microsoft.Health.Fhir.Ingest.Console.csproj +++ /dev/null @@ -1,8 +0,0 @@ - - - - Exe - netcoreapp3.1 - - - diff --git a/Microsoft.Health.Fhir.Ingest.Console/Program.cs b/Microsoft.Health.Fhir.Ingest.Console/Program.cs deleted file mode 100644 index 5e450f23..00000000 --- a/Microsoft.Health.Fhir.Ingest.Console/Program.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; - -namespace Microsoft.Health.Fhir.Ingest.Console -{ - class Program - { - static void Main(string[] args) - { - Console.WriteLine("Hello World!"); - } - } -} From 82824fee44bdcd30de0c7e840d8bc405bb227d45 Mon Sep 17 00:00:00 2001 From: Will Yochum Date: Mon, 7 Dec 2020 09:42:13 -0800 Subject: [PATCH 04/14] remove hardcoded mapping files --- src/console/devicecontent.json | 115 --------------------- src/console/fhirmapping.json | 176 --------------------------------- 2 files changed, 291 deletions(-) delete mode 100644 src/console/devicecontent.json delete mode 100644 src/console/fhirmapping.json diff --git a/src/console/devicecontent.json b/src/console/devicecontent.json deleted file mode 100644 index 99e7696b..00000000 --- a/src/console/devicecontent.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "templateType": "CollectionContent", - "template": [ - { - "templateType": "IotJsonPathContent", - "template": { - "typeName": "heartrate", - "typeMatchExpression": "$..[?(@Body.HeartRate)]", - "patientIdExpression": "$.SystemProperties.iothub-connection-device-id", - "values": [ - { - "required": "true", - "valueExpression": "$.Body.HeartRate", - "valueName": "hr" - } - ] - } - }, - { - "templateType": "IotJsonPathContent", - "template": { - "typeName": "respiratoryrate", - "typeMatchExpression": "$..[?(@Body.RespiratoryRate)]", - "patientIdExpression": "$.SystemProperties.iothub-connection-device-id", - "values": [ - { - "required": "true", - "valueExpression": "$.Body.RespiratoryRate", - "valueName": "respiratoryrate" - } - ] - } - }, - { - "templateType": "IotJsonPathContent", - "template": { - "typeName": "hrv", - "typeMatchExpression": "$..[?(@Body.HeartRateVariability)]", - "patientIdExpression": "$.SystemProperties.iothub-connection-device-id", - "values": [ - { - "required": "true", - "valueExpression": "$.Body.HeartRateVariability", - "valueName": "hrv" - } - ] - } - }, - { - "templateType": "IotJsonPathContent", - "template": { - "typeName": "bodytemperature", - "typeMatchExpression": "$..[?(@Body.BodyTemperature)]", - "patientIdExpression": "$.SystemProperties.iothub-connection-device-id", - "values": [ - { - "required": "true", - "valueExpression": "$.Body.BodyTemperature", - "valueName": "temp" - } - ] - } - }, - { - "templateType": "IotJsonPathContent", - "template": { - "typeName": "bp", - "typeMatchExpression": "$..[?(@Body.Systolic && @Body.Diastolic)]", - "patientIdExpression": "$.SystemProperties.iothub-connection-device-id", - "values": [ - { - "required": "true", - "valueExpression": "$.Body.Systolic", - "valueName": "systolic" - }, - { - "required": "true", - "valueExpression": "$.Body.Diastolic", - "valueName": "diastolic" - } - ] - } - }, - { - "templateType": "IotJsonPathContent", - "template": { - "typeName": "rangeofmotion", - "typeMatchExpression": "$..[?(@Body.RangeOfMotion)]", - "patientIdExpression": "$.SystemProperties.iothub-connection-device-id", - "values": [ - { - "required": "true", - "valueExpression": "$.Body.RangeOfMotion", - "valueName": "rangeofmotion" - } - ] - } - }, - { - "templateType": "IotJsonPathContent", - "template": { - "typeName": "kneebend", - "typeMatchExpression": "$..[?(@Body.KneeBend)]", - "patientIdExpression": "$.SystemProperties.iothub-connection-device-id", - "values": [ - { - "required": "true", - "valueExpression": "$.Body.KneeBend", - "valueName": "kneebend" - } - ] - } - } - ] -} \ No newline at end of file diff --git a/src/console/fhirmapping.json b/src/console/fhirmapping.json deleted file mode 100644 index b93574cf..00000000 --- a/src/console/fhirmapping.json +++ /dev/null @@ -1,176 +0,0 @@ -{ - "templateType": "CollectionFhir", - "template": [ - { - "templateType": "CodeValueFhir", - "template": { - "codes": [ - { - "code": "8867-4", - "system": "http://loinc.org", - "display": "Heart rate" - } - ], - "periodInterval": 60, - "typeName": "heartrate", - "value": { - "defaultPeriod": 30000, - "unit": "count/min", - "valueName": "hr", - "valueType": "SampledData" - } - } - }, - { - "templateType": "CodeValueFhir", - "template": { - "codes": [ - { - "code": "9279-1", - "system": "http://loinc.org", - "display": "Respiratory rate" - } - ], - "periodInterval": 60, - "typeName": "respiratoryrate", - "value": { - "defaultPeriod": 30000, - "unit": "breaths/minute", - "valueName": "respiratoryrate", - "valueType": "SampledData" - } - } - }, - { - "templateType": "CodeValueFhir", - "template": { - "codes": [ - { - "code": "80404-7", - "system": "http://loinc.org", - "display": "Heart rate variability" - } - ], - "periodInterval": 60, - "typeName": "hrv", - "value": { - "defaultPeriod": 30000, - "unit": "", - "valueName": "hrv", - "valueType": "SampledData" - } - } - }, - { - "templateType": "CodeValueFhir", - "template": { - "codes": [ - { - "code": "8310-5", - "system": "http://loinc.org", - "display": "Body temperature" - } - ], - "periodInterval": 60, - "typeName": "bodytemperature", - "value": { - "defaultPeriod": 30000, - "unit": "", - "valueName": "temp", - "valueType": "SampledData" - } - } - }, - { - "templateType": "CodeValueFhir", - "template": { - "codes": [ - { - "code": "85354-9", - "display": "Blood pressure panel", - "system": "http://loinc.org" - } - ], - "periodInterval": 60, - "typeName": "bloodpressure", - "components": [ - { - "codes": [ - { - "code": "8867-4", - "display": "Diastolic blood pressure", - "system": "http://loinc.org" - } - ], - "value": { - "defaultPeriod": 30000, - "unit": "mmHg", - "valueName": "diastolic", - "valueType": "SampledData" - } - }, - { - "codes": [ - { - "code": "8480-6", - "display": "Systolic blood pressure", - "system": "http://loinc.org" - }, - { - "code": "271649006", - "display": "Systolic blood pressure", - "system": "http://snomed.info/sct" - } - ], - "value": { - "defaultPeriod": 30000, - "unit": "mmHg", - "valueName": "systolic", - "valueType": "SampledData" - } - } - ] - } - }, - { - "templateType": "CodeValueFhir", - "template": { - "codes": [ - { - "code": "rangeOfMotion", - "system": "https://www.mydevice.com/v1", - "display": "Range Of Motion" - } - ], - "periodInterval": 60, - "typeName": "rangeofmotion", - "value": { - "defaultPeriod": 30000, - "unit": "", - "valueName": "rangeofmotion", - "valueType": "SampledData" - } - } - }, - { - "templateType": "CodeValueFhir", - "template": { - "codes": [ - { - "code": "kneeBend", - "system": "https://www.mydevice.com/v1", - "display": "Knee Bend" - } - ], - "periodInterval": 60, - "typeName": "kneebend", - "value": { - "defaultPeriod": 30000, - "unit": "", - "valueName": "kneebend", - "valueType": "SampledData" - } - } - } - ] -} \ No newline at end of file From fa32689e69fdf9d38910f2e475072d371872f3c0 Mon Sep 17 00:00:00 2001 From: Will Yochum Date: Mon, 7 Dec 2020 09:55:39 -0800 Subject: [PATCH 05/14] address pr feedback --- Microsoft.Health.Fhir.Ingest.sln | 7 + src/console/IomtLogging.cs | 34 +++ .../MeasurementCollectionToFhir/Processor.cs | 26 +- ...icrosoft.Health.Fhir.Ingest.Console.csproj | 13 +- src/console/Normalize/Processor.cs | 48 ++-- src/console/Program.cs | 48 +++- src/console/Template/BlobManager.cs | 42 +++ src/console/Template/ITemplateManager.cs | 14 + src/console/Template/TemplateOptions.cs | 17 ++ src/console/appsettings.json | 9 +- .../IomtConnectorFunctions.cs | 1 + .../Startup.cs | 1 + .../EventCheckpointing/ICheckpointClient.cs | 2 +- .../StorageCheckpointClient.cs | 38 ++- .../StorageCheckpointOptions.cs} | 4 +- .../EventConsumers/EventPrinter.cs | 7 +- .../EventConsumers/IEventConsumer.cs | 3 +- .../Service/EventBatchingService.cs | 28 +- .../Service/EventConsumerService.cs | 46 +++- .../Service/IEventConsumerService.cs | 4 +- .../Service/Infrastructure/EventQueue.cs | 21 +- .../EventHubProcessor/EventProcessor.cs | 29 +- .../Microsoft.Health.Events.csproj | 4 + .../Model/{Event.cs => EventMessage.cs} | 12 +- .../Model/EventMessageFactory.cs | 26 ++ .../Model/IEventMessage.cs | 26 ++ .../Model/MaximumWaitEvent.cs | 2 +- .../Telemetry/Metrics/EventMetrics.cs | 83 ++++++ .../MeasurementEventNormalizationService.cs | 1 + .../Service/MeasurementFhirImportService.cs | 5 +- .../Telemetry/ExceptionTelemetryProcessor.cs | 1 + .../Microsoft.Health.Logger.csproj | 16 ++ .../Telemetry/ITelemetryLogger.cs | 4 +- .../Telemetry/IomtTelemetryLogger.cs | 2 +- .../Metrics/MetricExtensionMethods.cs | 258 ++++++++++++++++++ ...asurementEventNormalizationServiceTests.cs | 2 +- .../MeasurementFhirImportServiceTests.cs | 1 + .../ExceptionTelemetryProcessorTests.cs | 1 + 38 files changed, 745 insertions(+), 141 deletions(-) create mode 100644 src/console/IomtLogging.cs create mode 100644 src/console/Template/BlobManager.cs create mode 100644 src/console/Template/ITemplateManager.cs create mode 100644 src/console/Template/TemplateOptions.cs rename src/lib/Microsoft.Health.Events/{Storage/StorageOptions.cs => EventCheckpointing/StorageCheckpointOptions.cs} (86%) rename src/lib/Microsoft.Health.Events/Model/{Event.cs => EventMessage.cs} (76%) create mode 100644 src/lib/Microsoft.Health.Events/Model/EventMessageFactory.cs create mode 100644 src/lib/Microsoft.Health.Events/Model/IEventMessage.cs create mode 100644 src/lib/Microsoft.Health.Events/Telemetry/Metrics/EventMetrics.cs create mode 100644 src/lib/Microsoft.Health.Logger/Microsoft.Health.Logger.csproj rename src/lib/{Microsoft.Health.Fhir.Ingest => Microsoft.Health.Logger}/Telemetry/ITelemetryLogger.cs (92%) rename src/lib/{Microsoft.Health.Fhir.Ingest => Microsoft.Health.Logger}/Telemetry/IomtTelemetryLogger.cs (96%) create mode 100644 src/lib/Microsoft.Health.Logger/Telemetry/Metrics/MetricExtensionMethods.cs diff --git a/Microsoft.Health.Fhir.Ingest.sln b/Microsoft.Health.Fhir.Ingest.sln index 59fb0fbd..31a7ffb3 100644 --- a/Microsoft.Health.Fhir.Ingest.sln +++ b/Microsoft.Health.Fhir.Ingest.sln @@ -83,6 +83,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "console", "console", "{1EF3 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.Ingest.Console", "src\console\Microsoft.Health.Fhir.Ingest.Console.csproj", "{927BC214-ABD9-4A1B-9F7C-75973513D141}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Logger", "src\lib\Microsoft.Health.Logger\Microsoft.Health.Logger.csproj", "{05123BAE-E96E-4C7E-95CB-C616DF940F17}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -161,6 +163,10 @@ Global {927BC214-ABD9-4A1B-9F7C-75973513D141}.Debug|Any CPU.Build.0 = Debug|Any CPU {927BC214-ABD9-4A1B-9F7C-75973513D141}.Release|Any CPU.ActiveCfg = Release|Any CPU {927BC214-ABD9-4A1B-9F7C-75973513D141}.Release|Any CPU.Build.0 = Release|Any CPU + {05123BAE-E96E-4C7E-95CB-C616DF940F17}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {05123BAE-E96E-4C7E-95CB-C616DF940F17}.Debug|Any CPU.Build.0 = Debug|Any CPU + {05123BAE-E96E-4C7E-95CB-C616DF940F17}.Release|Any CPU.ActiveCfg = Release|Any CPU + {05123BAE-E96E-4C7E-95CB-C616DF940F17}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -188,6 +194,7 @@ Global {EE072537-807D-4FE2-BFEB-424B64DCD7F9} = {FAF8B402-892E-4EA2-B4CF-69B0C70BA762} {22275DE3-859D-40F0-9547-7711568164C0} = {513D67B4-80E1-476D-955F-E7E7C79D144A} {927BC214-ABD9-4A1B-9F7C-75973513D141} = {1EF3584A-C437-4B45-8BF8-1597D5A8DBC7} + {05123BAE-E96E-4C7E-95CB-C616DF940F17} = {513D67B4-80E1-476D-955F-E7E7C79D144A} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A358924D-F948-4AE8-8CD0-A0F56225CE0C} diff --git a/src/console/IomtLogging.cs b/src/console/IomtLogging.cs new file mode 100644 index 00000000..1915fdf6 --- /dev/null +++ b/src/console/IomtLogging.cs @@ -0,0 +1,34 @@ +using EnsureThat; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Health.Logger.Telemetry; + +namespace Microsoft.Health.Fhir.Ingest.Console +{ + public class IomtLogging + { + public IomtLogging(IConfiguration configuration) + { + Configuration = configuration; + } + + public IConfiguration Configuration { get; } + + public void ConfigureServices(IServiceCollection services) + { + EnsureArg.IsNotNull(services, nameof(services)); + + var instrumentationKey = Configuration.GetSection("APPINSIGHTS_INSTRUMENTATIONKEY").Value; + + services.TryAddSingleton(sp => + { + var config = new TelemetryConfiguration(instrumentationKey); + var telemetryClient = new TelemetryClient(config); + return new IomtTelemetryLogger(telemetryClient); + }); + } + } +} diff --git a/src/console/MeasurementCollectionToFhir/Processor.cs b/src/console/MeasurementCollectionToFhir/Processor.cs index c5c062af..2c99e9bc 100644 --- a/src/console/MeasurementCollectionToFhir/Processor.cs +++ b/src/console/MeasurementCollectionToFhir/Processor.cs @@ -7,49 +7,45 @@ using System.IO; using System.Threading.Tasks; using EnsureThat; -using Microsoft.ApplicationInsights; -using Microsoft.ApplicationInsights.Extensibility; -using Microsoft.AspNetCore.Mvc; using Microsoft.Azure.WebJobs; using Microsoft.Health.Events.EventConsumers; using Microsoft.Health.Events.Model; +using Microsoft.Health.Fhir.Ingest.Console.Template; using Microsoft.Health.Fhir.Ingest.Host; using Microsoft.Health.Fhir.Ingest.Service; -using Microsoft.Health.Fhir.Ingest.Telemetry; +using Microsoft.Health.Logger.Telemetry; namespace Microsoft.Health.Fhir.Ingest.Console.MeasurementCollectionToFhir { public class Processor : IEventConsumer { + private ITemplateManager _templateManager; private MeasurementFhirImportService _measurementImportService; private string _templateDefinition; private ITelemetryLogger _logger; public Processor( [Blob("template/%Template:FhirMapping%", FileAccess.Read)] string templateDefinition, - [MeasurementFhirImport] MeasurementFhirImportService measurementImportService) + ITemplateManager templateManager, + [MeasurementFhirImport] MeasurementFhirImportService measurementImportService, + ITelemetryLogger logger) { _templateDefinition = templateDefinition; + _templateManager = templateManager; _measurementImportService = measurementImportService; - - // todo: inject logger - var config = new TelemetryConfiguration(); - var telemetryClient = new TelemetryClient(config); - _logger = new IomtTelemetryLogger(telemetryClient); + _logger = logger; } - public async Task ConsumeAsync(IEnumerable events) + public async Task ConsumeAsync(IEnumerable events) { EnsureArg.IsNotNull(events); try { - // todo: get template from blob container - string template = File.ReadAllText("./fhirmapping.json"); - _templateDefinition = template; + EnsureArg.IsNotNull(_templateDefinition); + var templateContent = _templateManager.GetTemplateAsString(_templateDefinition); await _measurementImportService.ProcessEventsAsync(events, _templateDefinition, _logger).ConfigureAwait(false); - return new AcceptedResult(); } catch { diff --git a/src/console/Microsoft.Health.Fhir.Ingest.Console.csproj b/src/console/Microsoft.Health.Fhir.Ingest.Console.csproj index eed0da10..f39568af 100644 --- a/src/console/Microsoft.Health.Fhir.Ingest.Console.csproj +++ b/src/console/Microsoft.Health.Fhir.Ingest.Console.csproj @@ -23,16 +23,5 @@ Always - - - - Always - - - - - - Always - - + diff --git a/src/console/Normalize/Processor.cs b/src/console/Normalize/Processor.cs index 4bc0e51f..760b52d9 100644 --- a/src/console/Normalize/Processor.cs +++ b/src/console/Normalize/Processor.cs @@ -1,6 +1,4 @@ -using Microsoft.ApplicationInsights; -using Microsoft.ApplicationInsights.Extensibility; -using Microsoft.AspNetCore.Mvc; +using EnsureThat; using Microsoft.Azure.EventHubs; using Microsoft.Azure.WebJobs; using Microsoft.Extensions.Configuration; @@ -8,10 +6,12 @@ using Microsoft.Health.Events.EventConsumers; using Microsoft.Health.Events.Model; using Microsoft.Health.Fhir.Ingest.Config; +using Microsoft.Health.Fhir.Ingest.Console.Template; using Microsoft.Health.Fhir.Ingest.Data; using Microsoft.Health.Fhir.Ingest.Service; using Microsoft.Health.Fhir.Ingest.Telemetry; using Microsoft.Health.Fhir.Ingest.Template; +using Microsoft.Health.Logger.Telemetry; using System.Collections.Generic; using System.IO; using System.Linq; @@ -22,32 +22,32 @@ namespace Microsoft.Health.Fhir.Ingest.Console.Normalize { public class Processor : IEventConsumer { - private string _templateDefinitions; + private string _templateDefinition; + private ITemplateManager _templateManager; private ITelemetryLogger _logger; private IConfiguration _env; private IOptions _options; public Processor( - [Blob("template/%Template:DeviceContent%", FileAccess.Read)] string templateDefinitions, + [Blob("template/%Template:DeviceContent%", FileAccess.Read)] string templateDefinition, + ITemplateManager templateManager, IConfiguration configuration, - IOptions options) + IOptions collectorOptions, + ITelemetryLogger logger) { - _templateDefinitions = templateDefinitions; - - var config = new TelemetryConfiguration(); - var telemetryClient = new TelemetryClient(config); - _logger = new IomtTelemetryLogger(telemetryClient); + _templateDefinition = templateDefinition; + _templateManager = templateManager; + _logger = logger; _env = configuration; - _options = options; + _options = collectorOptions; } - public async Task ConsumeAsync(IEnumerable events) + public async Task ConsumeAsync(IEnumerable events) { - // todo: get template from blob container - string readText = File.ReadAllText("./devicecontent.json"); - _templateDefinitions = readText; + EnsureArg.IsNotNull(_templateDefinition); + var templateContent = _templateManager.GetTemplateAsString(_templateDefinition); - var templateContext = CollectionContentTemplateFactory.Default.Create(_templateDefinitions); + var templateContext = CollectionContentTemplateFactory.Default.Create(templateContent); templateContext.EnsureValid(); var template = templateContext.Template; @@ -56,7 +56,8 @@ public async Task ConsumeAsync(IEnumerable events) events.Count()); IEnumerable eventHubEvents = events - .Select(x => { + .Select(x => + { var eventData = new EventData(x.Body.ToArray()); eventData.SystemProperties = new SystemPropertiesCollection( x.SequenceNumber, @@ -68,21 +69,20 @@ public async Task ConsumeAsync(IEnumerable events) { eventData.SystemProperties.TryAdd(entry.Key, entry.Value); } - + return eventData; - }) - .ToList(); + }); var dataNormalizationService = new MeasurementEventNormalizationService(_logger, template); + // todo: support managed identity var connectionString = _env.GetSection("OutputEventHub").Value; - var eventHubName = connectionString.Substring(connectionString.LastIndexOf('=') + 1); + var sb = new EventHubsConnectionStringBuilder(connectionString); + var eventHubName = sb.EntityPath; var collector = CreateCollector(eventHubName, connectionString, _options); await dataNormalizationService.ProcessAsync(eventHubEvents, collector).ConfigureAwait(false); - - return new AcceptedResult(); } private IAsyncCollector CreateCollector(string eventHubName, string connectionString, IOptions options) diff --git a/src/console/Program.cs b/src/console/Program.cs index 7e231d09..d0c50db4 100644 --- a/src/console/Program.cs +++ b/src/console/Program.cs @@ -9,9 +9,11 @@ using Microsoft.Health.Events.EventConsumers; using Microsoft.Health.Events.EventConsumers.Service; using Microsoft.Health.Events.EventHubProcessor; -using Microsoft.Health.Events.Storage; using Microsoft.Health.Fhir.Ingest.Config; +using Microsoft.Health.Fhir.Ingest.Console.Storage; +using Microsoft.Health.Fhir.Ingest.Console.Template; using Microsoft.Health.Fhir.Ingest.Service; +using Microsoft.Health.Logger.Telemetry; using System; using System.Collections.Generic; using System.Threading; @@ -25,7 +27,13 @@ public static async Task Main() { var config = GetEnvironmentConfig(); - var eventHub = config.GetSection("Console:EventHub").Value; + var eventHub = Environment.GetEnvironmentVariable("WEBJOBS_NAME"); + if (eventHub == null) + { + eventHub = config.GetSection("Console:EventHub").Value; + } + + System.Console.WriteLine($"reading from event hub: {eventHub}"); var eventHubOptions = GetEventHubInfo(config, eventHub); EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubConnectionString); @@ -34,11 +42,12 @@ public static async Task Main() var eventBatchingOptions = new EventBatchingOptions(); config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions); - var storageOptions = new StorageOptions(); - config.GetSection(StorageOptions.Settings).Bind(storageOptions); + var storageOptions = new StorageCheckpointOptions(); + config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions); var serviceProvider = GetRequiredServiceProvider(config, eventHub); - var eventConsumers = GetEventConsumers(config, eventHub, serviceProvider); + var logger = serviceProvider.GetRequiredService(); + var eventConsumers = GetEventConsumers(config, eventHub, serviceProvider, logger); var eventConsumerService = new EventConsumerService(eventConsumers); var checkpointClient = new StorageCheckpointClient(storageOptions); @@ -52,8 +61,8 @@ public static async Task Main() eventProcessorClientOptions.MaximumWaitTime = TimeSpan.FromSeconds(60); EventProcessorClient client = new EventProcessorClient(storageClient, consumerGroup, eventHubOptions.EventHubConnectionString, eventHubOptions.EventHubName, eventProcessorClientOptions); - var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient); - var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient); + var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger); + var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger); await eventHubReader.RunAsync(client, ct); } @@ -73,6 +82,10 @@ public static ServiceProvider GetRequiredServiceProvider(IConfiguration config, var serviceCollection = new ServiceCollection(); Normalize.ProcessorStartup startup = new Normalize.ProcessorStartup(config); startup.ConfigureServices(serviceCollection); + + var loggingService = new IomtLogging(config); + loggingService.ConfigureServices(serviceCollection); + var serviceProvider = serviceCollection.BuildServiceProvider(); return serviceProvider; } @@ -81,6 +94,10 @@ public static ServiceProvider GetRequiredServiceProvider(IConfiguration config, var serviceCollection = new ServiceCollection(); MeasurementCollectionToFhir.ProcessorStartup startup = new MeasurementCollectionToFhir.ProcessorStartup(config); startup.ConfigureServices(serviceCollection); + + var loggingService = new IomtLogging(config); + loggingService.ConfigureServices(serviceCollection); + var serviceProvider = serviceCollection.BuildServiceProvider(); return serviceProvider; } @@ -100,13 +117,24 @@ public static EventHubOptions GetEventHubInfo(IConfiguration config, string even return new EventHubOptions(connectionString, eventHubName); } - public static List GetEventConsumers(IConfiguration config, string inputEventHub, ServiceProvider sp) + public static List GetEventConsumers(IConfiguration config, string inputEventHub, ServiceProvider sp, ITelemetryLogger logger) { var eventConsumers = new List(); + var templateOptions = new TemplateOptions(); + config.GetSection(TemplateOptions.Settings).Bind(templateOptions); + + EnsureArg.IsNotNull(templateOptions); + EnsureArg.IsNotNull(templateOptions.BlobContainerName); + EnsureArg.IsNotNull(templateOptions.BlobStorageConnectionString); + + var templateManager = new BlobManager( + templateOptions.BlobStorageConnectionString, + templateOptions.BlobContainerName); + if (inputEventHub == "devicedata") { var template = config.GetSection("Template:DeviceContent").Value; - var deviceDataNormalization = new Normalize.Processor(template, config, sp.GetRequiredService>()); + var deviceDataNormalization = new Normalize.Processor(template, templateManager, config, sp.GetRequiredService>(), logger); eventConsumers.Add(deviceDataNormalization); } @@ -114,7 +142,7 @@ public static List GetEventConsumers(IConfiguration config, stri { var template = config.GetSection("Template:FhirMapping").Value; var measurementImportService = ResolveMeasurementService(sp); - var measurementToFhirConsumer = new MeasurementCollectionToFhir.Processor(template, measurementImportService); + var measurementToFhirConsumer = new MeasurementCollectionToFhir.Processor(template, templateManager, measurementImportService, logger); eventConsumers.Add(measurementToFhirConsumer); } diff --git a/src/console/Template/BlobManager.cs b/src/console/Template/BlobManager.cs new file mode 100644 index 00000000..2501539d --- /dev/null +++ b/src/console/Template/BlobManager.cs @@ -0,0 +1,42 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Azure.Storage.Blobs; +using EnsureThat; +using System.IO; +using System.Text; + +namespace Microsoft.Health.Fhir.Ingest.Console.Template +{ + public class BlobManager : ITemplateManager + { + private BlobContainerClient _blobContainer; + + public BlobManager(string connectionString, string blobContainerName) + { + EnsureArg.IsNotNull(connectionString); + EnsureArg.IsNotNull(blobContainerName); + + _blobContainer = new BlobContainerClient(connectionString, blobContainerName); + } + public byte[] GetTemplate(string templateName) + { + EnsureArg.IsNotNull(templateName); + + var blockBlob = _blobContainer.GetBlobClient(templateName); + var memoryStream = new MemoryStream(); + blockBlob.DownloadTo(memoryStream); + byte[] itemContent = memoryStream.ToArray(); + return itemContent; + } + + public string GetTemplateAsString(string templateName) + { + var templateBuffer = GetTemplate(templateName); + string templateContent = Encoding.UTF8.GetString(templateBuffer, 0, templateBuffer.Length); + return templateContent; + } + } +} diff --git a/src/console/Template/ITemplateManager.cs b/src/console/Template/ITemplateManager.cs new file mode 100644 index 00000000..c32dfe1e --- /dev/null +++ b/src/console/Template/ITemplateManager.cs @@ -0,0 +1,14 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Ingest.Console.Template +{ + public interface ITemplateManager + { + byte[] GetTemplate(string templateName); + + string GetTemplateAsString(string templateName); + } +} diff --git a/src/console/Template/TemplateOptions.cs b/src/console/Template/TemplateOptions.cs new file mode 100644 index 00000000..a6ef11d4 --- /dev/null +++ b/src/console/Template/TemplateOptions.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + + +namespace Microsoft.Health.Fhir.Ingest.Console.Storage +{ + public class TemplateOptions + { + public const string Settings = "TemplateStorage"; + + public string BlobStorageConnectionString { get; set; } + + public string BlobContainerName { get; set; } + } +} diff --git a/src/console/appsettings.json b/src/console/appsettings.json index d01a0efe..213e1783 100644 --- a/src/console/appsettings.json +++ b/src/console/appsettings.json @@ -1,11 +1,14 @@ { + "APPINSIGHTS_INSTRUMENTATIONKEY": "", "EventBatching:FlushTimespan": 300, "EventBatching:MaxEvents": 500, "Storage:BlobStorageConnectionString": "", "Storage:BlobContainerName": "", - "Storage:BlobPrefix": "testing", - "APPINSIGHTS_INSTRUMENTATIONKEY": "", - "Console:EventHub": "normalizeddata", + "Storage:BlobPrefix": "", + "TemplateStorage:BlobStorageConnectionString": "", + "TemplateStorage:BlobContainerName": "", + "Console:EventHub": "devicedata", + "FhirClient:UseManagedIdentity": "true", "FhirService:Authority": "", "FhirService:ClientId": "", "FhirService:ClientSecret": "", diff --git a/src/func/Microsoft.Health.Fhir.Ingest.Host/IomtConnectorFunctions.cs b/src/func/Microsoft.Health.Fhir.Ingest.Host/IomtConnectorFunctions.cs index 4548fc74..981854e7 100644 --- a/src/func/Microsoft.Health.Fhir.Ingest.Host/IomtConnectorFunctions.cs +++ b/src/func/Microsoft.Health.Fhir.Ingest.Host/IomtConnectorFunctions.cs @@ -17,6 +17,7 @@ using Microsoft.Health.Fhir.Ingest.Host; using Microsoft.Health.Fhir.Ingest.Telemetry; using Microsoft.Health.Fhir.Ingest.Template; +using Microsoft.Health.Logger.Telemetry; namespace Microsoft.Health.Fhir.Ingest.Service { diff --git a/src/func/Microsoft.Health.Fhir.Ingest.Host/Startup.cs b/src/func/Microsoft.Health.Fhir.Ingest.Host/Startup.cs index 59ea480e..99390eb3 100644 --- a/src/func/Microsoft.Health.Fhir.Ingest.Host/Startup.cs +++ b/src/func/Microsoft.Health.Fhir.Ingest.Host/Startup.cs @@ -11,6 +11,7 @@ using Microsoft.Azure.Functions.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Ingest.Telemetry; +using Microsoft.Health.Logger.Telemetry; namespace Microsoft.Health.Fhir.Ingest.Service { diff --git a/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs b/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs index 41453db5..6a313407 100644 --- a/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs +++ b/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs @@ -12,7 +12,7 @@ namespace Microsoft.Health.Events.EventCheckpointing { public interface ICheckpointClient { - Task SetCheckpointAsync(Event eventArg); + Task SetCheckpointAsync(IEventMessage eventArg); Task PublishCheckpointsAsync(CancellationToken cancellationToken); diff --git a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs index 6749f4bb..afa792a0 100644 --- a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs +++ b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs @@ -16,7 +16,6 @@ using Azure.Storage.Blobs.Models; using EnsureThat; using Microsoft.Health.Events.Model; -using Microsoft.Health.Events.Storage; namespace Microsoft.Health.Events.EventCheckpointing { @@ -26,9 +25,8 @@ public class StorageCheckpointClient : ICheckpointClient private BlobContainerClient _storageClient; private static System.Timers.Timer _publisherTimer; private int _publishTimerInterval = 10000; - private bool _canPublish = false; - public StorageCheckpointClient(StorageOptions options) + public StorageCheckpointClient(StorageCheckpointOptions options) { EnsureArg.IsNotNull(options); EnsureArg.IsNotNullOrWhiteSpace(options.BlobPrefix); @@ -138,18 +136,26 @@ Task> GetCheckpointsAsync() } } - public Task SetCheckpointAsync(Event eventArgs) + public Task SetCheckpointAsync(IEventMessage eventArgs) { EnsureArg.IsNotNull(eventArgs); EnsureArg.IsNotNullOrWhiteSpace(eventArgs.PartitionId); - _canPublish = true; - var checkpoint = new Checkpoint(); - checkpoint.LastProcessed = eventArgs.EnqueuedTime; - checkpoint.Id = eventArgs.PartitionId; - checkpoint.Prefix = BlobPrefix; + try + { + var checkpoint = new Checkpoint(); + checkpoint.LastProcessed = eventArgs.EnqueuedTime; + checkpoint.Id = eventArgs.PartitionId; + checkpoint.Prefix = BlobPrefix; - _checkpoints[eventArgs.PartitionId] = checkpoint; + _checkpoints[eventArgs.PartitionId] = checkpoint; + } +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 + { + Console.WriteLine($"Checkpointing error: {ex.Message}"); + } return Task.CompletedTask; } @@ -172,11 +178,15 @@ private void SetPublisherTimer() private async void OnTimedEvent(object source, ElapsedEventArgs e) { - if (_canPublish) + try + { + await PublishCheckpointsAsync(CancellationToken.None); + } +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 { - // await PublishCheckpointsAsync(CancellationToken.None); - await Task.Delay(1000); - _canPublish = false; + Console.WriteLine(ex.Message); } } } diff --git a/src/lib/Microsoft.Health.Events/Storage/StorageOptions.cs b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointOptions.cs similarity index 86% rename from src/lib/Microsoft.Health.Events/Storage/StorageOptions.cs rename to src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointOptions.cs index 01e44aac..4d32fdf2 100644 --- a/src/lib/Microsoft.Health.Events/Storage/StorageOptions.cs +++ b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointOptions.cs @@ -3,9 +3,9 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -namespace Microsoft.Health.Events.Storage +namespace Microsoft.Health.Events.EventCheckpointing { - public class StorageOptions + public class StorageCheckpointOptions { public const string Settings = "Storage"; diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/EventPrinter.cs b/src/lib/Microsoft.Health.Events/EventConsumers/EventPrinter.cs index 6aee5935..4b09e07b 100644 --- a/src/lib/Microsoft.Health.Events/EventConsumers/EventPrinter.cs +++ b/src/lib/Microsoft.Health.Events/EventConsumers/EventPrinter.cs @@ -8,24 +8,23 @@ using System.Text; using System.Threading.Tasks; using EnsureThat; -using Microsoft.AspNetCore.Mvc; using Microsoft.Health.Events.Model; namespace Microsoft.Health.Events.EventConsumers { public class EventPrinter : IEventConsumer { - public async Task ConsumeAsync(IEnumerable events) + public Task ConsumeAsync(IEnumerable events) { EnsureArg.IsNotNull(events); - foreach (Event evt in events) + foreach (EventMessage evt in events) { string message = Encoding.UTF8.GetString(evt.Body.ToArray()); var enqueuedTime = evt.EnqueuedTime.UtcDateTime; Console.WriteLine($"Enqueued Time: {enqueuedTime} Event Message: \"{message}\""); } - return await Task.FromResult(new AcceptedResult()); + return Task.CompletedTask; } } } diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/IEventConsumer.cs b/src/lib/Microsoft.Health.Events/EventConsumers/IEventConsumer.cs index a088a37b..a512bbfb 100644 --- a/src/lib/Microsoft.Health.Events/EventConsumers/IEventConsumer.cs +++ b/src/lib/Microsoft.Health.Events/EventConsumers/IEventConsumer.cs @@ -5,13 +5,12 @@ using System.Collections.Generic; using System.Threading.Tasks; -using Microsoft.AspNetCore.Mvc; using Microsoft.Health.Events.Model; namespace Microsoft.Health.Events.EventConsumers { public interface IEventConsumer { - Task ConsumeAsync(IEnumerable events); + Task ConsumeAsync(IEnumerable events); } } diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs index d704e4f8..8f6df9fb 100644 --- a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs @@ -11,26 +11,32 @@ using Microsoft.Health.Events.EventCheckpointing; using Microsoft.Health.Events.EventConsumers.Service.Infrastructure; using Microsoft.Health.Events.Model; +using Microsoft.Health.Logger.Telemetry; namespace Microsoft.Health.Events.EventConsumers.Service { public class EventBatchingService : IEventConsumerService { private ConcurrentDictionary _eventQueues; - private int _maxEvents = 100; - private TimeSpan _flushTimespan = TimeSpan.FromSeconds(300); + private int _maxEvents; + private TimeSpan _flushTimespan; private IEventConsumerService _eventConsumerService; private ICheckpointClient _checkpointClient; + private ITelemetryLogger _logger; private const int _timeBuffer = -5; - public EventBatchingService(IEventConsumerService eventConsumerService, EventBatchingOptions options, ICheckpointClient checkpointClient) + public EventBatchingService(IEventConsumerService eventConsumerService, EventBatchingOptions options, ICheckpointClient checkpointClient, ITelemetryLogger logger) { EnsureArg.IsNotNull(options); + EnsureArg.IsInt(options.MaxEvents); + EnsureArg.IsInt(options.FlushTimespan); + _eventQueues = new ConcurrentDictionary(); _eventConsumerService = eventConsumerService; _maxEvents = options.MaxEvents; _flushTimespan = TimeSpan.FromSeconds(options.FlushTimespan); _checkpointClient = checkpointClient; + _logger = logger; } public EventQueue GetQueue(string queueId) @@ -52,10 +58,10 @@ private bool EventQueueExists(string queueId) private EventQueue CreateQueueIfMissing(string queueId, DateTime initTime, TimeSpan flushTimespan) { - return _eventQueues.GetOrAdd(queueId, new EventQueue(queueId, initTime, flushTimespan)); + return _eventQueues.GetOrAdd(queueId, new EventQueue(queueId, initTime, flushTimespan, _logger)); } - public Task ConsumeEvent(Event eventArg) + public Task ConsumeEvent(IEventMessage eventArg) { EnsureArg.IsNotNull(eventArg); @@ -95,15 +101,15 @@ public Task ConsumeEvent(Event eventArg) // todo: fix -"Collection was modified; enumeration operation may not execute." private async void ThresholdCountReached(string queueId) { - Console.WriteLine($"The threshold count {_maxEvents} was reached."); + _logger.LogTrace($"Partition {queueId} threshold count {_maxEvents} was reached."); var events = await GetQueue(queueId).Flush(_maxEvents); await _eventConsumerService.ConsumeEvents(events); UpdateCheckpoint(events); } - private async void ThresholdTimeReached(string queueId, Event eventArg, DateTime windowEnd) + private async void ThresholdTimeReached(string queueId, IEventMessage eventArg, DateTime windowEnd) { - Console.WriteLine($"The threshold time {_eventQueues[queueId].GetQueueWindow()} was reached."); + _logger.LogTrace($"Partition {queueId} threshold time {_eventQueues[queueId].GetQueueWindow()} was reached."); var queue = GetQueue(queueId); var events = await queue.Flush(windowEnd); await _eventConsumerService.ConsumeEvents(events); @@ -115,14 +121,14 @@ private async void ThresholdWaitReached(string queueId, DateTime windowEnd) { if (windowEnd < DateTime.UtcNow.AddSeconds(_timeBuffer)) { - Console.WriteLine($"Threshold wait reached. Flushing {_eventQueues[queueId].GetQueueCount()} events up to: {windowEnd}"); + _logger.LogTrace($"Partition {queueId} threshold wait reached. Flushing {_eventQueues[queueId].GetQueueCount()} events up to: {windowEnd}"); var events = await GetQueue(queueId).Flush(windowEnd); await _eventConsumerService.ConsumeEvents(events); UpdateCheckpoint(events); } } - private async void UpdateCheckpoint(List events) + private async void UpdateCheckpoint(List events) { if (events.Count > 0) { @@ -131,7 +137,7 @@ private async void UpdateCheckpoint(List events) } } - public Task ConsumeEvents(IEnumerable events) + public Task ConsumeEvents(IEnumerable events) { throw new NotImplementedException(); } diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs index fef4477b..eb411c8d 100644 --- a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs @@ -3,9 +3,9 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using System.Collections.Generic; using System.Threading.Tasks; -using Microsoft.Health.Events.EventConsumers; using Microsoft.Health.Events.Model; namespace Microsoft.Health.Events.EventConsumers.Service @@ -13,25 +13,59 @@ namespace Microsoft.Health.Events.EventConsumers.Service public class EventConsumerService : IEventConsumerService { private readonly IEnumerable eventConsumers; + private const int _maximumBackoffMs = 32000; public EventConsumerService(IEnumerable eventConsumers) { this.eventConsumers = eventConsumers; } - public Task ConsumeEvent(Event eventArg) + public Task ConsumeEvent(IEventMessage eventArg) { - throw new System.NotImplementedException(); + throw new NotImplementedException(); } - public Task ConsumeEvents(IEnumerable events) + public async Task ConsumeEvents(IEnumerable events) { foreach (IEventConsumer eventConsumer in eventConsumers) { - eventConsumer.ConsumeAsync(events); + await OperationWithRetryAsync(eventConsumer, events); } + } + + private static async Task OperationWithRetryAsync(IEventConsumer eventConsumer, IEnumerable events) + { + int currentRetry = 0; + double backoffMs = 0; + Random random = new Random(); + bool operationComplete = false; + + while (!operationComplete) + { + try + { + if (currentRetry > 0 && backoffMs < _maximumBackoffMs) + { + int randomMs = random.Next(0, 1000); + backoffMs = Math.Pow(2000, currentRetry) + randomMs; + await Task.Delay((int)backoffMs); + } - return Task.CompletedTask; + await TryOperationAsync(eventConsumer, events).ConfigureAwait(false); + break; + } +#pragma warning disable CA1031 + catch (Exception e) +#pragma warning restore CA1031 + { + Console.WriteLine(e.Message); + } + } + } + + private static async Task TryOperationAsync(IEventConsumer eventConsumer, IEnumerable events) + { + await eventConsumer.ConsumeAsync(events); } } } diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/IEventConsumerService.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/IEventConsumerService.cs index 51f5f81a..d003006a 100644 --- a/src/lib/Microsoft.Health.Events/EventConsumers/Service/IEventConsumerService.cs +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/IEventConsumerService.cs @@ -11,8 +11,8 @@ namespace Microsoft.Health.Events.EventConsumers.Service { public interface IEventConsumerService { - Task ConsumeEvents(IEnumerable events); + Task ConsumeEvents(IEnumerable events); - Task ConsumeEvent(Event eventArg); + Task ConsumeEvent(IEventMessage eventArg); } } diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueue.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueue.cs index b4ad3c22..ededdac2 100644 --- a/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueue.cs +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueue.cs @@ -8,20 +8,23 @@ using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.Health.Events.Model; +using Microsoft.Health.Logger.Telemetry; namespace Microsoft.Health.Events.EventConsumers.Service.Infrastructure { public class EventQueue { private string _queueId; - private ConcurrentQueue _queue; + private ConcurrentQueue _queue; private EventQueueWindow _queueWindow; + private ITelemetryLogger _logger; - public EventQueue(string queueId, DateTime initDateTime, TimeSpan flushTimespan) + public EventQueue(string queueId, DateTime initDateTime, TimeSpan flushTimespan, ITelemetryLogger logger) { _queueId = queueId; - _queue = new ConcurrentQueue(); + _queue = new ConcurrentQueue(); _queueWindow = new EventQueueWindow(initDateTime, flushTimespan); + _logger = logger; } public void IncrementQueueWindow(DateTime dateTime) @@ -39,18 +42,18 @@ public int GetQueueCount() return _queue.Count; } - public void Enqueue(Event eventArg) + public void Enqueue(IEventMessage eventArg) { _queue.Enqueue(eventArg); } // flush a fixed number of events - public Task> Flush(int numEvents) + public Task> Flush(int numEvents) { Console.WriteLine($"Flushing {numEvents} events"); var count = 0; - var events = new List(); + var events = new List(); while (count < numEvents) { @@ -66,11 +69,9 @@ public Task> Flush(int numEvents) } // flush up to a date time - public Task> Flush(DateTime dateTime) + public Task> Flush(DateTime dateTime) { - Console.WriteLine($"Attempt to flush queue up to {dateTime}"); - - var events = new List(); + var events = new List(); while (_queue.TryPeek(out var eventData)) { var enqueuedUtc = eventData.EnqueuedTime.UtcDateTime; diff --git a/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs b/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs index b89bdca5..9fe7cb6a 100644 --- a/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs +++ b/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs @@ -13,45 +13,46 @@ using Microsoft.Health.Events.EventCheckpointing; using Microsoft.Health.Events.EventConsumers.Service; using Microsoft.Health.Events.Model; +using Microsoft.Health.Logger.Telemetry; namespace Microsoft.Health.Events.EventHubProcessor { public class EventProcessor { private IEventConsumerService _eventConsumerService; - private StorageCheckpointClient _checkpointClient; + private ICheckpointClient _checkpointClient; + private ITelemetryLogger _logger; - public EventProcessor(IEventConsumerService eventConsumerService, StorageCheckpointClient checkpointClient) + public EventProcessor(IEventConsumerService eventConsumerService, ICheckpointClient checkpointClient, ITelemetryLogger logger) { _eventConsumerService = eventConsumerService; _checkpointClient = checkpointClient; + _logger = logger; } public async Task RunAsync(EventProcessorClient processor, CancellationToken ct) { EnsureArg.IsNotNull(processor); + // Processes two types of events + // 1) Event hub events + // 2) Maximum wait events. These are generated when we have not received an event hub + // event for a certain time period and this event is used to flush events in the current window. Task ProcessEventHandler(ProcessEventArgs eventArgs) { try { + IEventMessage evt; if (eventArgs.HasEvent) { - var evt = new Event( - eventArgs.Partition.PartitionId, - eventArgs.Data.Body, - eventArgs.Data.Offset, - eventArgs.Data.SequenceNumber, - eventArgs.Data.EnqueuedTime.UtcDateTime, - eventArgs.Data.SystemProperties); - - _eventConsumerService.ConsumeEvent(evt); + evt = EventMessageFactory.CreateEvent(eventArgs); } else { - var evt = new MaximumWaitEvent(eventArgs.Partition.PartitionId, DateTime.UtcNow); - _eventConsumerService.ConsumeEvent(evt); + evt = new MaximumWaitEvent(eventArgs.Partition.PartitionId, DateTime.UtcNow); } + + _eventConsumerService.ConsumeEvent(evt); } catch { @@ -81,7 +82,7 @@ async Task ProcessInitializingHandler(PartitionInitializingEventArgs initArgs) if (checkpoint.Id == partitionId) { initArgs.DefaultStartingPosition = EventPosition.FromEnqueuedTime(checkpoint.LastProcessed); - Console.WriteLine($"Starting to read partition {partitionId} from checkpoint {checkpoint.LastProcessed}"); + _logger.LogTrace($"Starting to read partition {partitionId} from checkpoint {checkpoint.LastProcessed}"); break; } } diff --git a/src/lib/Microsoft.Health.Events/Microsoft.Health.Events.csproj b/src/lib/Microsoft.Health.Events/Microsoft.Health.Events.csproj index d9196a5c..9f87636a 100644 --- a/src/lib/Microsoft.Health.Events/Microsoft.Health.Events.csproj +++ b/src/lib/Microsoft.Health.Events/Microsoft.Health.Events.csproj @@ -34,4 +34,8 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/src/lib/Microsoft.Health.Events/Model/Event.cs b/src/lib/Microsoft.Health.Events/Model/EventMessage.cs similarity index 76% rename from src/lib/Microsoft.Health.Events/Model/Event.cs rename to src/lib/Microsoft.Health.Events/Model/EventMessage.cs index 1abb96ef..c539d72b 100644 --- a/src/lib/Microsoft.Health.Events/Model/Event.cs +++ b/src/lib/Microsoft.Health.Events/Model/EventMessage.cs @@ -8,20 +8,21 @@ namespace Microsoft.Health.Events.Model { - public class Event + public class EventMessage : IEventMessage { - public Event(string partitionId, DateTime dateTime) + public EventMessage(string partitionId, DateTime dateTime) { EnqueuedTime = dateTime; PartitionId = partitionId; } - public Event( + public EventMessage( string partitionId, ReadOnlyMemory body, long sequenceNumber, long offset, DateTimeOffset enqueuedTime, + IDictionary properties, IReadOnlyDictionary systemProperties) { PartitionId = partitionId; @@ -29,6 +30,7 @@ public Event( SequenceNumber = sequenceNumber; Offset = offset; EnqueuedTime = enqueuedTime; + Properties = new Dictionary(properties); SystemProperties = new Dictionary(systemProperties); } @@ -42,6 +44,8 @@ public Event( public DateTimeOffset EnqueuedTime { get; } - public Dictionary SystemProperties { get; } + public IDictionary Properties { get; } + + public IReadOnlyDictionary SystemProperties { get; } } } diff --git a/src/lib/Microsoft.Health.Events/Model/EventMessageFactory.cs b/src/lib/Microsoft.Health.Events/Model/EventMessageFactory.cs new file mode 100644 index 00000000..79fd8dfc --- /dev/null +++ b/src/lib/Microsoft.Health.Events/Model/EventMessageFactory.cs @@ -0,0 +1,26 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Azure.Messaging.EventHubs.Processor; + +namespace Microsoft.Health.Events.Model +{ + public static class EventMessageFactory + { + public static IEventMessage CreateEvent(ProcessEventArgs eventArgs) + { + var eventMessage = new EventMessage( + eventArgs.Partition.PartitionId, + eventArgs.Data.Body, + eventArgs.Data.Offset, + eventArgs.Data.SequenceNumber, + eventArgs.Data.EnqueuedTime.UtcDateTime, + eventArgs.Data.Properties, + eventArgs.Data.SystemProperties); + + return eventMessage; + } + } +} diff --git a/src/lib/Microsoft.Health.Events/Model/IEventMessage.cs b/src/lib/Microsoft.Health.Events/Model/IEventMessage.cs new file mode 100644 index 00000000..18c1dde2 --- /dev/null +++ b/src/lib/Microsoft.Health.Events/Model/IEventMessage.cs @@ -0,0 +1,26 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- +using System; +using System.Collections.Generic; + +namespace Microsoft.Health.Events.Model +{ + public interface IEventMessage + { + string PartitionId { get; } + + ReadOnlyMemory Body { get; } + + long SequenceNumber { get; } + + long Offset { get; } + + DateTimeOffset EnqueuedTime { get; } + + IDictionary Properties { get; } + + IReadOnlyDictionary SystemProperties { get; } + } +} diff --git a/src/lib/Microsoft.Health.Events/Model/MaximumWaitEvent.cs b/src/lib/Microsoft.Health.Events/Model/MaximumWaitEvent.cs index ee56b69b..a759da11 100644 --- a/src/lib/Microsoft.Health.Events/Model/MaximumWaitEvent.cs +++ b/src/lib/Microsoft.Health.Events/Model/MaximumWaitEvent.cs @@ -7,7 +7,7 @@ namespace Microsoft.Health.Events.Model { - public class MaximumWaitEvent : Event + public class MaximumWaitEvent : EventMessage { public MaximumWaitEvent(string partitionId, DateTime dateTime) : base(partitionId, dateTime) diff --git a/src/lib/Microsoft.Health.Events/Telemetry/Metrics/EventMetrics.cs b/src/lib/Microsoft.Health.Events/Telemetry/Metrics/EventMetrics.cs new file mode 100644 index 00000000..d6844dd6 --- /dev/null +++ b/src/lib/Microsoft.Health.Events/Telemetry/Metrics/EventMetrics.cs @@ -0,0 +1,83 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using Microsoft.Health.Common.Telemetry; + +namespace Microsoft.Health.Events.Telemetry +{ + /// + /// Defines known metrics and metric dimensions for use in Application Insights + /// + public static class EventMetrics + { + private static string _nameDimension = DimensionNames.Name; + private static string _categoryDimension = DimensionNames.Category; + + private static Metric _eventHubPartitionInitialized = new Metric( + "EventHubPartitionInitialized", + new Dictionary + { + { _nameDimension, "EventHubPartitionInitialized" }, + { _categoryDimension, Category.Traffic }, + }); + + private static Metric _eventBatchCreated = new Metric( + "EventBatchCreated", + new Dictionary + { + { _nameDimension, "EventBatchCreated" }, + { _categoryDimension, Category.Traffic }, + }); + + private static Metric _eventsFlushed = new Metric( + "EventsFlushed", + new Dictionary + { + { _nameDimension, "EventsFlushed" }, + { _categoryDimension, Category.Traffic }, + }); + + private static Metric _eventsConsumed = new Metric( + "EventsConsumed", + new Dictionary + { + { _nameDimension, "EventsConsumed" }, + { _categoryDimension, Category.Traffic }, + }); + + /// + /// Signals that an event hub partition has been intialized. + /// + public static Metric EventHubPartitionInitialized() + { + return _eventHubPartitionInitialized; + } + + /// + /// Signals that a batch of event hub events was created. + /// + public static Metric EventBatchCreated() + { + return _eventBatchCreated; + } + + /// + /// Signals that a batch of event hub events was flushed. + /// + public static Metric EventsFlushed() + { + return _eventsFlushed; + } + + /// + /// Signals that a batch of event hub events was consumed downstream. + /// + public static Metric EventsConsumed() + { + return _eventsConsumed; + } + } +} diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementEventNormalizationService.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementEventNormalizationService.cs index 0471f206..37838fa5 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementEventNormalizationService.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementEventNormalizationService.cs @@ -15,6 +15,7 @@ using Microsoft.Health.Fhir.Ingest.Data; using Microsoft.Health.Fhir.Ingest.Telemetry; using Microsoft.Health.Fhir.Ingest.Template; +using Microsoft.Health.Logger.Telemetry; using Newtonsoft.Json.Linq; namespace Microsoft.Health.Fhir.Ingest.Service diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs index 377cc853..025392e3 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs @@ -16,6 +16,7 @@ using Microsoft.Health.Fhir.Ingest.Data; using Microsoft.Health.Fhir.Ingest.Telemetry; using Microsoft.Health.Fhir.Ingest.Template; +using Microsoft.Health.Logger.Telemetry; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -39,7 +40,7 @@ public async Task ProcessStreamAsync(Stream data, string templateDefinition, ITe await ProcessMeasurementGroups(measurementGroups, template, log).ConfigureAwait(false); } - public async Task ProcessEventsAsync(IEnumerable events, string templateDefinition, ITelemetryLogger log) + public async Task ProcessEventsAsync(IEnumerable events, string templateDefinition, ITelemetryLogger log) { var template = BuildTemplate(templateDefinition, log); var measurementGroups = ParseEventData(events); @@ -107,7 +108,7 @@ private static async Task> ParseAsync(Stream data return measurementGroups; } - private static IEnumerable ParseEventData(IEnumerable data) + private static IEnumerable ParseEventData(IEnumerable data) { // Deserialize events into measurements and then group according to the device, type, and other factors var body = data.First().Body.ToArray(); diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ExceptionTelemetryProcessor.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ExceptionTelemetryProcessor.cs index c0850d37..e58f85a2 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ExceptionTelemetryProcessor.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ExceptionTelemetryProcessor.cs @@ -11,6 +11,7 @@ using Microsoft.Health.Fhir.Ingest.Data; using Microsoft.Health.Fhir.Ingest.Service; using Microsoft.Health.Fhir.Ingest.Template; +using Microsoft.Health.Logger.Telemetry; namespace Microsoft.Health.Fhir.Ingest.Telemetry { diff --git a/src/lib/Microsoft.Health.Logger/Microsoft.Health.Logger.csproj b/src/lib/Microsoft.Health.Logger/Microsoft.Health.Logger.csproj new file mode 100644 index 00000000..cc6efe0a --- /dev/null +++ b/src/lib/Microsoft.Health.Logger/Microsoft.Health.Logger.csproj @@ -0,0 +1,16 @@ + + + + netcoreapp3.1 + + + + + + + + + + + + diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ITelemetryLogger.cs b/src/lib/Microsoft.Health.Logger/Telemetry/ITelemetryLogger.cs similarity index 92% rename from src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ITelemetryLogger.cs rename to src/lib/Microsoft.Health.Logger/Telemetry/ITelemetryLogger.cs index 1ee00d37..80b17efc 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ITelemetryLogger.cs +++ b/src/lib/Microsoft.Health.Logger/Telemetry/ITelemetryLogger.cs @@ -3,10 +3,10 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; using Microsoft.Health.Common.Telemetry; +using System; -namespace Microsoft.Health.Fhir.Ingest.Telemetry +namespace Microsoft.Health.Logger.Telemetry { public interface ITelemetryLogger { diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/IomtTelemetryLogger.cs b/src/lib/Microsoft.Health.Logger/Telemetry/IomtTelemetryLogger.cs similarity index 96% rename from src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/IomtTelemetryLogger.cs rename to src/lib/Microsoft.Health.Logger/Telemetry/IomtTelemetryLogger.cs index c071e51a..c8e39555 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/IomtTelemetryLogger.cs +++ b/src/lib/Microsoft.Health.Logger/Telemetry/IomtTelemetryLogger.cs @@ -8,7 +8,7 @@ using Microsoft.ApplicationInsights; using Microsoft.Health.Fhir.Ingest.Telemetry.Metrics; -namespace Microsoft.Health.Fhir.Ingest.Telemetry +namespace Microsoft.Health.Logger.Telemetry { public class IomtTelemetryLogger : ITelemetryLogger { diff --git a/src/lib/Microsoft.Health.Logger/Telemetry/Metrics/MetricExtensionMethods.cs b/src/lib/Microsoft.Health.Logger/Telemetry/Metrics/MetricExtensionMethods.cs new file mode 100644 index 00000000..3e0e21e4 --- /dev/null +++ b/src/lib/Microsoft.Health.Logger/Telemetry/Metrics/MetricExtensionMethods.cs @@ -0,0 +1,258 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using EnsureThat; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Metrics; + +namespace Microsoft.Health.Fhir.Ingest.Telemetry.Metrics +{ + public static class MetricExtensionMethods + { + private static string _namespace = MetricIdentifier.DefaultMetricNamespace; + + public static void LogMetric(this Common.Telemetry.Metric metric, TelemetryClient telemetryClient, double metricValue) + { + EnsureArg.IsNotNull(metric); + EnsureArg.IsNotNull(telemetryClient); + + var metricName = metric.Name; + var dimensions = metric.Dimensions; + var dimensionNumber = metric.Dimensions.Count; + + if (dimensionNumber > 10) + { + telemetryClient.TrackException( + new Exception($"Metric {metricName} exceeds the amount of allowed dimensions")); + return; + } + + string[] dimNames = new string[dimensions.Count]; + dimensions.Keys.CopyTo(dimNames, 0); + + string[] dimValues = new string[dimNames.Length]; + int count = 0; + foreach (string dimName in dimNames) + { + dimValues[count] = dimensions[dimName].ToString(); + count++; + } + + dimensions.Values.CopyTo(dimValues, 0); + + switch (dimensionNumber) + { + case 0: + telemetryClient + .GetMetric( + metricName) + .TrackValue( + metricValue); + break; + case 1: + telemetryClient + .GetMetric( + metricName, + dimNames[0]) + .TrackValue( + metricValue, + dimValues[0]); + break; + case 2: + telemetryClient + .GetMetric( + metricName, + dimNames[0], + dimNames[1]) + .TrackValue( + metricValue, + dimValues[0], + dimValues[1]); + break; + case 3: + telemetryClient + .GetMetric( + metricName, + dimNames[0], + dimNames[1], + dimNames[2]) + .TrackValue( + metricValue, + dimValues[0], + dimValues[1], + dimValues[2]); + break; + case 4: + telemetryClient + .GetMetric( + metricName, + dimNames[0], + dimNames[1], + dimNames[2], + dimNames[3]) + .TrackValue( + metricValue, + dimValues[0], + dimValues[1], + dimValues[2], + dimValues[3]); + break; + case 5: + var metric5DId = new MetricIdentifier( + _namespace, + metricName, + dimNames[0], + dimNames[1], + dimNames[2], + dimNames[3], + dimNames[4]); + + telemetryClient + .GetMetric(metric5DId) + .TrackValue( + metricValue, + dimValues[0], + dimValues[1], + dimValues[2], + dimValues[3], + dimValues[4]); + break; + case 6: + var metric6DId = new MetricIdentifier( + _namespace, + metricName, + dimNames[0], + dimNames[1], + dimNames[2], + dimNames[3], + dimNames[4], + dimNames[5]); + + telemetryClient + .GetMetric(metric6DId) + .TrackValue( + metricValue, + dimValues[0], + dimValues[1], + dimValues[2], + dimValues[3], + dimValues[4], + dimValues[5]); + break; + case 7: + var metric7DId = new MetricIdentifier( + _namespace, + metricName, + dimNames[0], + dimNames[1], + dimNames[2], + dimNames[3], + dimNames[4], + dimNames[5], + dimNames[6]); + + telemetryClient + .GetMetric(metric7DId) + .TrackValue( + metricValue, + dimValues[0], + dimValues[1], + dimValues[2], + dimValues[3], + dimValues[4], + dimValues[5], + dimValues[6]); + break; + case 8: + var metric8DId = new MetricIdentifier( + _namespace, + metricName, + dimNames[0], + dimNames[1], + dimNames[2], + dimNames[3], + dimNames[4], + dimNames[5], + dimNames[6], + dimNames[7]); + + telemetryClient + .GetMetric(metric8DId) + .TrackValue( + metricValue, + dimValues[0], + dimValues[1], + dimValues[2], + dimValues[3], + dimValues[4], + dimValues[5], + dimValues[6], + dimValues[7]); + break; + case 9: + var metric9DId = new MetricIdentifier( + _namespace, + metricName, + dimNames[0], + dimNames[1], + dimNames[2], + dimNames[3], + dimNames[4], + dimNames[5], + dimNames[6], + dimNames[7], + dimNames[8]); + + telemetryClient + .GetMetric(metric9DId) + .TrackValue( + metricValue, + dimValues[0], + dimValues[1], + dimValues[2], + dimValues[3], + dimValues[4], + dimValues[5], + dimValues[6], + dimValues[7], + dimValues[8]); + break; + case 10: + var metric10DId = new MetricIdentifier( + _namespace, + metricName, + dimNames[0], + dimNames[1], + dimNames[2], + dimNames[3], + dimNames[4], + dimNames[5], + dimNames[6], + dimNames[7], + dimNames[8], + dimNames[9]); + + telemetryClient + .GetMetric(metric10DId) + .TrackValue( + metricValue, + dimValues[0], + dimValues[1], + dimValues[2], + dimValues[3], + dimValues[4], + dimValues[5], + dimValues[6], + dimValues[7], + dimValues[8], + dimValues[9]); + break; + default: + break; + } + } + } +} diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementEventNormalizationServiceTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementEventNormalizationServiceTests.cs index 7988457a..85b486ff 100644 --- a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementEventNormalizationServiceTests.cs +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementEventNormalizationServiceTests.cs @@ -9,8 +9,8 @@ using Microsoft.Azure.EventHubs; using Microsoft.Azure.WebJobs; using Microsoft.Health.Fhir.Ingest.Data; -using Microsoft.Health.Fhir.Ingest.Telemetry; using Microsoft.Health.Fhir.Ingest.Template; +using Microsoft.Health.Logger.Telemetry; using Newtonsoft.Json.Linq; using NSubstitute; using Xunit; diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementFhirImportServiceTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementFhirImportServiceTests.cs index 740e0578..35316f84 100644 --- a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementFhirImportServiceTests.cs +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementFhirImportServiceTests.cs @@ -14,6 +14,7 @@ using Microsoft.Health.Fhir.Ingest.Data; using Microsoft.Health.Fhir.Ingest.Telemetry; using Microsoft.Health.Fhir.Ingest.Template; +using Microsoft.Health.Logger.Telemetry; using Microsoft.Health.Tests.Common; using Newtonsoft.Json; using NSubstitute; diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Telemetry/ExceptionTelemetryProcessorTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Telemetry/ExceptionTelemetryProcessorTests.cs index fc98b96d..cb2021b4 100644 --- a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Telemetry/ExceptionTelemetryProcessorTests.cs +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Telemetry/ExceptionTelemetryProcessorTests.cs @@ -8,6 +8,7 @@ using Microsoft.Health.Extensions.Fhir; using Microsoft.Health.Fhir.Ingest.Service; using Microsoft.Health.Fhir.Ingest.Template; +using Microsoft.Health.Logger.Telemetry; using NSubstitute; using Xunit; From 8181b781b92c0cfb7fddc9e5a7d60042f30f8121 Mon Sep 17 00:00:00 2001 From: Will Yochum Date: Mon, 14 Dec 2020 14:53:02 -0800 Subject: [PATCH 06/14] refactor and address pr feedback --- src/console/Program.cs | 11 +++- .../{BlobManager.cs => TemplateManager.cs} | 25 +++----- .../EventCheckpointing/ICheckpointClient.cs | 2 + .../StorageCheckpointClient.cs | 53 +++++++++++++++- .../Service/EventBatchingService.cs | 62 +++++++++---------- .../{EventQueue.cs => EventPartition.cs} | 53 +++++++++------- .../Infrastructure/EventQueueWindow.cs | 34 ---------- .../EventHubProcessor/EventProcessor.cs | 25 ++++---- .../Repository/IRepositoryManager.cs | 12 ++++ .../Repository/StorageManager.cs | 38 ++++++++++++ 10 files changed, 192 insertions(+), 123 deletions(-) rename src/console/Template/{BlobManager.cs => TemplateManager.cs} (52%) rename src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/{EventQueue.cs => EventPartition.cs} (53%) delete mode 100644 src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueueWindow.cs create mode 100644 src/lib/Microsoft.Health.Events/Repository/IRepositoryManager.cs create mode 100644 src/lib/Microsoft.Health.Events/Repository/StorageManager.cs diff --git a/src/console/Program.cs b/src/console/Program.cs index d0c50db4..5a02899c 100644 --- a/src/console/Program.cs +++ b/src/console/Program.cs @@ -9,6 +9,7 @@ using Microsoft.Health.Events.EventConsumers; using Microsoft.Health.Events.EventConsumers.Service; using Microsoft.Health.Events.EventHubProcessor; +using Microsoft.Health.Events.Repository; using Microsoft.Health.Fhir.Ingest.Config; using Microsoft.Health.Fhir.Ingest.Console.Storage; using Microsoft.Health.Fhir.Ingest.Console.Template; @@ -27,13 +28,15 @@ public static async Task Main() { var config = GetEnvironmentConfig(); + // determine which event hub to read from var eventHub = Environment.GetEnvironmentVariable("WEBJOBS_NAME"); if (eventHub == null) { eventHub = config.GetSection("Console:EventHub").Value; } - System.Console.WriteLine($"reading from event hub: {eventHub}"); + System.Console.WriteLine($"Reading from event hub: {eventHub}"); + System.Console.WriteLine($"Logs and Metrics will be written to Application Insights"); var eventHubOptions = GetEventHubInfo(config, eventHub); EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubConnectionString); @@ -44,13 +47,13 @@ public static async Task Main() var storageOptions = new StorageCheckpointOptions(); config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions); + var checkpointClient = new StorageCheckpointClient(storageOptions); var serviceProvider = GetRequiredServiceProvider(config, eventHub); var logger = serviceProvider.GetRequiredService(); var eventConsumers = GetEventConsumers(config, eventHub, serviceProvider, logger); var eventConsumerService = new EventConsumerService(eventConsumers); - var checkpointClient = new StorageCheckpointClient(storageOptions); var ct = new CancellationToken(); @@ -127,10 +130,12 @@ public static List GetEventConsumers(IConfiguration config, stri EnsureArg.IsNotNull(templateOptions.BlobContainerName); EnsureArg.IsNotNull(templateOptions.BlobStorageConnectionString); - var templateManager = new BlobManager( + var storageManager = new StorageManager( templateOptions.BlobStorageConnectionString, templateOptions.BlobContainerName); + var templateManager = new TemplateManager(storageManager); + if (inputEventHub == "devicedata") { var template = config.GetSection("Template:DeviceContent").Value; diff --git a/src/console/Template/BlobManager.cs b/src/console/Template/TemplateManager.cs similarity index 52% rename from src/console/Template/BlobManager.cs rename to src/console/Template/TemplateManager.cs index 2501539d..d1d55278 100644 --- a/src/console/Template/BlobManager.cs +++ b/src/console/Template/TemplateManager.cs @@ -3,33 +3,22 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using Azure.Storage.Blobs; -using EnsureThat; -using System.IO; +using Microsoft.Health.Events.Repository; using System.Text; namespace Microsoft.Health.Fhir.Ingest.Console.Template { - public class BlobManager : ITemplateManager + public class TemplateManager : ITemplateManager { - private BlobContainerClient _blobContainer; - - public BlobManager(string connectionString, string blobContainerName) + private IRepositoryManager _respositoryManager; + public TemplateManager(IRepositoryManager repositoryManager) { - EnsureArg.IsNotNull(connectionString); - EnsureArg.IsNotNull(blobContainerName); - - _blobContainer = new BlobContainerClient(connectionString, blobContainerName); + _respositoryManager = repositoryManager; } + public byte[] GetTemplate(string templateName) { - EnsureArg.IsNotNull(templateName); - - var blockBlob = _blobContainer.GetBlobClient(templateName); - var memoryStream = new MemoryStream(); - blockBlob.DownloadTo(memoryStream); - byte[] itemContent = memoryStream.ToArray(); - return itemContent; + return _respositoryManager.GetItem(templateName); } public string GetTemplateAsString(string templateName) diff --git a/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs b/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs index 6a313407..14fefb3b 100644 --- a/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs +++ b/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs @@ -17,5 +17,7 @@ public interface ICheckpointClient Task PublishCheckpointsAsync(CancellationToken cancellationToken); Task> ListCheckpointsAsync(); + + Task GetCheckpointForPartitionAsync(string partitionId); } } diff --git a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs index afa792a0..f3d9301e 100644 --- a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs +++ b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.IO; @@ -21,7 +22,7 @@ namespace Microsoft.Health.Events.EventCheckpointing { public class StorageCheckpointClient : ICheckpointClient { - private Dictionary _checkpoints; + private ConcurrentDictionary _checkpoints; private BlobContainerClient _storageClient; private static System.Timers.Timer _publisherTimer; private int _publishTimerInterval = 10000; @@ -35,7 +36,7 @@ public StorageCheckpointClient(StorageCheckpointOptions options) BlobPrefix = options.BlobPrefix; - _checkpoints = new Dictionary(); + _checkpoints = new ConcurrentDictionary(); _storageClient = new BlobContainerClient(options.BlobStorageConnectionString, options.BlobContainerName); SetPublisherTimer(); @@ -136,6 +137,53 @@ Task> GetCheckpointsAsync() } } + public Task GetCheckpointForPartitionAsync(string partitionIdentifier) + { + var prefix = $"{BlobPrefix}/checkpoint/{partitionIdentifier}"; + + Task GetCheckpointAsync() + { + var checkpoint = new Checkpoint(); + + foreach (BlobItem blob in _storageClient.GetBlobs(traits: BlobTraits.Metadata, states: BlobStates.All, prefix: prefix, cancellationToken: CancellationToken.None)) + { + var partitionId = blob.Name.Split('/').Last(); + DateTimeOffset lastEventTimestamp = DateTime.MinValue; + + if (blob.Metadata.TryGetValue("LastProcessed", out var str)) + { + DateTimeOffset.TryParse(str, null, DateTimeStyles.AssumeUniversal, out lastEventTimestamp); + } + + checkpoint.Prefix = BlobPrefix; + checkpoint.Id = partitionId; + checkpoint.LastProcessed = lastEventTimestamp; + } + + return Task.FromResult(checkpoint); + } + + try + { + // todo: consider retries + return GetCheckpointAsync(); + } + catch (RequestFailedException ex) when (ex.ErrorCode == BlobErrorCode.ContainerNotFound) + { + // todo: log errors + throw; + } + catch + { + // todo: log errors + throw; + } + finally + { + // todo: log complete + } + } + public Task SetCheckpointAsync(IEventMessage eventArgs) { EnsureArg.IsNotNull(eventArgs); @@ -147,7 +195,6 @@ public Task SetCheckpointAsync(IEventMessage eventArgs) checkpoint.LastProcessed = eventArgs.EnqueuedTime; checkpoint.Id = eventArgs.PartitionId; checkpoint.Prefix = BlobPrefix; - _checkpoints[eventArgs.PartitionId] = checkpoint; } #pragma warning disable CA1031 diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs index 8f6df9fb..0dc920a9 100644 --- a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs @@ -17,7 +17,7 @@ namespace Microsoft.Health.Events.EventConsumers.Service { public class EventBatchingService : IEventConsumerService { - private ConcurrentDictionary _eventQueues; + private ConcurrentDictionary _eventPartitions; private int _maxEvents; private TimeSpan _flushTimespan; private IEventConsumerService _eventConsumerService; @@ -31,7 +31,7 @@ public EventBatchingService(IEventConsumerService eventConsumerService, EventBat EnsureArg.IsInt(options.MaxEvents); EnsureArg.IsInt(options.FlushTimespan); - _eventQueues = new ConcurrentDictionary(); + _eventPartitions = new ConcurrentDictionary(); _eventConsumerService = eventConsumerService; _maxEvents = options.MaxEvents; _flushTimespan = TimeSpan.FromSeconds(options.FlushTimespan); @@ -39,59 +39,59 @@ public EventBatchingService(IEventConsumerService eventConsumerService, EventBat _logger = logger; } - public EventQueue GetQueue(string queueId) + public EventPartition GetPartition(string partitionId) { - EnsureArg.IsNotNullOrWhiteSpace(queueId); + EnsureArg.IsNotNullOrWhiteSpace(partitionId); - if (!_eventQueues.ContainsKey(queueId)) + if (!_eventPartitions.ContainsKey(partitionId)) { - throw new Exception($"Queue with identifier {queueId} does not exist"); + throw new Exception($"Partition with identifier {partitionId} does not exist"); } - return _eventQueues[queueId]; + return _eventPartitions[partitionId]; } - private bool EventQueueExists(string queueId) + private bool EventPartitionExists(string partitionId) { - return _eventQueues.ContainsKey(queueId); + return _eventPartitions.ContainsKey(partitionId); } - private EventQueue CreateQueueIfMissing(string queueId, DateTime initTime, TimeSpan flushTimespan) + private EventPartition CreatePartitionIfMissing(string partitionId, DateTime initTime, TimeSpan flushTimespan) { - return _eventQueues.GetOrAdd(queueId, new EventQueue(queueId, initTime, flushTimespan, _logger)); + return _eventPartitions.GetOrAdd(partitionId, new EventPartition(partitionId, initTime, flushTimespan, _logger)); } public Task ConsumeEvent(IEventMessage eventArg) { EnsureArg.IsNotNull(eventArg); - var queueId = eventArg.PartitionId; + var partitionId = eventArg.PartitionId; var eventEnqueuedTime = eventArg.EnqueuedTime.UtcDateTime; if (eventArg is MaximumWaitEvent) { - if (EventQueueExists(queueId)) + if (EventPartitionExists(partitionId)) { - var windowThresholdTime = GetQueue(queueId).GetQueueWindow(); - ThresholdWaitReached(queueId, windowThresholdTime); + var windowThresholdTime = GetPartition(partitionId).GetPartitionWindow(); + ThresholdWaitReached(partitionId, windowThresholdTime); } } else { - var queue = CreateQueueIfMissing(queueId, eventEnqueuedTime, _flushTimespan); + var partition = CreatePartitionIfMissing(partitionId, eventEnqueuedTime, _flushTimespan); - queue.Enqueue(eventArg); + partition.Enqueue(eventArg); - var windowThresholdTime = queue.GetQueueWindow(); + var windowThresholdTime = partition.GetPartitionWindow(); if (eventEnqueuedTime > windowThresholdTime) { - ThresholdTimeReached(queueId, eventArg, windowThresholdTime); + ThresholdTimeReached(partitionId, eventArg, windowThresholdTime); return Task.CompletedTask; } - if (queue.GetQueueCount() >= _maxEvents) + if (partition.GetPartitionBatchCount() >= _maxEvents) { - ThresholdCountReached(queueId); + ThresholdCountReached(partitionId); } } @@ -99,30 +99,30 @@ public Task ConsumeEvent(IEventMessage eventArg) } // todo: fix -"Collection was modified; enumeration operation may not execute." - private async void ThresholdCountReached(string queueId) + private async void ThresholdCountReached(string partitionId) { - _logger.LogTrace($"Partition {queueId} threshold count {_maxEvents} was reached."); - var events = await GetQueue(queueId).Flush(_maxEvents); + _logger.LogTrace($"Partition {partitionId} threshold count {_maxEvents} was reached."); + var events = await GetPartition(partitionId).Flush(_maxEvents); await _eventConsumerService.ConsumeEvents(events); UpdateCheckpoint(events); } - private async void ThresholdTimeReached(string queueId, IEventMessage eventArg, DateTime windowEnd) + private async void ThresholdTimeReached(string partitionId, IEventMessage eventArg, DateTime windowEnd) { - _logger.LogTrace($"Partition {queueId} threshold time {_eventQueues[queueId].GetQueueWindow()} was reached."); - var queue = GetQueue(queueId); + _logger.LogTrace($"Partition {partitionId} threshold time {_eventPartitions[partitionId].GetPartitionWindow()} was reached."); + var queue = GetPartition(partitionId); var events = await queue.Flush(windowEnd); + queue.IncrementPartitionWindow(eventArg.EnqueuedTime.UtcDateTime); await _eventConsumerService.ConsumeEvents(events); - queue.IncrementQueueWindow(eventArg.EnqueuedTime.UtcDateTime); UpdateCheckpoint(events); } - private async void ThresholdWaitReached(string queueId, DateTime windowEnd) + private async void ThresholdWaitReached(string partitionId, DateTime windowEnd) { if (windowEnd < DateTime.UtcNow.AddSeconds(_timeBuffer)) { - _logger.LogTrace($"Partition {queueId} threshold wait reached. Flushing {_eventQueues[queueId].GetQueueCount()} events up to: {windowEnd}"); - var events = await GetQueue(queueId).Flush(windowEnd); + _logger.LogTrace($"Partition {partitionId} threshold wait reached. Flushing {_eventPartitions[partitionId].GetPartitionBatchCount()} events up to: {windowEnd}"); + var events = await GetPartition(partitionId).Flush(windowEnd); await _eventConsumerService.ConsumeEvents(events); UpdateCheckpoint(events); } diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueue.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventPartition.cs similarity index 53% rename from src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueue.cs rename to src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventPartition.cs index ededdac2..bd5f59cc 100644 --- a/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueue.cs +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventPartition.cs @@ -8,63 +8,69 @@ using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.Health.Events.Model; +using Microsoft.Health.Events.Telemetry; using Microsoft.Health.Logger.Telemetry; namespace Microsoft.Health.Events.EventConsumers.Service.Infrastructure { - public class EventQueue + public class EventPartition { - private string _queueId; - private ConcurrentQueue _queue; - private EventQueueWindow _queueWindow; + private string _partitionId; + private ConcurrentQueue _partition; + private DateTime _partitionWindow; + private TimeSpan _flushTimespan; private ITelemetryLogger _logger; - public EventQueue(string queueId, DateTime initDateTime, TimeSpan flushTimespan, ITelemetryLogger logger) + public EventPartition(string partitionId, DateTime initDateTime, TimeSpan flushTimespan, ITelemetryLogger logger) { - _queueId = queueId; - _queue = new ConcurrentQueue(); - _queueWindow = new EventQueueWindow(initDateTime, flushTimespan); + _partitionId = partitionId; + _partition = new ConcurrentQueue(); + _partitionWindow = initDateTime.Add(flushTimespan); + _flushTimespan = flushTimespan; _logger = logger; } - public void IncrementQueueWindow(DateTime dateTime) + public void Enqueue(IEventMessage eventArg) { - _queueWindow.IncrementWindow(dateTime); + _partition.Enqueue(eventArg); } - public DateTime GetQueueWindow() + public void IncrementPartitionWindow(DateTime dateTime) { - return _queueWindow.GetWindowEnd(); + // todo: consider computing instead of while loop. + while (dateTime >= _partitionWindow) + { + _partitionWindow = _partitionWindow.Add(_flushTimespan); + } } - public int GetQueueCount() + public DateTime GetPartitionWindow() { - return _queue.Count; + return _partitionWindow; } - public void Enqueue(IEventMessage eventArg) + public int GetPartitionBatchCount() { - _queue.Enqueue(eventArg); + return _partition.Count; } // flush a fixed number of events public Task> Flush(int numEvents) { - Console.WriteLine($"Flushing {numEvents} events"); - var count = 0; var events = new List(); while (count < numEvents) { - if (_queue.TryDequeue(out var dequeuedEvent)) + if (_partition.TryDequeue(out var dequeuedEvent)) { events.Add(dequeuedEvent); count++; } } - Console.WriteLine($"Current window {GetQueueWindow()}"); + _logger.LogTrace($"Flushed {events.Count} events on partition {_partitionId}"); + _logger.LogMetric(EventMetrics.EventsFlushed(), events.Count); return Task.FromResult(events); } @@ -72,12 +78,12 @@ public Task> Flush(int numEvents) public Task> Flush(DateTime dateTime) { var events = new List(); - while (_queue.TryPeek(out var eventData)) + while (_partition.TryPeek(out var eventData)) { var enqueuedUtc = eventData.EnqueuedTime.UtcDateTime; if (enqueuedUtc <= dateTime) { - _queue.TryDequeue(out var dequeuedEvent); + _partition.TryDequeue(out var dequeuedEvent); events.Add(dequeuedEvent); } else @@ -86,7 +92,8 @@ public Task> Flush(DateTime dateTime) } } - Console.WriteLine($"Flushed {events.Count} events up to {dateTime}"); + _logger.LogTrace($"Flushed {events.Count} events up to {dateTime} on partition {_partitionId}"); + _logger.LogMetric(EventMetrics.EventsFlushed(), events.Count); return Task.FromResult(events); } } diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueueWindow.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueueWindow.cs deleted file mode 100644 index c9d4126e..00000000 --- a/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventQueueWindow.cs +++ /dev/null @@ -1,34 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; - -namespace Microsoft.Health.Events.EventConsumers.Service.Infrastructure -{ - public class EventQueueWindow - { - private DateTime _windowEnd = DateTime.MinValue; - private TimeSpan _flushTimespan; - - public EventQueueWindow(DateTime initDateTime, TimeSpan flushTimespan) - { - _windowEnd = initDateTime.Add(flushTimespan); - _flushTimespan = flushTimespan; - } - - public void IncrementWindow(DateTime currentEnqueudTime) - { - while (currentEnqueudTime >= _windowEnd) - { - _windowEnd = _windowEnd.Add(_flushTimespan); - } - } - - public DateTime GetWindowEnd() - { - return _windowEnd; - } - } -} diff --git a/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs b/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs index 9fe7cb6a..53cc7168 100644 --- a/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs +++ b/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs @@ -13,6 +13,7 @@ using Microsoft.Health.Events.EventCheckpointing; using Microsoft.Health.Events.EventConsumers.Service; using Microsoft.Health.Events.Model; +using Microsoft.Health.Events.Telemetry; using Microsoft.Health.Logger.Telemetry; namespace Microsoft.Health.Events.EventHubProcessor @@ -71,20 +72,22 @@ Task ProcessErrorHandler(ProcessErrorEventArgs eventArgs) async Task ProcessInitializingHandler(PartitionInitializingEventArgs initArgs) { - Console.WriteLine($"Initializing partition {initArgs.PartitionId}"); - var partitionId = initArgs.PartitionId; + _logger.LogTrace($"Initializing partition {partitionId}"); - // todo: only get checkpoint for partition instead of listing them all - var checkpoints = await _checkpointClient.ListCheckpointsAsync(); - foreach (var checkpoint in checkpoints) + try { - if (checkpoint.Id == partitionId) - { - initArgs.DefaultStartingPosition = EventPosition.FromEnqueuedTime(checkpoint.LastProcessed); - _logger.LogTrace($"Starting to read partition {partitionId} from checkpoint {checkpoint.LastProcessed}"); - break; - } + var checkpoint = await _checkpointClient.GetCheckpointForPartitionAsync(partitionId); + initArgs.DefaultStartingPosition = EventPosition.FromEnqueuedTime(checkpoint.LastProcessed); + _logger.LogTrace($"Starting to read partition {partitionId} from checkpoint {checkpoint.LastProcessed}"); + _logger.LogMetric(EventMetrics.EventHubPartitionInitialized(), 1); + } +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 + { + _logger.LogTrace($"Failed to initialize partition {partitionId} from checkpoint"); + _logger.LogError(ex); } } diff --git a/src/lib/Microsoft.Health.Events/Repository/IRepositoryManager.cs b/src/lib/Microsoft.Health.Events/Repository/IRepositoryManager.cs new file mode 100644 index 00000000..84e218fe --- /dev/null +++ b/src/lib/Microsoft.Health.Events/Repository/IRepositoryManager.cs @@ -0,0 +1,12 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Events.Repository +{ + public interface IRepositoryManager + { + byte[] GetItem(string itemName); + } +} diff --git a/src/lib/Microsoft.Health.Events/Repository/StorageManager.cs b/src/lib/Microsoft.Health.Events/Repository/StorageManager.cs new file mode 100644 index 00000000..6547b0e4 --- /dev/null +++ b/src/lib/Microsoft.Health.Events/Repository/StorageManager.cs @@ -0,0 +1,38 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.IO; +using Azure.Storage.Blobs; +using EnsureThat; + +namespace Microsoft.Health.Events.Repository +{ + public class StorageManager : IRepositoryManager + { + private BlobContainerClient _blobContainer; + + public StorageManager(string connectionString, string blobContainerName) + { + EnsureArg.IsNotNull(connectionString); + EnsureArg.IsNotNull(blobContainerName); + + _blobContainer = new BlobContainerClient(connectionString, blobContainerName); + } + + public byte[] GetItem(string itemName) + { + EnsureArg.IsNotNull(itemName); + + var blockBlob = _blobContainer.GetBlobClient(itemName); + + using (var memoryStream = new MemoryStream()) + { + blockBlob.DownloadTo(memoryStream); + byte[] itemContent = memoryStream.ToArray(); + return itemContent; + } + } + } +} From 6b016d0711fda4944e3e51cc2d93d890357f773b Mon Sep 17 00:00:00 2001 From: Will Yochum Date: Tue, 15 Dec 2020 09:46:31 -0800 Subject: [PATCH 07/14] fix template bug --- src/console/MeasurementCollectionToFhir/Processor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/console/MeasurementCollectionToFhir/Processor.cs b/src/console/MeasurementCollectionToFhir/Processor.cs index 2c99e9bc..0313282d 100644 --- a/src/console/MeasurementCollectionToFhir/Processor.cs +++ b/src/console/MeasurementCollectionToFhir/Processor.cs @@ -45,7 +45,7 @@ public async Task ConsumeAsync(IEnumerable events) EnsureArg.IsNotNull(_templateDefinition); var templateContent = _templateManager.GetTemplateAsString(_templateDefinition); - await _measurementImportService.ProcessEventsAsync(events, _templateDefinition, _logger).ConfigureAwait(false); + await _measurementImportService.ProcessEventsAsync(events, templateContent, _logger).ConfigureAwait(false); } catch { From dbbf7034c1fbc10e8ee91fecedc3cbe574ba0022 Mon Sep 17 00:00:00 2001 From: Will Yochum Date: Tue, 15 Dec 2020 18:41:52 -0800 Subject: [PATCH 08/14] log measurement metrics --- .../Service/MeasurementFhirImportService.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs index 025392e3..48cd3fff 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs @@ -43,7 +43,7 @@ public async Task ProcessStreamAsync(Stream data, string templateDefinition, ITe public async Task ProcessEventsAsync(IEnumerable events, string templateDefinition, ITelemetryLogger log) { var template = BuildTemplate(templateDefinition, log); - var measurementGroups = ParseEventData(events); + var measurementGroups = ParseEventData(events, log); await ProcessMeasurementGroups(measurementGroups, template, log).ConfigureAwait(false); } @@ -108,7 +108,7 @@ private static async Task> ParseAsync(Stream data return measurementGroups; } - private static IEnumerable ParseEventData(IEnumerable data) + private static IEnumerable ParseEventData(IEnumerable data, ITelemetryLogger log) { // Deserialize events into measurements and then group according to the device, type, and other factors var body = data.First().Body.ToArray(); @@ -120,6 +120,7 @@ private static IEnumerable ParseEventData(IEnumerable { var measurements = g.ToList(); + _ = CalculateMetricsAsync(measurements, log).ConfigureAwait(false); return new MeasurementGroup { Data = measurements, From 521f4ac12729742085b6ee6ee5f43eeff2584e2f Mon Sep 17 00:00:00 2001 From: Will Yochum Date: Wed, 16 Dec 2020 16:10:32 -0800 Subject: [PATCH 09/14] remove timers from checkpointing --- .../MeasurementCollectionToFhir/Processor.cs | 14 +- src/console/Program.cs | 9 +- src/console/appsettings.json | 1 + .../Metrics/Dimensions/DimensionNames.cs | 10 ++ .../EventCheckpointing/ICheckpointClient.cs | 6 +- .../StorageCheckpointClient.cs | 147 +++++------------- .../StorageCheckpointOptions.cs | 2 + .../Service/EventBatchingService.cs | 8 +- .../EventHubProcessor/EventProcessor.cs | 21 +-- .../Telemetry/Metrics/EventMetrics.cs | 21 +++ 10 files changed, 90 insertions(+), 149 deletions(-) diff --git a/src/console/MeasurementCollectionToFhir/Processor.cs b/src/console/MeasurementCollectionToFhir/Processor.cs index 0313282d..38476ca4 100644 --- a/src/console/MeasurementCollectionToFhir/Processor.cs +++ b/src/console/MeasurementCollectionToFhir/Processor.cs @@ -39,18 +39,10 @@ public Processor( public async Task ConsumeAsync(IEnumerable events) { EnsureArg.IsNotNull(events); + EnsureArg.IsNotNull(_templateDefinition); - try - { - EnsureArg.IsNotNull(_templateDefinition); - var templateContent = _templateManager.GetTemplateAsString(_templateDefinition); - - await _measurementImportService.ProcessEventsAsync(events, templateContent, _logger).ConfigureAwait(false); - } - catch - { - throw; - } + var templateContent = _templateManager.GetTemplateAsString(_templateDefinition); + await _measurementImportService.ProcessEventsAsync(events, templateContent, _logger).ConfigureAwait(false); } } } diff --git a/src/console/Program.cs b/src/console/Program.cs index 5a02899c..05387819 100644 --- a/src/console/Program.cs +++ b/src/console/Program.cs @@ -45,14 +45,15 @@ public static async Task Main() var eventBatchingOptions = new EventBatchingOptions(); config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions); - var storageOptions = new StorageCheckpointOptions(); - config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions); - var checkpointClient = new StorageCheckpointClient(storageOptions); - var serviceProvider = GetRequiredServiceProvider(config, eventHub); var logger = serviceProvider.GetRequiredService(); var eventConsumers = GetEventConsumers(config, eventHub, serviceProvider, logger); + var storageOptions = new StorageCheckpointOptions(); + config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions); + storageOptions.BlobPrefix = eventHub; + var checkpointClient = new StorageCheckpointClient(storageOptions, logger); + var eventConsumerService = new EventConsumerService(eventConsumers); var ct = new CancellationToken(); diff --git a/src/console/appsettings.json b/src/console/appsettings.json index 213e1783..2816c87e 100644 --- a/src/console/appsettings.json +++ b/src/console/appsettings.json @@ -5,6 +5,7 @@ "Storage:BlobStorageConnectionString": "", "Storage:BlobContainerName": "", "Storage:BlobPrefix": "", + "Storage:CheckpointBatchCount": 5, "TemplateStorage:BlobStorageConnectionString": "", "TemplateStorage:BlobContainerName": "", "Console:EventHub": "devicedata", diff --git a/src/lib/Microsoft.Health.Common/Telemetry/Metrics/Dimensions/DimensionNames.cs b/src/lib/Microsoft.Health.Common/Telemetry/Metrics/Dimensions/DimensionNames.cs index d76f52a3..cdfae29b 100644 --- a/src/lib/Microsoft.Health.Common/Telemetry/Metrics/Dimensions/DimensionNames.cs +++ b/src/lib/Microsoft.Health.Common/Telemetry/Metrics/Dimensions/DimensionNames.cs @@ -22,6 +22,16 @@ public static class DimensionNames /// public static string Operation => nameof(DimensionNames.Operation); + /// + /// A metric dimension that represents a timestamp property of a metric. + /// + public static string Timestamp => nameof(DimensionNames.Timestamp); + + /// + /// A metric dimension that represents an identifier related to the metric emitted. + /// + public static string Identifier => nameof(DimensionNames.Identifier); + /// /// A metric dimension for a error type. /// diff --git a/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs b/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs index 14fefb3b..0023844f 100644 --- a/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs +++ b/src/lib/Microsoft.Health.Events/EventCheckpointing/ICheckpointClient.cs @@ -3,8 +3,6 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System.Collections.Generic; -using System.Threading; using System.Threading.Tasks; using Microsoft.Health.Events.Model; @@ -14,9 +12,7 @@ public interface ICheckpointClient { Task SetCheckpointAsync(IEventMessage eventArg); - Task PublishCheckpointsAsync(CancellationToken cancellationToken); - - Task> ListCheckpointsAsync(); + Task PublishCheckpointAsync(string partitionId); Task GetCheckpointForPartitionAsync(string partitionId); } diff --git a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs index f3d9301e..d752d006 100644 --- a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs +++ b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs @@ -11,35 +11,39 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using System.Timers; using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using EnsureThat; using Microsoft.Health.Events.Model; +using Microsoft.Health.Events.Telemetry; +using Microsoft.Health.Logger.Telemetry; namespace Microsoft.Health.Events.EventCheckpointing { public class StorageCheckpointClient : ICheckpointClient { private ConcurrentDictionary _checkpoints; + private ConcurrentDictionary _lastCheckpointTracker; + private int _lastCheckpointMaxCount; private BlobContainerClient _storageClient; - private static System.Timers.Timer _publisherTimer; - private int _publishTimerInterval = 10000; + private ITelemetryLogger _log; - public StorageCheckpointClient(StorageCheckpointOptions options) + public StorageCheckpointClient(StorageCheckpointOptions options, ITelemetryLogger log) { EnsureArg.IsNotNull(options); EnsureArg.IsNotNullOrWhiteSpace(options.BlobPrefix); EnsureArg.IsNotNullOrWhiteSpace(options.BlobStorageConnectionString); EnsureArg.IsNotNullOrWhiteSpace(options.BlobContainerName); + EnsureArg.IsNotNullOrWhiteSpace(options.CheckpointBatchCount); BlobPrefix = options.BlobPrefix; + _lastCheckpointMaxCount = int.Parse(options.CheckpointBatchCount); _checkpoints = new ConcurrentDictionary(); + _lastCheckpointTracker = new ConcurrentDictionary(); _storageClient = new BlobContainerClient(options.BlobStorageConnectionString, options.BlobContainerName); - - SetPublisherTimer(); + _log = log; } public string BlobPrefix { get; } @@ -71,69 +75,11 @@ public async Task UpdateCheckpointAsync(Checkpoint checkpoint) } } } - catch (RequestFailedException ex) when (ex.ErrorCode == BlobErrorCode.ContainerNotFound) - { - // todo: log - throw; - } - catch - { - // todo: log - throw; - } - finally - { - // todo: log - } - } - - public Task> ListCheckpointsAsync() - { - var prefix = $"{BlobPrefix}/checkpoint"; - - Task> GetCheckpointsAsync() - { - var checkpoints = new List(); - - foreach (BlobItem blob in _storageClient.GetBlobs(traits: BlobTraits.Metadata, states: BlobStates.All, prefix: prefix, cancellationToken: CancellationToken.None)) - { - var partitionId = blob.Name.Split('/').Last(); - DateTimeOffset lastEventTimestamp = DateTime.MinValue; - - if (blob.Metadata.TryGetValue("LastProcessed", out var str)) - { - DateTimeOffset.TryParse(str, null, DateTimeStyles.AssumeUniversal, out lastEventTimestamp); - } - - checkpoints.Add(new Checkpoint - { - Prefix = BlobPrefix, - Id = partitionId, - LastProcessed = lastEventTimestamp, - }); - } - - return Task.FromResult(checkpoints); - } - - try - { - // todo: consider retries - return GetCheckpointsAsync(); - } - catch (RequestFailedException ex) when (ex.ErrorCode == BlobErrorCode.ContainerNotFound) - { - // todo: log errors - throw; - } - catch - { - // todo: log errors - throw; - } - finally +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 { - // todo: log complete + _log.LogError(new Exception($"Unable to update checkpoint. {ex.Message}")); } } @@ -168,73 +114,52 @@ Task GetCheckpointAsync() // todo: consider retries return GetCheckpointAsync(); } - catch (RequestFailedException ex) when (ex.ErrorCode == BlobErrorCode.ContainerNotFound) - { - // todo: log errors - throw; - } - catch +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 { - // todo: log errors + _log.LogError(new Exception($"Unable to get checkpoint for partition. {ex.Message}")); throw; } - finally - { - // todo: log complete - } } - public Task SetCheckpointAsync(IEventMessage eventArgs) + public async Task SetCheckpointAsync(IEventMessage eventArgs) { EnsureArg.IsNotNull(eventArgs); EnsureArg.IsNotNullOrWhiteSpace(eventArgs.PartitionId); try { + Console.WriteLine(eventArgs.EnqueuedTime); + + var partitionId = eventArgs.PartitionId; var checkpoint = new Checkpoint(); checkpoint.LastProcessed = eventArgs.EnqueuedTime; - checkpoint.Id = eventArgs.PartitionId; + checkpoint.Id = partitionId; checkpoint.Prefix = BlobPrefix; + _checkpoints[eventArgs.PartitionId] = checkpoint; + var count = _lastCheckpointTracker.AddOrUpdate(partitionId, 1, (key, value) => value + 1); + + if (count >= _lastCheckpointMaxCount) + { + await PublishCheckpointAsync(partitionId); + _log.LogMetric(EventMetrics.EventWatermark(partitionId, eventArgs.EnqueuedTime.UtcDateTime), 1); + _lastCheckpointTracker[partitionId] = 0; + } } #pragma warning disable CA1031 catch (Exception ex) #pragma warning restore CA1031 { - Console.WriteLine($"Checkpointing error: {ex.Message}"); + _log.LogError(new Exception($"Unable to set checkpoint. {ex.Message}")); } - - return Task.CompletedTask; - } - - public async Task PublishCheckpointsAsync(CancellationToken ct) - { - foreach (KeyValuePair checkpoint in _checkpoints) - { - await UpdateCheckpointAsync(checkpoint.Value); - } - } - - private void SetPublisherTimer() - { - _publisherTimer = new System.Timers.Timer(_publishTimerInterval); - _publisherTimer.Elapsed += OnTimedEvent; - _publisherTimer.AutoReset = true; - _publisherTimer.Enabled = true; } - private async void OnTimedEvent(object source, ElapsedEventArgs e) + public async Task PublishCheckpointAsync(string partitionId) { - try - { - await PublishCheckpointsAsync(CancellationToken.None); - } -#pragma warning disable CA1031 - catch (Exception ex) -#pragma warning restore CA1031 - { - Console.WriteLine(ex.Message); - } + Checkpoint checkpoint = _checkpoints[partitionId]; + await UpdateCheckpointAsync(checkpoint); } } } diff --git a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointOptions.cs b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointOptions.cs index 4d32fdf2..a6b0ea5b 100644 --- a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointOptions.cs +++ b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointOptions.cs @@ -14,5 +14,7 @@ public class StorageCheckpointOptions public string BlobContainerName { get; set; } public string BlobPrefix { get; set; } + + public string CheckpointBatchCount { get; set; } } } diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs index 0dc920a9..bd5c47be 100644 --- a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs @@ -104,7 +104,7 @@ private async void ThresholdCountReached(string partitionId) _logger.LogTrace($"Partition {partitionId} threshold count {_maxEvents} was reached."); var events = await GetPartition(partitionId).Flush(_maxEvents); await _eventConsumerService.ConsumeEvents(events); - UpdateCheckpoint(events); + await UpdateCheckpoint(events); } private async void ThresholdTimeReached(string partitionId, IEventMessage eventArg, DateTime windowEnd) @@ -114,7 +114,7 @@ private async void ThresholdTimeReached(string partitionId, IEventMessage eventA var events = await queue.Flush(windowEnd); queue.IncrementPartitionWindow(eventArg.EnqueuedTime.UtcDateTime); await _eventConsumerService.ConsumeEvents(events); - UpdateCheckpoint(events); + await UpdateCheckpoint(events); } private async void ThresholdWaitReached(string partitionId, DateTime windowEnd) @@ -124,11 +124,11 @@ private async void ThresholdWaitReached(string partitionId, DateTime windowEnd) _logger.LogTrace($"Partition {partitionId} threshold wait reached. Flushing {_eventPartitions[partitionId].GetPartitionBatchCount()} events up to: {windowEnd}"); var events = await GetPartition(partitionId).Flush(windowEnd); await _eventConsumerService.ConsumeEvents(events); - UpdateCheckpoint(events); + await UpdateCheckpoint(events); } } - private async void UpdateCheckpoint(List events) + private async Task UpdateCheckpoint(List events) { if (events.Count > 0) { diff --git a/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs b/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs index 53cc7168..c0149a59 100644 --- a/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs +++ b/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs @@ -41,25 +41,18 @@ public async Task RunAsync(EventProcessorClient processor, CancellationToken ct) // event for a certain time period and this event is used to flush events in the current window. Task ProcessEventHandler(ProcessEventArgs eventArgs) { - try + IEventMessage evt; + if (eventArgs.HasEvent) { - IEventMessage evt; - if (eventArgs.HasEvent) - { - evt = EventMessageFactory.CreateEvent(eventArgs); - } - else - { - evt = new MaximumWaitEvent(eventArgs.Partition.PartitionId, DateTime.UtcNow); - } - - _eventConsumerService.ConsumeEvent(evt); + evt = EventMessageFactory.CreateEvent(eventArgs); } - catch + else { - throw; + evt = new MaximumWaitEvent(eventArgs.Partition.PartitionId, DateTime.UtcNow); } + _eventConsumerService.ConsumeEvent(evt); + return Task.CompletedTask; } diff --git a/src/lib/Microsoft.Health.Events/Telemetry/Metrics/EventMetrics.cs b/src/lib/Microsoft.Health.Events/Telemetry/Metrics/EventMetrics.cs index d6844dd6..bdf6bc95 100644 --- a/src/lib/Microsoft.Health.Events/Telemetry/Metrics/EventMetrics.cs +++ b/src/lib/Microsoft.Health.Events/Telemetry/Metrics/EventMetrics.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using System.Collections.Generic; using Microsoft.Health.Common.Telemetry; @@ -15,6 +16,8 @@ public static class EventMetrics { private static string _nameDimension = DimensionNames.Name; private static string _categoryDimension = DimensionNames.Category; + private static string _timeDimension = DimensionNames.Timestamp; + private static string _partitionDimension = DimensionNames.Identifier; private static Metric _eventHubPartitionInitialized = new Metric( "EventHubPartitionInitialized", @@ -79,5 +82,23 @@ public static Metric EventsConsumed() { return _eventsConsumed; } + + /// + /// Signals that a new watermark was published for a partition. + /// + /// The partition id of the event hub + /// The datetime of the watermark + public static Metric EventWatermark(string partitionId, DateTime dateTime) + { + return new Metric( + "EventsWatermarkUpdated", + new Dictionary + { + { _nameDimension, "EventsWatermarkUpdated" }, + { _timeDimension, dateTime }, + { _partitionDimension, partitionId }, + { _categoryDimension, Category.Latency }, + }); + } } } From 7cee82654c086fb08faf4971ad720c13351c3665 Mon Sep 17 00:00:00 2001 From: Will Yochum Date: Wed, 16 Dec 2020 18:57:19 -0800 Subject: [PATCH 10/14] testing build pipeline trigger --- src/console/appsettings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/console/appsettings.json b/src/console/appsettings.json index 2816c87e..2e22ba90 100644 --- a/src/console/appsettings.json +++ b/src/console/appsettings.json @@ -1,7 +1,7 @@ { "APPINSIGHTS_INSTRUMENTATIONKEY": "", "EventBatching:FlushTimespan": 300, - "EventBatching:MaxEvents": 500, + "EventBatching:MaxEvents": 300, "Storage:BlobStorageConnectionString": "", "Storage:BlobContainerName": "", "Storage:BlobPrefix": "", From 4d006aeb10b343b863b689853975e28cda85b06e Mon Sep 17 00:00:00 2001 From: Will <59618266+wi-y@users.noreply.github.com> Date: Tue, 5 Jan 2021 16:43:00 -0800 Subject: [PATCH 11/14] Personal/wiyochum/arm deployment event batching (#83) * template and script for deploying sa replacement as webjobs * fix template bug * log measurment metrics * fix params * update arm template and deploy script * set alwaysOn * fix errors/telemetry --- .../scripts/Create-IomtWebJobsEnvironment.ps1 | 113 +++ .../default-azuredeploy-webjobs.json | 746 ++++++++++++++++++ src/console/Program.cs | 3 +- .../StorageCheckpointClient.cs | 4 +- .../Service/EventConsumerService.cs | 21 +- .../EventHubProcessor/EventProcessor.cs | 4 +- .../Telemetry/Metrics/EventMetrics.cs | 2 +- 7 files changed, 879 insertions(+), 14 deletions(-) create mode 100644 deploy/scripts/Create-IomtWebJobsEnvironment.ps1 create mode 100644 deploy/templates/default-azuredeploy-webjobs.json diff --git a/deploy/scripts/Create-IomtWebJobsEnvironment.ps1 b/deploy/scripts/Create-IomtWebJobsEnvironment.ps1 new file mode 100644 index 00000000..e0bb19b2 --- /dev/null +++ b/deploy/scripts/Create-IomtWebJobsEnvironment.ps1 @@ -0,0 +1,113 @@ +<# +.SYNOPSIS +Creates a new IoMT FHIR Connector for Azure without using Stream Analytics +.DESCRIPTION +#> +param +( + [Parameter(Mandatory = $true)] + [string]$ResourceGroup, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [ValidateLength(5,12)] + [ValidateScript({ + if ("$_" -cmatch "(^([a-z]|\d)+$)") { + return $true + } + else { + throw "Environment name must be lowercase and numbers" + return $false + } + })] + [string]$EnvironmentName, + + [Parameter(Mandatory = $false)] + [ValidateSet('South Africa North', 'South Africa West', 'East Asia', 'Southeast Asia', 'Australia Central', 'Australia Central 2', 'Australia East', 'Australia Southeast', 'Brazil South', 'Brazil Southeast', 'Canada Central', 'Canada East', 'China East', 'China East 2', 'China North', 'China North 2', 'North Europe', 'West Europe', 'France Central', 'France South', 'Germany Central', 'Germany Northeast', 'Germany West Central', 'Central India', 'South India', 'West India', 'Japan East', 'Japan West', 'Korea Central', 'Korea South', 'Norway East', 'Switzerland North', 'Switzerland West', 'UAE Central', 'UAE North', 'UK West', 'UK South', 'Central US', 'East US', 'East US 2', 'North Central US', 'South Central US', 'West Central US', 'West US', 'West US 2')] + [string]$EnvironmentLocation = "North Central US", + [Parameter(Mandatory = $false)] + [ValidateSet('R4')] + [string]$FhirVersion = "R4", + + [Parameter(Mandatory = $false)] + [string]$SourceRepository = "https://github.com/microsoft/iomt-fhir", + + [Parameter(Mandatory = $false)] + [string]$SourceRevision = "master", + + [Parameter(Mandatory = $true)] + [string]$FhirServiceUrl, + + [Parameter(Mandatory = $true)] + [string]$FhirServiceAuthority, + + [Parameter(Mandatory = $true)] + [string]$FhirServiceClientId, + + [Parameter(Mandatory = $true)] + [string]$FhirServiceSecret, + + [Parameter(Mandatory = $false)] + [string]$EnvironmentDeploy = $true +) + +Function BuildPackage() { + try { + Push-Location $currentPath + cd ../../src/console/ + dotnet restore + dotnet build --output $buildPath /p:DeployOnBuild=true /p:DeployTarget=Package + } finally { + Pop-Location + } +} + +Function Deploy-WebJobs($DeviceDataWebJobName, $NormalizedDataWebJobName) { + try { + $tempPath = "$currentPath\Temp" + $webAppName = $EnvironmentName + $webJobType = "Continuous" + + Clear-Path -WebJobName $DeviceDataWebJobName + Clear-Path -WebJobName $NormalizedDataWebJobName + + $DeviceWebJobPath = "$tempPath\App_Data\jobs\$webJobType\$DeviceDataWebJobName" + $NormalizedWebJobPath = "$tempPath\App_Data\jobs\$webJobType\$NormalizedDataWebJobName" + Copy-Item "$buildPath\*" -Destination $DeviceWebJobPath -Recurse + Copy-Item "$buildPath\*" -Destination $NormalizedWebJobPath -Recurse + + Compress-Archive -Path "$tempPath\*" -DestinationPath "$currentPath\iomtwebjobs.zip" -Force + + Publish-AzWebApp -ArchivePath "$currentPath\iomtwebjobs.zip" -ResourceGroupName $ResourceGroup -Name $webAppName + } finally { + Pop-Location + } +} + +Function Clear-Path($WebJobName) { + $WebJobPath = "$tempPath\App_Data\jobs\$webJobType\$WebJobName" + Get-ChildItem -Path $WebJobPath -Recurse | Remove-Item -Force -Recurse + if( -Not (Test-Path -Path $WebJobPath ) ) + { + New-Item $WebJobPath -ItemType Directory + } +} + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# deploy event hubs, app service, key vaults, storage +if ($EnvironmentDeploy -eq $true) { + Write-Host "Deploying environment resources..." + $webjobTemplate = "..\templates\default-azuredeploy-webjobs.json" + New-AzResourceGroupDeployment -TemplateFile $webjobTemplate -ResourceGroupName $ResourceGroup -ServiceName $EnvironmentName -FhirServiceUrl $fhirServiceUrl -FhirServiceAuthority $FhirServiceAuthority -FhirServiceResource $fhirServiceUrl -FhirServiceClientId $FhirServiceClientId -FhirServiceClientSecret (ConvertTo-SecureString -String $FhirServiceSecret -AsPlainText -Force) -RepositoryUrl $SourceRepository -RepositoryBranch $SourceRevision -ResourceLocation $EnvironmentLocation +} + +# deploy the stream analytics replacement webjobs +Write-Host "Deploying WebJobs..." + +$currentPath = (Get-Location).Path +$buildPath = "$currentPath\OSS_Deployment" +BuildPackage +Deploy-WebJobs -DeviceDataWebJobName "devicedata" -NormalizedDataWebJobName "normalizeddata" + diff --git a/deploy/templates/default-azuredeploy-webjobs.json b/deploy/templates/default-azuredeploy-webjobs.json new file mode 100644 index 00000000..19b11e10 --- /dev/null +++ b/deploy/templates/default-azuredeploy-webjobs.json @@ -0,0 +1,746 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ServiceName": { + "type": "string", + "minLength": 3, + "maxLength": 20, + "metadata": { + "description": "Name for the service(s) being deployed. Name will applied to all relevant services being created." + } + }, + "RepositoryUrl": { + "type": "string", + "defaultValue": "https://github.com/Microsoft/iomt-fhir", + "metadata": { + "description": "Repository to pull source code from. If blank, source code will not be deployed." + } + }, + "RepositoryBranch": { + "type": "string", + "defaultValue": "master", + "metadata": { + "description": "Source code branch to deploy." + } + }, + "JobWindowUnit": { + "type": "string", + "allowedValues": [ + "SECOND" + ], + "metadata": { + "description": "The time period to collect events before sending them to the FHIR server." + }, + "defaultValue": "SECOND" + }, + "JobWindowMagnitude": { + "type": "int", + "minValue": 1, + "maxValue": 86400, + "metadata": { + "description": "The magnitude of time period to collect events before sending them to the FHIR server." + }, + "defaultValue": 60 + }, + "JobMaxEvents": { + "type": "int", + "minValue": 1, + "maxValue": 1000, + "metadata": { + "description": "The maximum number of events to collect before sending them to the FHIR server." + }, + "defaultValue": 500 + }, + "ThroughputUnits": { + "type": "int", + "minValue": 1, + "maxValue": 20, + "metadata": { + "description": "The throughput units reserved for the Event Hubs created." + }, + "defaultValue": 1 + }, + "AppServicePlanSku": { + "type": "string", + "allowedValues": [ + "F1", + "D1", + "B1", + "B2", + "B3", + "S1", + "S2", + "S3", + "P1", + "P2", + "P3", + "P4" + ], + "defaultValue": "S1", + "metadata": { + "description": "The app service plan tier to use for hosting the required Azure Functions." + } + }, + "ResourceLocation": { + "type": "string", + "allowedValues": [ + "South Africa North", + "South Africa West", + "East Asia", + "Southeast Asia", + "Australia Central", + "Australia Central 2", + "Australia East", + "Australia Southeast", + "Brazil South", + "Brazil Southeast", + "Canada Central", + "Canada East", + "China East", + "China East 2", + "China North", + "China North 2", + "North Europe", + "West Europe", + "France Central", + "France South", + "Germany Central", + "Germany Northeast", + "Germany West Central", + "Central India", + "South India", + "West India", + "Japan East", + "Japan West", + "Korea Central", + "Korea South", + "Norway East", + "Switzerland North", + "Switzerland West", + "UAE Central", + "UAE North", + "UK West", + "UK South", + "Central US", + "East US", + "East US 2", + "North Central US", + "South Central US", + "West Central US", + "West US", + "West US 2" + ], + "metadata": { + "description": "The location of the deployed resources." + } + }, + "FhirServiceUrl": { + "type": "string", + "metadata": { + "description": "Url of the FHIR server that IoMT will be written to." + } + }, + "FhirServiceAuthority": { + "type": "string", + "metadata": { + "description": "Authority of the FHIR to retrieve a token against." + } + }, + "FhirServiceResource": { + "type": "string", + "metadata": { + "description": "Resource/Audience representing the FHIR server on the provided authority." + } + }, + "FhirServiceClientId": { + "type": "string", + "metadata": { + "description": "Client Id to run services as for access to the FHIR server." + } + }, + "FhirServiceClientSecret": { + "type": "securestring", + "metadata": { + "description": "Client secret of the application for accessing a token." + } + }, + "FhirVersion": { + "type": "string", + "defaultValue": "R4", + "metadata": { + "description": "FHIR Version that the FHIR Server supports" + } + }, + "ResourceIdentityResolutionType": { + "type": "string", + "allowedValues": [ + "Lookup", + "Create", + "LookupWithEncounter" + ], + "defaultValue": "Lookup", + "metadata": { + "description": "Configures how patient, device, and other FHIR resource identities are resolved from the ingested data stream." + } + }, + "DefaultDeviceIdentifierSystem": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Default system to use when searching for device identities. If empty system is not used in the search." + } + } + }, + "variables": { + "asa_job_name": "[parameters('ServiceName')]", + "eventhub_namespace_name": "[parameters('ServiceName')]", + "normalizeddata_eventhub_name": "[concat(variables('eventhub_namespace_name'), '/normalizeddata')]", + "devicedata_eventhub_name": "[concat(variables('eventhub_namespace_name'), '/devicedata')]", + "storage_account_name": "[parameters('ServiceName')]", + "app_plan_name": "[concat(parameters('ServiceName'), 'plan')]", + "app_service_name": "[parameters('ServiceName')]", + "app_insights_name": "[parameters('ServiceName')]", + "key_vault_name": "[parameters('ServiceName')]", + "app_service_resource_id": "[resourceId('Microsoft.Web/sites', variables('app_service_name'))]", + "deploy_source_code": "[and(not(empty(parameters('repositoryUrl'))),not(empty(parameters('repositoryBranch'))))]", + "sender_role": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', '2b629674-e913-4c01-ae53-ef4638d8f975')]", + "receiver_role": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'a638d3c7-ab3a-418d-83e6-5f17a39d4fde')]" + }, + "resources": [ + { + "type": "Microsoft.EventHub/namespaces", + "apiVersion": "2017-04-01", + "name": "[variables('eventhub_namespace_name')]", + "location": "[parameters('ResourceLocation')]", + "tags": { + "IomtFhirConnector": "[parameters('ResourceIdentityResolutionType')]" + }, + "sku": { + "name": "Standard", + "tier": "Standard", + "capacity": "[parameters('ThroughputUnits')]" + }, + "properties": { + "zoneRedundant": true, + "isAutoInflateEnabled": false, + "maximumThroughputUnits": 0, + "kafkaEnabled": false + } + }, + { + "type": "Microsoft.EventHub/namespaces/AuthorizationRules", + "apiVersion": "2017-04-01", + "name": "[concat(variables('eventhub_namespace_name'), '/RootManageSharedAccessKey')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.EventHub/namespaces', variables('eventhub_namespace_name'))]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs', variables('eventhub_namespace_name'), 'devicedata')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs', variables('eventhub_namespace_name'), 'normalizeddata')]" + ], + "properties": { + "rights": [ + "Listen", + "Manage", + "Send" + ] + } + }, + { + "type": "Microsoft.EventHub/namespaces/eventhubs", + "apiVersion": "2017-04-01", + "name": "[concat(variables('eventhub_namespace_name'), '/devicedata')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.EventHub/namespaces', variables('eventhub_namespace_name'))]" + ], + "properties": { + "messageRetentionInDays": 1, + "partitionCount": 32, + "status": "Active" + } + }, + { + "type": "Microsoft.EventHub/namespaces/eventhubs", + "apiVersion": "2017-04-01", + "name": "[concat(variables('eventhub_namespace_name'), '/normalizeddata')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.EventHub/namespaces', variables('eventhub_namespace_name'))]" + ], + "properties": { + "messageRetentionInDays": 1, + "partitionCount": 32, + "status": "Active" + } + }, + { + "type": "Microsoft.EventHub/namespaces/eventhubs/authorizationRules", + "apiVersion": "2017-04-01", + "name": "[concat(variables('eventhub_namespace_name'), '/devicedata/reader')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.EventHub/namespaces/eventhubs', variables('eventhub_namespace_name'), 'devicedata')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs', variables('eventhub_namespace_name'), 'normalizeddata')]", + "[resourceId('Microsoft.EventHub/namespaces', variables('eventhub_namespace_name'))]", + "[resourceId('Microsoft.EventHub/namespaces/AuthorizationRules', variables('eventhub_namespace_name'), 'RootManageSharedAccessKey')]" + ], + "properties": { + "rights": [ + "Listen" + ] + } + }, + { + "type": "Microsoft.EventHub/namespaces/eventhubs/authorizationRules", + "apiVersion": "2017-04-01", + "name": "[concat(variables('eventhub_namespace_name'), '/devicedata/writer')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.EventHub/namespaces/eventhubs', variables('eventhub_namespace_name'), 'devicedata')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs', variables('eventhub_namespace_name'), 'normalizeddata')]", + "[resourceId('Microsoft.EventHub/namespaces', variables('eventhub_namespace_name'))]", + "[resourceId('Microsoft.EventHub/namespaces/AuthorizationRules', variables('eventhub_namespace_name'), 'RootManageSharedAccessKey')]" + ], + "properties": { + "rights": [ + "Send" + ] + } + }, + { + "type": "Microsoft.EventHub/namespaces/eventhubs/authorizationRules", + "apiVersion": "2017-04-01", + "name": "[concat(variables('eventhub_namespace_name'), '/normalizeddata/reader')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.EventHub/namespaces/eventhubs', variables('eventhub_namespace_name'), 'devicedata')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs', variables('eventhub_namespace_name'), 'normalizeddata')]", + "[resourceId('Microsoft.EventHub/namespaces', variables('eventhub_namespace_name'))]", + "[resourceId('Microsoft.EventHub/namespaces/AuthorizationRules', variables('eventhub_namespace_name'), 'RootManageSharedAccessKey')]" + ], + "properties": { + "rights": [ + "Listen" + ] + } + }, + { + "type": "Microsoft.EventHub/namespaces/eventhubs/authorizationRules", + "apiVersion": "2017-04-01", + "name": "[concat(variables('eventhub_namespace_name'), '/normalizeddata/writer')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.EventHub/namespaces/eventhubs', variables('eventhub_namespace_name'), 'devicedata')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs', variables('eventhub_namespace_name'), 'normalizeddata')]", + "[resourceId('Microsoft.EventHub/namespaces', variables('eventhub_namespace_name'))]", + "[resourceId('Microsoft.EventHub/namespaces/AuthorizationRules', variables('eventhub_namespace_name'), 'RootManageSharedAccessKey')]" + ], + "properties": { + "rights": [ + "Send", + "Listen" + ] + } + }, + { + "type": "Microsoft.EventHub/namespaces/eventhubs/consumergroups", + "apiVersion": "2017-04-01", + "name": "[concat(variables('eventhub_namespace_name'), '/devicedata/$Default')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.EventHub/namespaces/eventhubs', variables('eventhub_namespace_name'), 'devicedata')]", + "[resourceId('Microsoft.EventHub/namespaces', variables('eventhub_namespace_name'))]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', variables('eventhub_namespace_name'), 'devicedata', 'reader')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', variables('eventhub_namespace_name'), 'devicedata', 'writer')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', variables('eventhub_namespace_name'), 'normalizeddata', 'reader')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', variables('eventhub_namespace_name'), 'normalizeddata', 'writer')]" + ], + "properties": { + } + }, + { + "type": "Microsoft.EventHub/namespaces/eventhubs/consumergroups", + "apiVersion": "2017-04-01", + "name": "[concat(variables('eventhub_namespace_name'), '/normalizeddata/$Default')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.EventHub/namespaces/eventhubs', variables('eventhub_namespace_name'), 'normalizeddata')]", + "[resourceId('Microsoft.EventHub/namespaces', variables('eventhub_namespace_name'))]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', variables('eventhub_namespace_name'), 'devicedata', 'reader')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', variables('eventhub_namespace_name'), 'devicedata', 'writer')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', variables('eventhub_namespace_name'), 'normalizeddata', 'reader')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', variables('eventhub_namespace_name'), 'normalizeddata', 'writer')]" + ], + "properties": { + } + }, + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2019-04-01", + "name": "[variables('storage_account_name')]", + "location": "[parameters('ResourceLocation')]", + "tags": { + "IomtFhirConnector": "[parameters('ResourceIdentityResolutionType')]" + }, + "sku": { + "name": "Standard_RAGRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "virtualNetworkRules": [ + ], + "ipRules": [ + ], + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "enabled": true + }, + "blob": { + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + }, + "accessTier": "Hot" + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2019-04-01", + "name": "[concat(variables('storage_account_name'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('storage_account_name'))]" + ], + "properties": { + "cors": { + "corsRules": [ + ] + }, + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2019-04-01", + "name": "[concat(variables('storage_account_name'), '/default/template')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('storage_account_name'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('storage_account_name'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2019-04-01", + "name": "[concat(variables('storage_account_name'), '/default/checkpoint')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('storage_account_name'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('storage_account_name'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2016-09-01", + "name": "[variables('app_plan_name')]", + "location": "[parameters('ResourceLocation')]", + "tags": { + "IomtFhirConnector": "[variables('app_plan_name')]", + "IomtFhirVersion": "[parameters('FhirVersion')]" + }, + "sku": { + "name": "[parameters('AppServicePlanSku')]" + }, + "kind": "app", + "properties": { + "name": "[variables('app_plan_name')]", + "perSiteScaling": false, + "reserved": false, + "targetWorkerCount": 0, + "targetWorkerSizeId": 0 + } + }, + { + "type": "Microsoft.Web/sites", + "apiVersion": "2018-11-01", + "name": "[variables('app_service_name')]", + "location": "[parameters('ResourceLocation')]", + "kind": "app", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "enabled": true, + "hostNameSslStates": [ + { + "name": "[concat(variables('app_service_name'), '.azurewebsites.net')]", + "sslState": "Disabled", + "hostType": "Standard" + }, + { + "name": "[concat(variables('app_service_name'), '.scm.azurewebsites.net')]", + "sslState": "Disabled", + "hostType": "Repository" + } + ], + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('app_plan_name'))]", + "reserved": false, + "isXenon": false, + "hyperV": false, + "siteConfig": { + "alwaysOn": true + }, + "scmSiteAlsoStopped": false, + "clientAffinityEnabled": true, + "clientCertEnabled": false, + "hostNamesDisabled": false, + "containerSize": 0, + "dailyMemoryTimeQuota": 0, + "httpsOnly": false, + "redundancyMode": "None" + }, + "resources": [ + { + "name": "appsettings", + "dependsOn": [ + "[variables('app_service_resource_id')]", + "[resourceId('Microsoft.KeyVault/vaults/secrets', variables('key_vault_name'),'fhirserver-url')]", + "[resourceId('Microsoft.KeyVault/vaults/secrets', variables('key_vault_name'),'fhirserver-authority')]", + "[resourceId('Microsoft.KeyVault/vaults/secrets', variables('key_vault_name'),'fhirserver-resource')]", + "[resourceId('Microsoft.KeyVault/vaults/secrets', variables('key_vault_name'),'fhirserver-clientid')]", + "[resourceId('Microsoft.KeyVault/vaults/secrets', variables('key_vault_name'),'fhirserver-clientsecret')]" + ], + "type": "config", + "apiVersion": "2016-08-01", + "properties": { + "EventBatching:FlushTimespan": "[parameters('JobWindowMagnitude')]", + "EventBatching:MaxEvents": "[parameters('JobMaxEvents')]", + "Storage:BlobStorageConnectionString": "[concat('@Microsoft.KeyVault(SecretUri=', reference(resourceId('Microsoft.KeyVault/vaults/secrets', variables('key_vault_name'),'blob-storage-cs'), '2018-02-14').secretUriWithVersion, ')')]", + "Storage:BlobContainerName": "checkpoint", + "Storage:BlobPrefix": "", + "Storage:CheckpointBatchCount": 5, + "TemplateStorage:BlobStorageConnectionString": "[concat('@Microsoft.KeyVault(SecretUri=', reference(resourceId('Microsoft.KeyVault/vaults/secrets', variables('key_vault_name'),'blob-storage-cs'), '2018-02-14').secretUriWithVersion, ')')]", + "TemplateStorage:BlobContainerName": "template", + "InputEventHub": "[concat('@Microsoft.KeyVault(SecretUri=', reference(resourceId('Microsoft.KeyVault/vaults/secrets', variables('key_vault_name'),'device-input-connection'), '2018-02-14').secretUriWithVersion, ')')]", + "OutputEventHub": "[concat('@Microsoft.KeyVault(SecretUri=', reference(resourceId('Microsoft.KeyVault/vaults/secrets', variables('key_vault_name'),'device-output-connection'), '2018-02-14').secretUriWithVersion, ')')]", + "FhirVersion": "[parameters('FhirVersion')]", + "FhirService:Url": "[concat('@Microsoft.KeyVault(SecretUri=', reference(resourceId('Microsoft.KeyVault/vaults/secrets', variables('key_vault_name'),'fhirserver-url'), '2018-02-14').secretUriWithVersion, ')')]", + "FhirService:Authority": "[concat('@Microsoft.KeyVault(SecretUri=', reference(resourceId('Microsoft.KeyVault/vaults/secrets', variables('key_vault_name'),'fhirserver-authority'), '2018-02-14').secretUriWithVersion, ')')]", + "FhirService:Resource": "[concat('@Microsoft.KeyVault(SecretUri=', reference(resourceId('Microsoft.KeyVault/vaults/secrets', variables('key_vault_name'),'fhirserver-resource'), '2018-02-14').secretUriWithVersion, ')')]", + "FhirService:ClientId": "[concat('@Microsoft.KeyVault(SecretUri=', reference(resourceId('Microsoft.KeyVault/vaults/secrets', variables('key_vault_name'),'fhirserver-clientid'), '2018-02-14').secretUriWithVersion, ')')]", + "FhirService:ClientSecret": "[concat('@Microsoft.KeyVault(SecretUri=', reference(resourceId('Microsoft.KeyVault/vaults/secrets', variables('key_vault_name'),'fhirserver-clientsecret'), '2018-02-14').secretUriWithVersion, ')')]", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(concat('Microsoft.Insights/components/', variables('app_insights_name'))).InstrumentationKey]", + "Template:DeviceContent": "devicecontent.json", + "Template:FhirMapping": "fhirmapping.json", + "ResourceIdentity:ResourceIdentityResolutionType": "[parameters('ResourceIdentityResolutionType')]", + "ResourceIdentity:DefaultDeviceIdentifierSystem": "[parameters('DefaultDeviceIdentifierSystem')]" + } + } + ] + }, + { + "type": "microsoft.insights/components", + "apiVersion": "2015-05-01", + "name": "[variables('app_insights_name')]", + "location": "[parameters('ResourceLocation')]", + "tags": { + "IomtFhirConnector": "[parameters('ResourceIdentityResolutionType')]" + }, + "kind": "web", + "properties": { + "Application_Type": "web", + "Flow_Type": "Redfield", + "Request_Source": "IbizaAIExtension" + } + }, + { + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2016-10-01", + "name": "[variables('key_vault_name')]", + "location": "[parameters('ResourceLocation')]", + "tags": { + "IomtFhirConnector": "[parameters('ResourceIdentityResolutionType')]" + }, + "dependsOn": [ + "[variables('app_service_resource_id')]" + ], + "properties": { + "sku": { + "family": "A", + "name": "Standard" + }, + "tenantId": "[subscription().tenantId]", + "accessPolicies": [ + { + "tenantId": "[reference(variables('app_service_resource_id'), '2015-08-01', 'Full').Identity.tenantId]", + "objectId": "[reference(variables('app_service_resource_id'), '2015-08-01', 'Full').Identity.principalId]", + "permissions": { + "keys": [ + ], + "secrets": [ + "Get", + "List" + ], + "certificates": [ + ] + } + } + ], + "enabledForDeployment": false, + "enabledForDiskEncryption": false, + "enabledForTemplateDeployment": false + } + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2016-10-01", + "name": "[concat(variables('storage_account_name'), '/blob-storage-cs')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('key_vault_name'))]", + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('storage_account_name'), 'default')]" + ], + "properties": { + "contentType": "text/plain", + "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storage_account_name'), ';AccountKey=', listkeys(resourceId('Microsoft.Storage/storageAccounts', variables('storage_account_name')), '2019-04-01').keys[0].value)]", + "attributes": { + "enabled": true + } + } + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2016-10-01", + "name": "[concat(variables('key_vault_name'), '/fhirserver-authority')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('key_vault_name'))]" + ], + "properties": { + "contentType": "text/plain", + "value": "[parameters('FhirServiceAuthority')]", + "attributes": { + "enabled": true + } + } + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2016-10-01", + "name": "[concat(variables('key_vault_name'), '/fhirserver-clientid')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('key_vault_name'))]" + ], + "properties": { + "contentType": "text/plain", + "value": "[parameters('FhirServiceClientId')]", + "attributes": { + "enabled": true + } + } + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2016-10-01", + "name": "[concat(variables('key_vault_name'), '/fhirserver-clientsecret')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('key_vault_name'))]" + ], + "properties": { + "contentType": "text/plain", + "value": "[parameters('FhirServiceClientSecret')]", + "attributes": { + "enabled": true + } + } + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2016-10-01", + "name": "[concat(variables('key_vault_name'), '/fhirserver-resource')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('key_vault_name'))]" + ], + "properties": { + "contentType": "text/plain", + "value": "[parameters('FhirServiceResource')]", + "attributes": { + "enabled": true + } + } + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2016-10-01", + "name": "[concat(variables('key_vault_name'), '/fhirserver-url')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('key_vault_name'))]" + ], + "properties": { + "contentType": "text/plain", + "value": "[parameters('FhirServiceUrl')]", + "attributes": { + "enabled": true + } + } + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2016-10-01", + "name": "[concat(variables('key_vault_name'), '/device-input-connection')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('key_vault_name'))]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', variables('eventhub_namespace_name'), 'devicedata', 'reader')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs/consumergroups', variables('eventhub_namespace_name'), 'devicedata', '$Default')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs/consumergroups', variables('eventhub_namespace_name'), 'normalizeddata', '$Default')]" + ], + "properties": { + "contentType": "text/plain", + "value": "[listkeys(resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', variables('eventhub_namespace_name'), 'devicedata', 'reader'), '2017-04-01').primaryConnectionString]", + "attributes": { + "enabled": true + } + } + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2016-10-01", + "name": "[concat(variables('key_vault_name'), '/device-output-connection')]", + "location": "[parameters('ResourceLocation')]", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('key_vault_name'))]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', variables('eventhub_namespace_name'), 'normalizeddata', 'writer')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs/consumergroups', variables('eventhub_namespace_name'), 'devicedata', '$Default')]", + "[resourceId('Microsoft.EventHub/namespaces/eventhubs/consumergroups', variables('eventhub_namespace_name'), 'normalizeddata', '$Default')]" + ], + "properties": { + "contentType": "text/plain", + "value": "[listkeys(resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', variables('eventhub_namespace_name'), 'normalizeddata', 'writer'), '2017-04-01').primaryConnectionString]", + "attributes": { + "enabled": true + } + } + } + ], + "outputs": { + } +} diff --git a/src/console/Program.cs b/src/console/Program.cs index 05387819..86e2313d 100644 --- a/src/console/Program.cs +++ b/src/console/Program.cs @@ -54,7 +54,7 @@ public static async Task Main() storageOptions.BlobPrefix = eventHub; var checkpointClient = new StorageCheckpointClient(storageOptions, logger); - var eventConsumerService = new EventConsumerService(eventConsumers); + var eventConsumerService = new EventConsumerService(eventConsumers, logger); var ct = new CancellationToken(); @@ -74,6 +74,7 @@ public static IConfiguration GetEnvironmentConfig() { IConfiguration config = new ConfigurationBuilder() .AddJsonFile("appsettings.json", true, true) + .AddEnvironmentVariables() .Build(); return config; diff --git a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs index d752d006..7a3da28c 100644 --- a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs +++ b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs @@ -130,15 +130,13 @@ public async Task SetCheckpointAsync(IEventMessage eventArgs) try { - Console.WriteLine(eventArgs.EnqueuedTime); - var partitionId = eventArgs.PartitionId; var checkpoint = new Checkpoint(); checkpoint.LastProcessed = eventArgs.EnqueuedTime; checkpoint.Id = partitionId; checkpoint.Prefix = BlobPrefix; - _checkpoints[eventArgs.PartitionId] = checkpoint; + _checkpoints[partitionId] = checkpoint; var count = _lastCheckpointTracker.AddOrUpdate(partitionId, 1, (key, value) => value + 1); if (count >= _lastCheckpointMaxCount) diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs index eb411c8d..dc8a75f6 100644 --- a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs @@ -5,19 +5,23 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Microsoft.Health.Events.Model; +using Microsoft.Health.Logger.Telemetry; namespace Microsoft.Health.Events.EventConsumers.Service { public class EventConsumerService : IEventConsumerService { - private readonly IEnumerable eventConsumers; + private readonly IEnumerable _eventConsumers; private const int _maximumBackoffMs = 32000; + private ITelemetryLogger _logger; - public EventConsumerService(IEnumerable eventConsumers) + public EventConsumerService(IEnumerable eventConsumers, ITelemetryLogger logger) { - this.eventConsumers = eventConsumers; + _eventConsumers = eventConsumers; + _logger = logger; } public Task ConsumeEvent(IEventMessage eventArg) @@ -27,13 +31,16 @@ public Task ConsumeEvent(IEventMessage eventArg) public async Task ConsumeEvents(IEnumerable events) { - foreach (IEventConsumer eventConsumer in eventConsumers) + if (events.Any()) { - await OperationWithRetryAsync(eventConsumer, events); + foreach (IEventConsumer eventConsumer in _eventConsumers) + { + await OperationWithRetryAsync(eventConsumer, events); + } } } - private static async Task OperationWithRetryAsync(IEventConsumer eventConsumer, IEnumerable events) + private async Task OperationWithRetryAsync(IEventConsumer eventConsumer, IEnumerable events) { int currentRetry = 0; double backoffMs = 0; @@ -58,7 +65,7 @@ private static async Task OperationWithRetryAsync(IEventConsumer eventConsumer, catch (Exception e) #pragma warning restore CA1031 { - Console.WriteLine(e.Message); + _logger.LogError(e); } } } diff --git a/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs b/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs index c0149a59..112f10c3 100644 --- a/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs +++ b/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs @@ -56,10 +56,10 @@ Task ProcessEventHandler(ProcessEventArgs eventArgs) return Task.CompletedTask; } + // todo: consider retry Task ProcessErrorHandler(ProcessErrorEventArgs eventArgs) { - // todo: add an error processor - Console.WriteLine(eventArgs.Exception.Message); + _logger.LogError(eventArgs.Exception); return Task.CompletedTask; } diff --git a/src/lib/Microsoft.Health.Events/Telemetry/Metrics/EventMetrics.cs b/src/lib/Microsoft.Health.Events/Telemetry/Metrics/EventMetrics.cs index bdf6bc95..725b08e8 100644 --- a/src/lib/Microsoft.Health.Events/Telemetry/Metrics/EventMetrics.cs +++ b/src/lib/Microsoft.Health.Events/Telemetry/Metrics/EventMetrics.cs @@ -95,7 +95,7 @@ public static Metric EventWatermark(string partitionId, DateTime dateTime) new Dictionary { { _nameDimension, "EventsWatermarkUpdated" }, - { _timeDimension, dateTime }, + { _timeDimension, dateTime.ToString() }, { _partitionDimension, partitionId }, { _categoryDimension, Category.Latency }, }); From 020f6291fd773e0031469147d8e46d115f70b691 Mon Sep 17 00:00:00 2001 From: Will Yochum Date: Tue, 5 Jan 2021 16:50:20 -0800 Subject: [PATCH 12/14] remove debugging --- .../Service/MeasurementFhirImportService.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs index 48cd3fff..bf51135f 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs @@ -111,10 +111,6 @@ private static async Task> ParseAsync(Stream data private static IEnumerable ParseEventData(IEnumerable data, ITelemetryLogger log) { // Deserialize events into measurements and then group according to the device, type, and other factors - var body = data.First().Body.ToArray(); - var text = System.Text.Encoding.Default.GetString(body); - var measurement = JsonConvert.DeserializeObject(text); - return data.Select(e => JsonConvert.DeserializeObject(System.Text.Encoding.Default.GetString(e.Body.ToArray()))) .GroupBy(m => $"{m.DeviceId}-{m.Type}-{m.PatientId}-{m.EncounterId}-{m.CorrelationId}") .Select(g => From ab78a3ae45bd76979abf8762fd21e5fa5533c034 Mon Sep 17 00:00:00 2001 From: Will Yochum Date: Tue, 5 Jan 2021 16:54:02 -0800 Subject: [PATCH 13/14] add null check --- .../EventCheckpointing/StorageCheckpointClient.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs index d752d006..a3396c1d 100644 --- a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs +++ b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs @@ -52,13 +52,14 @@ public async Task UpdateCheckpointAsync(Checkpoint checkpoint) { EnsureArg.IsNotNull(checkpoint); EnsureArg.IsNotNullOrWhiteSpace(checkpoint.Id); + var lastProcessed = EnsureArg.IsNotNullOrWhiteSpace(checkpoint.LastProcessed.DateTime.ToString("MM/dd/yyyy hh:mm:ss.fff tt")); var blobName = $"{BlobPrefix}/checkpoint/{checkpoint.Id}"; var blobClient = _storageClient.GetBlobClient(blobName); var metadata = new Dictionary() { - { "LastProcessed", checkpoint.LastProcessed.DateTime.ToString("MM/dd/yyyy hh:mm:ss.fff tt") }, + { "LastProcessed", lastProcessed }, }; try From 4d3b85f454b6adb64603b3e0618548c10d7bb190 Mon Sep 17 00:00:00 2001 From: Will Yochum Date: Tue, 5 Jan 2021 17:22:23 -0800 Subject: [PATCH 14/14] update namespace --- Microsoft.Health.Fhir.Ingest.sln | 2 +- src/console/{IomtLogging.cs => IomtLogger.cs} | 6 +++--- src/console/MeasurementCollectionToFhir/Processor.cs | 2 +- src/console/Normalize/Processor.cs | 2 +- src/console/Program.cs | 6 +++--- .../IomtConnectorFunctions.cs | 2 +- src/func/Microsoft.Health.Fhir.Ingest.Host/Startup.cs | 3 +-- .../EventCheckpointing/StorageCheckpointClient.cs | 2 +- .../EventConsumers/Service/EventBatchingService.cs | 2 +- .../EventConsumers/Service/EventConsumerService.cs | 2 +- .../EventConsumers/Service/Infrastructure/EventPartition.cs | 2 +- .../EventHubProcessor/EventProcessor.cs | 2 +- .../Microsoft.Health.Events/Microsoft.Health.Events.csproj | 2 +- .../Service/MeasurementEventNormalizationService.cs | 2 +- .../Service/MeasurementFhirImportService.cs | 2 +- .../Telemetry/ExceptionTelemetryProcessor.cs | 2 +- ...Health.Logger.csproj => Microsoft.Health.Logging.csproj} | 0 .../Microsoft.Health.Logger/Telemetry/ITelemetryLogger.cs | 2 +- .../Telemetry/IomtTelemetryLogger.cs | 4 ++-- .../Telemetry/Metrics/MetricExtensionMethods.cs | 2 +- .../Service/MeasurementEventNormalizationServiceTests.cs | 2 +- .../Service/MeasurementFhirImportServiceTests.cs | 2 +- .../Telemetry/ExceptionTelemetryProcessorTests.cs | 2 +- 23 files changed, 27 insertions(+), 28 deletions(-) rename src/console/{IomtLogging.cs => IomtLogger.cs} (88%) rename src/lib/Microsoft.Health.Logger/{Microsoft.Health.Logger.csproj => Microsoft.Health.Logging.csproj} (100%) diff --git a/Microsoft.Health.Fhir.Ingest.sln b/Microsoft.Health.Fhir.Ingest.sln index 31a7ffb3..e16e95b7 100644 --- a/Microsoft.Health.Fhir.Ingest.sln +++ b/Microsoft.Health.Fhir.Ingest.sln @@ -83,7 +83,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "console", "console", "{1EF3 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.Ingest.Console", "src\console\Microsoft.Health.Fhir.Ingest.Console.csproj", "{927BC214-ABD9-4A1B-9F7C-75973513D141}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Logger", "src\lib\Microsoft.Health.Logger\Microsoft.Health.Logger.csproj", "{05123BAE-E96E-4C7E-95CB-C616DF940F17}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Logging", "src\lib\Microsoft.Health.Logger\Microsoft.Health.Logging.csproj", "{05123BAE-E96E-4C7E-95CB-C616DF940F17}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/src/console/IomtLogging.cs b/src/console/IomtLogger.cs similarity index 88% rename from src/console/IomtLogging.cs rename to src/console/IomtLogger.cs index 1915fdf6..215f52b0 100644 --- a/src/console/IomtLogging.cs +++ b/src/console/IomtLogger.cs @@ -4,13 +4,13 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; namespace Microsoft.Health.Fhir.Ingest.Console { - public class IomtLogging + public class IomtLogger { - public IomtLogging(IConfiguration configuration) + public IomtLogger(IConfiguration configuration) { Configuration = configuration; } diff --git a/src/console/MeasurementCollectionToFhir/Processor.cs b/src/console/MeasurementCollectionToFhir/Processor.cs index 38476ca4..fee4369e 100644 --- a/src/console/MeasurementCollectionToFhir/Processor.cs +++ b/src/console/MeasurementCollectionToFhir/Processor.cs @@ -13,7 +13,7 @@ using Microsoft.Health.Fhir.Ingest.Console.Template; using Microsoft.Health.Fhir.Ingest.Host; using Microsoft.Health.Fhir.Ingest.Service; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; namespace Microsoft.Health.Fhir.Ingest.Console.MeasurementCollectionToFhir { diff --git a/src/console/Normalize/Processor.cs b/src/console/Normalize/Processor.cs index 760b52d9..998d5d3e 100644 --- a/src/console/Normalize/Processor.cs +++ b/src/console/Normalize/Processor.cs @@ -11,7 +11,7 @@ using Microsoft.Health.Fhir.Ingest.Service; using Microsoft.Health.Fhir.Ingest.Telemetry; using Microsoft.Health.Fhir.Ingest.Template; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; using System.Collections.Generic; using System.IO; using System.Linq; diff --git a/src/console/Program.cs b/src/console/Program.cs index 86e2313d..1ddd6228 100644 --- a/src/console/Program.cs +++ b/src/console/Program.cs @@ -14,7 +14,7 @@ using Microsoft.Health.Fhir.Ingest.Console.Storage; using Microsoft.Health.Fhir.Ingest.Console.Template; using Microsoft.Health.Fhir.Ingest.Service; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; using System; using System.Collections.Generic; using System.Threading; @@ -88,7 +88,7 @@ public static ServiceProvider GetRequiredServiceProvider(IConfiguration config, Normalize.ProcessorStartup startup = new Normalize.ProcessorStartup(config); startup.ConfigureServices(serviceCollection); - var loggingService = new IomtLogging(config); + var loggingService = new IomtLogger(config); loggingService.ConfigureServices(serviceCollection); var serviceProvider = serviceCollection.BuildServiceProvider(); @@ -100,7 +100,7 @@ public static ServiceProvider GetRequiredServiceProvider(IConfiguration config, MeasurementCollectionToFhir.ProcessorStartup startup = new MeasurementCollectionToFhir.ProcessorStartup(config); startup.ConfigureServices(serviceCollection); - var loggingService = new IomtLogging(config); + var loggingService = new IomtLogger(config); loggingService.ConfigureServices(serviceCollection); var serviceProvider = serviceCollection.BuildServiceProvider(); diff --git a/src/func/Microsoft.Health.Fhir.Ingest.Host/IomtConnectorFunctions.cs b/src/func/Microsoft.Health.Fhir.Ingest.Host/IomtConnectorFunctions.cs index 981854e7..85b5ee63 100644 --- a/src/func/Microsoft.Health.Fhir.Ingest.Host/IomtConnectorFunctions.cs +++ b/src/func/Microsoft.Health.Fhir.Ingest.Host/IomtConnectorFunctions.cs @@ -17,7 +17,7 @@ using Microsoft.Health.Fhir.Ingest.Host; using Microsoft.Health.Fhir.Ingest.Telemetry; using Microsoft.Health.Fhir.Ingest.Template; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; namespace Microsoft.Health.Fhir.Ingest.Service { diff --git a/src/func/Microsoft.Health.Fhir.Ingest.Host/Startup.cs b/src/func/Microsoft.Health.Fhir.Ingest.Host/Startup.cs index 99390eb3..574470fb 100644 --- a/src/func/Microsoft.Health.Fhir.Ingest.Host/Startup.cs +++ b/src/func/Microsoft.Health.Fhir.Ingest.Host/Startup.cs @@ -10,8 +10,7 @@ using Microsoft.ApplicationInsights.Extensibility; using Microsoft.Azure.Functions.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Health.Fhir.Ingest.Telemetry; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; namespace Microsoft.Health.Fhir.Ingest.Service { diff --git a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs index 0cf0427d..dfee3ff5 100644 --- a/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs +++ b/src/lib/Microsoft.Health.Events/EventCheckpointing/StorageCheckpointClient.cs @@ -17,7 +17,7 @@ using EnsureThat; using Microsoft.Health.Events.Model; using Microsoft.Health.Events.Telemetry; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; namespace Microsoft.Health.Events.EventCheckpointing { diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs index bd5c47be..9354a40e 100644 --- a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventBatchingService.cs @@ -11,7 +11,7 @@ using Microsoft.Health.Events.EventCheckpointing; using Microsoft.Health.Events.EventConsumers.Service.Infrastructure; using Microsoft.Health.Events.Model; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; namespace Microsoft.Health.Events.EventConsumers.Service { diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs index dc8a75f6..f483bf13 100644 --- a/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/EventConsumerService.cs @@ -8,7 +8,7 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Health.Events.Model; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; namespace Microsoft.Health.Events.EventConsumers.Service { diff --git a/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventPartition.cs b/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventPartition.cs index bd5f59cc..00ca635f 100644 --- a/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventPartition.cs +++ b/src/lib/Microsoft.Health.Events/EventConsumers/Service/Infrastructure/EventPartition.cs @@ -9,7 +9,7 @@ using System.Threading.Tasks; using Microsoft.Health.Events.Model; using Microsoft.Health.Events.Telemetry; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; namespace Microsoft.Health.Events.EventConsumers.Service.Infrastructure { diff --git a/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs b/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs index 112f10c3..a256d1b0 100644 --- a/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs +++ b/src/lib/Microsoft.Health.Events/EventHubProcessor/EventProcessor.cs @@ -14,7 +14,7 @@ using Microsoft.Health.Events.EventConsumers.Service; using Microsoft.Health.Events.Model; using Microsoft.Health.Events.Telemetry; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; namespace Microsoft.Health.Events.EventHubProcessor { diff --git a/src/lib/Microsoft.Health.Events/Microsoft.Health.Events.csproj b/src/lib/Microsoft.Health.Events/Microsoft.Health.Events.csproj index 9f87636a..22d52d98 100644 --- a/src/lib/Microsoft.Health.Events/Microsoft.Health.Events.csproj +++ b/src/lib/Microsoft.Health.Events/Microsoft.Health.Events.csproj @@ -36,6 +36,6 @@ - + diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementEventNormalizationService.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementEventNormalizationService.cs index 37838fa5..0114ac84 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementEventNormalizationService.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementEventNormalizationService.cs @@ -15,7 +15,7 @@ using Microsoft.Health.Fhir.Ingest.Data; using Microsoft.Health.Fhir.Ingest.Telemetry; using Microsoft.Health.Fhir.Ingest.Template; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; using Newtonsoft.Json.Linq; namespace Microsoft.Health.Fhir.Ingest.Service diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs index bf51135f..818c2e7c 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementFhirImportService.cs @@ -16,7 +16,7 @@ using Microsoft.Health.Fhir.Ingest.Data; using Microsoft.Health.Fhir.Ingest.Telemetry; using Microsoft.Health.Fhir.Ingest.Template; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; using Newtonsoft.Json; using Newtonsoft.Json.Linq; diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ExceptionTelemetryProcessor.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ExceptionTelemetryProcessor.cs index e58f85a2..630598cf 100644 --- a/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ExceptionTelemetryProcessor.cs +++ b/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/ExceptionTelemetryProcessor.cs @@ -11,7 +11,7 @@ using Microsoft.Health.Fhir.Ingest.Data; using Microsoft.Health.Fhir.Ingest.Service; using Microsoft.Health.Fhir.Ingest.Template; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; namespace Microsoft.Health.Fhir.Ingest.Telemetry { diff --git a/src/lib/Microsoft.Health.Logger/Microsoft.Health.Logger.csproj b/src/lib/Microsoft.Health.Logger/Microsoft.Health.Logging.csproj similarity index 100% rename from src/lib/Microsoft.Health.Logger/Microsoft.Health.Logger.csproj rename to src/lib/Microsoft.Health.Logger/Microsoft.Health.Logging.csproj diff --git a/src/lib/Microsoft.Health.Logger/Telemetry/ITelemetryLogger.cs b/src/lib/Microsoft.Health.Logger/Telemetry/ITelemetryLogger.cs index 80b17efc..fc2f597c 100644 --- a/src/lib/Microsoft.Health.Logger/Telemetry/ITelemetryLogger.cs +++ b/src/lib/Microsoft.Health.Logger/Telemetry/ITelemetryLogger.cs @@ -6,7 +6,7 @@ using Microsoft.Health.Common.Telemetry; using System; -namespace Microsoft.Health.Logger.Telemetry +namespace Microsoft.Health.Logging.Telemetry { public interface ITelemetryLogger { diff --git a/src/lib/Microsoft.Health.Logger/Telemetry/IomtTelemetryLogger.cs b/src/lib/Microsoft.Health.Logger/Telemetry/IomtTelemetryLogger.cs index c8e39555..5647c75a 100644 --- a/src/lib/Microsoft.Health.Logger/Telemetry/IomtTelemetryLogger.cs +++ b/src/lib/Microsoft.Health.Logger/Telemetry/IomtTelemetryLogger.cs @@ -6,9 +6,9 @@ using System; using EnsureThat; using Microsoft.ApplicationInsights; -using Microsoft.Health.Fhir.Ingest.Telemetry.Metrics; +using Microsoft.Health.Logging.Metrics.Telemetry; -namespace Microsoft.Health.Logger.Telemetry +namespace Microsoft.Health.Logging.Telemetry { public class IomtTelemetryLogger : ITelemetryLogger { diff --git a/src/lib/Microsoft.Health.Logger/Telemetry/Metrics/MetricExtensionMethods.cs b/src/lib/Microsoft.Health.Logger/Telemetry/Metrics/MetricExtensionMethods.cs index 3e0e21e4..6d82df48 100644 --- a/src/lib/Microsoft.Health.Logger/Telemetry/Metrics/MetricExtensionMethods.cs +++ b/src/lib/Microsoft.Health.Logger/Telemetry/Metrics/MetricExtensionMethods.cs @@ -8,7 +8,7 @@ using Microsoft.ApplicationInsights; using Microsoft.ApplicationInsights.Metrics; -namespace Microsoft.Health.Fhir.Ingest.Telemetry.Metrics +namespace Microsoft.Health.Logging.Metrics.Telemetry { public static class MetricExtensionMethods { diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementEventNormalizationServiceTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementEventNormalizationServiceTests.cs index 85b486ff..8ad0ef70 100644 --- a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementEventNormalizationServiceTests.cs +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementEventNormalizationServiceTests.cs @@ -10,7 +10,7 @@ using Microsoft.Azure.WebJobs; using Microsoft.Health.Fhir.Ingest.Data; using Microsoft.Health.Fhir.Ingest.Template; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; using Newtonsoft.Json.Linq; using NSubstitute; using Xunit; diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementFhirImportServiceTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementFhirImportServiceTests.cs index 35316f84..ffac86fd 100644 --- a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementFhirImportServiceTests.cs +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Service/MeasurementFhirImportServiceTests.cs @@ -14,7 +14,7 @@ using Microsoft.Health.Fhir.Ingest.Data; using Microsoft.Health.Fhir.Ingest.Telemetry; using Microsoft.Health.Fhir.Ingest.Template; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; using Microsoft.Health.Tests.Common; using Newtonsoft.Json; using NSubstitute; diff --git a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Telemetry/ExceptionTelemetryProcessorTests.cs b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Telemetry/ExceptionTelemetryProcessorTests.cs index cb2021b4..f8f32ca5 100644 --- a/test/Microsoft.Health.Fhir.Ingest.UnitTests/Telemetry/ExceptionTelemetryProcessorTests.cs +++ b/test/Microsoft.Health.Fhir.Ingest.UnitTests/Telemetry/ExceptionTelemetryProcessorTests.cs @@ -8,7 +8,7 @@ using Microsoft.Health.Extensions.Fhir; using Microsoft.Health.Fhir.Ingest.Service; using Microsoft.Health.Fhir.Ingest.Template; -using Microsoft.Health.Logger.Telemetry; +using Microsoft.Health.Logging.Telemetry; using NSubstitute; using Xunit;