From db82dede236dc24ab921885a89c319f26f9ea84c Mon Sep 17 00:00:00 2001 From: edward-miller-lcg <119338797+edward-miller-lcg@users.noreply.github.com> Date: Fri, 7 Jun 2024 11:03:53 -0500 Subject: [PATCH 1/4] Convert saving of submission to blob Use blob storage as opposed to the container's file system. --- .../Shared/Settings/ConfigurationConstants.cs | 1 + .../Interfaces/IBlobStorageRepository.cs | 8 ++++ .../Repositories/BlobStorageRepository.cs | 39 +++++++++++++++++++ .../Listeners/SubmitReportListener.cs | 32 ++++++--------- DotNet/Submission/Program.cs | 3 +- .../Settings/BlobStorageSettings.cs | 7 ++++ DotNet/Submission/appsettings.json | 4 ++ 7 files changed, 73 insertions(+), 21 deletions(-) create mode 100644 DotNet/Submission/Application/Interfaces/IBlobStorageRepository.cs create mode 100644 DotNet/Submission/Application/Repositories/BlobStorageRepository.cs create mode 100644 DotNet/Submission/Settings/BlobStorageSettings.cs diff --git a/DotNet/Shared/Settings/ConfigurationConstants.cs b/DotNet/Shared/Settings/ConfigurationConstants.cs index 7be1275b2..d52b179ff 100644 --- a/DotNet/Shared/Settings/ConfigurationConstants.cs +++ b/DotNet/Shared/Settings/ConfigurationConstants.cs @@ -15,6 +15,7 @@ public static class AppSettings public const string DataProtection = "DataProtection"; public const string LinkTokenService = "LinkTokenService"; public const string SecretManagement = "SecretManagement"; + public const string BlobStorageSettings = "BlobStorageSettings"; } public static class DatabaseConnections diff --git a/DotNet/Submission/Application/Interfaces/IBlobStorageRepository.cs b/DotNet/Submission/Application/Interfaces/IBlobStorageRepository.cs new file mode 100644 index 000000000..133743078 --- /dev/null +++ b/DotNet/Submission/Application/Interfaces/IBlobStorageRepository.cs @@ -0,0 +1,8 @@ +namespace LantanaGroup.Link.Submission.Application.Interfaces; + +public interface IBlobStorageRepository +{ + Task UploadBlobAsync(string directoryName, string blobName, string blob); + Task DownloadBlobAsync(string blobName); + Task DeleteBlobAsync(string blobName); +} diff --git a/DotNet/Submission/Application/Repositories/BlobStorageRepository.cs b/DotNet/Submission/Application/Repositories/BlobStorageRepository.cs new file mode 100644 index 000000000..b99ad5b2a --- /dev/null +++ b/DotNet/Submission/Application/Repositories/BlobStorageRepository.cs @@ -0,0 +1,39 @@ +using Azure.Storage.Blobs; +using LantanaGroup.Link.Submission.Application.Interfaces; +using LantanaGroup.Link.Submission.Settings; +using Microsoft.Extensions.Options; + +namespace LantanaGroup.Link.Submission.Application.Repositories; + +public class BlobStorageRepository : IBlobStorageRepository +{ + private readonly ILogger _logger; + private readonly BlobStorageSettings _blobStorageSettings; + + public BlobStorageRepository(IOptions blobStorageSettings, ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _blobStorageSettings = blobStorageSettings.Value ?? throw new ArgumentNullException(nameof(blobStorageSettings)); + } + + public Task DeleteBlobAsync(string blobName) + { + throw new NotImplementedException(); + } + + public Task DownloadBlobAsync(string blobName) + { + throw new NotImplementedException(); + } + + public async Task UploadBlobAsync(string directoryName, string blobName, string blob) + { + BlobContainerClient containerClient = new BlobContainerClient(_blobStorageSettings.ConnectionString, _blobStorageSettings.ContainerName); + containerClient.CreateIfNotExists(); + BlobClient blobClient = containerClient.GetBlobClient($"{directoryName}/{blobName}"); + + await blobClient.UploadAsync(BinaryData.FromString(blob), overwrite: true); + } + + +} diff --git a/DotNet/Submission/Listeners/SubmitReportListener.cs b/DotNet/Submission/Listeners/SubmitReportListener.cs index ce8e9e5e5..031829828 100644 --- a/DotNet/Submission/Listeners/SubmitReportListener.cs +++ b/DotNet/Submission/Listeners/SubmitReportListener.cs @@ -15,6 +15,7 @@ using StackExchange.Redis; using System.Text.Json; using Task = System.Threading.Tasks.Task; +using LantanaGroup.Link.Submission.Application.Interfaces; namespace LantanaGroup.Link.Submission.Listeners { @@ -30,6 +31,8 @@ public class SubmitReportListener : BackgroundService private readonly ITransientExceptionHandler _transientExceptionHandler; private readonly IDeadLetterExceptionHandler _deadLetterExceptionHandler; + private readonly IBlobStorageRepository _blobStorageRepository; + private string Name => this.GetType().Name; public SubmitReportListener(ILogger logger, @@ -37,7 +40,8 @@ public SubmitReportListener(ILogger logger, IMediator mediator, IOptions submissionConfig, IOptions fileSystemConfig, IHttpClientFactory httpClient, ITransientExceptionHandler transientExceptionHandler, - IDeadLetterExceptionHandler deadLetterExceptionHandler) + IDeadLetterExceptionHandler deadLetterExceptionHandler, + IBlobStorageRepository blobStorageRepository) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _kafkaConsumerFactory = kafkaConsumerFactory ?? throw new ArgumentException(nameof(kafkaConsumerFactory)); @@ -56,6 +60,7 @@ public SubmitReportListener(ILogger logger, _deadLetterExceptionHandler.ServiceName = "Submission"; _deadLetterExceptionHandler.Topic = nameof(KafkaTopic.SubmitReport) + "-Error"; + _blobStorageRepository = blobStorageRepository ?? throw new ArgumentNullException(nameof(blobStorageRepository)); } protected override Task ExecuteAsync(CancellationToken stoppingToken) @@ -219,13 +224,6 @@ await consumer.ConsumeWithInstrumentation(async (result, cancellationToken) => var fhirSerializer = new FhirJsonSerializer(); try { - if (Directory.Exists(submissionDirectory)) - { - Directory.Delete(submissionDirectory, true); - } - - Directory.CreateDirectory(submissionDirectory); - #region Device Hl7.Fhir.Model.Device device = new Device(); @@ -237,8 +235,7 @@ await consumer.ConsumeWithInstrumentation(async (result, cancellationToken) => fileName = "sending-device.json"; contents = await fhirSerializer.SerializeToStringAsync(device); - await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents, - cancellationToken); + await _blobStorageRepository.UploadBlobAsync(submissionDirectory, fileName, contents); #endregion @@ -247,8 +244,7 @@ await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents, fileName = "sending-organization.json"; contents = await fhirSerializer.SerializeToStringAsync(value.Organization); - await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents, - cancellationToken); + await _blobStorageRepository.UploadBlobAsync(submissionDirectory, fileName, contents); #endregion @@ -257,8 +253,7 @@ await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents, fileName = "patient-list.json"; contents = await fhirSerializer.SerializeToStringAsync(admittedPatients); - await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents, - cancellationToken); + await _blobStorageRepository.UploadBlobAsync(submissionDirectory, fileName, contents); #endregion @@ -267,8 +262,7 @@ await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents, fileName = "query-plan.json"; contents = queryPlans; - await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents, - cancellationToken); + await _blobStorageRepository.UploadBlobAsync(submissionDirectory, fileName, contents); #endregion @@ -280,8 +274,7 @@ await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents, fileName = $"aggregate-{measureShortName}.json"; contents = await fhirSerializer.SerializeToStringAsync(aggregate); - await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents, - cancellationToken); + await _blobStorageRepository.UploadBlobAsync(submissionDirectory, fileName, contents); } #endregion @@ -349,8 +342,7 @@ await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents, fileName = "other-resources.json"; contents = await fhirSerializer.SerializeToStringAsync(otherResourcesBundle); - await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents, - cancellationToken); + await _blobStorageRepository.UploadBlobAsync(submissionDirectory, fileName, contents); #endregion } diff --git a/DotNet/Submission/Program.cs b/DotNet/Submission/Program.cs index 49debc96a..c57d2e68c 100644 --- a/DotNet/Submission/Program.cs +++ b/DotNet/Submission/Program.cs @@ -97,6 +97,7 @@ static void RegisterServices(WebApplicationBuilder builder) builder.Services.Configure(builder.Configuration.GetRequiredSection(nameof(SubmissionServiceConfig))); builder.Services.Configure(builder.Configuration.GetRequiredSection(nameof(ConsumerSettings))); builder.Services.Configure(builder.Configuration.GetSection(ConfigurationConstants.AppSettings.CORS)); + builder.Services.Configure(builder.Configuration.GetSection(ConfigurationConstants.AppSettings.BlobStorageSettings)); // Add services to the container. builder.Services.AddHttpClient(); @@ -137,7 +138,7 @@ static void RegisterServices(WebApplicationBuilder builder) builder.Services.AddTransient(); // Add repositories - // TODO + builder.Services.AddTransient(); #region Exception Handling //Report Scheduled Listener diff --git a/DotNet/Submission/Settings/BlobStorageSettings.cs b/DotNet/Submission/Settings/BlobStorageSettings.cs new file mode 100644 index 000000000..7629f859c --- /dev/null +++ b/DotNet/Submission/Settings/BlobStorageSettings.cs @@ -0,0 +1,7 @@ +namespace LantanaGroup.Link.Submission.Settings; + +public class BlobStorageSettings +{ + public string ConnectionString { get; set; } + public string ContainerName { get; set; } +} diff --git a/DotNet/Submission/appsettings.json b/DotNet/Submission/appsettings.json index b44106226..79e4f5930 100644 --- a/DotNet/Submission/appsettings.json +++ b/DotNet/Submission/appsettings.json @@ -12,6 +12,10 @@ "MongoDB": { "ConnectionString": "mongodb://localhost:27017", "DatabaseName": "linkReportDb" + } + "BlobStorageSettings": { + "ContainerName": "", + "ConnectionString": "" }, "SubmissionServiceConfig": { "ReportServiceUrl": "", From 90ee98c806a8cec3efc4254184c1b5f762c27201 Mon Sep 17 00:00:00 2001 From: edward-miller-lcg <119338797+edward-miller-lcg@users.noreply.github.com> Date: Fri, 7 Jun 2024 11:09:43 -0500 Subject: [PATCH 2/4] Update Submission.csproj --- DotNet/Submission/Submission.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/DotNet/Submission/Submission.csproj b/DotNet/Submission/Submission.csproj index ceb5213e6..a01bdc501 100644 --- a/DotNet/Submission/Submission.csproj +++ b/DotNet/Submission/Submission.csproj @@ -31,6 +31,7 @@ + From 1946418d37c7621243ce05fe91ca2b272d5d3ab5 Mon Sep 17 00:00:00 2001 From: edward-miller-lcg <119338797+edward-miller-lcg@users.noreply.github.com> Date: Fri, 7 Jun 2024 12:21:43 -0500 Subject: [PATCH 3/4] Update appsettings.json --- DotNet/Submission/appsettings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DotNet/Submission/appsettings.json b/DotNet/Submission/appsettings.json index 79e4f5930..96c5c5706 100644 --- a/DotNet/Submission/appsettings.json +++ b/DotNet/Submission/appsettings.json @@ -12,7 +12,7 @@ "MongoDB": { "ConnectionString": "mongodb://localhost:27017", "DatabaseName": "linkReportDb" - } + }, "BlobStorageSettings": { "ContainerName": "", "ConnectionString": "" From 1c3b814b836123513192e299512bd13e0b1ef6a4 Mon Sep 17 00:00:00 2001 From: Keith Kissal <99497673+kissalk@users.noreply.github.com> Date: Tue, 5 Nov 2024 07:56:31 -0500 Subject: [PATCH 4/4] Update Dockerfile --- DotNet/Account/Dockerfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/DotNet/Account/Dockerfile b/DotNet/Account/Dockerfile index 4291259be..9d7987bdf 100644 --- a/DotNet/Account/Dockerfile +++ b/DotNet/Account/Dockerfile @@ -1,12 +1,5 @@ #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging. -# ARG ACCESS_TOKEN="mbygihet4uek5omhnuatzdxu5npiqz2rw3i6i7bgywaqwtshafaq" -# ARG ARTIFACTS_ENDPOINT="https://pkgs.dev.azure.com/lantanagroup/nhsnlink/_packaging/Shared_BOTW_Feed/nuget/v3/index.json" - -# Configure the environment variables -# ENV NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED true -# ENV VSS_NUGET_EXTERNAL_FEED_ENDPOINTS "{\"endpointCredentials\": [{\"endpoint\":\"${ARTIFACTS_ENDPOINT}\", \"password\":\"${ACCESS_TOKEN}\"}]}" - FROM mcr.microsoft.com/dotnet/aspnet:8.0-jammy-amd64 AS base WORKDIR /app