Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert saving of submission to blob storage #268

Draft
wants to merge 4 commits into
base: dev
Choose a base branch
from
Draft

Convert saving of submission to blob storage #268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
db82ded
Convert saving of submission to blob
edward-miller-lcg Jun 7, 2024
90ee98c
Update Submission.csproj
edward-miller-lcg Jun 7, 2024
1946418
Update appsettings.json
edward-miller-lcg Jun 7, 2024
1c3b814
Update Dockerfile
kissalk Nov 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions DotNet/Account/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,5 @@
#See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging.

# ARG ACCESS_TOKEN="mbygihet4uek5omhnuatzdxu5npiqz2rw3i6i7bgywaqwtshafaq"
# ARG ARTIFACTS_ENDPOINT="https://pkgs.dev.azure.com/lantanagroup/nhsnlink/_packaging/Shared_BOTW_Feed/nuget/v3/index.json"

# Configure the environment variables
# ENV NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED true
# ENV VSS_NUGET_EXTERNAL_FEED_ENDPOINTS "{\"endpointCredentials\": [{\"endpoint\":\"${ARTIFACTS_ENDPOINT}\", \"password\":\"${ACCESS_TOKEN}\"}]}"

FROM mcr.microsoft.com/dotnet/aspnet:8.0-jammy-amd64 AS base
WORKDIR /app

Expand Down
1 change: 1 addition & 0 deletions DotNet/Shared/Settings/ConfigurationConstants.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ public static class AppSettings
public const string DataProtection = "DataProtection";
public const string LinkTokenService = "LinkTokenService";
public const string SecretManagement = "SecretManagement";
public const string BlobStorageSettings = "BlobStorageSettings";
}

public static class DatabaseConnections
Expand Down
8 changes: 8 additions & 0 deletions DotNet/Submission/Application/Interfaces/IBlobStorageRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
namespace LantanaGroup.Link.Submission.Application.Interfaces;

public interface IBlobStorageRepository
{
Task UploadBlobAsync(string directoryName, string blobName, string blob);
Task<Stream> DownloadBlobAsync(string blobName);
Task DeleteBlobAsync(string blobName);
}
39 changes: 39 additions & 0 deletions DotNet/Submission/Application/Repositories/BlobStorageRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
using Azure.Storage.Blobs;
using LantanaGroup.Link.Submission.Application.Interfaces;
using LantanaGroup.Link.Submission.Settings;
using Microsoft.Extensions.Options;

namespace LantanaGroup.Link.Submission.Application.Repositories;

public class BlobStorageRepository : IBlobStorageRepository
{
private readonly ILogger<BlobStorageRepository> _logger;
private readonly BlobStorageSettings _blobStorageSettings;

public BlobStorageRepository(IOptions<BlobStorageSettings> blobStorageSettings, ILogger<BlobStorageRepository> logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_blobStorageSettings = blobStorageSettings.Value ?? throw new ArgumentNullException(nameof(blobStorageSettings));
}

public Task DeleteBlobAsync(string blobName)
{
throw new NotImplementedException();
}

public Task<Stream> DownloadBlobAsync(string blobName)
{
throw new NotImplementedException();
}

public async Task UploadBlobAsync(string directoryName, string blobName, string blob)
{
BlobContainerClient containerClient = new BlobContainerClient(_blobStorageSettings.ConnectionString, _blobStorageSettings.ContainerName);
containerClient.CreateIfNotExists();
BlobClient blobClient = containerClient.GetBlobClient($"{directoryName}/{blobName}");

await blobClient.UploadAsync(BinaryData.FromString(blob), overwrite: true);
}


}
32 changes: 12 additions & 20 deletions DotNet/Submission/Listeners/SubmitReportListener.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
using StackExchange.Redis;
using System.Text.Json;
using Task = System.Threading.Tasks.Task;
using LantanaGroup.Link.Submission.Application.Interfaces;

namespace LantanaGroup.Link.Submission.Listeners
{
Expand All @@ -30,14 +31,17 @@ public class SubmitReportListener : BackgroundService
private readonly ITransientExceptionHandler<SubmitReportKey, SubmitReportValue> _transientExceptionHandler;
private readonly IDeadLetterExceptionHandler<SubmitReportKey, SubmitReportValue> _deadLetterExceptionHandler;

private readonly IBlobStorageRepository _blobStorageRepository;

private string Name => this.GetType().Name;

public SubmitReportListener(ILogger<SubmitReportListener> logger,
IKafkaConsumerFactory<SubmitReportKey, SubmitReportValue> kafkaConsumerFactory,
IMediator mediator, IOptions<SubmissionServiceConfig> submissionConfig,
IOptions<FileSystemConfig> fileSystemConfig, IHttpClientFactory httpClient,
ITransientExceptionHandler<SubmitReportKey, SubmitReportValue> transientExceptionHandler,
IDeadLetterExceptionHandler<SubmitReportKey, SubmitReportValue> deadLetterExceptionHandler)
IDeadLetterExceptionHandler<SubmitReportKey, SubmitReportValue> deadLetterExceptionHandler,
IBlobStorageRepository blobStorageRepository)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_kafkaConsumerFactory = kafkaConsumerFactory ?? throw new ArgumentException(nameof(kafkaConsumerFactory));
Expand All @@ -56,6 +60,7 @@ public SubmitReportListener(ILogger<SubmitReportListener> logger,

_deadLetterExceptionHandler.ServiceName = "Submission";
_deadLetterExceptionHandler.Topic = nameof(KafkaTopic.SubmitReport) + "-Error";
_blobStorageRepository = blobStorageRepository ?? throw new ArgumentNullException(nameof(blobStorageRepository));
}

protected override Task ExecuteAsync(CancellationToken stoppingToken)
Expand Down Expand Up @@ -219,13 +224,6 @@ await consumer.ConsumeWithInstrumentation(async (result, cancellationToken) =>
var fhirSerializer = new FhirJsonSerializer();
try
{
if (Directory.Exists(submissionDirectory))
{
Directory.Delete(submissionDirectory, true);
}

Directory.CreateDirectory(submissionDirectory);

#region Device

Hl7.Fhir.Model.Device device = new Device();
Expand All @@ -237,8 +235,7 @@ await consumer.ConsumeWithInstrumentation(async (result, cancellationToken) =>
fileName = "sending-device.json";
contents = await fhirSerializer.SerializeToStringAsync(device);

await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents,
cancellationToken);
await _blobStorageRepository.UploadBlobAsync(submissionDirectory, fileName, contents);

#endregion

Expand All @@ -247,8 +244,7 @@ await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents,
fileName = "sending-organization.json";
contents = await fhirSerializer.SerializeToStringAsync(value.Organization);

await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents,
cancellationToken);
await _blobStorageRepository.UploadBlobAsync(submissionDirectory, fileName, contents);

#endregion

Expand All @@ -257,8 +253,7 @@ await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents,
fileName = "patient-list.json";
contents = await fhirSerializer.SerializeToStringAsync(admittedPatients);

await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents,
cancellationToken);
await _blobStorageRepository.UploadBlobAsync(submissionDirectory, fileName, contents);

#endregion

Expand All @@ -267,8 +262,7 @@ await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents,
fileName = "query-plan.json";
contents = queryPlans;

await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents,
cancellationToken);
await _blobStorageRepository.UploadBlobAsync(submissionDirectory, fileName, contents);

#endregion

Expand All @@ -280,8 +274,7 @@ await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents,
fileName = $"aggregate-{measureShortName}.json";
contents = await fhirSerializer.SerializeToStringAsync(aggregate);

await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents,
cancellationToken);
await _blobStorageRepository.UploadBlobAsync(submissionDirectory, fileName, contents);
}

#endregion
Expand Down Expand Up @@ -349,8 +342,7 @@ await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents,
fileName = "other-resources.json";
contents = await fhirSerializer.SerializeToStringAsync(otherResourcesBundle);

await File.WriteAllTextAsync(submissionDirectory + "/" + fileName, contents,
cancellationToken);
await _blobStorageRepository.UploadBlobAsync(submissionDirectory, fileName, contents);

#endregion
}
Expand Down
3 changes: 2 additions & 1 deletion DotNet/Submission/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ static void RegisterServices(WebApplicationBuilder builder)
builder.Services.Configure<SubmissionServiceConfig>(builder.Configuration.GetRequiredSection(nameof(SubmissionServiceConfig)));
builder.Services.Configure<ConsumerSettings>(builder.Configuration.GetRequiredSection(nameof(ConsumerSettings)));
builder.Services.Configure<CorsSettings>(builder.Configuration.GetSection(ConfigurationConstants.AppSettings.CORS));
builder.Services.Configure<BlobStorageSettings>(builder.Configuration.GetSection(ConfigurationConstants.AppSettings.BlobStorageSettings));

// Add services to the container.
builder.Services.AddHttpClient();
Expand Down Expand Up @@ -137,7 +138,7 @@ static void RegisterServices(WebApplicationBuilder builder)
builder.Services.AddTransient<IRetryEntityFactory, RetryEntityFactory>();

// Add repositories
// TODO
builder.Services.AddTransient<IBlobStorageRepository, BlobStorageRepository>();

#region Exception Handling
//Report Scheduled Listener
Expand Down
7 changes: 7 additions & 0 deletions DotNet/Submission/Settings/BlobStorageSettings.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
namespace LantanaGroup.Link.Submission.Settings;

public class BlobStorageSettings
{
public string ConnectionString { get; set; }
public string ContainerName { get; set; }
}
1 change: 1 addition & 0 deletions DotNet/Submission/Submission.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
<ItemGroup>
<PackageReference Include="AspNetCore.HealthChecks.UI.Client" Version="8.*" />
<PackageReference Include="Azure.Identity" Version="1.11.2" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.20.0" />
<PackageReference Include="Confluent.Kafka" Version="2.*" />
<PackageReference Include="Confluent.Kafka.Extensions.OpenTelemetry" Version="0.*" />
<PackageReference Include="MediatR" Version="12.*" />
Expand Down
4 changes: 4 additions & 0 deletions DotNet/Submission/appsettings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@
"ConnectionString": "mongodb://localhost:27017",
"DatabaseName": "linkReportDb"
},
"BlobStorageSettings": {
"ContainerName": "",
"ConnectionString": ""
},
"SubmissionServiceConfig": {
"ReportServiceUrl": "",
"CensusUrl": "",
Expand Down