diff --git a/ManagedCode.Storage.Aws/AWSStorage.cs b/ManagedCode.Storage.Aws/AWSStorage.cs index 0dd4156..51a9ebd 100644 --- a/ManagedCode.Storage.Aws/AWSStorage.cs +++ b/ManagedCode.Storage.Aws/AWSStorage.cs @@ -13,6 +13,7 @@ using ManagedCode.Storage.Core; using ManagedCode.Storage.Core.Models; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace ManagedCode.Storage.Aws; @@ -90,6 +91,13 @@ public override async IAsyncEnumerable GetBlobMetadataListAsync(st } while (objectsRequest is not null); } + public override async Task> GetStreamAsync(string fileName, CancellationToken cancellationToken = default) + { + Stream stream = await StorageClient.GetObjectStreamAsync(StorageOptions.Bucket, fileName, null, + cancellationToken); + return Result.Succeed(stream); + } + protected override IAmazonS3 CreateStorageClient() { return new AmazonS3Client(new BasicAWSCredentials(StorageOptions.PublicKey, StorageOptions.SecretKey), StorageOptions.OriginalOptions); diff --git a/ManagedCode.Storage.Azure.DataLake/AzureDataLakeStorage.cs b/ManagedCode.Storage.Azure.DataLake/AzureDataLakeStorage.cs index 1fe4e22..26bc852 100644 --- a/ManagedCode.Storage.Azure.DataLake/AzureDataLakeStorage.cs +++ b/ManagedCode.Storage.Azure.DataLake/AzureDataLakeStorage.cs @@ -277,4 +277,16 @@ private DataLakeFileClient GetFileClient(BaseOptions options) _ => StorageClient.GetDirectoryClient(options.Directory).GetFileClient(options.FileName) }; } + + public override async Task> GetStreamAsync(string fileName, CancellationToken cancellationToken = default) + { + return await OpenReadStreamAsync( + new OpenReadStreamOptions() + { + FileName = fileName, + Position = 0, + BufferSize = 4096 + }, + cancellationToken); + } } \ No newline at end of file diff --git a/ManagedCode.Storage.Azure/AzureStorage.cs b/ManagedCode.Storage.Azure/AzureStorage.cs index 0fe8fcd..12374e3 100644 --- a/ManagedCode.Storage.Azure/AzureStorage.cs +++ b/ManagedCode.Storage.Azure/AzureStorage.cs @@ -376,4 +376,9 @@ public async Task SetStorageOptions(Action options, return await CreateContainerAsync(cancellationToken); } + + public async override Task> GetStreamAsync(string fileName, CancellationToken cancellationToken = default) + { + return await OpenReadStreamAsync(fileName, cancellationToken); + } } \ No newline at end of file diff --git a/ManagedCode.Storage.Client/IStorageClient.cs b/ManagedCode.Storage.Client/IStorageClient.cs index 2f7725b..6ea9826 100644 --- a/ManagedCode.Storage.Client/IStorageClient.cs +++ b/ManagedCode.Storage.Client/IStorageClient.cs @@ -8,7 +8,7 @@ namespace ManagedCode.Storage.Client; -public interface IStorageClient : IUploader, IDownloader +public interface IStorageClient { void SetChunkSize(long size); @@ -20,6 +20,24 @@ public interface IStorageClient : IUploader, IDownloader /// This includes the file name, progress percentage, total bytes, transferred bytes, elapsed time, remaining time, speed, and any error message. /// event EventHandler OnProgressStatusChanged; + + Task> UploadFile(string base64, string apiUrl, string contentName, CancellationToken cancellationToken = default); + + Task> UploadFile(byte[] bytes, string apiUrl, string contentName, CancellationToken cancellationToken = default); + + Task> UploadFile(FileInfo fileInfo, string apiUrl, string contentName, CancellationToken cancellationToken = default); + + Task> UploadFile(Stream stream, string apiUrl, string contentName, CancellationToken cancellationToken = default); + + Task> DownloadFile(string fileName, string apiUrl, string? path = null, CancellationToken cancellationToken = default); + + Task> UploadLargeFile(Stream file, + string uploadApiUrl, + string completeApiUrl, + Action? onProgressChanged, + CancellationToken cancellationToken = default); + + Task> GetFileStream(string fileName, string apiUrl, CancellationToken cancellationToken = default); } diff --git a/ManagedCode.Storage.Client/StorageClient.cs b/ManagedCode.Storage.Client/StorageClient.cs index f6d05ec..2e92407 100644 --- a/ManagedCode.Storage.Client/StorageClient.cs +++ b/ManagedCode.Storage.Client/StorageClient.cs @@ -1,4 +1,6 @@ -using System; +using ManagedCode.Communication; +using ManagedCode.Storage.Core.Models; +using System; using System.Collections.Generic; using System.IO; using System.Net; @@ -6,8 +8,6 @@ using System.Net.Http.Json; using System.Threading; using System.Threading.Tasks; -using ManagedCode.Communication; -using ManagedCode.Storage.Core.Models; namespace ManagedCode.Storage.Client; @@ -191,78 +191,26 @@ public async Task> UploadLargeFile(Stream file, return await mergeResult.Content.ReadFromJsonAsync>(cancellationToken: cancellationToken); } - public Task> UploadAsync(Stream stream, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public Task> UploadAsync(byte[] data, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public Task> UploadAsync(string content, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public Task> UploadAsync(FileInfo fileInfo, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public Task> UploadAsync(Stream stream, UploadOptions options, CancellationToken cancellationToken = default) + public async Task> GetFileStream(string fileName, string apiUrl, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); - } - - public Task> UploadAsync(byte[] data, UploadOptions options, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public Task> UploadAsync(string content, UploadOptions options, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public Task> UploadAsync(FileInfo fileInfo, UploadOptions options, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public Task> UploadAsync(Stream stream, Action action, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public Task> UploadAsync(byte[] data, Action action, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public Task> UploadAsync(string content, Action action, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public Task> UploadAsync(FileInfo fileInfo, Action action, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public Task> DownloadAsync(string fileName, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public Task> DownloadAsync(DownloadOptions options, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } + try + { + var response = await _httpClient.GetAsync($"{apiUrl}/{fileName}"); + if (response.IsSuccessStatusCode) + { + var stream = await response.Content.ReadAsStreamAsync(); + return Result.Succeed(stream); + } - public Task> DownloadAsync(Action action, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); + return Result.Fail(response.StatusCode); + } + catch (HttpRequestException e) when (e.StatusCode != null) + { + return Result.Fail(e.StatusCode.Value); + } + catch (Exception) + { + return Result.Fail(HttpStatusCode.InternalServerError); + } } } \ No newline at end of file diff --git a/ManagedCode.Storage.Core/BaseStorage.cs b/ManagedCode.Storage.Core/BaseStorage.cs index 0e3fa73..b0c7ee7 100644 --- a/ManagedCode.Storage.Core/BaseStorage.cs +++ b/ManagedCode.Storage.Core/BaseStorage.cs @@ -261,6 +261,8 @@ public Task SetStorageOptions(Action options, CancellationToke return CreateContainerAsync(cancellationToken); } + public abstract Task> GetStreamAsync(string fileName, CancellationToken cancellationToken = default); + public T StorageClient { get; protected set; } protected abstract T CreateStorageClient(); diff --git a/ManagedCode.Storage.Core/IStorage.cs b/ManagedCode.Storage.Core/IStorage.cs index a9d22ba..a7d5c1f 100644 --- a/ManagedCode.Storage.Core/IStorage.cs +++ b/ManagedCode.Storage.Core/IStorage.cs @@ -15,7 +15,6 @@ public interface IStorage : IStorage where TOptions : IStorageO Task SetStorageOptions(Action options, CancellationToken cancellationToken = default); } - public interface IDownloader { /// @@ -34,6 +33,14 @@ public interface IDownloader Task> DownloadAsync(Action action, CancellationToken cancellationToken = default); } +public interface IStreamer +{ + /// + /// Gets file stream. + /// + Task> GetStreamAsync(string fileName, CancellationToken cancellationToken = default); +} + public interface IUploader { /// @@ -186,7 +193,7 @@ public interface IStorageOperations Task> HasLegalHoldAsync(Action action, CancellationToken cancellationToken = default); } -public interface IStorage : IUploader, IDownloader, IStorageOperations +public interface IStorage : IUploader, IDownloader, IStreamer, IStorageOperations { /// /// Create a container if it does not already exist. diff --git a/ManagedCode.Storage.FileSystem/FileSystemStorage.cs b/ManagedCode.Storage.FileSystem/FileSystemStorage.cs index 6f972ed..402c956 100644 --- a/ManagedCode.Storage.FileSystem/FileSystemStorage.cs +++ b/ManagedCode.Storage.FileSystem/FileSystemStorage.cs @@ -10,6 +10,7 @@ using ManagedCode.Storage.Core; using ManagedCode.Storage.Core.Models; using ManagedCode.Storage.FileSystem.Options; +using Microsoft.Extensions.Options; namespace ManagedCode.Storage.FileSystem; @@ -236,4 +237,15 @@ private void EnsureDirectoryExist(string directory) Directory.CreateDirectory(path); } } + + public override async Task> GetStreamAsync(string fileName, CancellationToken cancellationToken = default) + { + await EnsureContainerExist(); + + var filePath = GetPathFromOptions(new DownloadOptions() { FileName = fileName }); + + return File.Exists(filePath) + ? Result.Succeed(new FileStream(filePath, FileMode.Open, FileAccess.Read)) + : Result.Fail("File not found"); + } } \ No newline at end of file diff --git a/ManagedCode.Storage.Google/GCPStorage.cs b/ManagedCode.Storage.Google/GCPStorage.cs index d7b2881..ef7784a 100644 --- a/ManagedCode.Storage.Google/GCPStorage.cs +++ b/ManagedCode.Storage.Google/GCPStorage.cs @@ -4,6 +4,8 @@ using System.IO; using System.Linq; using System.Net; +using System.Net.Http; +using System.Security.AccessControl; using System.Threading; using System.Threading.Tasks; using Google; @@ -19,10 +21,15 @@ namespace ManagedCode.Storage.Google; public class GCPStorage : BaseStorage, IGCPStorage { private readonly ILogger? _logger; + private UrlSigner urlSigner; public GCPStorage(GCPStorageOptions options, ILogger? logger = null) : base(options) { _logger = logger; + if (options.GoogleCredential != null) + { + urlSigner = UrlSigner.FromCredential(options.GoogleCredential); + } } public override async Task RemoveContainerAsync(CancellationToken cancellationToken = default) @@ -282,4 +289,22 @@ protected override async Task> HasLegalHoldInternalAsync(LegalHoldO return Result.Fail(ex); } } + + public override async Task> GetStreamAsync(string fileName, CancellationToken cancellationToken = default) + { + await EnsureContainerExist(); + + if (urlSigner == null) + { + return Result.Fail("Google credentials are required to get stream"); + } + + string signedUrl = urlSigner.Sign(StorageOptions.BucketOptions.Bucket, fileName, TimeSpan.FromHours(1), HttpMethod.Get); + + using (HttpClient httpClient = new HttpClient()) + { + Stream stream = await httpClient.GetStreamAsync(signedUrl, cancellationToken); + return Result.Succeed(stream); + } + } } \ No newline at end of file diff --git a/ManagedCode.Storage.IntegrationTests/Constants/ApiEndpoints.cs b/ManagedCode.Storage.IntegrationTests/Constants/ApiEndpoints.cs index 781d840..6ee714c 100644 --- a/ManagedCode.Storage.IntegrationTests/Constants/ApiEndpoints.cs +++ b/ManagedCode.Storage.IntegrationTests/Constants/ApiEndpoints.cs @@ -9,5 +9,6 @@ public static class Base public const string UploadFile = "{0}/upload"; public const string UploadLargeFile = "{0}/upload-chunks"; public const string DownloadFile = "{0}/download"; + public const string StreamFile = "{0}/stream"; } } \ No newline at end of file diff --git a/ManagedCode.Storage.IntegrationTests/StorageTestApplication.cs b/ManagedCode.Storage.IntegrationTests/StorageTestApplication.cs index a225b1d..3694901 100644 --- a/ManagedCode.Storage.IntegrationTests/StorageTestApplication.cs +++ b/ManagedCode.Storage.IntegrationTests/StorageTestApplication.cs @@ -27,7 +27,7 @@ public class StorageTestApplication : WebApplicationFactory, IC public StorageTestApplication() { _azuriteContainer = new AzuriteBuilder() - .WithImage("mcr.microsoft.com/azure-storage/azurite:3.26.0") + .WithImage("mcr.microsoft.com/azure-storage/azurite:3.29.0") .Build(); _azuriteContainer.StartAsync().Wait(); diff --git a/ManagedCode.Storage.IntegrationTests/TestApp/Controllers/Base/BaseTestController.cs b/ManagedCode.Storage.IntegrationTests/TestApp/Controllers/Base/BaseTestController.cs index 02eb07e..ff0c45f 100644 --- a/ManagedCode.Storage.IntegrationTests/TestApp/Controllers/Base/BaseTestController.cs +++ b/ManagedCode.Storage.IntegrationTests/TestApp/Controllers/Base/BaseTestController.cs @@ -1,6 +1,5 @@ using Amazon.Runtime.Internal; using ManagedCode.Communication; -using ManagedCode.Storage.Aws; using ManagedCode.Storage.Core; using ManagedCode.Storage.Core.Helpers; using ManagedCode.Storage.Core.Models; @@ -45,7 +44,17 @@ public async Task DownloadFileAsync([FromRoute] string fileName) return result.Value!; } - + + [HttpGet("stream/{fileName}")] + public async Task StreamFileAsync([FromRoute] string fileName) + { + var result = await Storage.GetStreamAsync(fileName); + var metadataAsync = await Storage.GetBlobMetadataAsync(fileName); + + result.ThrowIfFail(); + return Results.Stream(result.Value, metadataAsync.Value?.MimeType ?? "application/octet-stream", fileName); + } + [HttpPost("upload-chunks/upload")] public async Task UploadLargeFile([FromForm] FileUploadPayload file, CancellationToken cancellationToken = default) { diff --git a/ManagedCode.Storage.IntegrationTests/Tests/Azure/AzureStreamControllerTests.cs b/ManagedCode.Storage.IntegrationTests/Tests/Azure/AzureStreamControllerTests.cs new file mode 100644 index 0000000..e2f29e1 --- /dev/null +++ b/ManagedCode.Storage.IntegrationTests/Tests/Azure/AzureStreamControllerTests.cs @@ -0,0 +1,10 @@ +using ManagedCode.Storage.IntegrationTests.Constants; + +namespace ManagedCode.Storage.IntegrationTests.Tests.Azure; + +public class AzureStreamControllerTests : BaseStreamControllerTests +{ + public AzureStreamControllerTests(StorageTestApplication testApplication) : base(testApplication, ApiEndpoints.Azure) + { + } +} \ No newline at end of file diff --git a/ManagedCode.Storage.IntegrationTests/Tests/BaseStreamControllerTests.cs b/ManagedCode.Storage.IntegrationTests/Tests/BaseStreamControllerTests.cs new file mode 100644 index 0000000..4e61243 --- /dev/null +++ b/ManagedCode.Storage.IntegrationTests/Tests/BaseStreamControllerTests.cs @@ -0,0 +1,61 @@ +using System.Net; +using FluentAssertions; +using ManagedCode.Storage.Core.Helpers; +using ManagedCode.Storage.Core.Models; +using ManagedCode.Storage.IntegrationTests.Constants; +using Org.BouncyCastle.Crypto.IO; +using Xunit; + +namespace ManagedCode.Storage.IntegrationTests.Tests; + +public abstract class BaseStreamControllerTests : BaseControllerTests +{ + private readonly string _streamEndpoint; + private readonly string _uploadEndpoint; + + protected BaseStreamControllerTests(StorageTestApplication testApplication, string apiEndpoint) : base(testApplication, apiEndpoint) + { + _streamEndpoint = string.Format(ApiEndpoints.Base.StreamFile, ApiEndpoint); + _uploadEndpoint = string.Format(ApiEndpoints.Base.UploadFile, ApiEndpoint); + } + + [Fact] + public async Task StreamFile_WhenFileExists_SaveToTempStorage_ReturnSuccess() + { + // Arrange + var storageClient = GetStorageClient(); + var contentName = "file"; + var extension = ".txt"; + await using var localFile = LocalFile.FromRandomNameWithExtension(extension); + FileHelper.GenerateLocalFile(localFile, 1); + var fileCRC = Crc32Helper.Calculate(await localFile.ReadAllBytesAsync()); + var uploadFileBlob = await storageClient.UploadFile(localFile.FileStream, _uploadEndpoint, contentName); + + // Act + var streamFileResult = await storageClient.GetFileStream(uploadFileBlob.Value.FullName, _streamEndpoint); + + // Assert + streamFileResult.IsSuccess.Should().BeTrue(); + streamFileResult.Should().NotBeNull(); + + await using var stream = streamFileResult.Value; + await using var newLocalFile = await LocalFile.FromStreamAsync(stream, Path.GetTempPath(), Guid.NewGuid().ToString("N") + extension); + + var streamedFileCRC = Crc32Helper.CalculateFileCRC(newLocalFile.FilePath); + streamedFileCRC.Should().Be(fileCRC); + } + + [Fact] + public async Task StreamFile_WhenFileDoNotExist_ReturnFail() + { + // Arrange + var storageClient = GetStorageClient(); ; + + // Act + var streamFileResult = await storageClient.GetFileStream(Guid.NewGuid().ToString(), _streamEndpoint); + + // Assert + streamFileResult.IsFailed.Should().BeTrue(); + streamFileResult.GetError().Value.ErrorCode.Should().Be(HttpStatusCode.InternalServerError.ToString()); + } +} \ No newline at end of file