Merge remote-tracking branch 'origin/shliakhetskyi/get_stream_from_bl…

…ob' into storage-client
managedcode · Jun 2, 2024 · 3ccaa9a · 3ccaa9a
2 parents 57f3764 + c7df239
commit 3ccaa9a
Show file tree

Hide file tree

Showing 14 changed files with 198 additions and 80 deletions.
diff --git a/ManagedCode.Storage.Aws/AWSStorage.cs b/ManagedCode.Storage.Aws/AWSStorage.cs
@@ -13,6 +13,7 @@
 using ManagedCode.Storage.Core;
 using ManagedCode.Storage.Core.Models;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 
 namespace ManagedCode.Storage.Aws;
 
@@ -90,6 +91,13 @@ public override async IAsyncEnumerable<BlobMetadata> GetBlobMetadataListAsync(st
         } while (objectsRequest is not null);
     }
 
+    public override async Task<Result<Stream>> GetStreamAsync(string fileName, CancellationToken cancellationToken = default)
+    {
+        Stream stream = await StorageClient.GetObjectStreamAsync(StorageOptions.Bucket, fileName, null,
+                cancellationToken);
+        return Result<Stream>.Succeed(stream);
+    }
+
     protected override IAmazonS3 CreateStorageClient()
     {
         return new AmazonS3Client(new BasicAWSCredentials(StorageOptions.PublicKey, StorageOptions.SecretKey), StorageOptions.OriginalOptions);

diff --git a/ManagedCode.Storage.Azure.DataLake/AzureDataLakeStorage.cs b/ManagedCode.Storage.Azure.DataLake/AzureDataLakeStorage.cs
@@ -277,4 +277,16 @@ private DataLakeFileClient GetFileClient(BaseOptions options)
             _ => StorageClient.GetDirectoryClient(options.Directory).GetFileClient(options.FileName)
         };
     }
+
+    public override async Task<Result<Stream>> GetStreamAsync(string fileName, CancellationToken cancellationToken = default)
+    {
+        return await OpenReadStreamAsync(
+            new OpenReadStreamOptions()
+            {
+                FileName = fileName,
+                Position = 0,
+                BufferSize = 4096
+            },
+            cancellationToken);
+    }
 }
diff --git a/ManagedCode.Storage.Azure/AzureStorage.cs b/ManagedCode.Storage.Azure/AzureStorage.cs
@@ -376,4 +376,9 @@ public async Task<Result> SetStorageOptions(Action<IStorageOptions> options,
 
         return await CreateContainerAsync(cancellationToken);
     }
+
+    public async override Task<Result<Stream>> GetStreamAsync(string fileName, CancellationToken cancellationToken = default)
+    {
+        return await OpenReadStreamAsync(fileName, cancellationToken);
+    }
 }
diff --git a/ManagedCode.Storage.Client/IStorageClient.cs b/ManagedCode.Storage.Client/IStorageClient.cs
@@ -8,7 +8,7 @@
 
 namespace ManagedCode.Storage.Client;
 
-public interface IStorageClient : IUploader, IDownloader
+public interface IStorageClient
 {
     void SetChunkSize(long size);
 
@@ -20,6 +20,24 @@ public interface IStorageClient : IUploader, IDownloader
     ///     This includes the file name, progress percentage, total bytes, transferred bytes, elapsed time, remaining time, speed, and any error message.
     /// </remarks>
     event EventHandler<ProgressStatus> OnProgressStatusChanged;
+
+    Task<Result<BlobMetadata>> UploadFile(string base64, string apiUrl, string contentName, CancellationToken cancellationToken = default);
+
+    Task<Result<BlobMetadata>> UploadFile(byte[] bytes, string apiUrl, string contentName, CancellationToken cancellationToken = default);
+
+    Task<Result<BlobMetadata>> UploadFile(FileInfo fileInfo, string apiUrl, string contentName, CancellationToken cancellationToken = default);
+
+    Task<Result<BlobMetadata>> UploadFile(Stream stream, string apiUrl, string contentName, CancellationToken cancellationToken = default);
+
+    Task<Result<LocalFile>> DownloadFile(string fileName, string apiUrl, string? path = null, CancellationToken cancellationToken = default);
+
+    Task<Result<uint>> UploadLargeFile(Stream file,
+        string uploadApiUrl,
+        string completeApiUrl,
+        Action<double>? onProgressChanged,
+        CancellationToken cancellationToken = default);
+
+    Task<Result<Stream>> GetFileStream(string fileName, string apiUrl, CancellationToken cancellationToken = default);
 }
 
 

diff --git a/ManagedCode.Storage.Client/StorageClient.cs b/ManagedCode.Storage.Client/StorageClient.cs
@@ -1,13 +1,13 @@
-using System;
+using ManagedCode.Communication;
+using ManagedCode.Storage.Core.Models;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Net;
 using System.Net.Http;
 using System.Net.Http.Json;
 using System.Threading;
 using System.Threading.Tasks;
-using ManagedCode.Communication;
-using ManagedCode.Storage.Core.Models;
 
 namespace ManagedCode.Storage.Client;
 
@@ -191,78 +191,26 @@ public async Task<Result<uint>> UploadLargeFile(Stream file,
         return await mergeResult.Content.ReadFromJsonAsync<Result<uint>>(cancellationToken: cancellationToken);
     }
 
-    public Task<Result<BlobMetadata>> UploadAsync(Stream stream, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<Result<BlobMetadata>> UploadAsync(byte[] data, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<Result<BlobMetadata>> UploadAsync(string content, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<Result<BlobMetadata>> UploadAsync(FileInfo fileInfo, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<Result<BlobMetadata>> UploadAsync(Stream stream, UploadOptions options, CancellationToken cancellationToken = default)
+    public async Task<Result<Stream>> GetFileStream(string fileName, string apiUrl, CancellationToken cancellationToken = default)
     {
-        throw new NotImplementedException();
-    }
-
-    public Task<Result<BlobMetadata>> UploadAsync(byte[] data, UploadOptions options, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<Result<BlobMetadata>> UploadAsync(string content, UploadOptions options, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<Result<BlobMetadata>> UploadAsync(FileInfo fileInfo, UploadOptions options, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<Result<BlobMetadata>> UploadAsync(Stream stream, Action<UploadOptions> action, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<Result<BlobMetadata>> UploadAsync(byte[] data, Action<UploadOptions> action, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<Result<BlobMetadata>> UploadAsync(string content, Action<UploadOptions> action, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<Result<BlobMetadata>> UploadAsync(FileInfo fileInfo, Action<UploadOptions> action, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<Result<LocalFile>> DownloadAsync(string fileName, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
-
-    public Task<Result<LocalFile>> DownloadAsync(DownloadOptions options, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
-    }
+        try
+        {
+            var response = await _httpClient.GetAsync($"{apiUrl}/{fileName}");
+            if (response.IsSuccessStatusCode)
+            {
+                var stream = await response.Content.ReadAsStreamAsync();
+                return Result<Stream>.Succeed(stream);
+            }
 
-    public Task<Result<LocalFile>> DownloadAsync(Action<DownloadOptions> action, CancellationToken cancellationToken = default)
-    {
-        throw new NotImplementedException();
+            return Result<Stream>.Fail(response.StatusCode);
+        }
+        catch (HttpRequestException e) when (e.StatusCode != null)
+        {
+            return Result<Stream>.Fail(e.StatusCode.Value);
+        }
+        catch (Exception)
+        {
+            return Result<Stream>.Fail(HttpStatusCode.InternalServerError);
+        }
     }
 }
diff --git a/ManagedCode.Storage.Core/BaseStorage.cs b/ManagedCode.Storage.Core/BaseStorage.cs
@@ -261,6 +261,8 @@ public Task<Result> SetStorageOptions(Action<TOptions> options, CancellationToke
         return CreateContainerAsync(cancellationToken);
     }
 
+    public abstract Task<Result<Stream>> GetStreamAsync(string fileName, CancellationToken cancellationToken = default);
+
     public T StorageClient { get; protected set; }
 
     protected abstract T CreateStorageClient();

diff --git a/ManagedCode.Storage.Core/IStorage.cs b/ManagedCode.Storage.Core/IStorage.cs
@@ -15,7 +15,6 @@ public interface IStorage<out T, TOptions> : IStorage where TOptions : IStorageO
     Task<Result> SetStorageOptions(Action<TOptions> options, CancellationToken cancellationToken = default);
 }
 
-
 public interface IDownloader
 {
     /// <summary>
@@ -34,6 +33,14 @@ public interface IDownloader
     Task<Result<LocalFile>> DownloadAsync(Action<DownloadOptions> action, CancellationToken cancellationToken = default);
 }
 
+public interface IStreamer
+{
+    /// <summary>
+    ///     Gets file stream.
+    /// </summary>
+    Task<Result<Stream>> GetStreamAsync(string fileName, CancellationToken cancellationToken = default);
+}
+
 public interface IUploader
 {
     /// <summary>
@@ -186,7 +193,7 @@ public interface IStorageOperations
     Task<Result<bool>> HasLegalHoldAsync(Action<LegalHoldOptions> action, CancellationToken cancellationToken = default);
 }
 
-public interface IStorage : IUploader, IDownloader, IStorageOperations
+public interface IStorage : IUploader, IDownloader, IStreamer, IStorageOperations
 {
     /// <summary>
     ///     Create a container if it does not already exist.

diff --git a/ManagedCode.Storage.FileSystem/FileSystemStorage.cs b/ManagedCode.Storage.FileSystem/FileSystemStorage.cs
@@ -10,6 +10,7 @@
 using ManagedCode.Storage.Core;
 using ManagedCode.Storage.Core.Models;
 using ManagedCode.Storage.FileSystem.Options;
+using Microsoft.Extensions.Options;
 
 namespace ManagedCode.Storage.FileSystem;
 
@@ -236,4 +237,15 @@ private void EnsureDirectoryExist(string directory)
             Directory.CreateDirectory(path);
         }
     }
+
+    public override async Task<Result<Stream>> GetStreamAsync(string fileName, CancellationToken cancellationToken = default)
+    {
+        await EnsureContainerExist();
+
+        var filePath = GetPathFromOptions(new DownloadOptions() { FileName = fileName });
+
+        return File.Exists(filePath)
+            ? Result<Stream>.Succeed(new FileStream(filePath, FileMode.Open, FileAccess.Read))
+            : Result<Stream>.Fail("File not found");
+    }
 }
diff --git a/ManagedCode.Storage.Google/GCPStorage.cs b/ManagedCode.Storage.Google/GCPStorage.cs
@@ -4,6 +4,8 @@
 using System.IO;
 using System.Linq;
 using System.Net;
+using System.Net.Http;
+using System.Security.AccessControl;
 using System.Threading;
 using System.Threading.Tasks;
 using Google;
@@ -19,10 +21,15 @@ namespace ManagedCode.Storage.Google;
 public class GCPStorage : BaseStorage<StorageClient, GCPStorageOptions>, IGCPStorage
 {
     private readonly ILogger<GCPStorage>? _logger;
+    private UrlSigner urlSigner;
 
     public GCPStorage(GCPStorageOptions options, ILogger<GCPStorage>? logger = null) : base(options)
     {
         _logger = logger;
+        if (options.GoogleCredential != null)
+        {
+            urlSigner = UrlSigner.FromCredential(options.GoogleCredential);
+        }
     }
 
     public override async Task<Result> RemoveContainerAsync(CancellationToken cancellationToken = default)
@@ -282,4 +289,22 @@ protected override async Task<Result<bool>> HasLegalHoldInternalAsync(LegalHoldO
             return Result<bool>.Fail(ex);
         }
     }
+
+    public override async Task<Result<Stream>> GetStreamAsync(string fileName, CancellationToken cancellationToken = default)
+    {
+        await EnsureContainerExist();
+
+        if (urlSigner == null)
+        {
+            return Result<Stream>.Fail("Google credentials are required to get stream");
+        }
+
+        string signedUrl = urlSigner.Sign(StorageOptions.BucketOptions.Bucket, fileName, TimeSpan.FromHours(1), HttpMethod.Get);
+
+        using (HttpClient httpClient = new HttpClient())
+        {
+            Stream stream = await httpClient.GetStreamAsync(signedUrl, cancellationToken);
+            return Result<Stream>.Succeed(stream);
+        }
+    }
 }
diff --git a/ManagedCode.Storage.IntegrationTests/Constants/ApiEndpoints.cs b/ManagedCode.Storage.IntegrationTests/Constants/ApiEndpoints.cs
@@ -9,5 +9,6 @@ public static class Base
         public const string UploadFile = "{0}/upload";
         public const string UploadLargeFile = "{0}/upload-chunks";
         public const string DownloadFile = "{0}/download";
+        public const string StreamFile = "{0}/stream";
     }
 }
diff --git a/ManagedCode.Storage.IntegrationTests/StorageTestApplication.cs b/ManagedCode.Storage.IntegrationTests/StorageTestApplication.cs
@@ -27,7 +27,7 @@ public class StorageTestApplication : WebApplicationFactory<HttpHostProgram>, IC
     public StorageTestApplication()
     {
         _azuriteContainer = new AzuriteBuilder()
-            .WithImage("mcr.microsoft.com/azure-storage/azurite:3.26.0")
+            .WithImage("mcr.microsoft.com/azure-storage/azurite:3.29.0")
             .Build();
 
         _azuriteContainer.StartAsync().Wait();

diff --git a/ManagedCode.Storage.IntegrationTests/TestApp/Controllers/Base/BaseTestController.cs b/ManagedCode.Storage.IntegrationTests/TestApp/Controllers/Base/BaseTestController.cs
@@ -1,6 +1,5 @@
 using Amazon.Runtime.Internal;
 using ManagedCode.Communication;
-using ManagedCode.Storage.Aws;
 using ManagedCode.Storage.Core;
 using ManagedCode.Storage.Core.Helpers;
 using ManagedCode.Storage.Core.Models;
@@ -45,7 +44,17 @@ public async Task<FileResult> DownloadFileAsync([FromRoute] string fileName)
 
         return result.Value!;
     }
-
+
+    [HttpGet("stream/{fileName}")]
+    public async Task<Microsoft.AspNetCore.Http.IResult> StreamFileAsync([FromRoute] string fileName)
+    {
+        var result = await Storage.GetStreamAsync(fileName);
+        var metadataAsync = await Storage.GetBlobMetadataAsync(fileName);
+
+        result.ThrowIfFail();
+        return Results.Stream(result.Value, metadataAsync.Value?.MimeType ?? "application/octet-stream", fileName);
+    }
+
     [HttpPost("upload-chunks/upload")]
     public async Task<Result> UploadLargeFile([FromForm] FileUploadPayload file, CancellationToken cancellationToken = default)
     {

diff --git a/ManagedCode.Storage.IntegrationTests/Tests/Azure/AzureStreamControllerTests.cs b/ManagedCode.Storage.IntegrationTests/Tests/Azure/AzureStreamControllerTests.cs
@@ -0,0 +1,10 @@
+using ManagedCode.Storage.IntegrationTests.Constants;
+
+namespace ManagedCode.Storage.IntegrationTests.Tests.Azure;
+
+public class AzureStreamControllerTests : BaseStreamControllerTests
+{
+    public AzureStreamControllerTests(StorageTestApplication testApplication) : base(testApplication, ApiEndpoints.Azure)
+    {
+    }
+}