diff --git a/.docs/content/0.installation/5.version-migration.md b/.docs/content/0.installation/5.version-migration.md new file mode 100644 index 000000000..c8b25674a --- /dev/null +++ b/.docs/content/0.installation/5.version-migration.md @@ -0,0 +1,51 @@ +# How to migrate ArmoniK.Core dependencies during upgrade ? + +## 0.28.x -> 0.29.x + +### Database + +This version changes the structure of a Result in the database. It introduces a new field called `OpaqueId` which holds the identifier of its associated value in the Object Storage. Previously, the ResultId was used. The following MongoDB request converts the ResultId into the OpaqueId to support the new implementation. + +```js +db.Result.updateMany({}, + [{ + $addFields: { + OpaqueId: { + $function: { + body: function(data) { + const base64Chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + const bytes = []; + for (let i = 0; i < data.length; i++) { + bytes.push(data.charCodeAt(i)); + } + + let base64 = ''; + let i = 0; + while (i < bytes.length) { + let byte1 = bytes[i++] || 0; + let byte2 = bytes[i++] || 0; + let byte3 = bytes[i++] || 0; + + let enc1 = byte1 >> 2; + let enc2 = ((byte1 & 3) << 4) | (byte2 >> 4); + let enc3 = ((byte2 & 15) << 2) | (byte3 >> 6); + let enc4 = byte3 & 63; + + if (isNaN(byte2)) { + enc3 = enc4 = 64; + } else if (isNaN(byte3)) { + enc4 = 64; + } + + base64 += base64Chars[enc1] + base64Chars[enc2] + base64Chars[enc3] + base64Chars[enc4]; + } + + return BinData(0, base64); + }, + args: ["$_id"], + lang: "js" + } + } + } + }]) +``` diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 75dbbfd5d..22b2dfc06 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,6 +40,7 @@ jobs: - Adaptors/MongoDB/tests - Adaptors/Memory/tests - Adaptors/S3/tests + - Adaptors/Embed/tests os: - ubuntu-latest fail-fast: false @@ -179,6 +180,7 @@ jobs: - Common/tests - Adaptors/MongoDB/tests - Adaptors/Memory/tests + - Adaptors/Embed/tests fail-fast: false runs-on: windows-latest steps: @@ -189,12 +191,10 @@ jobs: submodules: true - name: Dotnet Restore - run: | - dotnet restore + run: dotnet restore - name: Dotnet Build - run: | - dotnet build + run: dotnet build - name: Run tests run: | @@ -440,6 +440,7 @@ jobs: object: - redis - minio + - embed log-level: - Information - Verbose diff --git a/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj new file mode 100644 index 000000000..05f2744e5 --- /dev/null +++ b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj @@ -0,0 +1,74 @@ + + + net8.0 + win-x64;linux-x64;linux-arm64 + True + ANEO + Copyright (C) ANEO, 2021-2022 + AGPL-3.0-or-later + True + true + enable + true + + + + Embedded + true + DEBUG;TRACE + + + + true + true + snupkg + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + runtime + + + runtime + + + runtime + + + false + runtime + + + runtime + + + runtime + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + runtime + + + + + + false + runtime + + + false + runtime + + + + diff --git a/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj.DotSettings b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj.DotSettings new file mode 100644 index 000000000..89316e414 --- /dev/null +++ b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj.DotSettings @@ -0,0 +1,2 @@ + + Library \ No newline at end of file diff --git a/Adaptors/MongoDB/src/Options/QueueStorage.cs b/Adaptors/Embed/src/ObjectBuilder.cs similarity index 51% rename from Adaptors/MongoDB/src/Options/QueueStorage.cs rename to Adaptors/Embed/src/ObjectBuilder.cs index 8ca8cb351..28f7063ef 100644 --- a/Adaptors/MongoDB/src/Options/QueueStorage.cs +++ b/Adaptors/Embed/src/ObjectBuilder.cs @@ -1,4 +1,4 @@ -// This file is part of the ArmoniK project +// This file is part of the ArmoniK project // // Copyright (C) ANEO, 2021-2024. All rights reserved. // @@ -15,20 +15,27 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -using System; +using ArmoniK.Core.Base; +using ArmoniK.Core.Utils; using JetBrains.Annotations; -namespace ArmoniK.Core.Adapters.MongoDB.Options; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +namespace ArmoniK.Core.Adapters.Embed; + +/// +/// Class for building Embed instance and Object interfaces through Dependency Injection +/// [PublicAPI] -public class QueueStorage +public class ObjectBuilder : IDependencyInjectionBuildable { - public const string SettingSection = nameof(MongoDB) + ":" + nameof(QueueStorage); - - public TimeSpan LockRefreshPeriodicity { get; set; } = TimeSpan.FromMinutes(2); - - public TimeSpan PollPeriodicity { get; set; } = TimeSpan.FromSeconds(5); - - public TimeSpan LockRefreshExtension { get; set; } = TimeSpan.FromMinutes(5); + /// + [PublicAPI] + public void Build(IServiceCollection serviceCollection, + ConfigurationManager configuration, + ILogger logger) + => serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); } diff --git a/Adaptors/Embed/src/ObjectStorage.cs b/Adaptors/Embed/src/ObjectStorage.cs new file mode 100644 index 000000000..8bb28d693 --- /dev/null +++ b/Adaptors/Embed/src/ObjectStorage.cs @@ -0,0 +1,86 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using ArmoniK.Core.Base; +using ArmoniK.Core.Base.DataStructures; + +using JetBrains.Annotations; + +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +namespace ArmoniK.Core.Adapters.Embed; + +[UsedImplicitly] +public class ObjectStorage : IObjectStorage +{ + private readonly ILogger logger_; + private bool isInitialized_; + + /// + /// implementation for Redis + /// + /// Logger used to print logs + public ObjectStorage(ILogger logger) + => logger_ = logger; + + /// + public Task Init(CancellationToken cancellationToken) + { + isInitialized_ = true; + return Task.CompletedTask; + } + + /// + public Task Check(HealthCheckTag tag) + => Task.FromResult(isInitialized_ + ? HealthCheckResult.Healthy() + : HealthCheckResult.Unhealthy("Object storage not initialized yet.")); + + /// + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) + { + var array = new List(); + + await foreach (var val in valueChunks.WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + array.AddRange(val.Span); + } + + return (array.ToArray(), array.Count); + } + + /// + public IAsyncEnumerable GetValuesAsync(byte[] id, + CancellationToken cancellationToken = default) + => AsyncEnumerable.Repeat(id, + 1); + + /// + public Task TryDeleteAsync(IEnumerable ids, + CancellationToken cancellationToken = default) + => Task.CompletedTask; +} diff --git a/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj b/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj new file mode 100644 index 000000000..975cca170 --- /dev/null +++ b/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj @@ -0,0 +1,28 @@ + + + net8.0 + win-x64;linux-x64;linux-arm64 + false + enable + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + diff --git a/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj.DotSettings b/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj.DotSettings new file mode 100644 index 000000000..89316e414 --- /dev/null +++ b/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj.DotSettings @@ -0,0 +1,2 @@ + + Library \ No newline at end of file diff --git a/Adaptors/Embed/tests/ObjectStorageTests.cs b/Adaptors/Embed/tests/ObjectStorageTests.cs new file mode 100644 index 000000000..472d36a4a --- /dev/null +++ b/Adaptors/Embed/tests/ObjectStorageTests.cs @@ -0,0 +1,78 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System.Collections.Generic; +using System.IO; + +using ArmoniK.Core.Base; +using ArmoniK.Core.Common.Injection; +using ArmoniK.Core.Common.Injection.Options; +using ArmoniK.Core.Common.Tests.TestBase; +using ArmoniK.Core.Common.Utils; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +using NUnit.Framework; + +namespace ArmoniK.Core.Adapters.Embed.Tests; + +[TestFixture] +public class ObjectStorageTests : ObjectStorageTestBase +{ + private static readonly string SolutionRoot = + Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(typeof(ObjectStorageTests) + .Assembly + .Location))))) ?? + string.Empty)); + + private static readonly string S3Path = + $"{Path.DirectorySeparatorChar}src{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}Debug{Path.DirectorySeparatorChar}net8.0{Path.DirectorySeparatorChar}ArmoniK.Core.Adapters.Embed.dll"; + + + protected override void GetObjectStorageInstance() + { + Dictionary minimalConfig = new() + { + { + "Components:ObjectStorageAdaptorSettings:ClassName", "ArmoniK.Core.Adapters.Embed.ObjectBuilder" + }, + { + "Components:ObjectStorageAdaptorSettings:AdapterAbsolutePath", $"{SolutionRoot}{S3Path}" + }, + }; + + var configuration = new ConfigurationManager(); + configuration.AddInMemoryCollection(minimalConfig); + + var services = new ServiceCollection(); + services.AddLogging(); + var logger = new LoggerInit(configuration); + + services.AddAdapter(configuration, + nameof(Components.ObjectStorageAdaptorSettings), + logger.GetLogger()); + + var provider = services.BuildServiceProvider(new ServiceProviderOptions + { + ValidateOnBuild = true, + }); + + ObjectStorage = provider.GetRequiredService(); + RunTests = true; + } +} diff --git a/Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj b/Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj index 9c88183b8..f369c28fe 100644 --- a/Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj +++ b/Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj @@ -1,4 +1,4 @@ - + net8.0 win-x64;linux-x64;linux-arm64 @@ -10,10 +10,18 @@ True true enable + true - + + false + runtime + + + false + runtime + diff --git a/Adaptors/LocalStorage/src/ServiceCollectionExt.cs b/Adaptors/LocalStorage/src/ObjectBuilder.cs similarity index 68% rename from Adaptors/LocalStorage/src/ServiceCollectionExt.cs rename to Adaptors/LocalStorage/src/ObjectBuilder.cs index fb69ba594..7de022ad8 100644 --- a/Adaptors/LocalStorage/src/ServiceCollectionExt.cs +++ b/Adaptors/LocalStorage/src/ObjectBuilder.cs @@ -15,9 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -using ArmoniK.Api.Common.Utils; -using ArmoniK.Core.Common.Injection.Options; -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base; using ArmoniK.Core.Utils; using JetBrains.Annotations; @@ -28,34 +26,27 @@ namespace ArmoniK.Core.Adapters.LocalStorage; -public static class ServiceCollectionExt +/// +/// Class for building Local Storage instance and Object interfaces through Dependency Injection +/// +[PublicAPI] +public class ObjectBuilder : IDependencyInjectionBuildable { + /// [PublicAPI] - public static IServiceCollection AddLocalStorage(this IServiceCollection serviceCollection, - ConfigurationManager configuration, - ILogger logger) + public void Build(IServiceCollection serviceCollection, + ConfigurationManager configuration, + ILogger logger) { - var components = configuration.GetSection(Components.SettingSection); - - if (components["ObjectStorage"] != "ArmoniK.Adapters.LocalStorage.ObjectStorage") - { - return serviceCollection; - } - serviceCollection.AddOption(configuration, Options.LocalStorage.SettingSection, out Options.LocalStorage storageOptions); - using var _ = logger.BeginNamedScope("Object Local configuration", - ("Path", storageOptions.Path)); - logger.LogDebug("setup local storage"); serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage), sp => new ObjectStorage(storageOptions.Path, storageOptions.ChunkSize, sp.GetRequiredService>())); - - return serviceCollection; } } diff --git a/Adaptors/LocalStorage/src/ObjectStorage.cs b/Adaptors/LocalStorage/src/ObjectStorage.cs index 43607022c..7ef35013f 100644 --- a/Adaptors/LocalStorage/src/ObjectStorage.cs +++ b/Adaptors/LocalStorage/src/ObjectStorage.cs @@ -19,13 +19,13 @@ using System.Collections.Generic; using System.IO; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; -using ArmoniK.Api.Common.Utils; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base.Exceptions; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; @@ -94,15 +94,16 @@ public Task Check(HealthCheckTag tag) }; /// - public async Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) { long size = 0; + var key = Guid.NewGuid() + .ToString(); var filename = Path.Combine(path_, key); - using var _ = logger_.LogFunction(filename); // Write to temporary file await using var file = File.Open(filename, @@ -135,18 +136,18 @@ public async Task AddOrUpdateAsync(string await file.FlushAsync(cancellationToken) .ConfigureAwait(false); - return size; + return (Encoding.UTF8.GetBytes(key), size); } /// - public async IAsyncEnumerable GetValuesAsync(string key, + public async IAsyncEnumerable GetValuesAsync(byte[] id, [EnumeratorCancellation] CancellationToken cancellationToken = default) { + var key = Encoding.UTF8.GetString(id); + var filename = Path.Combine(path_, key); - using var _ = logger_.LogFunction(filename); - if (!File.Exists(filename)) { throw new ObjectDataNotFoundException($"The object {key} has not been found in {path_}"); @@ -187,29 +188,24 @@ public async IAsyncEnumerable GetValuesAsync(string } /// - public async Task TryDeleteAsync(IEnumerable keys, + public async Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) { - foreach (var key in keys) + foreach (var id in ids) { + var key = Encoding.UTF8.GetString(id); await TryDeleteAsync(key, cancellationToken) .ConfigureAwait(false); } } - /// - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => throw new NotImplementedException(); - public Task TryDeleteAsync(string key, CancellationToken cancellationToken = default) { var filename = Path.Combine(path_, key); - using var _ = logger_.LogFunction(filename); - File.Delete(filename); return Task.FromResult(true); diff --git a/Adaptors/Memory/src/ObjectStorage.cs b/Adaptors/Memory/src/ObjectStorage.cs new file mode 100644 index 000000000..1e91b7e52 --- /dev/null +++ b/Adaptors/Memory/src/ObjectStorage.cs @@ -0,0 +1,104 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +using ArmoniK.Core.Base; +using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; +using ArmoniK.Utils; + +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace ArmoniK.Core.Adapters.Memory; + +public class ObjectStorage : IObjectStorage +{ + private readonly ConcurrentDictionary store_ = new(); + private bool isInitialized_; + + /// + public Task Init(CancellationToken cancellationToken) + { + isInitialized_ = true; + return Task.CompletedTask; + } + + /// + public Task Check(HealthCheckTag tag) + => Task.FromResult(isInitialized_ + ? HealthCheckResult.Healthy() + : HealthCheckResult.Unhealthy()); + + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) + { + var array = new List(); + + var key = Guid.NewGuid() + .ToString(); + + await foreach (var val in valueChunks.WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + array.AddRange(val.ToArray()); + } + + store_[key] = array.ToArray(); + + return (Encoding.UTF8.GetBytes(key), array.Count); + } + +#pragma warning disable CS1998 + public async IAsyncEnumerable GetValuesAsync(byte[] id, +#pragma warning restore CS1998 + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var key = Encoding.UTF8.GetString(id); + if (!store_.TryGetValue(key, + out var value)) + { + throw new ObjectDataNotFoundException(); + } + + foreach (var chunk in value.ToChunks(100)) + { + yield return chunk; + } + } + + public Task TryDeleteAsync(IEnumerable ids, + CancellationToken cancellationToken = default) + { + foreach (var id in ids) + { + var key = Encoding.UTF8.GetString(id); + + store_.TryRemove(key, + out _); + } + + return Task.CompletedTask; + } +} diff --git a/Adaptors/Memory/src/ResultTable.cs b/Adaptors/Memory/src/ResultTable.cs index 9934fe0e3..dd93aa36d 100644 --- a/Adaptors/Memory/src/ResultTable.cs +++ b/Adaptors/Memory/src/ResultTable.cs @@ -25,6 +25,7 @@ using System.Threading.Tasks; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Storage; diff --git a/Adaptors/MongoDB/tests/ObjectStorageTests.cs b/Adaptors/Memory/tests/ObjectStorageTests.cs similarity index 68% rename from Adaptors/MongoDB/tests/ObjectStorageTests.cs rename to Adaptors/Memory/tests/ObjectStorageTests.cs index 0ac4b90ec..2a813a797 100644 --- a/Adaptors/MongoDB/tests/ObjectStorageTests.cs +++ b/Adaptors/Memory/tests/ObjectStorageTests.cs @@ -15,14 +15,11 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -using ArmoniK.Core.Common.Storage; using ArmoniK.Core.Common.Tests.TestBase; -using Microsoft.Extensions.DependencyInjection; - using NUnit.Framework; -namespace ArmoniK.Core.Adapters.MongoDB.Tests; +namespace ArmoniK.Core.Adapters.Memory.Tests; [TestFixture] public class ObjectStorageTests : ObjectStorageTestBase @@ -30,17 +27,13 @@ public class ObjectStorageTests : ObjectStorageTestBase public override void TearDown() { ObjectStorage = null; - tableProvider_?.Dispose(); - RunTests = false; + RunTests = false; } - private MongoDatabaseProvider? tableProvider_; protected override void GetObjectStorageInstance() { - tableProvider_ = new MongoDatabaseProvider(serviceConfigurator: collection => collection.AddSingleton()); - var provider = tableProvider_.GetServiceProvider(); - ObjectStorage = provider.GetRequiredService(); + ObjectStorage = new ObjectStorage(); RunTests = true; } } diff --git a/Adaptors/MongoDB/src/Object/ObjectDataModelMapping.cs b/Adaptors/MongoDB/src/Object/ObjectDataModelMapping.cs deleted file mode 100644 index fa9560aeb..000000000 --- a/Adaptors/MongoDB/src/Object/ObjectDataModelMapping.cs +++ /dev/null @@ -1,85 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using System; -using System.Threading.Tasks; - -using ArmoniK.Core.Adapters.MongoDB.Common; - -using MongoDB.Bson.Serialization.Attributes; -using MongoDB.Driver; - -namespace ArmoniK.Core.Adapters.MongoDB.Object; - -public class ObjectDataModelMapping : IMongoDataModelMapping -{ - public const string Collection = "Object"; - - [BsonId] - public string Id - => $"{Key}.{ChunkIdx}"; - - [BsonElement] - public string Key { get; set; } = ""; - - [BsonElement] - public byte[] Chunk { get; set; } = Array.Empty(); - - [BsonElement] - public int ChunkIdx { get; set; } - - /// - [BsonIgnore] - public string CollectionName { get; } = Collection; - - /// - public async Task InitializeIndexesAsync(IClientSessionHandle sessionHandle, - IMongoCollection collection, - Options.MongoDB options) - { - var keyIndex = Builders.IndexKeys.Hashed(model => model.Key); - var chunkIdxIndex = Builders.IndexKeys.Hashed(model => model.ChunkIdx); - var iDIndex = Builders.IndexKeys.Hashed(model => model.Id); - - var indexModels = new CreateIndexModel[] - { - new(iDIndex, - new CreateIndexOptions - { - Name = nameof(iDIndex), - }), - new(keyIndex, - new CreateIndexOptions - { - Name = nameof(keyIndex), - }), - new(chunkIdxIndex, - new CreateIndexOptions - { - Name = nameof(chunkIdxIndex), - }), - }; - - await collection.Indexes.CreateManyAsync(sessionHandle, - indexModels) - .ConfigureAwait(false); - } - - public Task ShardCollectionAsync(IClientSessionHandle sessionHandle, - Options.MongoDB options) - => Task.CompletedTask; -} diff --git a/Adaptors/MongoDB/src/ObjectStorage.cs b/Adaptors/MongoDB/src/ObjectStorage.cs deleted file mode 100644 index 0b84fcee9..000000000 --- a/Adaptors/MongoDB/src/ObjectStorage.cs +++ /dev/null @@ -1,199 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; - -using ArmoniK.Api.Common.Utils; -using ArmoniK.Core.Adapters.MongoDB.Common; -using ArmoniK.Core.Adapters.MongoDB.Object; -using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; -using ArmoniK.Core.Common.Storage; -using ArmoniK.Utils; - -using JetBrains.Annotations; - -using Microsoft.Extensions.Diagnostics.HealthChecks; -using Microsoft.Extensions.Logging; - -using MongoDB.Driver; -using MongoDB.Driver.Linq; - -namespace ArmoniK.Core.Adapters.MongoDB; - -[PublicAPI] -public class ObjectStorage : IObjectStorage -{ - private readonly ILogger logger_; - private readonly MongoCollectionProvider objectCollectionProvider_; - private readonly string objectStorageName_; - private readonly SessionProvider sessionProvider_; - private bool isInitialized_; - - public ObjectStorage(SessionProvider sessionProvider, - MongoCollectionProvider objectCollectionProvider, - ILogger logger, - Options.ObjectStorage options) - { - if (options.ChunkSize == 0) - { - throw new ArgumentOutOfRangeException(nameof(options), - $"Minimum value for {nameof(Options.ObjectStorage.ChunkSize)} is 1."); - } - - sessionProvider_ = sessionProvider; - objectCollectionProvider_ = objectCollectionProvider; - objectStorageName_ = "storage/"; - ChunkSize = options.ChunkSize; - logger_ = logger; - } - - public int ChunkSize { get; } - - /// - public async Task Init(CancellationToken cancellationToken) - { - if (!isInitialized_) - { - await sessionProvider_.Init(cancellationToken) - .ConfigureAwait(false); - sessionProvider_.Get(); - - await objectCollectionProvider_.Init(cancellationToken) - .ConfigureAwait(false); - objectCollectionProvider_.Get(); - } - - isInitialized_ = true; - } - - /// - public Task Check(HealthCheckTag tag) - => Task.FromResult(isInitialized_ - ? HealthCheckResult.Healthy() - : HealthCheckResult.Unhealthy()); - - /// - public async Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) - { - long size = 0; - var dbKey = objectStorageName_ + key; - using var _ = logger_.LogFunction(dbKey); - var objectCollection = objectCollectionProvider_.Get(); - - var taskList = new List(); - - var idx = 0; - await foreach (var chunk in valueChunks.WithCancellation(cancellationToken) - .ConfigureAwait(false)) - { - size += chunk.Length; - taskList.Add(objectCollection.InsertOneAsync(new ObjectDataModelMapping - { - Chunk = chunk.ToArray(), - ChunkIdx = idx, - Key = dbKey, - }, - cancellationToken: cancellationToken)); - ++idx; - } - - // If there was no chunks, add an empty chunk, just so that it could be found in the future - if (idx == 0) - { - taskList.Add(objectCollection.InsertOneAsync(new ObjectDataModelMapping - { - Chunk = Array.Empty(), - ChunkIdx = idx, - Key = dbKey, - }, - cancellationToken: cancellationToken)); - } - - await taskList.WhenAll() - .ConfigureAwait(false); - - return size; - } - - /// - async IAsyncEnumerable IObjectStorage.GetValuesAsync(string key, - [EnumeratorCancellation] CancellationToken cancellationToken) - { - using var _ = logger_.LogFunction(objectStorageName_ + key); - var sessionHandle = sessionProvider_.Get(); - var objectCollection = objectCollectionProvider_.Get(); - - var throwException = true; - await foreach (var chunk in objectCollection.AsQueryable(sessionHandle) - .Where(odm => odm.Key == objectStorageName_ + key) - .OrderBy(odm => odm.ChunkIdx) - .Select(odm => odm.Chunk) - .ToAsyncEnumerable(cancellationToken) - .ConfigureAwait(false)) - { - throwException = false; - yield return chunk; - } - - if (throwException) - { - throw new ObjectDataNotFoundException($"Result {key} not found"); - } - } - - /// - public async Task TryDeleteAsync(IEnumerable keys, - CancellationToken cancellationToken = default) - - { - using var _ = logger_.LogFunction(objectStorageName_); - var objectCollection = objectCollectionProvider_.Get(); - - var names = keys.Select(key => objectStorageName_ + key); - - await objectCollection.DeleteManyAsync(odm => names.Contains(odm.Key), - cancellationToken) - .ConfigureAwait(false); - logger_.LogInformation("Deleted data with {resultIds}", - keys); - } - - /// - public async IAsyncEnumerable ListKeysAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) - { - using var _ = logger_.LogFunction(); - var sessionHandle = sessionProvider_.Get(); - var objectCollection = objectCollectionProvider_.Get(); - - await foreach (var key in objectCollection.AsQueryable(sessionHandle) - .Where(odm => odm.ChunkIdx == 0) - .Select(odm => odm.Key) - .ToAsyncEnumerable(cancellationToken) - .ConfigureAwait(false)) - { - yield return key; - } - } -} diff --git a/Adaptors/MongoDB/src/Options/MongoDB.cs b/Adaptors/MongoDB/src/Options/MongoDB.cs index dcbf8dfc7..403112a6d 100644 --- a/Adaptors/MongoDB/src/Options/MongoDB.cs +++ b/Adaptors/MongoDB/src/Options/MongoDB.cs @@ -56,10 +56,6 @@ public class MongoDB public TableStorage TableStorage { get; set; } = new(); - public ObjectStorage ObjectStorage { get; set; } = new(); - - public QueueStorage QueueStorage { get; set; } = new(); - public int MaxConnectionPoolSize { get; set; } = 500; public TimeSpan ServerSelectionTimeout { get; set; } = TimeSpan.FromMinutes(2); diff --git a/Adaptors/MongoDB/src/ResultTable.cs b/Adaptors/MongoDB/src/ResultTable.cs index a4de3c194..b0a4f357e 100644 --- a/Adaptors/MongoDB/src/ResultTable.cs +++ b/Adaptors/MongoDB/src/ResultTable.cs @@ -27,6 +27,7 @@ using ArmoniK.Core.Adapters.MongoDB.Common; using ArmoniK.Core.Adapters.MongoDB.Table.DataModel; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; diff --git a/Adaptors/MongoDB/src/ServiceCollectionExt.cs b/Adaptors/MongoDB/src/ServiceCollectionExt.cs index 9fa9e90a7..a38fd6e3d 100644 --- a/Adaptors/MongoDB/src/ServiceCollectionExt.cs +++ b/Adaptors/MongoDB/src/ServiceCollectionExt.cs @@ -77,14 +77,6 @@ public static IServiceCollection AddMongoStorages(this IServiceCollection servic .AddSingleton(); } - if (components["ObjectStorage"] == "ArmoniK.Adapters.MongoDB.ObjectStorage") - { - services.AddOption(configuration, - Options.ObjectStorage.SettingSection) - .AddSingleton() - .AddSingleton(); - } - services.AddOption(configuration, Options.MongoDB.SettingSection, out var mongoOptions); diff --git a/Adaptors/MongoDB/src/Table/DataModel/ResultDataModelMapping.cs b/Adaptors/MongoDB/src/Table/DataModel/ResultDataModelMapping.cs index 1621b252c..f4ba0cd91 100644 --- a/Adaptors/MongoDB/src/Table/DataModel/ResultDataModelMapping.cs +++ b/Adaptors/MongoDB/src/Table/DataModel/ResultDataModelMapping.cs @@ -53,7 +53,7 @@ public ResultDataModelMapping() cm.MapProperty(nameof(Result.Size)) .SetIgnoreIfDefault(true) .SetDefaultValue(0); - cm.MapProperty(nameof(Result.Data)) + cm.MapProperty(nameof(Result.OpaqueId)) .SetIsRequired(true); cm.SetIgnoreExtraElements(true); cm.MapCreator(model => new Result(model.SessionId, @@ -65,7 +65,7 @@ public ResultDataModelMapping() model.DependentTasks, model.CreationDate, model.Size, - model.Data)); + model.OpaqueId)); }); } } diff --git a/Adaptors/MongoDB/src/TaskTable.cs b/Adaptors/MongoDB/src/TaskTable.cs index 9823caf54..2f42590f7 100644 --- a/Adaptors/MongoDB/src/TaskTable.cs +++ b/Adaptors/MongoDB/src/TaskTable.cs @@ -27,6 +27,7 @@ using ArmoniK.Core.Adapters.MongoDB.Options; using ArmoniK.Core.Adapters.MongoDB.Table.DataModel; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; diff --git a/Adaptors/MongoDB/tests/BsonSerializerTest.cs b/Adaptors/MongoDB/tests/BsonSerializerTest.cs index 0f5d39d83..213ed7aaf 100644 --- a/Adaptors/MongoDB/tests/BsonSerializerTest.cs +++ b/Adaptors/MongoDB/tests/BsonSerializerTest.cs @@ -119,7 +119,7 @@ public void SerializeResultDataModel() deserialized.DependentTasks); Assert.AreEqual(rdm.CreationDate, deserialized.CreationDate); - Assert.IsTrue(rdm.Data.SequenceEqual(deserialized.Data)); + Assert.IsTrue(rdm.OpaqueId.SequenceEqual(deserialized.OpaqueId)); } [Test] diff --git a/Adaptors/MongoDB/tests/InjectionTests.cs b/Adaptors/MongoDB/tests/InjectionTests.cs index d46387388..a55a71130 100644 --- a/Adaptors/MongoDB/tests/InjectionTests.cs +++ b/Adaptors/MongoDB/tests/InjectionTests.cs @@ -20,7 +20,6 @@ using System.Diagnostics; using ArmoniK.Core.Adapters.MongoDB.Options; -using ArmoniK.Core.Common.Storage; using ArmoniK.Core.Utils; using Microsoft.Extensions.Configuration; @@ -87,9 +86,6 @@ public void SetUp() { $"{Options.MongoDB.SettingSection}:{nameof(Options.MongoDB.TableStorage)}:PollingDelayMax", "00:00:20" }, - { - $"{Options.MongoDB.SettingSection}:{nameof(Options.MongoDB.ObjectStorage)}:ChunkSize", "100000" - }, }; var logger = NullLogger.Instance; @@ -233,23 +229,6 @@ public void ReadTablePollingMaxDelay() options.PollingDelayMax); } - [Test] - public void ObjectOptionsNotNull() - { - var options = provider_!.GetRequiredService(); - - Assert.NotNull(options); - } - - [Test] - public void ReadObjectChunkSize() - { - var options = provider_!.GetRequiredService(); - - Assert.AreEqual(100000, - options.ChunkSize); - } - [Test] public void BuildTableStorage() { @@ -275,22 +254,4 @@ public void TableStorageHasPollingDelayMax() Assert.AreEqual(TimeSpan.FromSeconds(20), table.PollingDelayMax); } - - [Test] - public void BuildObjectStorage() - { - var objectStorage = provider_!.GetRequiredService(); - - Assert.NotNull(objectStorage); - } - - [Test] - public void ObjectStorageFactoryHasBindingToObjectStorage() - { - var objectStorage = provider_!.GetRequiredService(); - - Assert.NotNull(objectStorage); - Assert.AreEqual(typeof(ObjectStorage), - objectStorage.GetType()); - } } diff --git a/Adaptors/MongoDB/tests/MongoDatabaseProvider.cs b/Adaptors/MongoDB/tests/MongoDatabaseProvider.cs index 3d6232496..7d3e75e38 100644 --- a/Adaptors/MongoDB/tests/MongoDatabaseProvider.cs +++ b/Adaptors/MongoDB/tests/MongoDatabaseProvider.cs @@ -82,9 +82,6 @@ public MongoDatabaseProvider(bool useSingleNodeReplicaSet { $"{Components.SettingSection}:{nameof(Components.TableStorage)}", "ArmoniK.Adapters.MongoDB.TableStorage" }, - { - $"{Components.SettingSection}:{nameof(Components.ObjectStorage)}", "ArmoniK.Adapters.MongoDB.ObjectStorage" - }, { $"{Components.SettingSection}:{nameof(Components.AuthenticationStorage)}", "ArmoniK.Adapters.MongoDB.AuthenticationTable" @@ -96,10 +93,6 @@ public MongoDatabaseProvider(bool useSingleNodeReplicaSet $"{Options.MongoDB.SettingSection}:{nameof(Options.MongoDB.TableStorage)}:{nameof(Options.MongoDB.TableStorage.PollingDelayMax)}", "00:00:10" }, - { - $"{Options.MongoDB.SettingSection}:{nameof(Options.MongoDB.ObjectStorage)}:{nameof(Options.MongoDB.ObjectStorage.ChunkSize)}", - "140000" - }, }; var configuration = new ConfigurationManager(); diff --git a/Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj b/Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj index d729d4bf1..da557fc65 100644 --- a/Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj +++ b/Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj @@ -1,4 +1,4 @@ - + net8.0 win-x64;linux-x64;linux-arm64 @@ -9,6 +9,19 @@ True true enable + true + + + + Embedded + true + DEBUG;TRACE + + + + true + true + snupkg @@ -20,7 +33,43 @@ - + + runtime + + + runtime + + + runtime + + + false + runtime + + + runtime + + + runtime + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + runtime + + + + + + false + runtime + + + false + runtime + diff --git a/Adaptors/Redis/src/ObjectBuilder.cs b/Adaptors/Redis/src/ObjectBuilder.cs new file mode 100644 index 000000000..0374b48c9 --- /dev/null +++ b/Adaptors/Redis/src/ObjectBuilder.cs @@ -0,0 +1,117 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using System; +using System.IO; +using System.Security.Cryptography.X509Certificates; + +using ArmoniK.Core.Base; +using ArmoniK.Core.Utils; + +using JetBrains.Annotations; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +using StackExchange.Redis; + +namespace ArmoniK.Core.Adapters.Redis; + +/// +/// Class for building Redis instance and Object interfaces through Dependency Injection +/// +[PublicAPI] +public class ObjectBuilder : IDependencyInjectionBuildable +{ + /// + [PublicAPI] + public void Build(IServiceCollection serviceCollection, + ConfigurationManager configuration, + ILogger logger) + { + // ReSharper disable once InlineOutVariableDeclaration + Options.Redis redisOptions; + serviceCollection.AddOption(configuration, + Options.Redis.SettingSection, + out redisOptions); + + if (!string.IsNullOrEmpty(redisOptions.CredentialsPath)) + { + configuration.AddJsonFile(redisOptions.CredentialsPath, + false, + false); + + serviceCollection.AddOption(configuration, + Options.Redis.SettingSection, + out redisOptions); + + logger.LogTrace("Loaded Redis credentials from file {path}", + redisOptions.CredentialsPath); + } + + if (!string.IsNullOrEmpty(redisOptions.CaPath)) + { + var localTrustStore = new X509Store(StoreName.Root); + var certificateCollection = new X509Certificate2Collection(); + try + { + certificateCollection.ImportFromPemFile(redisOptions.CaPath); + localTrustStore.Open(OpenFlags.ReadWrite); + localTrustStore.AddRange(certificateCollection); + logger.LogTrace("Imported Redis certificate from file {path}", + redisOptions.CaPath); + } + catch (Exception ex) + { + logger.LogError("Root certificate import failed: {error}", + ex.Message); + throw; + } + finally + { + localTrustStore.Close(); + } + } + + var config = new ConfigurationOptions + { + ClientName = redisOptions.ClientName, + ReconnectRetryPolicy = new ExponentialRetry(10), + Ssl = redisOptions.Ssl, + AbortOnConnectFail = true, + SslHost = redisOptions.SslHost, + Password = redisOptions.Password, + User = redisOptions.User, + }; + config.EndPoints.Add(redisOptions.EndpointUrl); + + if (redisOptions.Timeout > 0) + { + config.ConnectTimeout = redisOptions.Timeout; + } + + logger.LogDebug("setup connection to Redis at {EndpointUrl} with user {user}", + redisOptions.EndpointUrl, + redisOptions.User); + + serviceCollection.AddSingleton(_ => ConnectionMultiplexer.Connect(config, + TextWriter.Null) + .GetDatabase()); + serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); + } +} diff --git a/Adaptors/Redis/src/ObjectStorage.cs b/Adaptors/Redis/src/ObjectStorage.cs index 9acf36ee0..bb45932c1 100644 --- a/Adaptors/Redis/src/ObjectStorage.cs +++ b/Adaptors/Redis/src/ObjectStorage.cs @@ -19,13 +19,13 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; -using ArmoniK.Api.Common.Utils; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Utils; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -87,13 +87,14 @@ public Task Check(HealthCheckTag tag) }; /// - public async Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) { - long size = 0; - var storageNameKey = objectStorageName_ + key; - using var _ = logger_.LogFunction(storageNameKey); + var key = Guid.NewGuid() + .ToString(); + var storageNameKey = objectStorageName_ + key; + long size = 0; var idx = 0; var taskList = new List(); @@ -112,14 +113,14 @@ public async Task AddOrUpdateAsync(string await taskList.WhenAll() .ConfigureAwait(false); - return size; + return (Encoding.UTF8.GetBytes(key), size); } /// - public async IAsyncEnumerable GetValuesAsync(string key, + public async IAsyncEnumerable GetValuesAsync(byte[] id, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - using var _ = logger_.LogFunction(objectStorageName_ + key); + var key = Encoding.UTF8.GetString(id); var value = await PerformActionWithRetry(() => redis_.StringGetAsync(objectStorageName_ + key + "_count")) .ConfigureAwait(false); @@ -145,20 +146,17 @@ public async IAsyncEnumerable GetValuesAsync(string } /// - public async Task TryDeleteAsync(IEnumerable keys, + public async Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) - => await keys.ParallelForEach(key => TryDeleteAsync(key, - cancellationToken)) - .ConfigureAwait(false); - - /// - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => throw new NotImplementedException(); + => await ids.ParallelForEach(id => TryDeleteAsync(id, + cancellationToken)) + .ConfigureAwait(false); - private async Task TryDeleteAsync(string key, + private async Task TryDeleteAsync(byte[] id, CancellationToken cancellationToken = default) { - using var _ = logger_.LogFunction(objectStorageName_ + key); + var key = Encoding.UTF8.GetString(id); + var value = await PerformActionWithRetry(() => redis_.StringGetAsync(objectStorageName_ + key + "_count")) .ConfigureAwait(false); diff --git a/Adaptors/Redis/src/ServiceCollectionExt.cs b/Adaptors/Redis/src/ServiceCollectionExt.cs deleted file mode 100644 index 79c8ed094..000000000 --- a/Adaptors/Redis/src/ServiceCollectionExt.cs +++ /dev/null @@ -1,124 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using System; -using System.IO; -using System.Security.Cryptography.X509Certificates; - -using ArmoniK.Api.Common.Utils; -using ArmoniK.Core.Common.Injection.Options; -using ArmoniK.Core.Common.Storage; -using ArmoniK.Core.Utils; - -using JetBrains.Annotations; - -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -using StackExchange.Redis; - -namespace ArmoniK.Core.Adapters.Redis; - -public static class ServiceCollectionExt -{ - [PublicAPI] - public static IServiceCollection AddRedis(this IServiceCollection serviceCollection, - ConfigurationManager configuration, - ILogger logger) - { - var components = configuration.GetSection(Components.SettingSection); - - if (components["ObjectStorage"] == "ArmoniK.Adapters.Redis.ObjectStorage") - { - // ReSharper disable once InlineOutVariableDeclaration - Options.Redis redisOptions; - serviceCollection.AddOption(configuration, - Options.Redis.SettingSection, - out redisOptions); - - using var _ = logger.BeginNamedScope("Redis configuration", - ("EndpointUrl", redisOptions.EndpointUrl)); - - if (!string.IsNullOrEmpty(redisOptions.CredentialsPath)) - { - configuration.AddJsonFile(redisOptions.CredentialsPath, - false, - false); - - serviceCollection.AddOption(configuration, - Options.Redis.SettingSection, - out redisOptions); - - logger.LogTrace("Loaded Redis credentials from file {path}", - redisOptions.CredentialsPath); - } - - if (!string.IsNullOrEmpty(redisOptions.CaPath)) - { - var localTrustStore = new X509Store(StoreName.Root); - var certificateCollection = new X509Certificate2Collection(); - try - { - certificateCollection.ImportFromPemFile(redisOptions.CaPath); - localTrustStore.Open(OpenFlags.ReadWrite); - localTrustStore.AddRange(certificateCollection); - logger.LogTrace("Imported Redis certificate from file {path}", - redisOptions.CaPath); - } - catch (Exception ex) - { - logger.LogError("Root certificate import failed: {error}", - ex.Message); - throw; - } - finally - { - localTrustStore.Close(); - } - } - - var config = new ConfigurationOptions - { - ClientName = redisOptions.ClientName, - ReconnectRetryPolicy = new ExponentialRetry(10), - Ssl = redisOptions.Ssl, - AbortOnConnectFail = true, - SslHost = redisOptions.SslHost, - Password = redisOptions.Password, - User = redisOptions.User, - }; - config.EndPoints.Add(redisOptions.EndpointUrl); - - if (redisOptions.Timeout > 0) - { - config.ConnectTimeout = redisOptions.Timeout; - } - - logger.LogDebug("setup connection to Redis at {EndpointUrl} with user {user}", - redisOptions.EndpointUrl, - redisOptions.User); - - serviceCollection.AddSingleton(_ => ConnectionMultiplexer.Connect(config, - TextWriter.Null) - .GetDatabase()); - serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); - } - - return serviceCollection; - } -} diff --git a/Adaptors/Redis/tests/ObjectStorageTests.cs b/Adaptors/Redis/tests/ObjectStorageTests.cs index a133b4753..cd43ef46f 100644 --- a/Adaptors/Redis/tests/ObjectStorageTests.cs +++ b/Adaptors/Redis/tests/ObjectStorageTests.cs @@ -19,7 +19,7 @@ using System.Collections.Generic; using System.IO; -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base; using ArmoniK.Core.Common.Tests.TestBase; using ArmoniK.Core.Utils; @@ -53,9 +53,6 @@ protected override void GetObjectStorageInstance() // Minimal set of configurations to operate on a toy DB Dictionary minimalConfig = new() { - { - "Components:ObjectStorage", "ArmoniK.Adapters.Redis.ObjectStorage" - }, { "Redis:MaxRetry", "5" }, diff --git a/Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj b/Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj index 512eba0cb..d2064ca6e 100644 --- a/Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj +++ b/Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj @@ -2,8 +2,14 @@ net8.0 win-x64;linux-x64;linux-arm64 - enable + ANEO + Copyright (C) ANEO, 2021-2021 + AGPL-3.0-or-later + True + True + true enable + true @@ -18,7 +24,43 @@ - + + runtime + + + runtime + + + runtime + + + false + runtime + + + runtime + + + runtime + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + runtime + + + + + + false + runtime + + + false + runtime + diff --git a/Adaptors/S3/src/ObjectBuilder.cs b/Adaptors/S3/src/ObjectBuilder.cs new file mode 100644 index 000000000..599a64846 --- /dev/null +++ b/Adaptors/S3/src/ObjectBuilder.cs @@ -0,0 +1,77 @@ +// This file is part of the ArmoniK project +// +// Copyright (C) ANEO, 2021-2024. All rights reserved. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY, without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +using Amazon.S3; + +using ArmoniK.Core.Base; +using ArmoniK.Core.Utils; + +using JetBrains.Annotations; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace ArmoniK.Core.Adapters.S3; + +/// +/// Class for building S3 instance and Object interfaces through Dependency Injection +/// +[PublicAPI] +public class ObjectBuilder : IDependencyInjectionBuildable +{ + /// + [PublicAPI] + public void Build(IServiceCollection serviceCollection, + ConfigurationManager configuration, + ILogger logger) + { + // ReSharper disable once InlineOutVariableDeclaration + Options.S3 s3Options; + serviceCollection.AddOption(configuration, + Options.S3.SettingSection, + out s3Options); + + logger.LogInformation("setup connection to S3 at {EndpointUrl} with user {user} with option ForcePathStyle = {ForcePathStyle} with BucketName = {BucketName}", + s3Options.EndpointUrl, + s3Options.Login, + s3Options.MustForcePathStyle, + s3Options.BucketName); + + var s3Config = new AmazonS3Config + { + ForcePathStyle = s3Options.MustForcePathStyle, + ServiceURL = s3Options.EndpointUrl, + }; + + AmazonS3Client s3Client; + if (string.IsNullOrWhiteSpace(s3Options.Login)) + { + s3Client = new AmazonS3Client(s3Config); + } + else + { + s3Client = new AmazonS3Client(s3Options.Login, + s3Options.Password, + s3Config); + } + + + serviceCollection.AddSingleton(_ => s3Client); + serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); + } +} diff --git a/Adaptors/S3/src/ObjectStorage.cs b/Adaptors/S3/src/ObjectStorage.cs index d8bb7bf0a..d63acb17a 100644 --- a/Adaptors/S3/src/ObjectStorage.cs +++ b/Adaptors/S3/src/ObjectStorage.cs @@ -15,16 +15,22 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; using Amazon.S3; using Amazon.S3.Model; using Amazon.S3.Util; -using ArmoniK.Api.Common.Utils; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Utils; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -90,13 +96,12 @@ public Task Check(HealthCheckTag tag) }; /// - public async IAsyncEnumerable GetValuesAsync(string key, + public async IAsyncEnumerable GetValuesAsync(byte[] id, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var objectStorageFullName = $"{objectStorageName_}{key}"; - using var loggerFunction = logger_.LogFunction(objectStorageFullName); - using var loggerContext = logger_.BeginPropertyScope(("objectKey", key), - ("@S3Options", options_.Confidential())); + var key = Encoding.UTF8.GetString(id); + var objectStorageFullName = $"{objectStorageName_}{key}"; + try { await s3Client_.GetObjectAsync(options_.BucketName, @@ -160,30 +165,19 @@ await s3Client_.GetObjectAsync(options_.BucketName, } /// - public async Task TryDeleteAsync(IEnumerable keys, - CancellationToken cancellationToken = default) - => await keys.ParallelForEach(key => TryDeleteAsync(key, - cancellationToken)) - .ConfigureAwait(false); - - /// - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => throw new NotImplementedException(); - - /// - public async Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) + public async Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) { long[] sizeBox = { 0, }; - var objectStorageFullName = $"{objectStorageName_}{key}"; - using var loggerFunction = logger_.LogFunction(objectStorageFullName); - using var loggerContext = logger_.BeginPropertyScope(("objectKey", key), - ("@S3Options", options_.Confidential())); + var key = Guid.NewGuid() + .ToString(); + var objectStorageFullName = $"{objectStorageName_}{key}"; + logger_.LogDebug("Upload object"); var initRequest = new InitiateMultipartUploadRequest { @@ -234,16 +228,22 @@ await s3Client_.AbortMultipartUploadAsync(abortMpuRequest, throw; } - return sizeBox[0]; + return (Encoding.UTF8.GetBytes(key), sizeBox[0]); } - private async Task TryDeleteAsync(string key, + /// + public async Task TryDeleteAsync(IEnumerable ids, + CancellationToken cancellationToken = default) + => await ids.ParallelForEach(id => TryDeleteAsync(id, + cancellationToken)) + .ConfigureAwait(false); + + private async Task TryDeleteAsync(byte[] id, CancellationToken cancellationToken = default) { - var objectStorageFullName = $"{objectStorageName_}{key}"; - using var loggerFunction = logger_.LogFunction(objectStorageFullName); - using var loggerContext = logger_.BeginPropertyScope(("objectKey", key), - ("@S3Options", options_.Confidential())); + var key = Encoding.UTF8.GetString(id); + var objectStorageFullName = $"{objectStorageName_}{key}"; + var objectDeleteRequest = new DeleteObjectRequest { BucketName = options_.BucketName, diff --git a/Adaptors/S3/src/ServiceCollectionExt.cs b/Adaptors/S3/src/ServiceCollectionExt.cs deleted file mode 100644 index 97d415ba8..000000000 --- a/Adaptors/S3/src/ServiceCollectionExt.cs +++ /dev/null @@ -1,84 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using Amazon.S3; - -using ArmoniK.Api.Common.Utils; -using ArmoniK.Core.Common.Injection.Options; -using ArmoniK.Core.Common.Storage; -using ArmoniK.Core.Utils; - -using JetBrains.Annotations; - -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -namespace ArmoniK.Core.Adapters.S3; - -public static class ServiceCollectionExt -{ - [PublicAPI] - public static IServiceCollection AddS3(this IServiceCollection serviceCollection, - ConfigurationManager configuration, - ILogger logger) - { - var components = configuration.GetSection(Components.SettingSection); - - if (components["ObjectStorage"] == "ArmoniK.Adapters.S3.ObjectStorage") - { - // ReSharper disable once InlineOutVariableDeclaration - Options.S3 s3Options; - serviceCollection.AddOption(configuration, - Options.S3.SettingSection, - out s3Options); - - using var _ = logger.BeginNamedScope("S3 configuration", - ("EndpointUrl", s3Options.EndpointUrl)); - - logger.LogInformation("setup connection to S3 at {EndpointUrl} with user {user} with option ForcePathStyle = {ForcePathStyle} with BucketName = {BucketName}", - s3Options.EndpointUrl, - s3Options.Login, - s3Options.MustForcePathStyle, - s3Options.BucketName); - - var s3Config = new AmazonS3Config - { - ForcePathStyle = s3Options.MustForcePathStyle, - ServiceURL = s3Options.EndpointUrl, - }; - - AmazonS3Client s3Client; - if (string.IsNullOrWhiteSpace(s3Options.Login)) - { - s3Client = new AmazonS3Client(s3Config); - } - else - { - s3Client = new AmazonS3Client(s3Options.Login, - s3Options.Password, - s3Config); - } - - - serviceCollection.AddSingleton(_ => s3Client); - serviceCollection.AddSingletonWithHealthCheck(nameof(IObjectStorage)); - } - - return serviceCollection; - } -} diff --git a/Adaptors/S3/tests/ArmoniK.Core.Adapters.S3.Tests.csproj b/Adaptors/S3/tests/ArmoniK.Core.Adapters.S3.Tests.csproj index 41d99c8ab..ae9d8bc16 100644 --- a/Adaptors/S3/tests/ArmoniK.Core.Adapters.S3.Tests.csproj +++ b/Adaptors/S3/tests/ArmoniK.Core.Adapters.S3.Tests.csproj @@ -27,7 +27,6 @@ - diff --git a/Adaptors/S3/tests/ObjectStorageTests.cs b/Adaptors/S3/tests/ObjectStorageTests.cs index 05996f044..db6615666 100644 --- a/Adaptors/S3/tests/ObjectStorageTests.cs +++ b/Adaptors/S3/tests/ObjectStorageTests.cs @@ -15,7 +15,9 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base; +using ArmoniK.Core.Common.Injection; +using ArmoniK.Core.Common.Injection.Options; using ArmoniK.Core.Common.Tests.TestBase; using ArmoniK.Core.Common.Utils; @@ -35,12 +37,25 @@ public override void TearDown() RunTests = false; } + private static readonly string SolutionRoot = + Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(typeof(ObjectStorageTests) + .Assembly + .Location))))) ?? + string.Empty)); + + private static readonly string S3Path = + $"{Path.DirectorySeparatorChar}src{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}Debug{Path.DirectorySeparatorChar}net8.0{Path.DirectorySeparatorChar}ArmoniK.Core.Adapters.S3.dll"; + + protected override void GetObjectStorageInstance() { Dictionary minimalConfig = new() { { - "Components:ObjectStorage", "ArmoniK.Adapters.S3.ObjectStorage" + "Components:ObjectStorageAdaptorSettings:ClassName", "ArmoniK.Core.Adapters.S3.ObjectBuilder" + }, + { + "Components:ObjectStorageAdaptorSettings:AdapterAbsolutePath", $"{SolutionRoot}{S3Path}" }, { "S3:BucketName", "miniobucket" @@ -66,10 +81,9 @@ protected override void GetObjectStorageInstance() services.AddLogging(); var logger = new LoggerInit(configuration); - services.AddS3(configuration, - logger.GetLogger()); - - services.AddSingleton(); + services.AddAdapter(configuration, + nameof(Components.ObjectStorageAdaptorSettings), + logger.GetLogger()); var provider = services.BuildServiceProvider(new ServiceProviderOptions { diff --git a/ArmoniK.Core.sln b/ArmoniK.Core.sln index 05b8c8138..fb1b1af96 100644 --- a/ArmoniK.Core.sln +++ b/ArmoniK.Core.sln @@ -101,6 +101,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArmoniK.Core.Tests.Connecti EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArmoniK.Core.Adapters.SQS", "Adaptors\SQS\src\ArmoniK.Core.Adapters.SQS.csproj", "{399C779C-CE8D-4757-8098-7F055467D96C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArmoniK.Core.Adapters.Embed", "Adaptors\Embed\src\ArmoniK.Core.Adapters.Embed.csproj", "{F95A1BF7-B660-4789-9F2D-1749FC104EEE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArmoniK.Core.Adapters.Embed.Tests", "Adaptors\Embed\tests\ArmoniK.Core.Adapters.Embed.Tests.csproj", "{2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -413,6 +417,22 @@ Global {399C779C-CE8D-4757-8098-7F055467D96C}.Release|Any CPU.Build.0 = Release|Any CPU {399C779C-CE8D-4757-8098-7F055467D96C}.Release|x64.ActiveCfg = Release|Any CPU {399C779C-CE8D-4757-8098-7F055467D96C}.Release|x64.Build.0 = Release|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Debug|x64.ActiveCfg = Debug|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Debug|x64.Build.0 = Debug|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Release|Any CPU.Build.0 = Release|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Release|x64.ActiveCfg = Release|Any CPU + {F95A1BF7-B660-4789-9F2D-1749FC104EEE}.Release|x64.Build.0 = Release|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Debug|x64.ActiveCfg = Debug|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Debug|x64.Build.0 = Debug|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Release|Any CPU.Build.0 = Release|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Release|x64.ActiveCfg = Release|Any CPU + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -465,6 +485,8 @@ Global {9F19E6DD-D9CD-43EC-B5CC-0081997ED8B8} = {6EE499A5-760F-4823-8AB6-DBDE8C5B5F45} {6ABF29BB-2F42-4088-AE93-E41416CB3271} = {CD412C3D-63D0-4726-B4C3-FEF701E4DCAF} {399C779C-CE8D-4757-8098-7F055467D96C} = {1A9BCE53-79D0-4761-B3A2-6967B610FA94} + {F95A1BF7-B660-4789-9F2D-1749FC104EEE} = {1A9BCE53-79D0-4761-B3A2-6967B610FA94} + {2DC798DD-F147-4245-BD63-E0E6E5BB5E9E} = {2D548C40-6260-4A4E-9C56-E8A80C639E13} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {1285A466-2AF6-43E6-8DCC-2F93A5D5F02E} diff --git a/Adaptors/MongoDB/src/Options/ObjectStorage.cs b/Base/src/DataStructures/ObjectData.cs similarity index 65% rename from Adaptors/MongoDB/src/Options/ObjectStorage.cs rename to Base/src/DataStructures/ObjectData.cs index d50a98e53..0a6719d77 100644 --- a/Adaptors/MongoDB/src/Options/ObjectStorage.cs +++ b/Base/src/DataStructures/ObjectData.cs @@ -1,4 +1,4 @@ -// This file is part of the ArmoniK project +// This file is part of the ArmoniK project // // Copyright (C) ANEO, 2021-2024. All rights reserved. // @@ -15,14 +15,20 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -using JetBrains.Annotations; +namespace ArmoniK.Core.Base.DataStructures; -namespace ArmoniK.Core.Adapters.MongoDB.Options; - -[PublicAPI] -public class ObjectStorage +/// +/// Metadata given to the +/// +public record ObjectData { - public const string SettingSection = nameof(MongoDB) + ":" + nameof(ObjectStorage); + /// + /// Id of the result + /// + public required string ResultId; - public int ChunkSize { get; set; } = 14500000; + /// + /// Id of the session + /// + public required string SessionId; } diff --git a/Common/src/Exceptions/ArmoniKException.cs b/Base/src/Exceptions/ArmoniKException.cs similarity index 97% rename from Common/src/Exceptions/ArmoniKException.cs rename to Base/src/Exceptions/ArmoniKException.cs index 1412f565b..339229974 100644 --- a/Common/src/Exceptions/ArmoniKException.cs +++ b/Base/src/Exceptions/ArmoniKException.cs @@ -17,7 +17,7 @@ using System; -namespace ArmoniK.Core.Common.Exceptions; +namespace ArmoniK.Core.Base.Exceptions; /// /// Base exception for exceptions in ArmoniK core diff --git a/Common/src/Exceptions/ObjectDataNotFoundException.cs b/Base/src/Exceptions/ObjectDataNotFoundException.cs similarity index 96% rename from Common/src/Exceptions/ObjectDataNotFoundException.cs rename to Base/src/Exceptions/ObjectDataNotFoundException.cs index bcf9ebe07..7a8c1ba4f 100644 --- a/Common/src/Exceptions/ObjectDataNotFoundException.cs +++ b/Base/src/Exceptions/ObjectDataNotFoundException.cs @@ -17,7 +17,7 @@ using System; -namespace ArmoniK.Core.Common.Exceptions; +namespace ArmoniK.Core.Base.Exceptions; [Serializable] public class ObjectDataNotFoundException : ArmoniKException diff --git a/Common/src/Storage/IObjectStorage.cs b/Base/src/IObjectStorage.cs similarity index 68% rename from Common/src/Storage/IObjectStorage.cs rename to Base/src/IObjectStorage.cs index 87315c2c4..13257773f 100644 --- a/Common/src/Storage/IObjectStorage.cs +++ b/Base/src/IObjectStorage.cs @@ -20,10 +20,10 @@ using System.Threading; using System.Threading.Tasks; -using ArmoniK.Core.Base; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; -namespace ArmoniK.Core.Common.Storage; +namespace ArmoniK.Core.Base; /// /// Object Storage interface @@ -35,47 +35,40 @@ public interface IObjectStorage : IInitializable /// Add the given data in the storage at the given key /// Update data if it already exists /// - /// Key representing the object /// Chunks of data that will be stored in the Object Storage /// Token used to cancel the execution of the method /// - /// The size of the object that has been uploaded. + /// The opaque unique identifier representing the object and the size of the object that has been uploaded. /// + /// + /// Opaque ID will be used to refer to the object. The implementation of this interface should generate it. + /// /// the key is not found - Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default); + Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData data, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default); /// /// Get object in the Object Storage /// - /// Key representing the object to be retrieved + /// Opaque unique identifier representing the object /// Token used to cancel the execution of the method /// /// Byte arrays representing the object chunked /// /// the key is not found - IAsyncEnumerable GetValuesAsync(string key, + IAsyncEnumerable GetValuesAsync(byte[] id, CancellationToken cancellationToken = default); /// /// Delete data in the object storage /// - /// Keys representing the objects to delete + /// Opaque unique identifiers representing the objects to delete /// Token used to cancel the execution of the method /// /// Task representing the asynchronous execution of the method /// /// the key is not found - Task TryDeleteAsync(IEnumerable keys, + Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default); - - /// - /// List data in the Object Storage - /// - /// Token used to cancel the execution of the method - /// - /// The keys representing data found in the Object Storage - /// - IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default); } diff --git a/Common/src/Auth/Authentication/Authenticator.cs b/Common/src/Auth/Authentication/Authenticator.cs index 6a2df81b6..9d76f39ed 100644 --- a/Common/src/Auth/Authentication/Authenticator.cs +++ b/Common/src/Auth/Authentication/Authenticator.cs @@ -22,8 +22,8 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authorization; -using ArmoniK.Core.Common.Exceptions; using JetBrains.Annotations; diff --git a/Common/src/Exceptions/InvalidSessionTransitionException.cs b/Common/src/Exceptions/InvalidSessionTransitionException.cs index 79abb5156..9d7a6c9a5 100644 --- a/Common/src/Exceptions/InvalidSessionTransitionException.cs +++ b/Common/src/Exceptions/InvalidSessionTransitionException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/PartitionNotFoundException.cs b/Common/src/Exceptions/PartitionNotFoundException.cs index 257c8d57a..bace4fc57 100644 --- a/Common/src/Exceptions/PartitionNotFoundException.cs +++ b/Common/src/Exceptions/PartitionNotFoundException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/ResultNotFoundException.cs b/Common/src/Exceptions/ResultNotFoundException.cs index 74ce230f3..6d76cc1a3 100644 --- a/Common/src/Exceptions/ResultNotFoundException.cs +++ b/Common/src/Exceptions/ResultNotFoundException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/SessionNotFoundException.cs b/Common/src/Exceptions/SessionNotFoundException.cs index c58e67f09..443f19675 100644 --- a/Common/src/Exceptions/SessionNotFoundException.cs +++ b/Common/src/Exceptions/SessionNotFoundException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/SubmissionClosedException.cs b/Common/src/Exceptions/SubmissionClosedException.cs index 2894f5e9c..89b2db80a 100644 --- a/Common/src/Exceptions/SubmissionClosedException.cs +++ b/Common/src/Exceptions/SubmissionClosedException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskAlreadyExistsException.cs b/Common/src/Exceptions/TaskAlreadyExistsException.cs index 8dcd080c7..6cf3003a9 100644 --- a/Common/src/Exceptions/TaskAlreadyExistsException.cs +++ b/Common/src/Exceptions/TaskAlreadyExistsException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskAlreadyInFinalStateException.cs b/Common/src/Exceptions/TaskAlreadyInFinalStateException.cs index 0df0b44c6..5c6c98d3a 100644 --- a/Common/src/Exceptions/TaskAlreadyInFinalStateException.cs +++ b/Common/src/Exceptions/TaskAlreadyInFinalStateException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskCanceledException.cs b/Common/src/Exceptions/TaskCanceledException.cs index eab3eb11b..ba7258efe 100644 --- a/Common/src/Exceptions/TaskCanceledException.cs +++ b/Common/src/Exceptions/TaskCanceledException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskNotFoundException.cs b/Common/src/Exceptions/TaskNotFoundException.cs index fa0851a8a..fe348ba75 100644 --- a/Common/src/Exceptions/TaskNotFoundException.cs +++ b/Common/src/Exceptions/TaskNotFoundException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TaskPausedException.cs b/Common/src/Exceptions/TaskPausedException.cs index 474f33049..8e959912f 100644 --- a/Common/src/Exceptions/TaskPausedException.cs +++ b/Common/src/Exceptions/TaskPausedException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Exceptions/TimeoutException.cs b/Common/src/Exceptions/TimeoutException.cs index 06042151a..762f9f365 100644 --- a/Common/src/Exceptions/TimeoutException.cs +++ b/Common/src/Exceptions/TimeoutException.cs @@ -17,6 +17,8 @@ using System; +using ArmoniK.Core.Base.Exceptions; + namespace ArmoniK.Core.Common.Exceptions; [Serializable] diff --git a/Common/src/Injection/Options/Components.cs b/Common/src/Injection/Options/Components.cs index 905666df0..835134dc6 100644 --- a/Common/src/Injection/Options/Components.cs +++ b/Common/src/Injection/Options/Components.cs @@ -43,7 +43,7 @@ public class Components /// /// Represents which object storage is used to store data for tasks /// - public string? ObjectStorage { get; set; } + public AdapterSettings ObjectStorageAdaptorSettings { get; set; } = new(); /// /// Represents which database is used for authentication diff --git a/Common/src/Injection/ServiceCollectionExt.cs b/Common/src/Injection/ServiceCollectionExt.cs index c4e080f9a..3799e182b 100644 --- a/Common/src/Injection/ServiceCollectionExt.cs +++ b/Common/src/Injection/ServiceCollectionExt.cs @@ -80,35 +80,37 @@ public static IServiceCollection AddArmoniKWorkerConnection(this IServiceCollect return services; } - public static IServiceCollection AddQueue(this IServiceCollection services, - ConfigurationManager configuration, - ILogger logger) + public static IServiceCollection AddAdapter(this IServiceCollection services, + ConfigurationManager configuration, + string storage, + ILogger logger) { - var queueSettings = configuration.GetRequiredValue($"{Components.SettingSection}:{nameof(Components.QueueAdaptorSettings)}"); + var settings = configuration.GetRequiredValue($"{Components.SettingSection}:{storage}"); - logger.LogInformation("Queue settings for loading adapter {@queueSettings}", - queueSettings); + logger.LogInformation("{storage} settings for loading adapter {@settings}", + storage, + settings); logger.LogDebug("{path}", - queueSettings.AdapterAbsolutePath); + settings.AdapterAbsolutePath); logger.LogDebug("{class}", - queueSettings.ClassName); + settings.ClassName); - if (string.IsNullOrEmpty(queueSettings.AdapterAbsolutePath)) + if (string.IsNullOrEmpty(settings.AdapterAbsolutePath)) { - throw new InvalidOperationException($"{nameof(queueSettings.AdapterAbsolutePath)} should not be null or empty."); + throw new InvalidOperationException($"{nameof(settings.AdapterAbsolutePath)} should not be null or empty."); } - if (string.IsNullOrEmpty(queueSettings.ClassName)) + if (string.IsNullOrEmpty(settings.ClassName)) { - throw new InvalidOperationException($"{nameof(queueSettings.ClassName)} should not be null or empty."); + throw new InvalidOperationException($"{nameof(settings.ClassName)} should not be null or empty."); } - var assembly = Assembly.LoadFrom(queueSettings.AdapterAbsolutePath); + var assembly = Assembly.LoadFrom(settings.AdapterAbsolutePath); logger.LogInformation("Loaded assembly {assemblyName}", assembly.FullName); - var type = assembly.GetType(queueSettings.ClassName, + var type = assembly.GetType(settings.ClassName, true, true); diff --git a/Common/src/Pollster/Agent.cs b/Common/src/Pollster/Agent.cs index 9a5f476ee..780b840c1 100644 --- a/Common/src/Pollster/Agent.cs +++ b/Common/src/Pollster/Agent.cs @@ -26,6 +26,8 @@ using ArmoniK.Api.Common.Utils; using ArmoniK.Api.gRPC.V1; using ArmoniK.Core.Base; +using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Storage; @@ -45,16 +47,16 @@ namespace ArmoniK.Core.Common.Pollster; /// public sealed class Agent : IAgent { - private readonly List createdTasks_; - private readonly ILogger logger_; - private readonly IObjectStorage objectStorage_; - private readonly IPushQueueStorage pushQueueStorage_; - private readonly IResultTable resultTable_; - private readonly Dictionary sentResults_; - private readonly SessionData sessionData_; - private readonly ISubmitter submitter_; - private readonly TaskData taskData_; - private readonly ITaskTable taskTable_; + private readonly List createdTasks_; + private readonly ILogger logger_; + private readonly IObjectStorage objectStorage_; + private readonly IPushQueueStorage pushQueueStorage_; + private readonly IResultTable resultTable_; + private readonly Dictionary sentResults_; + private readonly SessionData sessionData_; + private readonly ISubmitter submitter_; + private readonly TaskData taskData_; + private readonly ITaskTable taskTable_; /// /// Initializes a new instance of the @@ -64,8 +66,8 @@ public sealed class Agent : IAgent /// Interface to put tasks in the queue /// Interface to manage result states /// Interface to manage task states - /// Data of the session - /// Data of the task + /// OpaqueId of the session + /// OpaqueId of the task /// Shared folder between Agent and Worker /// Token send to the worker to identify the running task /// Logger used to produce logs for this class @@ -87,7 +89,7 @@ public Agent(ISubmitter submitter, taskTable_ = taskTable; logger_ = logger; createdTasks_ = new List(); - sentResults_ = new Dictionary(); + sentResults_ = new Dictionary(); sessionData_ = sessionData; taskData_ = taskData; Folder = folder; @@ -120,11 +122,12 @@ await submitter_.FinalizeTaskCreation(createdTasks_, cancellationToken) .ConfigureAwait(false); - foreach (var (result, size) in sentResults_) + foreach (var (result, (id, size)) in sentResults_) { await resultTable_.CompleteResult(taskData_.SessionId, result, size, + id, cancellationToken) .ConfigureAwait(false); } @@ -155,29 +158,32 @@ public async Task GetResourceData(string token, ThrowIfInvalidToken(token); + try { - await using (var fs = new FileStream(Path.Combine(Folder, - resultId), - FileMode.OpenOrCreate)) + var opaqueId = (await resultTable_.GetResult(resultId, + cancellationToken) + .ConfigureAwait(false)).OpaqueId; + + await using var fs = new FileStream(Path.Combine(Folder, + resultId), + FileMode.OpenOrCreate); + await using var w = new BinaryWriter(fs); + await foreach (var chunk in objectStorage_.GetValuesAsync(opaqueId, + cancellationToken) + .ConfigureAwait(false)) { - await using var w = new BinaryWriter(fs); - await foreach (var chunk in objectStorage_.GetValuesAsync(resultId, - cancellationToken) - .ConfigureAwait(false)) - { - w.Write(chunk); - } + w.Write(chunk); } return resultId; } - catch (ObjectDataNotFoundException) + catch (Exception ex) when (ex is ResultNotFoundException or ObjectDataNotFoundException) { throw new RpcException(new Status(StatusCode.NotFound, - "Data not found"), - "Data not found"); + "ResultId not found"), + "ResultId not found"); } } @@ -262,42 +268,48 @@ public async Task> CreateResults(string var results = await requests.Select(async rc => { - var resultId = Guid.NewGuid() - .ToString(); - - var size = await objectStorage_.AddOrUpdateAsync(resultId, - new List> - { - rc.data, - }.ToAsyncEnumerable(), - cancellationToken) - .ConfigureAwait(false); - - return new Result(rc.request.SessionId, - resultId, - rc.request.Name, - taskData_.TaskId, - "", - ResultStatus.Created, - new List(), - DateTime.UtcNow, - size, - Array.Empty()); + var result = new Result(rc.request.SessionId, + Guid.NewGuid() + .ToString(), + rc.request.Name, + taskData_.TaskId, + "", + ResultStatus.Created, + new List(), + DateTime.UtcNow, + 0, + Array.Empty()); + + var add = await objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = result.ResultId, + SessionId = rc.request.SessionId, + }, + new List> + { + rc.data, + }.ToAsyncEnumerable(), + cancellationToken) + .ConfigureAwait(false); + + return (result, add); }) .WhenAll() .ConfigureAwait(false); - await resultTable_.Create(results, + var ids = results.ViewSelect(tuple => tuple.result); + + await resultTable_.Create(ids, cancellationToken) .ConfigureAwait(false); foreach (var result in results) { - sentResults_.Add(result.ResultId, - result.Size); + sentResults_.Add(result.result.ResultId, + result.add); } - return results; + return ids; } /// @@ -315,19 +327,21 @@ public async Task> NotifyResultData(string toke using var r = new BinaryReader(fs); var channel = Channel.CreateUnbounded>(); - var add = objectStorage_.AddOrUpdateAsync(result, - channel.Reader.ReadAllAsync(cancellationToken), - cancellationToken); + var addTask = objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = result, + SessionId = sessionData_.SessionId, + }, + channel.Reader.ReadAllAsync(cancellationToken), + cancellationToken); - long size = 0; - int read; + int read; do { var buffer = new byte[PayloadConfiguration.MaxChunkSize]; read = r.Read(buffer, 0, PayloadConfiguration.MaxChunkSize); - size += read; if (read > 0) { await channel.Writer.WriteAsync(buffer.AsMemory(0, @@ -339,9 +353,9 @@ await channel.Writer.WriteAsync(buffer.AsMemory(0, channel.Writer.Complete(); - await add.ConfigureAwait(false); + var add = await addTask.ConfigureAwait(false); sentResults_.Add(result, - size); + add); } return resultIds; diff --git a/Common/src/Pollster/DataPrefetcher.cs b/Common/src/Pollster/DataPrefetcher.cs index cb6ca1f5a..ae566f9f6 100644 --- a/Common/src/Pollster/DataPrefetcher.cs +++ b/Common/src/Pollster/DataPrefetcher.cs @@ -15,6 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Threading; @@ -23,7 +24,7 @@ using ArmoniK.Api.Common.Utils; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Storage; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -39,6 +40,7 @@ public class DataPrefetcher : IInitializable private readonly ActivitySource? activitySource_; private readonly ILogger logger_; private readonly IObjectStorage objectStorage_; + private readonly IResultTable resultTable_; private bool isInitialized_; @@ -46,13 +48,16 @@ public class DataPrefetcher : IInitializable /// Create data prefetcher for tasks /// /// Interface to manage data + /// Interface to manage results metadata /// Activity source for tracing /// Logger used to print logs public DataPrefetcher(IObjectStorage objectStorage, + IResultTable resultTable, ActivitySource? activitySource, ILogger logger) { objectStorage_ = objectStorage; + resultTable_ = resultTable; logger_ = logger; activitySource_ = activitySource; } @@ -93,28 +98,27 @@ public async Task PrefetchDataAsync(TaskData taskData, activity?.AddEvent(new ActivityEvent("Load payload")); - - await using (var fs = new FileStream(Path.Combine(folder, - taskData.PayloadId), - FileMode.OpenOrCreate)) - { - await using var w = new BinaryWriter(fs); - await foreach (var chunk in objectStorage_.GetValuesAsync(taskData.PayloadId, - cancellationToken) - .ConfigureAwait(false)) - { - w.Write(chunk); - } - } + var dependencies = new List + { + taskData.PayloadId, + }; + dependencies.AddRange(taskData.DataDependencies); - foreach (var dataDependency in taskData.DataDependencies) + await foreach (var id in resultTable_.GetResults(data => dependencies.Contains(data.ResultId), + r => new + { + r.ResultId, + r.OpaqueId, + }, + cancellationToken) + .ConfigureAwait(false)) { await using var fs = new FileStream(Path.Combine(folder, - dataDependency), + id.ResultId), FileMode.OpenOrCreate); await using var w = new BinaryWriter(fs); - await foreach (var chunk in objectStorage_.GetValuesAsync(dataDependency, + await foreach (var chunk in objectStorage_.GetValuesAsync(id.OpaqueId, cancellationToken) .ConfigureAwait(false)) { diff --git a/Common/src/Pollster/TaskHandler.cs b/Common/src/Pollster/TaskHandler.cs index ecb57e541..73daabfc8 100644 --- a/Common/src/Pollster/TaskHandler.cs +++ b/Common/src/Pollster/TaskHandler.cs @@ -26,6 +26,7 @@ using ArmoniK.Api.Common.Utils; using ArmoniK.Core.Base; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Meter; diff --git a/Common/src/Storage/Result.cs b/Common/src/Storage/Result.cs index f0975b352..bb62abd48 100644 --- a/Common/src/Storage/Result.cs +++ b/Common/src/Storage/Result.cs @@ -31,7 +31,7 @@ namespace ArmoniK.Core.Common.Storage; /// List of tasks that depend on this result. /// Date of creation of the current object. /// Size of the result. -/// Data for the current +/// Opaque Identifier used by the object storage to refer to this result's data public record Result(string SessionId, string ResultId, string Name, @@ -41,7 +41,7 @@ public record Result(string SessionId, List DependentTasks, DateTime CreationDate, long Size, - byte[] Data) + byte[] OpaqueId) { /// /// Creates a copy of a and modify it according to given updates diff --git a/Common/src/Storage/ResultLifeCycleHelper.cs b/Common/src/Storage/ResultLifeCycleHelper.cs index 81e630e40..36c8d22d9 100644 --- a/Common/src/Storage/ResultLifeCycleHelper.cs +++ b/Common/src/Storage/ResultLifeCycleHelper.cs @@ -22,6 +22,7 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Utils; using Microsoft.Extensions.Logging; @@ -156,8 +157,8 @@ public static async Task PurgeResultsAsync(IResultTable resultTable, string sessionId, CancellationToken cancellationToken) { - await foreach (var ids in resultTable.GetResults(result => result.SessionId == sessionId, - result => result.ResultId, + await foreach (var ids in resultTable.GetResults(result => result.SessionId == sessionId && result.OpaqueId.Length > 0, + result => result.OpaqueId, cancellationToken) .ToChunksAsync(500, Timeout.InfiniteTimeSpan, @@ -171,7 +172,9 @@ await objectStorage.TryDeleteAsync(ids, await resultTable.UpdateManyResults(result => result.SessionId == sessionId, new UpdateDefinition().Set(result => result.Status, - ResultStatus.DeletedData), + ResultStatus.DeletedData) + .Set(result => result.OpaqueId, + Array.Empty()), cancellationToken) .ConfigureAwait(false); } diff --git a/Common/src/Storage/ResultTableExtensions.cs b/Common/src/Storage/ResultTableExtensions.cs index 5475a6432..cbb0506c8 100644 --- a/Common/src/Storage/ResultTableExtensions.cs +++ b/Common/src/Storage/ResultTableExtensions.cs @@ -101,6 +101,7 @@ public static Task BulkUpdateResults(this IResultTable /// id of the session containing the results /// Id of the result to complete /// Size of the result to complete + /// Opaque unique identifier representing the object /// Token used to cancel the execution of the method /// /// The new version of the result metadata @@ -110,13 +111,16 @@ public static async Task CompleteResult(this IResultTable resultTable, string sessionId, string resultId, long size, + byte[] opaqueId, CancellationToken cancellationToken = default) { var result = await resultTable.UpdateOneResult(resultId, new UpdateDefinition().Set(data => data.Status, ResultStatus.Completed) .Set(data => data.Size, - size), + size) + .Set(data => data.OpaqueId, + opaqueId), cancellationToken) .ConfigureAwait(false); @@ -128,49 +132,10 @@ public static async Task CompleteResult(this IResultTable resultTable, { Status = ResultStatus.Completed, Size = size, + OpaqueId = opaqueId, }; } - /// - /// Update result with small payload - /// - /// Interface to manage results - /// id of the session containing the results - /// id of the task owning the result - /// id of the result to be modified - /// payload data - /// Token used to cancel the execution of the method - /// - /// Task representing the asynchronous execution of the method - /// - public static async Task SetResult(this IResultTable resultTable, - string sessionId, - string ownerTaskId, - string resultId, - byte[] smallPayload, - CancellationToken cancellationToken = default) - { - var count = await resultTable.UpdateManyResults(result => result.ResultId == resultId && result.OwnerTaskId == ownerTaskId, - new UpdateDefinition().Set(result => result.Status, - ResultStatus.Completed) - .Set(data => data.Size, - smallPayload.Length) - .Set(result => result.Data, - smallPayload), - cancellationToken) - .ConfigureAwait(false); - - resultTable.Logger.LogDebug("Update {result} from {owner} to {status}", - resultId, - ownerTaskId, - ResultStatus.Completed); - - if (count == 0) - { - throw new ResultNotFoundException($"Result '{resultId}' was not found for '{ownerTaskId}'"); - } - } - /// /// Update result /// @@ -188,13 +153,16 @@ public static async Task SetResult(this IResultTable resultTable, string ownerTaskId, string resultId, long size, + byte[] opaqueId, CancellationToken cancellationToken = default) { var count = await resultTable.UpdateManyResults(result => result.ResultId == resultId && result.OwnerTaskId == ownerTaskId, new UpdateDefinition().Set(result => result.Status, ResultStatus.Completed) .Set(result => result.Size, - size), + size) + .Set(result => result.OpaqueId, + opaqueId), cancellationToken) .ConfigureAwait(false); diff --git a/Common/src/Storage/TaskLifeCycleHelper.cs b/Common/src/Storage/TaskLifeCycleHelper.cs index 484d76879..99c337cb0 100644 --- a/Common/src/Storage/TaskLifeCycleHelper.cs +++ b/Common/src/Storage/TaskLifeCycleHelper.cs @@ -24,6 +24,7 @@ using ArmoniK.Api.Common.Utils; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Utils; diff --git a/Common/src/Storage/TaskTableExtensions.cs b/Common/src/Storage/TaskTableExtensions.cs index 7a14a88be..e299bb68b 100644 --- a/Common/src/Storage/TaskTableExtensions.cs +++ b/Common/src/Storage/TaskTableExtensions.cs @@ -22,6 +22,7 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using Microsoft.Extensions.Logging; diff --git a/Common/src/Stream/Worker/WorkerStreamHandler.cs b/Common/src/Stream/Worker/WorkerStreamHandler.cs index 115de8541..0a5a209c8 100644 --- a/Common/src/Stream/Worker/WorkerStreamHandler.cs +++ b/Common/src/Stream/Worker/WorkerStreamHandler.cs @@ -23,7 +23,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Worker; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.gRPC.Convertors; using ArmoniK.Core.Common.Injection.Options; using ArmoniK.Core.Common.Storage; diff --git a/Common/src/gRPC/Convertors/TaskTableExt.cs b/Common/src/gRPC/Convertors/TaskTableExt.cs index 2f566c744..72bf7c68a 100644 --- a/Common/src/gRPC/Convertors/TaskTableExt.cs +++ b/Common/src/gRPC/Convertors/TaskTableExt.cs @@ -20,7 +20,7 @@ using System.Threading.Tasks; using ArmoniK.Api.gRPC.V1.Submitter; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Storage; using TaskStatus = ArmoniK.Core.Common.Storage.TaskStatus; diff --git a/Common/src/gRPC/RpcExt.cs b/Common/src/gRPC/RpcExt.cs index 9885482f8..386be962e 100644 --- a/Common/src/gRPC/RpcExt.cs +++ b/Common/src/gRPC/RpcExt.cs @@ -1,4 +1,4 @@ -// This file is part of the ArmoniK project +// This file is part of the ArmoniK project // // Copyright (C) ANEO, 2021-2024. All rights reserved. // @@ -22,7 +22,7 @@ using System.Threading.Tasks; using ArmoniK.Api.gRPC.V1; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using Grpc.Core; diff --git a/Common/src/gRPC/Services/GrpcAuthService.cs b/Common/src/gRPC/Services/GrpcAuthService.cs index 78ee74944..9d1f0cb8e 100644 --- a/Common/src/gRPC/Services/GrpcAuthService.cs +++ b/Common/src/gRPC/Services/GrpcAuthService.cs @@ -20,10 +20,10 @@ using System.Threading.Tasks; using ArmoniK.Api.gRPC.V1.Auth; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Auth.Authorization.Permissions; -using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Meter; using Grpc.Core; diff --git a/Common/src/gRPC/Services/GrpcResultsService.cs b/Common/src/gRPC/Services/GrpcResultsService.cs index e3cf424fc..8d99b6ccc 100644 --- a/Common/src/gRPC/Services/GrpcResultsService.cs +++ b/Common/src/gRPC/Services/GrpcResultsService.cs @@ -25,6 +25,8 @@ using ArmoniK.Api.gRPC.V1.Results; using ArmoniK.Api.gRPC.V1.SortDirection; using ArmoniK.Core.Base; +using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Exceptions; @@ -171,35 +173,40 @@ public override async Task CreateResults(CreateResultsReq var resultId = Guid.NewGuid() .ToString(); - var size = await objectStorage_.AddOrUpdateAsync(resultId, - new List> - { - rc.Data.Memory, - }.ToAsyncEnumerable(), - context.CancellationToken) - .ConfigureAwait(false); - - return new Result(request.SessionId, - resultId, - rc.Name, - "", - request.SessionId, - ResultStatus.Created, - new List(), - DateTime.UtcNow, - size, - Array.Empty()); + var (id, size) = await objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = resultId, + SessionId = request.SessionId, + }, + new List> + { + rc.Data.Memory, + }.ToAsyncEnumerable(), + context.CancellationToken) + .ConfigureAwait(false); + + return (new Result(request.SessionId, + resultId, + rc.Name, + "", + request.SessionId, + ResultStatus.Created, + new List(), + DateTime.UtcNow, + size, + Array.Empty()), id); }) .WhenAll() .ConfigureAwait(false); - await resultTable_.Create(results, + await resultTable_.Create(results.ViewSelect(tuple => tuple.Item1), context.CancellationToken) .ConfigureAwait(false); var resultList = await results.Select(async r => await resultTable_.CompleteResult(request.SessionId, - r.ResultId, - r.Size) + r.Item1.ResultId, + r.Item1.Size, + r.id) .ConfigureAwait(false)) .WhenAll() .ConfigureAwait(false); @@ -221,7 +228,13 @@ public override async Task DeleteResultsData(DeleteRe using var measure = meter_.CountAndTime(); try { - await objectStorage_.TryDeleteAsync(request.ResultId, + var opaqueIds = await resultTable_.GetResults(result => request.ResultId.Contains(result.ResultId), + result => result.OpaqueId, + context.CancellationToken) + .ToListAsync(context.CancellationToken) + .ConfigureAwait(false); + + await objectStorage_.TryDeleteAsync(opaqueIds, context.CancellationToken) .ConfigureAwait(false); @@ -236,7 +249,14 @@ await objectStorage_.TryDeleteAsync(request.ResultId, catch (ObjectDataNotFoundException e) { logger_.LogWarning(e, - "Error while downloading results"); + "Error while deleting results"); + throw new RpcException(new Status(StatusCode.NotFound, + "Result data not found")); + } + catch (ResultNotFoundException e) + { + logger_.LogWarning(e, + "Error while deleting results"); throw new RpcException(new Status(StatusCode.NotFound, "Result data not found")); } @@ -251,7 +271,10 @@ public override async Task DownloadResultData(DownloadResultDataRequest using var measure = meter_.CountAndTime(); try { - await foreach (var chunk in objectStorage_.GetValuesAsync(request.ResultId, + var id = (await resultTable_.GetResult(request.ResultId) + .ConfigureAwait(false)).OpaqueId; + + await foreach (var chunk in objectStorage_.GetValuesAsync(id, context.CancellationToken) .ConfigureAwait(false)) { @@ -269,6 +292,13 @@ await responseStream.WriteAsync(new DownloadResultDataResponse throw new RpcException(new Status(StatusCode.NotFound, "Result data not found")); } + catch (ResultNotFoundException e) + { + logger_.LogWarning(e, + "Error while downloading results"); + throw new RpcException(new Status(StatusCode.NotFound, + "Result data not found")); + } } [RequiresPermission(typeof(GrpcResultsService), @@ -311,17 +341,16 @@ public override async Task UploadResultData(IAsyncStre var sessionTask = sessionTable_.GetSessionAsync(id.SessionId, context.CancellationToken); - long size = 0; - await objectStorage_.AddOrUpdateAsync(id.ResultId, - requestStream.ReadAllAsync(context.CancellationToken) - .Select(r => - { - size += r.DataChunk.Length; - return r.DataChunk.Memory; - }), - context.CancellationToken) - .ConfigureAwait(false); + var (opaqueId, size) = await objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = id.ResultId, + SessionId = id.SessionId, + }, + requestStream.ReadAllAsync(context.CancellationToken) + .Select(r => r.DataChunk.Memory), + context.CancellationToken) + .ConfigureAwait(false); try { @@ -330,6 +359,7 @@ await objectStorage_.AddOrUpdateAsync(id.ResultId, var resultData = await resultTable_.CompleteResult(id.SessionId, id.ResultId, size, + opaqueId, context.CancellationToken) .ConfigureAwait(false); diff --git a/Common/src/gRPC/Services/GrpcSessionsService.cs b/Common/src/gRPC/Services/GrpcSessionsService.cs index ddad9d9d7..04fd4e829 100644 --- a/Common/src/gRPC/Services/GrpcSessionsService.cs +++ b/Common/src/gRPC/Services/GrpcSessionsService.cs @@ -23,6 +23,7 @@ using ArmoniK.Api.gRPC.V1.SortDirection; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Exceptions; diff --git a/Common/src/gRPC/Services/GrpcSubmitterService.cs b/Common/src/gRPC/Services/GrpcSubmitterService.cs index 7c6c5f0d0..0ea46f946 100644 --- a/Common/src/gRPC/Services/GrpcSubmitterService.cs +++ b/Common/src/gRPC/Services/GrpcSubmitterService.cs @@ -22,6 +22,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Exceptions; diff --git a/Common/src/gRPC/Services/GrpcTasksService.cs b/Common/src/gRPC/Services/GrpcTasksService.cs index 0118b5979..7f5527edb 100644 --- a/Common/src/gRPC/Services/GrpcTasksService.cs +++ b/Common/src/gRPC/Services/GrpcTasksService.cs @@ -25,6 +25,7 @@ using ArmoniK.Api.gRPC.V1.SortDirection; using ArmoniK.Api.gRPC.V1.Tasks; using ArmoniK.Core.Base; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Auth.Authorization; using ArmoniK.Core.Common.Exceptions; diff --git a/Common/src/gRPC/Services/Submitter.cs b/Common/src/gRPC/Services/Submitter.cs index d9ac4f932..fab7bfb2c 100644 --- a/Common/src/gRPC/Services/Submitter.cs +++ b/Common/src/gRPC/Services/Submitter.cs @@ -27,7 +27,8 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; using ArmoniK.Core.Base; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.gRPC.Convertors; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; @@ -196,7 +197,7 @@ await responseStream.WriteAsync(new ResultReply } } - await foreach (var chunk in objectStorage_.GetValuesAsync(request.ResultId, + await foreach (var chunk in objectStorage_.GetValuesAsync(result.OpaqueId, cancellationToken) .ConfigureAwait(false)) { @@ -394,15 +395,20 @@ public async Task SetResult(string sessionId, { using var activity = activitySource_.StartActivity($"{nameof(SetResult)}"); - var size = await objectStorage_.AddOrUpdateAsync(key, - chunks, - cancellationToken) - .ConfigureAwait(false); + var (id, size) = await objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = key, + SessionId = sessionId, + }, + chunks, + cancellationToken) + .ConfigureAwait(false); await resultTable_.SetResult(sessionId, ownerTaskId, key, size, + id, cancellationToken) .ConfigureAwait(false); } @@ -432,7 +438,7 @@ public async Task> CreateTasks(string ("PartitionId", options.PartitionId)); var requests = new List(); - var payloadUploadTasks = new List>(); + var payloadUploadTasks = new List>(); await foreach (var taskRequest in taskRequests.WithCancellation(cancellationToken) .ConfigureAwait(false)) @@ -447,7 +453,11 @@ public async Task> CreateTasks(string options, taskRequest.ExpectedOutputKeys.ToList(), taskRequest.DataDependencies.ToList())); - payloadUploadTasks.Add(objectStorage_.AddOrUpdateAsync(payloadId, + payloadUploadTasks.Add(objectStorage_.AddOrUpdateAsync(new ObjectData + { + ResultId = payloadId, + SessionId = sessionId, + }, taskRequest.PayloadChunks, cancellationToken)); } @@ -457,18 +467,18 @@ public async Task> CreateTasks(string await resultTable_.Create(requests.Zip(payloadSizes, (request, - size) => new Result(sessionId, - request.PayloadId, - "", - parentTaskId.Equals(sessionId) - ? "" - : parentTaskId, - parentTaskId, - ResultStatus.Completed, - new List(), - DateTime.UtcNow, - size, - Array.Empty())) + r) => new Result(sessionId, + request.PayloadId, + "", + parentTaskId.Equals(sessionId) + ? "" + : parentTaskId, + parentTaskId, + ResultStatus.Completed, + new List(), + DateTime.UtcNow, + r.size, + r.id)) .AsICollection(), cancellationToken) .ConfigureAwait(false); @@ -513,12 +523,20 @@ await taskTable_.SetTaskSuccessAsync(taskDataEnd, if (submitterOptions_.DeletePayload) { //Discard value is used to remove warnings CS4014 !! - _ = Task.Factory.StartNew(async () => await objectStorage_.TryDeleteAsync(new[] - { - taskData.PayloadId, - }, - CancellationToken.None) - .ConfigureAwait(false), + _ = Task.Factory.StartNew(async () => + { + var opaqueId = (await resultTable_.GetResult(taskData.PayloadId, + CancellationToken.None) + .ConfigureAwait(false)).OpaqueId; + + + await objectStorage_.TryDeleteAsync(new[] + { + opaqueId, + }, + CancellationToken.None) + .ConfigureAwait(false); + }, cancellationToken); await resultTable_.MarkAsDeleted(taskData.PayloadId, @@ -609,12 +627,20 @@ await taskTable_.SetTaskTimeoutAsync(taskDataEnd, .ConfigureAwait(false); //Discard value is used to remove warnings CS4014 !! - _ = Task.Factory.StartNew(async () => await objectStorage_.TryDeleteAsync(new[] - { - taskData.TaskId, - }, - CancellationToken.None) - .ConfigureAwait(false), + _ = Task.Factory.StartNew(async () => + { + var opaqueId = (await resultTable_.GetResult(taskData.PayloadId, + CancellationToken.None) + .ConfigureAwait(false)).OpaqueId; + + + await objectStorage_.TryDeleteAsync(new[] + { + opaqueId, + }, + CancellationToken.None) + .ConfigureAwait(false); + }, cancellationToken); logger_.LogInformation("Remove input payload of {task}", diff --git a/Common/tests/AdapterLoading/AdapterLoadingTest.cs b/Common/tests/AdapterLoading/AdapterLoadingTest.cs index 7a204bd19..5c65d3fb0 100644 --- a/Common/tests/AdapterLoading/AdapterLoadingTest.cs +++ b/Common/tests/AdapterLoading/AdapterLoadingTest.cs @@ -78,8 +78,9 @@ public static void BuildServiceProvider(Dictionary config) configuration.AddInMemoryCollection(config); var serviceCollection = new ServiceCollection(); - serviceCollection.AddQueue(configuration, - logger); + serviceCollection.AddAdapter(configuration, + nameof(Components.QueueAdaptorSettings), + logger); serviceCollection.BuildServiceProvider(); } diff --git a/Common/tests/Helpers/SimpleObjectStorage.cs b/Common/tests/Helpers/SimpleObjectStorage.cs index 20226720c..782f269cd 100644 --- a/Common/tests/Helpers/SimpleObjectStorage.cs +++ b/Common/tests/Helpers/SimpleObjectStorage.cs @@ -22,8 +22,8 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Storage; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -37,26 +37,19 @@ public Task Check(HealthCheckTag tag) public Task Init(CancellationToken cancellationToken) => Task.CompletedTask; - public Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) - => Task.FromResult((long)0); - - public IAsyncEnumerable GetValuesAsync(string key, - CancellationToken cancellationToken = default) - => new List - { - Encoding.UTF8.GetBytes(key), - }.ToAsyncEnumerable(); - - public Task TryDeleteAsync(IEnumerable keys, + public Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) => Task.CompletedTask; - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => new List + public Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) + => Task.FromResult((Encoding.UTF8.GetBytes("id"), (long)0)); + + public IAsyncEnumerable GetValuesAsync(byte[] id, + CancellationToken cancellationToken = default) + => new List { - "key1", - "key2", + id, }.ToAsyncEnumerable(); } diff --git a/Common/tests/Helpers/TestDatabaseProvider.cs b/Common/tests/Helpers/TestDatabaseProvider.cs index 388fe18c0..e7ed060de 100644 --- a/Common/tests/Helpers/TestDatabaseProvider.cs +++ b/Common/tests/Helpers/TestDatabaseProvider.cs @@ -21,8 +21,10 @@ using System.Threading; using ArmoniK.Api.Common.Options; +using ArmoniK.Core.Adapters.Memory; using ArmoniK.Core.Adapters.MongoDB; using ArmoniK.Core.Adapters.MongoDB.Common; +using ArmoniK.Core.Base; using ArmoniK.Core.Common.Auth.Authentication; using ArmoniK.Core.Common.Injection; using ArmoniK.Core.Common.Storage; @@ -107,10 +109,6 @@ public TestDatabaseProvider(Action? collectionConfigurato $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, @@ -140,6 +138,7 @@ public TestDatabaseProvider(Action? collectionConfigurato out _) .Configure(o => o.CopyFrom(AuthenticatorOptions.DefaultNoAuth)) .AddLogging() + .AddSingleton() .AddSingleton(loggerProvider.CreateLogger("root")) .AddSingleton(ActivitySource) .AddSingleton(_ => client_); diff --git a/Common/tests/Helpers/TestPollingAgentProvider.cs b/Common/tests/Helpers/TestPollingAgentProvider.cs index f5b110d7c..ef15e8dcc 100644 --- a/Common/tests/Helpers/TestPollingAgentProvider.cs +++ b/Common/tests/Helpers/TestPollingAgentProvider.cs @@ -89,10 +89,6 @@ public TestPollingAgentProvider(IWorkerStreamHandler workerStreamHandler) $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, @@ -117,6 +113,7 @@ public TestPollingAgentProvider(IWorkerStreamHandler workerStreamHandler) .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton(workerStreamHandler); var computePlanOptions = builder.Configuration.GetRequiredValue(ComputePlane.SettingSection); diff --git a/Common/tests/Helpers/TestPollsterProvider.cs b/Common/tests/Helpers/TestPollsterProvider.cs index f6c1e9533..859dd2966 100644 --- a/Common/tests/Helpers/TestPollsterProvider.cs +++ b/Common/tests/Helpers/TestPollsterProvider.cs @@ -113,10 +113,6 @@ public TestPollsterProvider(IWorkerStreamHandler workerStreamHandler, $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, @@ -180,6 +176,7 @@ acquireTimeout is null .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton() .AddOption(builder.Configuration, Injection.Options.Pollster.SettingSection) diff --git a/Common/tests/Helpers/TestTaskHandlerProvider.cs b/Common/tests/Helpers/TestTaskHandlerProvider.cs index ee8291db9..b03035556 100644 --- a/Common/tests/Helpers/TestTaskHandlerProvider.cs +++ b/Common/tests/Helpers/TestTaskHandlerProvider.cs @@ -22,6 +22,7 @@ using System.Threading; using ArmoniK.Api.Common.Options; +using ArmoniK.Core.Adapters.Memory; using ArmoniK.Core.Adapters.MongoDB; using ArmoniK.Core.Base; using ArmoniK.Core.Common.gRPC.Services; @@ -109,10 +110,6 @@ public TestTaskHandlerProvider(IWorkerStreamHandler workerStreamHandler, $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, @@ -160,6 +157,7 @@ public TestTaskHandlerProvider(IWorkerStreamHandler workerStreamHandler, .AddOption(builder.Configuration, Injection.Options.Pollster.SettingSection) .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() diff --git a/Common/tests/Pollster/AgentTest.cs b/Common/tests/Pollster/AgentTest.cs index 8f35e294f..4420526a9 100644 --- a/Common/tests/Pollster/AgentTest.cs +++ b/Common/tests/Pollster/AgentTest.cs @@ -406,15 +406,6 @@ await holder.Agent.NotifyResultData(holder.Token, CancellationToken.None) .ConfigureAwait(false); - var datAsyncEnumerable = holder.ObjectStorage.GetValuesAsync(ExpectedOutput1, - CancellationToken.None); - - var dataStored = await datAsyncEnumerable.SingleAsync(CancellationToken.None) - .ConfigureAwait(false); - - Assert.AreEqual(data, - dataStored); - await holder.Agent.FinalizeTaskCreation(CancellationToken.None) .ConfigureAwait(false); @@ -431,6 +422,15 @@ await holder.Agent.FinalizeTaskCreation(CancellationToken.None) Assert.AreEqual(data.Length, resultData.Size); + var datAsyncEnumerable = holder.ObjectStorage.GetValuesAsync(resultData.OpaqueId, + CancellationToken.None); + + var dataStored = await datAsyncEnumerable.SingleAsync(CancellationToken.None) + .ConfigureAwait(false); + + Assert.AreEqual(data, + dataStored); + var dependents = await holder.ResultTable.GetDependents(holder.Session, ExpectedOutput1, CancellationToken.None) @@ -605,14 +605,33 @@ public async Task GetResourceDataShouldSucceed() { using var holder = new AgentHolder(); - await holder.ObjectStorage.AddOrUpdateAsync("ResourceData", - new List - { - Encoding.ASCII.GetBytes("Data1"), - Encoding.ASCII.GetBytes("Data2"), - }.Select(bytes => new ReadOnlyMemory(bytes)) - .ToAsyncEnumerable(), - CancellationToken.None) + var (id, size) = await holder.ObjectStorage.AddOrUpdateAsync(new ObjectData + { + ResultId = "ResultId", + SessionId = "SessionId", + }, + new List + { + Encoding.ASCII.GetBytes("Data1"), + Encoding.ASCII.GetBytes("Data2"), + }.Select(bytes => new ReadOnlyMemory(bytes)) + .ToAsyncEnumerable(), + CancellationToken.None) + .ConfigureAwait(false); + + await holder.ResultTable.Create(new List + { + new("SessionId", + "ResourceData", + "", + "", + "", + ResultStatus.Completed, + new List(), + DateTime.UtcNow, + size, + id), + }) .ConfigureAwait(false); await holder.Agent.GetResourceData(holder.Token, @@ -657,14 +676,6 @@ public async Task CreateResultsShouldSucceed() resultMetadata.Status); Assert.AreEqual(holder.TaskData.TaskId, resultMetadata.CreatedBy); - - var bytes = (await holder.ObjectStorage.GetValuesAsync(result.ResultId) - .ToListAsync() - .ConfigureAwait(false)).Single(); - - Assert.AreEqual(ByteString.CopyFromUtf8(result.Name) - .ToByteArray(), - bytes); } await holder.Agent.FinalizeTaskCreation(CancellationToken.None) @@ -684,6 +695,14 @@ await holder.Agent.FinalizeTaskCreation(CancellationToken.None) resultMetadata.Status); Assert.AreEqual(7, resultMetadata.Size); + + var bytes = (await holder.ObjectStorage.GetValuesAsync(resultMetadata.OpaqueId) + .ToListAsync() + .ConfigureAwait(false)).Single(); + + Assert.AreEqual(ByteString.CopyFromUtf8(result.Name) + .ToByteArray(), + bytes); } } diff --git a/Common/tests/Pollster/DataPrefetcherTest.cs b/Common/tests/Pollster/DataPrefetcherTest.cs index 492099a1f..032c5b83c 100644 --- a/Common/tests/Pollster/DataPrefetcherTest.cs +++ b/Common/tests/Pollster/DataPrefetcherTest.cs @@ -20,15 +20,20 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Linq.Expressions; +using System.Text; using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Common.Pollster; using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Common.Tests.Helpers; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Moq; @@ -52,13 +57,105 @@ public virtual void TearDown() private ActivitySource? activitySource_; + private class CustomGetResultTable : IResultTable + { + private readonly List resultIds_; + private readonly string sessionId_; + + + public CustomGetResultTable(string sessionId, + List resultIds) + { + sessionId_ = sessionId; + resultIds_ = resultIds; + } + + public Task Check(HealthCheckTag tag) + => throw new NotImplementedException(); + + public Task Init(CancellationToken cancellationToken) + => throw new NotImplementedException(); + + public ILogger Logger + => NullLogger.Instance; + + public Task ChangeResultOwnership(string oldTaskId, + IEnumerable requests, + CancellationToken cancellationToken) + => throw new NotImplementedException(); + + public Task Create(ICollection results, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task AddTaskDependencies(IDictionary> dependencies, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task DeleteResult(string key, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task DeleteResults(string sessionId, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public IAsyncEnumerable GetResults(Expression> filter, + Expression> convertor, + CancellationToken cancellationToken = default) + => resultIds_.Select(s => convertor.Compile() + .Invoke(new Result(sessionId_, + s, + "", + "", + "", + ResultStatus.Completed, + new List(), + DateTime.UtcNow, + 100, + Encoding.UTF8.GetBytes(s)))) + .ToAsyncEnumerable(); + + public Task<(IEnumerable results, int totalCount)> ListResultsAsync(Expression> filter, + Expression> orderField, + bool ascOrder, + int page, + int pageSize, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task SetTaskOwnership(ICollection<(string resultId, string taskId)> requests, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task UpdateOneResult(string resultId, + UpdateDefinition updates, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + + public Task UpdateManyResults(Expression> filter, + UpdateDefinition updates, + CancellationToken cancellationToken = default) + => throw new NotImplementedException(); + } + [Test] public async Task EmptyPayloadAndOneDependency() { + const string sessionId = "SessionId"; + const string parentTaskId = "ParentTaskId"; + const string taskId = "TaskId"; + const string output1 = "Output1"; + const string dependency1 = "Dependency1"; + const string podId = "PodId"; + const string podName = "PodName"; + const string payloadId = "PayloadId"; + const string createdBy = "CreatedBy"; + var mockObjectStorage = new Mock(); - mockObjectStorage.Setup(x => x.GetValuesAsync(It.IsAny(), + mockObjectStorage.Setup(x => x.GetValuesAsync(It.IsAny(), CancellationToken.None)) - .Returns((string _, + .Returns((byte[] _, CancellationToken _) => new List { Convert.FromBase64String("1111"), @@ -66,21 +163,20 @@ public async Task EmptyPayloadAndOneDependency() Convert.FromBase64String("3333"), Convert.FromBase64String("4444"), }.ToAsyncEnumerable()); + var loggerFactory = new LoggerFactory(); var dataPrefetcher = new DataPrefetcher(mockObjectStorage.Object, + new CustomGetResultTable(sessionId, + new List + { + dependency1, + payloadId, + }), activitySource_, loggerFactory.CreateLogger()); - const string sessionId = "SessionId"; - const string parentTaskId = "ParentTaskId"; - const string taskId = "TaskId"; - const string output1 = "Output1"; - const string dependency1 = "Dependency1"; - const string podId = "PodId"; - const string podName = "PodName"; - const string payloadId = "PayloadId"; - const string createdBy = "CreatedBy"; + var sharedFolder = Path.Combine(Path.GetTempPath(), "data"); var internalFolder = Path.Combine(Path.GetTempPath(), @@ -135,10 +231,22 @@ await dataPrefetcher.PrefetchDataAsync(new TaskData(sessionId, [Test] public async Task EmptyPayloadAndNoDependenciesStateMachine() { + const string sessionId = "SessionId"; + const string parentTaskId = "ParentTaskId"; + const string taskId = "TaskId"; + const string output1 = "Output1"; + const string dependency1 = "Dependency1"; + const string dependency2 = "Dependency2"; + const string podId = "PodId"; + const string podName = "PodName"; + const string payloadId = "PayloadId"; + const string createdBy = "CreatedBy"; + + var mockObjectStorage = new Mock(); - mockObjectStorage.Setup(x => x.GetValuesAsync(It.IsAny(), + mockObjectStorage.Setup(x => x.GetValuesAsync(It.IsAny(), CancellationToken.None)) - .Returns((string _, + .Returns((byte[] _, CancellationToken _) => new List { Convert.FromBase64String("1111"), @@ -150,19 +258,17 @@ public async Task EmptyPayloadAndNoDependenciesStateMachine() var loggerFactory = new LoggerFactory(); var dataPrefetcher = new DataPrefetcher(mockObjectStorage.Object, + new CustomGetResultTable(sessionId, + new List + { + dependency1, + dependency2, + payloadId, + }), activitySource_, loggerFactory.CreateLogger()); - const string sessionId = "SessionId"; - const string parentTaskId = "ParentTaskId"; - const string taskId = "TaskId"; - const string output1 = "Output1"; - const string dependency1 = "Dependency1"; - const string dependency2 = "Dependency2"; - const string podId = "PodId"; - const string podName = "PodName"; - const string payloadId = "PayloadId"; - const string createdBy = "CreatedBy"; + var sharedFolder = Path.Combine(Path.GetTempPath(), "data"); var internalFolder = Path.Combine(Path.GetTempPath(), @@ -219,6 +325,7 @@ public async Task InitShouldSucceed() var loggerFactory = new LoggerFactory(); var dataPrefetcher = new DataPrefetcher(mockObjectStorage.Object, + new SimpleResultTable(), activitySource_, loggerFactory.CreateLogger()); diff --git a/Common/tests/Pollster/PollsterTest.cs b/Common/tests/Pollster/PollsterTest.cs index f9ea55cea..f36b4d238 100644 --- a/Common/tests/Pollster/PollsterTest.cs +++ b/Common/tests/Pollster/PollsterTest.cs @@ -25,7 +25,7 @@ using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Pollster; using ArmoniK.Core.Common.Storage; diff --git a/Common/tests/Pollster/TaskHandlerTest.cs b/Common/tests/Pollster/TaskHandlerTest.cs index 3867092d6..093f308d0 100644 --- a/Common/tests/Pollster/TaskHandlerTest.cs +++ b/Common/tests/Pollster/TaskHandlerTest.cs @@ -29,6 +29,7 @@ using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Meter; @@ -1784,21 +1785,18 @@ public Task Check(HealthCheckTag tag) public Task Init(CancellationToken cancellationToken) => Task.CompletedTask; - public Task AddOrUpdateAsync(string key, - IAsyncEnumerable> valueChunks, - CancellationToken cancellationToken = default) - => Task.FromResult(42); - - public IAsyncEnumerable GetValuesAsync(string key, + public IAsyncEnumerable GetValuesAsync(byte[] id, CancellationToken cancellationToken = default) => throw new ObjectDataNotFoundException(); - public Task TryDeleteAsync(IEnumerable keys, + public Task TryDeleteAsync(IEnumerable ids, CancellationToken cancellationToken = default) => Task.CompletedTask; - public IAsyncEnumerable ListKeysAsync(CancellationToken cancellationToken = default) - => throw new NotImplementedException(); + public Task<(byte[] id, long size)> AddOrUpdateAsync(ObjectData metaData, + IAsyncEnumerable> valueChunks, + CancellationToken cancellationToken = default) + => Task.FromResult<(byte[] id, long size)>((Encoding.UTF8.GetBytes("forty-two"), 42)); } [Test] diff --git a/Common/tests/Submitter/GrpcSubmitterServiceTests.cs b/Common/tests/Submitter/GrpcSubmitterServiceTests.cs index 312fe17ad..ccc8ff0b9 100644 --- a/Common/tests/Submitter/GrpcSubmitterServiceTests.cs +++ b/Common/tests/Submitter/GrpcSubmitterServiceTests.cs @@ -25,6 +25,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Storage; diff --git a/Common/tests/Submitter/IntegrationGrpcSubmitterServiceTest.cs b/Common/tests/Submitter/IntegrationGrpcSubmitterServiceTest.cs index c1f0f4bc6..0e04b6ad4 100644 --- a/Common/tests/Submitter/IntegrationGrpcSubmitterServiceTest.cs +++ b/Common/tests/Submitter/IntegrationGrpcSubmitterServiceTest.cs @@ -26,6 +26,7 @@ using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Storage; diff --git a/Common/tests/Submitter/SubmitterTests.cs b/Common/tests/Submitter/SubmitterTests.cs index fe9f173de..5bba0080d 100644 --- a/Common/tests/Submitter/SubmitterTests.cs +++ b/Common/tests/Submitter/SubmitterTests.cs @@ -26,6 +26,7 @@ using ArmoniK.Api.Common.Options; using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Submitter; +using ArmoniK.Core.Adapters.Memory; using ArmoniK.Core.Adapters.MongoDB; using ArmoniK.Core.Base; using ArmoniK.Core.Common.Exceptions; @@ -94,10 +95,6 @@ public async Task SetUp() $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.TableStorage.PollingDelayMin)}", "00:00:10" }, - { - $"{Adapters.MongoDB.Options.MongoDB.SettingSection}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage)}:{nameof(Adapters.MongoDB.Options.MongoDB.ObjectStorage.ChunkSize)}", - "14000" - }, { $"{ComputePlane.SettingSection}:{nameof(ComputePlane.MessageBatchSize)}", "1" }, @@ -126,6 +123,7 @@ public async Task SetUp() .AddSingleton(client_) .AddLogging(builder => builder.AddProvider(loggerProvider)) .AddSingleton() + .AddSingleton() .AddOption(configuration, Injection.Options.Submitter.SettingSection) .AddSingleton(pushQueueStorage_); @@ -969,10 +967,7 @@ await submitter_.TryGetResult(resultRequest, Assert.AreEqual(ResultReply.TypeOneofCase.Result, writer.Messages[1] .TypeCase); - Assert.AreEqual(ResultReply.TypeOneofCase.Result, - writer.Messages[2] - .TypeCase); - Assert.IsTrue(writer.Messages[2] + Assert.IsTrue(writer.Messages[1] .Result.DataComplete); } diff --git a/Common/tests/TaskLifeCycleHelperTest.cs b/Common/tests/TaskLifeCycleHelperTest.cs index 04421b3f5..6535c0fa7 100644 --- a/Common/tests/TaskLifeCycleHelperTest.cs +++ b/Common/tests/TaskLifeCycleHelperTest.cs @@ -427,7 +427,8 @@ await TaskLifeCycleHelper.FinalizeTaskCreation(holder.TaskTable, await holder.ResultTable.CompleteResult(sessionId, results[1] .ResultId, - 10) + 10, + Encoding.UTF8.GetBytes("first data dependency")) .ConfigureAwait(false); await TaskLifeCycleHelper.ResolveDependencies(holder.TaskTable, @@ -455,7 +456,8 @@ await TaskLifeCycleHelper.ResolveDependencies(holder.TaskTable, await holder.ResultTable.CompleteResult(sessionId, results[2] .ResultId, - 10) + 10, + Encoding.UTF8.GetBytes("second data dependency")) .ConfigureAwait(false); await TaskLifeCycleHelper.ResolveDependencies(holder.TaskTable, @@ -613,12 +615,14 @@ await TaskLifeCycleHelper.FinalizeTaskCreation(holder.TaskTable, await holder.ResultTable.CompleteResult(sessionId, results[1] .ResultId, - 10) + 10, + Encoding.UTF8.GetBytes("first data dependency")) .ConfigureAwait(false); await holder.ResultTable.CompleteResult(sessionId, results[2] .ResultId, - 10) + 10, + Encoding.UTF8.GetBytes("second data dependency")) .ConfigureAwait(false); await TaskLifeCycleHelper.ResolveDependencies(holder.TaskTable, holder.ResultTable, diff --git a/Common/tests/TestBase/ObjectStorageTestBase.cs b/Common/tests/TestBase/ObjectStorageTestBase.cs index 72248c1b4..ef1b37cd3 100644 --- a/Common/tests/TestBase/ObjectStorageTestBase.cs +++ b/Common/tests/TestBase/ObjectStorageTestBase.cs @@ -22,9 +22,9 @@ using System.Threading; using System.Threading.Tasks; +using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; -using ArmoniK.Core.Common.Exceptions; -using ArmoniK.Core.Common.Storage; +using ArmoniK.Core.Base.Exceptions; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -55,25 +55,37 @@ public async Task SetUp() Encoding.ASCII.GetBytes("CCCC"), Encoding.ASCII.GetBytes("DDDD"), }; - await ObjectStorage!.AddOrUpdateAsync("dataKey1", - dataBytesList.ToAsyncEnumerable()) - .ConfigureAwait(false); + datakey1_ = (await ObjectStorage!.AddOrUpdateAsync(new ObjectData + { + ResultId = "ResultId", + SessionId = "SessionId", + }, + dataBytesList.ToAsyncEnumerable()) + .ConfigureAwait(false)).id; dataBytesList = new List> { Encoding.ASCII.GetBytes("AAAABBBB"), }; - await ObjectStorage.AddOrUpdateAsync("dataKey2", - dataBytesList.ToAsyncEnumerable()) - .ConfigureAwait(false); + datakey2_ = (await ObjectStorage.AddOrUpdateAsync(new ObjectData + { + ResultId = "ResultId", + SessionId = "SessionId", + }, + dataBytesList.ToAsyncEnumerable()) + .ConfigureAwait(false)).id; dataBytesList = new List> { Array.Empty(), }; - await ObjectStorage.AddOrUpdateAsync("dataKeyEmpty", - dataBytesList.ToAsyncEnumerable()) - .ConfigureAwait(false); + datakeyEmpty_ = (await ObjectStorage.AddOrUpdateAsync(new ObjectData + { + ResultId = "ResultId", + SessionId = "SessionId", + }, + dataBytesList.ToAsyncEnumerable()) + .ConfigureAwait(false)).id; } [TearDown] @@ -94,7 +106,10 @@ private static bool CheckForSkipSetup() /* Boolean to control that tests are executed in * an instance of this class */ - protected bool RunTests; + protected bool RunTests; + private byte[]? datakey1_; + private byte[]? datakey2_; + private byte[]? datakeyEmpty_; /* Function be override so it returns the suitable instance * of TaskTable to the corresponding interface implementation */ @@ -138,11 +153,15 @@ public async Task AddValuesAsyncWithoutChunkShouldWork() { if (RunTests) { - var size = await ObjectStorage!.AddOrUpdateAsync("dataKeyNoChunk", - AsyncEnumerable.Empty>()) - .ConfigureAwait(false); + var (id, size) = await ObjectStorage!.AddOrUpdateAsync(new ObjectData + { + ResultId = "ResultId", + SessionId = "SessionId", + }, + AsyncEnumerable.Empty>()) + .ConfigureAwait(false); var data = new List(); - await foreach (var chunk in ObjectStorage!.GetValuesAsync("dataKeyNoChunk") + await foreach (var chunk in ObjectStorage!.GetValuesAsync(id) .ConfigureAwait(false)) { data.AddRange(chunk); @@ -174,13 +193,17 @@ public async Task AddValuesAsyncShouldWork(params string[] inputChunks) { if (RunTests) { - var size = await ObjectStorage!.AddOrUpdateAsync("dataKeyTest", - inputChunks.ToAsyncEnumerable() - .Select(s => (ReadOnlyMemory)Encoding.ASCII.GetBytes(s))) - .ConfigureAwait(false); + var (id, size) = await ObjectStorage!.AddOrUpdateAsync(new ObjectData + { + ResultId = "ResultId", + SessionId = "SessionId", + }, + inputChunks.ToAsyncEnumerable() + .Select(s => (ReadOnlyMemory)Encoding.ASCII.GetBytes(s))) + .ConfigureAwait(false); var data = new List(); - await foreach (var chunk in ObjectStorage!.GetValuesAsync("dataKeyTest") + await foreach (var chunk in ObjectStorage!.GetValuesAsync(id) .ConfigureAwait(false)) { data.AddRange(chunk); @@ -197,13 +220,33 @@ public async Task AddValuesAsyncShouldWork(params string[] inputChunks) } [Test] - public void GetValuesAsyncShouldFail() + public async Task GetValuesAsyncShouldFail() { if (RunTests) { - var res = ObjectStorage!.GetValuesAsync("dataKeyNotExist"); - Assert.ThrowsAsync(async () => await res.FirstAsync() - .ConfigureAwait(false)); + var id = Encoding.UTF8.GetBytes("IdThatShouldFail"); + var data = new List(); + + try + { + var res = ObjectStorage!.GetValuesAsync(id); + + + await foreach (var chunk in res.ConfigureAwait(false)) + { + data.AddRange(chunk); + } + + Assert.AreEqual(id, + data.ToArray()); + } + catch (ObjectDataNotFoundException) + { + } + catch (Exception) + { + Assert.Fail("When value should not be available, it should either throw or return the value"); + } } } @@ -212,7 +255,7 @@ public async Task PayloadShouldBeEqual() { if (RunTests) { - var res = ObjectStorage!.GetValuesAsync("dataKey2"); + var res = ObjectStorage!.GetValuesAsync(datakey2_!); var data = new List(); foreach (var item in await res.ToListAsync() .ConfigureAwait(false)) @@ -222,7 +265,8 @@ public async Task PayloadShouldBeEqual() var str = Encoding.ASCII.GetString(data.ToArray()); Console.WriteLine(str); - Assert.IsTrue(str.SequenceEqual("AAAABBBB")); + Assert.AreEqual("AAAABBBB", + str); } } @@ -231,7 +275,7 @@ public async Task Payload2ShouldBeEqual() { if (RunTests) { - var res = ObjectStorage!.GetValuesAsync("dataKey1"); + var res = ObjectStorage!.GetValuesAsync(datakey1_!); // var data = await res.AggregateAsync((bytes1, bytes2) => bytes1.Concat(bytes2).ToArray()); var data = new List(); foreach (var item in await res.ToListAsync() @@ -242,7 +286,8 @@ public async Task Payload2ShouldBeEqual() var str = Encoding.ASCII.GetString(data.ToArray()); Console.WriteLine(str); - Assert.IsTrue(str.SequenceEqual("AAAABBBBCCCCDDDD")); + Assert.AreEqual("AAAABBBBCCCCDDDD", + str); } } @@ -251,7 +296,7 @@ public async Task EmptyPayload() { if (RunTests) { - var res = await ObjectStorage!.GetValuesAsync("dataKeyEmpty") + var res = await ObjectStorage!.GetValuesAsync(datakeyEmpty_!) .ToListAsync() .ConfigureAwait(false); Console.WriteLine(res.Count); @@ -267,44 +312,6 @@ public async Task EmptyPayload() } } - [Test] - public async Task DeleteKeysAndGetValuesAsyncShouldFail() - { - if (RunTests) - { - var listChunks = new List> - { - Encoding.ASCII.GetBytes("Armonik Payload chunk"), - Encoding.ASCII.GetBytes("Data 1"), - Encoding.ASCII.GetBytes("Data 2"), - Encoding.ASCII.GetBytes("Data 3"), - Encoding.ASCII.GetBytes("Data 4"), - }; - - await ObjectStorage!.AddOrUpdateAsync("dataKey", - listChunks.ToAsyncEnumerable()) - .ConfigureAwait(false); - - var res = await ObjectStorage!.GetValuesAsync("dataKey") - .ToListAsync() - .ConfigureAwait(false); - Assert.AreEqual(string.Join("", - listChunks.Select(chunk => Encoding.ASCII.GetString(chunk.ToArray()))), - string.Join("", - res.Select(chunk => Encoding.ASCII.GetString(chunk)))); - - await ObjectStorage!.TryDeleteAsync(new[] - { - "dataKey", - }) - .ConfigureAwait(false); - - Assert.ThrowsAsync(async () => await ObjectStorage!.GetValuesAsync("dataKey") - .FirstAsync() - .ConfigureAwait(false)); - } - } - [Test] public async Task DeleteDeleteTwiceShouldSucceed() { @@ -319,19 +326,23 @@ public async Task DeleteDeleteTwiceShouldSucceed() Encoding.ASCII.GetBytes("Data 4"), }; - await ObjectStorage!.AddOrUpdateAsync("dataKey", - listChunks.ToAsyncEnumerable()) - .ConfigureAwait(false); + var (id, size) = await ObjectStorage!.AddOrUpdateAsync(new ObjectData + { + ResultId = "ResultId", + SessionId = "SessionId", + }, + listChunks.ToAsyncEnumerable()) + .ConfigureAwait(false); await ObjectStorage!.TryDeleteAsync(new[] { - "dataKey", + id, }) .ConfigureAwait(false); await ObjectStorage!.TryDeleteAsync(new[] { - "dataKey", + id, }) .ConfigureAwait(false); } diff --git a/Common/tests/TestBase/ResultTableTestBase.cs b/Common/tests/TestBase/ResultTableTestBase.cs index 797fd1976..1c39e1068 100644 --- a/Common/tests/TestBase/ResultTableTestBase.cs +++ b/Common/tests/TestBase/ResultTableTestBase.cs @@ -23,6 +23,7 @@ using System.Threading.Tasks; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.Storage; using ArmoniK.Utils; @@ -323,48 +324,25 @@ public async Task SetResultShouldSucceed() { if (RunTests) { - await ResultTable!.SetResult((string)"SessionId", - (string)"OwnerId", - (string)"ResultIsNotAvailable", - 5, - CancellationToken.None) - .ConfigureAwait(false); - - var result = await ResultTable!.GetResult("ResultIsNotAvailable", - CancellationToken.None) - .ConfigureAwait(false); - - Assert.IsTrue(result.ResultId == "ResultIsNotAvailable"); - Assert.AreEqual(5, - result.Size); - } - } - - [Test] - public async Task SetResultSmallPayloadShouldSucceed() - { - if (RunTests) - { - var smallPayload = new[] - { - (byte)1, - (byte)2, - }; - + var id = Encoding.UTF8.GetBytes("OpaqueId"); await ResultTable!.SetResult("SessionId", "OwnerId", "ResultIsNotAvailable", - smallPayload, + 5, + id, CancellationToken.None) .ConfigureAwait(false); + var result = await ResultTable!.GetResult("ResultIsNotAvailable", CancellationToken.None) .ConfigureAwait(false); - Assert.AreEqual(result.Data, - smallPayload); - Assert.AreEqual(smallPayload.Length, + Assert.AreEqual("ResultIsNotAvailable", + result.ResultId); + Assert.AreEqual(5, result.Size); + Assert.AreEqual(id, + result.OpaqueId); } } @@ -769,6 +747,7 @@ public async Task CompleteResultShouldSucceed() .ToString(); var sessionId = Guid.NewGuid() .ToString(); + var id = Encoding.UTF8.GetBytes("OpaqueId"); await ResultTable!.Create(new List { new(sessionId, @@ -788,11 +767,16 @@ public async Task CompleteResultShouldSucceed() var result = await ResultTable.CompleteResult(sessionId, resultId, 5, + id, CancellationToken.None) .ConfigureAwait(false); Assert.AreEqual(ResultStatus.Completed, result.Status); + Assert.AreEqual(id, + result.OpaqueId); + Assert.AreEqual(5, + result.Size); result = await ResultTable.GetResult(resultId, CancellationToken.None) @@ -802,6 +786,8 @@ public async Task CompleteResultShouldSucceed() result.Status); Assert.AreEqual(5, result.Size); + Assert.AreEqual(id, + result.OpaqueId); } } @@ -810,9 +796,11 @@ public void CompleteResultShouldThrow() { if (RunTests) { + var id = Encoding.UTF8.GetBytes("OpaqueId"); Assert.ThrowsAsync(async () => await ResultTable!.CompleteResult("SessionId", "NotExistingResult111", 5, + id, CancellationToken.None) .ConfigureAwait(false)); } diff --git a/Common/tests/TestBase/TaskTableTestBase.cs b/Common/tests/TestBase/TaskTableTestBase.cs index c197133e5..c84fa8030 100644 --- a/Common/tests/TestBase/TaskTableTestBase.cs +++ b/Common/tests/TestBase/TaskTableTestBase.cs @@ -32,6 +32,7 @@ using ArmoniK.Api.gRPC.V1.Submitter; using ArmoniK.Api.gRPC.V1.Tasks; using ArmoniK.Core.Base.DataStructures; +using ArmoniK.Core.Base.Exceptions; using ArmoniK.Core.Common.Exceptions; using ArmoniK.Core.Common.gRPC; using ArmoniK.Core.Common.gRPC.Convertors; diff --git a/Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj b/Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj index de55cc27f..ccc01cead 100644 --- a/Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj +++ b/Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj @@ -36,10 +36,7 @@ - - - diff --git a/Compute/PollingAgent/src/Program.cs b/Compute/PollingAgent/src/Program.cs index 48f2d58c5..7e8c133dd 100644 --- a/Compute/PollingAgent/src/Program.cs +++ b/Compute/PollingAgent/src/Program.cs @@ -22,15 +22,13 @@ using System.Threading; using System.Threading.Tasks; -using ArmoniK.Core.Adapters.LocalStorage; using ArmoniK.Core.Adapters.MongoDB; -using ArmoniK.Core.Adapters.Redis; -using ArmoniK.Core.Adapters.S3; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Common.DynamicLoading; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Injection; +using ArmoniK.Core.Common.Injection.Options; using ArmoniK.Core.Common.Meter; using ArmoniK.Core.Common.Pollster; using ArmoniK.Core.Common.Pollster.TaskProcessingChecker; @@ -93,14 +91,12 @@ public static async Task Main(string[] args) .AddArmoniKWorkerConnection(builder.Configuration) .AddMongoComponents(builder.Configuration, logger.GetLogger()) - .AddQueue(builder.Configuration, - logger.GetLogger()) - .AddRedis(builder.Configuration, - logger.GetLogger()) - .AddS3(builder.Configuration, - logger.GetLogger()) - .AddLocalStorage(builder.Configuration, - logger.GetLogger()) + .AddAdapter(builder.Configuration, + nameof(Components.QueueAdaptorSettings), + logger.GetLogger()) + .AddAdapter(builder.Configuration, + nameof(Components.ObjectStorageAdaptorSettings), + logger.GetLogger()) .AddHostedService() .AddHostedService() .AddHostedService() diff --git a/Control/Metrics/src/ArmoniK.Core.Control.Metrics.csproj b/Control/Metrics/src/ArmoniK.Core.Control.Metrics.csproj index 10d4781b6..53290c993 100644 --- a/Control/Metrics/src/ArmoniK.Core.Control.Metrics.csproj +++ b/Control/Metrics/src/ArmoniK.Core.Control.Metrics.csproj @@ -35,10 +35,7 @@ - - - diff --git a/Control/PartitionMetrics/src/ArmoniK.Core.Control.PartitionMetrics.csproj b/Control/PartitionMetrics/src/ArmoniK.Core.Control.PartitionMetrics.csproj index 6610aa411..5a70d5b96 100644 --- a/Control/PartitionMetrics/src/ArmoniK.Core.Control.PartitionMetrics.csproj +++ b/Control/PartitionMetrics/src/ArmoniK.Core.Control.PartitionMetrics.csproj @@ -37,10 +37,7 @@ - - - diff --git a/Control/Submitter/src/ArmoniK.Core.Control.Submitter.csproj b/Control/Submitter/src/ArmoniK.Core.Control.Submitter.csproj index 9177894ff..72201f23b 100644 --- a/Control/Submitter/src/ArmoniK.Core.Control.Submitter.csproj +++ b/Control/Submitter/src/ArmoniK.Core.Control.Submitter.csproj @@ -38,9 +38,6 @@ - - - diff --git a/Control/Submitter/src/Program.cs b/Control/Submitter/src/Program.cs index d5c5da458..2172d04d2 100644 --- a/Control/Submitter/src/Program.cs +++ b/Control/Submitter/src/Program.cs @@ -22,10 +22,7 @@ using System.Threading; using System.Threading.Tasks; -using ArmoniK.Core.Adapters.LocalStorage; using ArmoniK.Core.Adapters.MongoDB; -using ArmoniK.Core.Adapters.Redis; -using ArmoniK.Core.Adapters.S3; using ArmoniK.Core.Base; using ArmoniK.Core.Base.DataStructures; using ArmoniK.Core.Common.Auth.Authentication; @@ -33,6 +30,7 @@ using ArmoniK.Core.Common.gRPC; using ArmoniK.Core.Common.gRPC.Services; using ArmoniK.Core.Common.Injection; +using ArmoniK.Core.Common.Injection.Options; using ArmoniK.Core.Common.Meter; using ArmoniK.Core.Common.Pollster; using ArmoniK.Core.Common.Storage; @@ -91,14 +89,12 @@ public static async Task Main(string[] args) .AddHttpClient() .AddMongoComponents(builder.Configuration, logger.GetLogger()) - .AddQueue(builder.Configuration, - logger.GetLogger()) - .AddRedis(builder.Configuration, - logger.GetLogger()) - .AddLocalStorage(builder.Configuration, - logger.GetLogger()) - .AddS3(builder.Configuration, - logger.GetLogger()) + .AddAdapter(builder.Configuration, + nameof(Components.QueueAdaptorSettings), + logger.GetLogger()) + .AddAdapter(builder.Configuration, + nameof(Components.ObjectStorageAdaptorSettings), + logger.GetLogger()) .AddSingleton() .AddSingletonWithHealthCheck(nameof(ExceptionInterceptor)) .AddOption(builder.Configuration, diff --git a/Dockerfile b/Dockerfile index f8918d636..780b0e63f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ COPY ["Adaptors/PubSub/src/ArmoniK.Core.Adapters.PubSub.csproj", "Adaptors/PubSu COPY ["Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj", "Adaptors/Redis/src/"] COPY ["Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj", "Adaptors/S3/src/"] COPY ["Adaptors/SQS/src/ArmoniK.Core.Adapters.SQS.csproj", "Adaptors/SQS/src/"] +COPY ["Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj", "Adaptors/Embed/src/"] COPY ["Base/src/ArmoniK.Core.Base.csproj", "Base/src/"] COPY ["Common/src/ArmoniK.Core.Common.csproj", "Common/src/"] COPY ["Compute/PollingAgent/src/ArmoniK.Core.Compute.PollingAgent.csproj", "Compute/PollingAgent/src/"] @@ -40,6 +41,10 @@ RUN dotnet restore -a "${TARGETARCH}" "Adaptors/Amqp/src/ArmoniK.Core.Adapters.A RUN dotnet restore -a "${TARGETARCH}" "Adaptors/RabbitMQ/src/ArmoniK.Core.Adapters.RabbitMQ.csproj" RUN dotnet restore -a "${TARGETARCH}" "Adaptors/PubSub/src/ArmoniK.Core.Adapters.PubSub.csproj" RUN dotnet restore -a "${TARGETARCH}" "Adaptors/SQS/src/ArmoniK.Core.Adapters.SQS.csproj" +RUN dotnet restore -a "${TARGETARCH}" "Adaptors/S3/src/ArmoniK.Core.Adapters.S3.csproj" +RUN dotnet restore -a "${TARGETARCH}" "Adaptors/LocalStorage/src/ArmoniK.Core.Adapters.LocalStorage.csproj" +RUN dotnet restore -a "${TARGETARCH}" "Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj" +RUN dotnet restore -a "${TARGETARCH}" "Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj" # git ls-tree -r HEAD --name-only --full-tree | grep "csproj$" | xargs -I % sh -c "export D=\$(dirname %) ; echo COPY [\\\"\$D\\\", \\\"\$D\\\"]" COPY ["Adaptors/Amqp/src", "Adaptors/Amqp/src"] @@ -52,6 +57,7 @@ COPY ["Adaptors/PubSub/src", "Adaptors/PubSub/src"] COPY ["Adaptors/Redis/src", "Adaptors/Redis/src"] COPY ["Adaptors/S3/src", "Adaptors/S3/src"] COPY ["Adaptors/SQS/src", "Adaptors/SQS/src"] +COPY ["Adaptors/Embed/src", "Adaptors/Embed/src"] COPY ["Base/src", "Base/src"] COPY ["Common/src", "Common/src"] COPY ["Compute/PollingAgent/src", "Compute/PollingAgent/src"] @@ -72,6 +78,18 @@ RUN dotnet publish "ArmoniK.Core.Adapters.Amqp.csproj" -a "${TARGETARCH}" --no-r WORKDIR /src/Adaptors/RabbitMQ/src RUN dotnet publish "ArmoniK.Core.Adapters.RabbitMQ.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/rabbit /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION +WORKDIR /src/Adaptors/LocalStorage/src +RUN dotnet publish "ArmoniK.Core.Adapters.LocalStorage.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/local_storage /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION + +WORKDIR /src/Adaptors/Embed/src +RUN dotnet publish "ArmoniK.Core.Adapters.Embed.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/embed /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION + +WORKDIR /src/Adaptors/Redis/src +RUN dotnet publish "ArmoniK.Core.Adapters.Redis.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/redis /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION + +WORKDIR /src/Adaptors/S3/src +RUN dotnet publish "ArmoniK.Core.Adapters.S3.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/s3 /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION + WORKDIR /src/Compute/PollingAgent/src RUN dotnet publish "ArmoniK.Core.Compute.PollingAgent.csproj" -a "${TARGETARCH}" --no-restore -o /app/publish/polling_agent /p:UseAppHost=false -p:RunAnalyzers=false -p:WarningLevel=0 -p:PackageVersion=$VERSION -p:Version=$VERSION @@ -94,6 +112,14 @@ WORKDIR /adapters/queue/amqp COPY --from=build /app/publish/amqp . WORKDIR /adapters/queue/rabbit COPY --from=build /app/publish/rabbit . +WORKDIR /adapters/object/local_storage +COPY --from=build /app/publish/local_storage . +WORKDIR /adapters/object/redis +COPY --from=build /app/publish/redis . +WORKDIR /adapters/object/embed +COPY --from=build /app/publish/embed . +WORKDIR /adapters/object/s3 +COPY --from=build /app/publish/s3 . WORKDIR /app COPY --from=build /app/publish/polling_agent . ENV ASPNETCORE_URLS http://+:1080 @@ -126,6 +152,14 @@ WORKDIR /adapters/queue/amqp COPY --from=build /app/publish/amqp . WORKDIR /adapters/queue/rabbit COPY --from=build /app/publish/rabbit . +WORKDIR /adapters/object/local_storage +COPY --from=build /app/publish/local_storage . +WORKDIR /adapters/object/redis +COPY --from=build /app/publish/redis . +WORKDIR /adapters/object/embed +COPY --from=build /app/publish/embed . +WORKDIR /adapters/object/s3 +COPY --from=build /app/publish/s3 . WORKDIR /app COPY --from=build /app/publish/submitter . ENV ASPNETCORE_URLS http://+:1080, http://+:1081 diff --git a/justfile b/justfile index 8b255cc74..dad19ad66 100644 --- a/justfile +++ b/justfile @@ -62,6 +62,8 @@ object_storage := if object == "redis" { '{ name = "redis", image = "redis:bullseye" ' } else if object == "minio" { '{ name = "minio", image = "quay.io/minio/minio" ' +} else if object == "embed" { + '{ name = "embed"' } else { '{ name = "local", image = "" ' } @@ -179,6 +181,7 @@ _usage: redis: to use redis for object storage (default) minio: to use minio for object storage. local: to mount a local volume for object storage + embed: to use the database as an object storage replicas: Number of polling agents / worker to be replicated (default = 3) diff --git a/terraform/locals.tf b/terraform/locals.tf index e5e81099e..074743fc6 100644 --- a/terraform/locals.tf +++ b/terraform/locals.tf @@ -16,7 +16,7 @@ locals { worker = merge(var.compute_plane.worker, { image = var.worker_image }) queue = one(concat(module.queue_activemq, module.queue_rabbitmq, module.queue_artemis, module.queue_pubsub, module.queue_sqs, module.queue_none)) database = module.database - object = one(concat(module.object_redis, module.object_minio, module.object_local)) + object = one(concat(module.object_redis, module.object_minio, module.object_local, module.object_embed)) env_maps = concat([ local.queue.generated_env_vars, local.object.generated_env_vars, diff --git a/terraform/main.tf b/terraform/main.tf index 56f9b4b2b..b20d0b968 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -60,6 +60,11 @@ module "object_local" { local_path = var.object_storage.local_storage_path } +module "object_embed" { + source = "./modules/storage/object/embed" + count = var.object_storage.name == "embed" ? 1 : 0 +} + module "queue_rabbitmq" { source = "./modules/storage/queue/rabbitmq" count = var.queue_storage.name == "rabbitmq" ? 1 : 0 diff --git a/terraform/modules/storage/object/embed/output.tf b/terraform/modules/storage/object/embed/output.tf new file mode 100644 index 000000000..3109defe1 --- /dev/null +++ b/terraform/modules/storage/object/embed/output.tf @@ -0,0 +1,11 @@ +output "generated_env_vars" { + value = ({ + "Components__ObjectStorageAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.Embed.ObjectBuilder" + "Components__ObjectStorageAdaptorSettings__AdapterAbsolutePath" = "/adapters/object/embed/ArmoniK.Core.Adapters.Embed.dll" + }) +} + +output "volumes" { + description = "Volumes that agents and submitters must mount to access the object storage" + value = {} +} diff --git a/terraform/modules/storage/object/local/output.tf b/terraform/modules/storage/object/local/output.tf index 0892e2b89..bb9ad423b 100644 --- a/terraform/modules/storage/object/local/output.tf +++ b/terraform/modules/storage/object/local/output.tf @@ -1,7 +1,8 @@ output "generated_env_vars" { value = ({ - "Components__ObjectStorage" = "ArmoniK.Adapters.LocalStorage.ObjectStorage", - "LocalStorage__Path" = var.local_path + "Components__ObjectStorageAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.LocalStorage.ObjectBuilder" + "Components__ObjectStorageAdaptorSettings__AdapterAbsolutePath" = "/adapters/object/local_storage/ArmoniK.Core.Adapters.LocalStorage.dll" + "LocalStorage__Path" = var.local_path }) } diff --git a/terraform/modules/storage/object/minio/outputs.tf b/terraform/modules/storage/object/minio/outputs.tf index 6f9947d26..ad1c280ad 100644 --- a/terraform/modules/storage/object/minio/outputs.tf +++ b/terraform/modules/storage/object/minio/outputs.tf @@ -1,11 +1,12 @@ output "generated_env_vars" { value = ({ - "Components__ObjectStorage" = "ArmoniK.Adapters.S3.ObjectStorage", - "S3__EndpointUrl" = "http://${var.host}:${var.port}" - "S3__BucketName" = var.bucket_name - "S3__Login" = var.login - "S3__Password" = var.password - "S3__MustForcePathStyle" = true + "Components__ObjectStorageAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.S3.ObjectBuilder" + "Components__ObjectStorageAdaptorSettings__AdapterAbsolutePath" = "/adapters/object/s3/ArmoniK.Core.Adapters.S3.dll" + "S3__EndpointUrl" = "http://${var.host}:${var.port}" + "S3__BucketName" = var.bucket_name + "S3__Login" = var.login + "S3__Password" = var.password + "S3__MustForcePathStyle" = true }) } diff --git a/terraform/modules/storage/object/redis/outputs.tf b/terraform/modules/storage/object/redis/outputs.tf index 7ffd47b7f..82b09244f 100644 --- a/terraform/modules/storage/object/redis/outputs.tf +++ b/terraform/modules/storage/object/redis/outputs.tf @@ -1,7 +1,8 @@ output "generated_env_vars" { value = ({ - "Components__ObjectStorage" = "ArmoniK.Adapters.Redis.ObjectStorage", - "Redis__EndpointUrl" = "object:${var.exposed_port}" + "Components__ObjectStorageAdaptorSettings__ClassName" = "ArmoniK.Core.Adapters.Redis.ObjectBuilder" + "Components__ObjectStorageAdaptorSettings__AdapterAbsolutePath" = "/adapters/object/redis/ArmoniK.Core.Adapters.Redis.dll" + "Redis__EndpointUrl" = "object:${var.exposed_port}" }) } output "volumes" { diff --git a/terraform/variables.tf b/terraform/variables.tf index ea1eb649a..23efed83c 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -62,8 +62,8 @@ variable "object_storage" { local_storage_path = optional(string, "/local_storage") }) validation { - condition = can(regex("^(redis|local|minio)$", var.object_storage.name)) - error_message = "Must be redis, minio, or local" + condition = can(regex("^(redis|local|minio|embed)$", var.object_storage.name)) + error_message = "Must be redis, minio, embed, or local" } default = {} }