Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migration tools to migrate serialized data from 3.x to 7.0 #8214

Draft
wants to merge 5 commits into
base: 3.x
Choose a base branch
from
Draft

Migration tools to migrate serialized data from 3.x to 7.0 #8214

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
af5ce19
Utilities to serialize some type in a 7.0 compliant way
benjaminpetit Nov 17, 2022
78af76d
Added base list operation on IGrainStorage
benjaminpetit Dec 7, 2022
5da476b
WIP
benjaminpetit Dec 7, 2022
e7c4ebf
WIP AzureBlob migration provider
benjaminpetit Dec 9, 2022
cf9cd2e
Hack to make build pass
benjaminpetit Jan 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions Orleans.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DistributedTests.Client", "
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DistributedTests.Server", "test\DistributedTests\DistributedTests.Server\DistributedTests.Server.csproj", "{E8335DC9-9A7F-45C1-AFA3-0AA93ABD4FA5}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Orleans.Persistence.Migration", "src\Orleans.Persistence.Migration\Orleans.Persistence.Migration.csproj", "{E0B94181-200A-4CC6-94CA-D3D81CA6720A}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Migration.Tests", "test\Extensions\Migration.Tests\Migration.Tests.csproj", "{57285DA6-6EF1-4B46-A178-1ABB48CD0B57}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Orleans.Persistence.AzureStorage.Migration", "src\Azure\Orleans.Persistence.AzureStorage.Migration\Orleans.Persistence.AzureStorage.Migration.csproj", "{0073C9D9-1906-452A-873E-9A2639FFB041}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tester.AzureUtils.Migration", "test\Extensions\Tester.AzureUtils.Migration\Tester.AzureUtils.Migration.csproj", "{C5529A77-1931-469B-AF8C-9CEEAE555FEA}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -1318,6 +1326,54 @@ Global
{E8335DC9-9A7F-45C1-AFA3-0AA93ABD4FA5}.Release|x64.Build.0 = Release|Any CPU
{E8335DC9-9A7F-45C1-AFA3-0AA93ABD4FA5}.Release|x86.ActiveCfg = Release|Any CPU
{E8335DC9-9A7F-45C1-AFA3-0AA93ABD4FA5}.Release|x86.Build.0 = Release|Any CPU
{E0B94181-200A-4CC6-94CA-D3D81CA6720A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E0B94181-200A-4CC6-94CA-D3D81CA6720A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E0B94181-200A-4CC6-94CA-D3D81CA6720A}.Debug|x64.ActiveCfg = Debug|Any CPU
{E0B94181-200A-4CC6-94CA-D3D81CA6720A}.Debug|x64.Build.0 = Debug|Any CPU
{E0B94181-200A-4CC6-94CA-D3D81CA6720A}.Debug|x86.ActiveCfg = Debug|Any CPU
{E0B94181-200A-4CC6-94CA-D3D81CA6720A}.Debug|x86.Build.0 = Debug|Any CPU
{E0B94181-200A-4CC6-94CA-D3D81CA6720A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E0B94181-200A-4CC6-94CA-D3D81CA6720A}.Release|Any CPU.Build.0 = Release|Any CPU
{E0B94181-200A-4CC6-94CA-D3D81CA6720A}.Release|x64.ActiveCfg = Release|Any CPU
{E0B94181-200A-4CC6-94CA-D3D81CA6720A}.Release|x64.Build.0 = Release|Any CPU
{E0B94181-200A-4CC6-94CA-D3D81CA6720A}.Release|x86.ActiveCfg = Release|Any CPU
{E0B94181-200A-4CC6-94CA-D3D81CA6720A}.Release|x86.Build.0 = Release|Any CPU
{57285DA6-6EF1-4B46-A178-1ABB48CD0B57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{57285DA6-6EF1-4B46-A178-1ABB48CD0B57}.Debug|Any CPU.Build.0 = Debug|Any CPU
{57285DA6-6EF1-4B46-A178-1ABB48CD0B57}.Debug|x64.ActiveCfg = Debug|Any CPU
{57285DA6-6EF1-4B46-A178-1ABB48CD0B57}.Debug|x64.Build.0 = Debug|Any CPU
{57285DA6-6EF1-4B46-A178-1ABB48CD0B57}.Debug|x86.ActiveCfg = Debug|Any CPU
{57285DA6-6EF1-4B46-A178-1ABB48CD0B57}.Debug|x86.Build.0 = Debug|Any CPU
{57285DA6-6EF1-4B46-A178-1ABB48CD0B57}.Release|Any CPU.ActiveCfg = Release|Any CPU
{57285DA6-6EF1-4B46-A178-1ABB48CD0B57}.Release|Any CPU.Build.0 = Release|Any CPU
{57285DA6-6EF1-4B46-A178-1ABB48CD0B57}.Release|x64.ActiveCfg = Release|Any CPU
{57285DA6-6EF1-4B46-A178-1ABB48CD0B57}.Release|x64.Build.0 = Release|Any CPU
{57285DA6-6EF1-4B46-A178-1ABB48CD0B57}.Release|x86.ActiveCfg = Release|Any CPU
{57285DA6-6EF1-4B46-A178-1ABB48CD0B57}.Release|x86.Build.0 = Release|Any CPU
{0073C9D9-1906-452A-873E-9A2639FFB041}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0073C9D9-1906-452A-873E-9A2639FFB041}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0073C9D9-1906-452A-873E-9A2639FFB041}.Debug|x64.ActiveCfg = Debug|Any CPU
{0073C9D9-1906-452A-873E-9A2639FFB041}.Debug|x64.Build.0 = Debug|Any CPU
{0073C9D9-1906-452A-873E-9A2639FFB041}.Debug|x86.ActiveCfg = Debug|Any CPU
{0073C9D9-1906-452A-873E-9A2639FFB041}.Debug|x86.Build.0 = Debug|Any CPU
{0073C9D9-1906-452A-873E-9A2639FFB041}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0073C9D9-1906-452A-873E-9A2639FFB041}.Release|Any CPU.Build.0 = Release|Any CPU
{0073C9D9-1906-452A-873E-9A2639FFB041}.Release|x64.ActiveCfg = Release|Any CPU
{0073C9D9-1906-452A-873E-9A2639FFB041}.Release|x64.Build.0 = Release|Any CPU
{0073C9D9-1906-452A-873E-9A2639FFB041}.Release|x86.ActiveCfg = Release|Any CPU
{0073C9D9-1906-452A-873E-9A2639FFB041}.Release|x86.Build.0 = Release|Any CPU
{C5529A77-1931-469B-AF8C-9CEEAE555FEA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C5529A77-1931-469B-AF8C-9CEEAE555FEA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C5529A77-1931-469B-AF8C-9CEEAE555FEA}.Debug|x64.ActiveCfg = Debug|Any CPU
{C5529A77-1931-469B-AF8C-9CEEAE555FEA}.Debug|x64.Build.0 = Debug|Any CPU
{C5529A77-1931-469B-AF8C-9CEEAE555FEA}.Debug|x86.ActiveCfg = Debug|Any CPU
{C5529A77-1931-469B-AF8C-9CEEAE555FEA}.Debug|x86.Build.0 = Debug|Any CPU
{C5529A77-1931-469B-AF8C-9CEEAE555FEA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C5529A77-1931-469B-AF8C-9CEEAE555FEA}.Release|Any CPU.Build.0 = Release|Any CPU
{C5529A77-1931-469B-AF8C-9CEEAE555FEA}.Release|x64.ActiveCfg = Release|Any CPU
{C5529A77-1931-469B-AF8C-9CEEAE555FEA}.Release|x64.Build.0 = Release|Any CPU
{C5529A77-1931-469B-AF8C-9CEEAE555FEA}.Release|x86.ActiveCfg = Release|Any CPU
{C5529A77-1931-469B-AF8C-9CEEAE555FEA}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -1427,6 +1483,10 @@ Global
{7F6A75BB-72AE-4509-8E82-55C764D0C070} = {FFEC9FEE-FEDF-4510-B7D2-0B0B3374ED2F}
{25D20278-8901-47CC-AD1D-F3C4BEB845BF} = {FFEC9FEE-FEDF-4510-B7D2-0B0B3374ED2F}
{E8335DC9-9A7F-45C1-AFA3-0AA93ABD4FA5} = {FFEC9FEE-FEDF-4510-B7D2-0B0B3374ED2F}
{E0B94181-200A-4CC6-94CA-D3D81CA6720A} = {FE2E08C6-9C3B-4AEE-AE07-CCA387580D7A}
{57285DA6-6EF1-4B46-A178-1ABB48CD0B57} = {082D25DB-70CA-48F4-93E0-EC3455F494B8}
{0073C9D9-1906-452A-873E-9A2639FFB041} = {4C5D66BF-EE1C-4DD8-8551-D1B7F3768A34}
{C5529A77-1931-469B-AF8C-9CEEAE555FEA} = {082D25DB-70CA-48F4-93E0-EC3455F494B8}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {7BFB3429-B5BB-4DB1-95B4-67D77A864952}
Expand Down
2 changes: 1 addition & 1 deletion global.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"sdk": {
"rollForward": "feature",
"version": "6.0.400"
"version": "7.0.100"
}
}
2 changes: 2 additions & 0 deletions src/AWS/Orleans.Persistence.DynamoDB/Provider/DynamoDBGrainStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,8 @@ internal void ConvertToStorageFormat(object grainState, GrainStateRecord entity)
throw new ArgumentOutOfRangeException("GrainState.Size", msg);
}
}

public IAsyncEnumerable<StorageEntry> GetAll(CancellationToken cancellationToken) => throw new NotImplementedException();
}

public static class DynamoDBGrainStorageFactory
Expand Down
3 changes: 3 additions & 0 deletions src/AdoNet/Orleans.Persistence.AdoNet/Storage/Provider/AdoNetGrainStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
using Orleans.Configuration;
using Orleans.Configuration.Overrides;
using Orleans.Runtime.Configuration;
using System.Runtime.CompilerServices;

namespace Orleans.Storage
{
Expand Down Expand Up @@ -614,5 +615,7 @@ private ICollection<IStorageSerializer> ConfigureSerializers(AdoNetGrainStorageO

return serializers;
}

public IAsyncEnumerable<StorageEntry> GetAll(CancellationToken cancellationToken) => throw new NotImplementedException();
}
}
121 changes: 121 additions & 0 deletions src/Azure/Orleans.Persistence.AzureStorage.Migration/AzureBlobSiloBuilderExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using Orleans.Configuration;
using Orleans.Hosting;
using Orleans.Providers;
using Orleans.Runtime;
using Orleans.Storage;
using Orleans.Storage.Migration.AzureStorage;

namespace Orleans.Hosting
{
public static class AzureBlobSiloBuilderExtensions
{
/// <summary>
/// Configure silo to use azure blob storage as the default grain storage.
/// </summary>
public static ISiloHostBuilder AddMigrationAzureBlobGrainStorageAsDefault(this ISiloHostBuilder builder, Action<AzureBlobStorageOptions> configureOptions)
{
return builder.AddMigrationAzureBlobGrainStorage(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, configureOptions);
}

/// <summary>
/// Configure silo to use azure blob storage for grain storage.
/// </summary>
public static ISiloHostBuilder AddMigrationAzureBlobGrainStorage(this ISiloHostBuilder builder, string name, Action<AzureBlobStorageOptions> configureOptions)
{
return builder.ConfigureServices(services => services.AddMigrationAzureBlobGrainStorage(name, configureOptions));
}

/// <summary>
/// Configure silo to use azure blob storage as the default grain storage.
/// </summary>
public static ISiloHostBuilder AddMigrationAzureBlobGrainStorageAsDefault(this ISiloHostBuilder builder, Action<OptionsBuilder<AzureBlobStorageOptions>> configureOptions = null)
{
return builder.AddMigrationAzureBlobGrainStorage(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, configureOptions);
}

/// <summary>
/// Configure silo to use azure blob storage for grain storage.
/// </summary>
public static ISiloHostBuilder AddMigrationAzureBlobGrainStorage(this ISiloHostBuilder builder, string name, Action<OptionsBuilder<AzureBlobStorageOptions>> configureOptions = null)
{
return builder.ConfigureServices(services => services.AddMigrationAzureBlobGrainStorage(name, configureOptions));
}

/// <summary>
/// Configure silo to use azure blob storage as the default grain storage.
/// </summary>
public static ISiloBuilder AddMigrationAzureBlobGrainStorageAsDefault(this ISiloBuilder builder, Action<AzureBlobStorageOptions> configureOptions)
{
return builder.AddMigrationAzureBlobGrainStorage(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, configureOptions);
}

/// <summary>
/// Configure silo to use azure blob storage for grain storage.
/// </summary>
public static ISiloBuilder AddMigrationAzureBlobGrainStorage(this ISiloBuilder builder, string name, Action<AzureBlobStorageOptions> configureOptions)
{
return builder.ConfigureServices(services => services.AddMigrationAzureBlobGrainStorage(name, configureOptions));
}

/// <summary>
/// Configure silo to use azure blob storage as the default grain storage.
/// </summary>
public static ISiloBuilder AddMigrationAzureBlobGrainStorageAsDefault(this ISiloBuilder builder, Action<OptionsBuilder<AzureBlobStorageOptions>> configureOptions = null)
{
return builder.AddMigrationAzureBlobGrainStorage(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, configureOptions);
}

/// <summary>
/// Configure silo to use azure blob storage for grain storage.
/// </summary>
public static ISiloBuilder AddMigrationAzureBlobGrainStorage(this ISiloBuilder builder, string name, Action<OptionsBuilder<AzureBlobStorageOptions>> configureOptions = null)
{
return builder.ConfigureServices(services => services.AddMigrationAzureBlobGrainStorage(name, configureOptions));
}

/// <summary>
/// Configure silo to use azure blob storage as the default grain storage.
/// </summary>
public static IServiceCollection AddMigrationAzureBlobGrainStorageAsDefault(this IServiceCollection services, Action<AzureBlobStorageOptions> configureOptions)
{
return services.AddMigrationAzureBlobGrainStorage(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, ob => ob.Configure(configureOptions));
}

/// <summary>
/// Configure silo to use azure blob storage for grain storage.
/// </summary>
public static IServiceCollection AddMigrationAzureBlobGrainStorage(this IServiceCollection services, string name, Action<AzureBlobStorageOptions> configureOptions)
{
return services.AddMigrationAzureBlobGrainStorage(name, ob => ob.Configure(configureOptions));
}

/// <summary>
/// Configure silo to use azure blob storage as the default grain storage.
/// </summary>
public static IServiceCollection AddMigrationAzureBlobGrainStorageAsDefault(this IServiceCollection services, Action<OptionsBuilder<AzureBlobStorageOptions>> configureOptions = null)
{
return services.AddMigrationAzureBlobGrainStorage(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, configureOptions);
}

/// <summary>
/// Configure silo to use azure blob storage for grain storage.
/// </summary>
public static IServiceCollection AddMigrationAzureBlobGrainStorage(this IServiceCollection services, string name,
Action<OptionsBuilder<AzureBlobStorageOptions>> configureOptions = null)
{
configureOptions?.Invoke(services.AddOptions<AzureBlobStorageOptions>(name));
services.AddTransient<IConfigurationValidator>(sp => new AzureBlobStorageOptionsValidator(sp.GetRequiredService<IOptionsMonitor<AzureBlobStorageOptions>>().Get(name), name));
services.ConfigureNamedOptionForLogging<AzureBlobStorageOptions>(name);
if (string.Equals(name, ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, StringComparison.Ordinal))
{
services.TryAddSingleton(sp => sp.GetServiceByName<IGrainStorage>(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME));
}
return services.AddSingletonNamedService<IGrainStorage>(name, MigrationAzureBlobGrainStorageFactory.Create)
.AddSingletonNamedService<ILifecycleParticipant<ISiloLifecycle>>(name, (s, n) => (ILifecycleParticipant<ISiloLifecycle>)s.GetRequiredServiceByName<IGrainStorage>(n));
}
}
}
Loading