diff --git a/Orleans.sln b/Orleans.sln index de03b2d8f4..e87bc8bf51 100644 --- a/Orleans.sln +++ b/Orleans.sln @@ -192,10 +192,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Orleans.TestingHost.Legacy" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Orleans.TestingHost.AppDomain", "test\TestInfrastructure\Orleans.TestingHost.AppDomain\Orleans.TestingHost.AppDomain.csproj", "{DF911257-3617-4B5C-9B78-AED17BA6DC9C}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Orleans.Transactions.DynamoDB", "src\AWS\Orleans.Transactions.DynamoDB\Orleans.Transactions.DynamoDB.csproj", "{17A7F27C-DBDE-4339-A7F5-18BD80C3F205}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Orleans.Transactions.DynamoDB.Test", "test\Transactions\Orleans.Transactions.DynamoDB.Test\Orleans.Transactions.DynamoDB.Test.csproj", "{8B3EEA6B-BE00-482D-9625-15A70B534183}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DependencyInjection.Tests", "test\DependencyInjection.Tests\DependencyInjection.Tests.csproj", "{F23930FE-A219-49E1-8ECB-5A94F271EDC1}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Grains", "Grains", "{2A128E88-B281-4BFB-ADEB-E515437F2385}" @@ -1252,30 +1248,6 @@ Global {DF911257-3617-4B5C-9B78-AED17BA6DC9C}.Release|x64.Build.0 = Release|Any CPU {DF911257-3617-4B5C-9B78-AED17BA6DC9C}.Release|x86.ActiveCfg = Release|Any CPU {DF911257-3617-4B5C-9B78-AED17BA6DC9C}.Release|x86.Build.0 = Release|Any CPU - {17A7F27C-DBDE-4339-A7F5-18BD80C3F205}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {17A7F27C-DBDE-4339-A7F5-18BD80C3F205}.Debug|Any CPU.Build.0 = Debug|Any CPU - {17A7F27C-DBDE-4339-A7F5-18BD80C3F205}.Debug|x64.ActiveCfg = Debug|Any CPU - {17A7F27C-DBDE-4339-A7F5-18BD80C3F205}.Debug|x64.Build.0 = Debug|Any CPU - {17A7F27C-DBDE-4339-A7F5-18BD80C3F205}.Debug|x86.ActiveCfg = Debug|Any CPU - {17A7F27C-DBDE-4339-A7F5-18BD80C3F205}.Debug|x86.Build.0 = Debug|Any CPU - {17A7F27C-DBDE-4339-A7F5-18BD80C3F205}.Release|Any CPU.ActiveCfg = Release|Any CPU - {17A7F27C-DBDE-4339-A7F5-18BD80C3F205}.Release|Any CPU.Build.0 = Release|Any CPU - {17A7F27C-DBDE-4339-A7F5-18BD80C3F205}.Release|x64.ActiveCfg = Release|Any CPU - {17A7F27C-DBDE-4339-A7F5-18BD80C3F205}.Release|x64.Build.0 = Release|Any CPU - {17A7F27C-DBDE-4339-A7F5-18BD80C3F205}.Release|x86.ActiveCfg = Release|Any CPU - {17A7F27C-DBDE-4339-A7F5-18BD80C3F205}.Release|x86.Build.0 = Release|Any CPU - {8B3EEA6B-BE00-482D-9625-15A70B534183}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {8B3EEA6B-BE00-482D-9625-15A70B534183}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8B3EEA6B-BE00-482D-9625-15A70B534183}.Debug|x64.ActiveCfg = Debug|Any CPU - {8B3EEA6B-BE00-482D-9625-15A70B534183}.Debug|x64.Build.0 = Debug|Any CPU - {8B3EEA6B-BE00-482D-9625-15A70B534183}.Debug|x86.ActiveCfg = Debug|Any CPU - {8B3EEA6B-BE00-482D-9625-15A70B534183}.Debug|x86.Build.0 = Debug|Any CPU - {8B3EEA6B-BE00-482D-9625-15A70B534183}.Release|Any CPU.ActiveCfg = Release|Any CPU - {8B3EEA6B-BE00-482D-9625-15A70B534183}.Release|Any CPU.Build.0 = Release|Any CPU - {8B3EEA6B-BE00-482D-9625-15A70B534183}.Release|x64.ActiveCfg = Release|Any CPU - {8B3EEA6B-BE00-482D-9625-15A70B534183}.Release|x64.Build.0 = Release|Any CPU - {8B3EEA6B-BE00-482D-9625-15A70B534183}.Release|x86.ActiveCfg = Release|Any CPU - {8B3EEA6B-BE00-482D-9625-15A70B534183}.Release|x86.Build.0 = Release|Any CPU {F23930FE-A219-49E1-8ECB-5A94F271EDC1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {F23930FE-A219-49E1-8ECB-5A94F271EDC1}.Debug|Any CPU.Build.0 = Debug|Any CPU {F23930FE-A219-49E1-8ECB-5A94F271EDC1}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -1385,8 +1357,6 @@ Global {262A898E-0EED-4235-908B-322A699CDD01} = {A6573187-FD0D-4DF7-91D1-03E07E470C0A} {AC640F05-E013-4D80-A1A4-E577D6883D16} = {B4E538C2-F1C4-4314-84EE-188C59579DED} {DF911257-3617-4B5C-9B78-AED17BA6DC9C} = {B4E538C2-F1C4-4314-84EE-188C59579DED} - {17A7F27C-DBDE-4339-A7F5-18BD80C3F205} = {DA8E126B-BCDB-4E8F-BFB9-2DBFD41F8F70} - {8B3EEA6B-BE00-482D-9625-15A70B534183} = {E4550469-BCFB-4F3E-B778-3769DE18F45A} {F23930FE-A219-49E1-8ECB-5A94F271EDC1} = {A6573187-FD0D-4DF7-91D1-03E07E470C0A} {2A128E88-B281-4BFB-ADEB-E515437F2385} = {A6573187-FD0D-4DF7-91D1-03E07E470C0A} {082D25DB-70CA-48F4-93E0-EC3455F494B8} = {A6573187-FD0D-4DF7-91D1-03E07E470C0A} diff --git a/Test.cmd b/Test.cmd index 17a8ccb979..0e334c1d9b 100644 --- a/Test.cmd +++ b/Test.cmd @@ -35,7 +35,6 @@ set TESTS=^ %CMDHOME%\test\RuntimeCodeGen.Tests,^ %CMDHOME%\test\Transactions\Orleans.Transactions.Tests,^ %CMDHOME%\test\Transactions\Orleans.Transactions.Azure.Test,^ -%CMDHOME%\test\Transactions\Orleans.Transactions.DynamoDB.Test,^ %CMDHOME%\test\TestInfrastructure\Orleans.TestingHost.Tests,^ %CMDHOME%\test\DependencyInjection.Tests diff --git a/src/AWS/Orleans.Transactions.DynamoDB/Directory.Build.props b/src/AWS/Orleans.Transactions.DynamoDB/Directory.Build.props deleted file mode 100644 index 05abe39f6f..0000000000 --- a/src/AWS/Orleans.Transactions.DynamoDB/Directory.Build.props +++ /dev/null @@ -1,35 +0,0 @@ - - - <_ParentDirectoryBuildPropsPath Condition="'$(_DirectoryBuildPropsFile)' != ''">$([System.IO.Path]::Combine('..', '$(_DirectoryBuildPropsFile)')) - - - - - - false - - - - true - - - - - true - https://github.com/dotnet/orleans - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/AWS/Orleans.Transactions.DynamoDB/Hosting/SiloBuilderExtensions.cs b/src/AWS/Orleans.Transactions.DynamoDB/Hosting/SiloBuilderExtensions.cs deleted file mode 100644 index 355857e428..0000000000 --- a/src/AWS/Orleans.Transactions.DynamoDB/Hosting/SiloBuilderExtensions.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System; -using Microsoft.Extensions.DependencyInjection; -using Orleans.Configuration; -using Orleans.Transactions.DynamoDB; - -namespace Orleans.Hosting -{ - public static class SiloBuilderExtensions - { - /// - /// Configure cluster to use dynamoDB transaction log using configure action. - /// - public static ISiloHostBuilder UseDynamoDBTransactionLog(this ISiloHostBuilder builder, Action configureOptions) - { - return builder.UseDynamoDBTransactionLog(ob => ob.Configure(configureOptions)); - } - - /// - /// Configure cluster to use dynamoDB transaction log using configuration builder. - /// - public static ISiloHostBuilder UseDynamoDBTransactionLog(this ISiloHostBuilder builder, Action> configureOptions) - { - return builder.ConfigureServices(services => services.UseDynamoDBTransactionLog(configureOptions)); - } - - /// - /// Configure cluster service to use dynamoDB transaction log using configure action. - /// - public static IServiceCollection UseDynamoDBTransactionLog(this IServiceCollection services, Action configureOptions) - { - return services.UseDynamoDBTransactionLog(ob => ob.Configure(configureOptions)); - } - - /// - /// Configure cluster service to use dynamoDB transaction log using configuration builder. - /// - public static IServiceCollection UseDynamoDBTransactionLog(this IServiceCollection services, - Action> configureOptions) - { - configureOptions?.Invoke(services.AddOptions()); - services.AddTransient(); - services.AddTransient(DynamoDBTransactionLogStorage.Create); - return services; - } - } -} diff --git a/src/AWS/Orleans.Transactions.DynamoDB/Orleans.Transactions.DynamoDB.csproj b/src/AWS/Orleans.Transactions.DynamoDB/Orleans.Transactions.DynamoDB.csproj deleted file mode 100644 index 9040e20df7..0000000000 --- a/src/AWS/Orleans.Transactions.DynamoDB/Orleans.Transactions.DynamoDB.csproj +++ /dev/null @@ -1,27 +0,0 @@ - - - Microsoft.Orleans.Transactions.DynamoDB - Microsoft Orleans Transactions on DynamoDB - DynamoDB Transaction library of Microsoft Orleans used on the server. - $(PackageTags) DynamoDB Transactions - - - - Orleans.Transactions.DynamoDB - Orleans.Transactions.DynamoDB - $(DefineConstants);TRANSACTIONS_DYNAMODB - - - - - - - - - - - - - - - diff --git a/src/AWS/Orleans.Transactions.DynamoDB/Properties/AssemblyInfo.cs b/src/AWS/Orleans.Transactions.DynamoDB/Properties/AssemblyInfo.cs deleted file mode 100644 index bf170eabdf..0000000000 --- a/src/AWS/Orleans.Transactions.DynamoDB/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("AWSUtils.Tests")] diff --git a/src/AWS/Orleans.Transactions.DynamoDB/Storage/DynamoDBTransactionLogOptions.cs b/src/AWS/Orleans.Transactions.DynamoDB/Storage/DynamoDBTransactionLogOptions.cs deleted file mode 100644 index 4b66e48794..0000000000 --- a/src/AWS/Orleans.Transactions.DynamoDB/Storage/DynamoDBTransactionLogOptions.cs +++ /dev/null @@ -1,67 +0,0 @@ -using Microsoft.Extensions.Options; -using Orleans.Runtime; -using Orleans.Transactions.DynamoDB; - -namespace Orleans.Configuration -{ - public class DynamoDBTransactionLogOptions - { - /// - /// AccessKey string for DynamoDB Storage - /// - [Redact] - public string AccessKey { get; set; } - - /// - /// Secret key for DynamoDB storage - /// - [Redact] - public string SecretKey { get; set; } - - /// - /// DynamoDB Service name - /// - public string Service { get; set; } - - /// - /// Read capacity unit for DynamoDB storage - /// - public int ReadCapacityUnits { get; set; } = DynamoDBStorage.DefaultReadCapacityUnits; - - /// - /// Write capacity unit for DynamoDB storage - /// - public int WriteCapacityUnits { get; set; } = DynamoDBStorage.DefaultWriteCapacityUnits; - - /// - /// DynamoDB table name. - /// Defaults to 'TransactionLog'. - /// - public string TableName { get; set; } = "TransactionLog"; - } - - public class DynamoDBTransactionLogOptionsValidator : IConfigurationValidator - { - private readonly DynamoDBTransactionLogOptions options; - - public DynamoDBTransactionLogOptionsValidator(IOptions configurationOptions) - { - this.options = configurationOptions.Value; - } - - public void ValidateConfiguration() - { - if (string.IsNullOrWhiteSpace(this.options.TableName)) - throw new OrleansConfigurationException( - $"Configuration for DynamoDBTransactionLogStorage is invalid. {nameof(this.options.TableName)} is not valid."); - - if (this.options.ReadCapacityUnits == 0) - throw new OrleansConfigurationException( - $"Configuration for DynamoDBTransactionLogStorage is invalid. {nameof(this.options.ReadCapacityUnits)} is not valid."); - - if (this.options.WriteCapacityUnits == 0) - throw new OrleansConfigurationException( - $"Configuration for DynamoDBTransactionLogStorage is invalid. {nameof(this.options.WriteCapacityUnits)} is not valid."); - } - } -} diff --git a/src/AWS/Orleans.Transactions.DynamoDB/Storage/DynamoDBTransactionLogStorage.cs b/src/AWS/Orleans.Transactions.DynamoDB/Storage/DynamoDBTransactionLogStorage.cs deleted file mode 100644 index f6df373684..0000000000 --- a/src/AWS/Orleans.Transactions.DynamoDB/Storage/DynamoDBTransactionLogStorage.cs +++ /dev/null @@ -1,407 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.DependencyInjection; -using Orleans.Configuration; -using Orleans.Serialization; -using Orleans.Transactions.Abstractions; -using Microsoft.Extensions.Logging; -using Amazon.DynamoDBv2.Model; -using Amazon.DynamoDBv2; -using System.Globalization; -using System.IO; - -namespace Orleans.Transactions.DynamoDB -{ - public class DynamoDBTransactionLogStorage : ITransactionLogStorage - { - - private const string RowKey = "RowKey"; - private const string PartitionKey = "PartitionKey"; - private const string AllocatedTransactionIdsKey = "AllocatedTransactionIds"; - private const string TransactionsKey = "Transactions"; - private const string RowKeyAlias = ":RowKey"; - private const string PartitionKeyAlias = ":PartitionKey"; - - private const int BatchOperationLimit = 25; - private const int CommitRecordsPerRow = 40; - private const string CommitRecordPartitionKey = "0"; - - private const string StartRowPartitionKey = "1"; - private static readonly AttributeValue StartRowRowKey = new AttributeValue { N = "0" }; - - //TODO: jbragg - Do not use serializationManager for persistent data!! - private readonly SerializationManager serializationManager; - private readonly DynamoDBTransactionLogOptions options; - private readonly ILoggerFactory loggerFactory; - - private DynamoDBStorage storage; - - private long startRecordValue; - private long nextLogSequenceNumber; - - // Log iteration indexes, reused between operations - private Dictionary currentLastEvaluatedKey; - private List currentQueryResult; - private int currentQueryResultIndex; - private List currentRowTransactions; - private int currentRowTransactionsIndex; - - public DynamoDBTransactionLogStorage(SerializationManager serializationManager, IOptions configurationOptions, ILoggerFactory loggerFactory) - { - this.serializationManager = serializationManager; - this.options = configurationOptions.Value; - this.loggerFactory = loggerFactory; - } - - public async Task Initialize() - { - storage = new DynamoDBStorage(this.loggerFactory, this.options.Service, this.options.AccessKey, this.options.SecretKey, - this.options.ReadCapacityUnits, this.options.WriteCapacityUnits); - await storage.InitializeTable(this.options.TableName, - new List - { - new KeySchemaElement { AttributeName = PartitionKey, KeyType = KeyType.HASH }, - new KeySchemaElement { AttributeName = RowKey, KeyType = KeyType.RANGE } - }, - new List - { - new AttributeDefinition { AttributeName = PartitionKey, AttributeType = ScalarAttributeType.S }, - new AttributeDefinition { AttributeName = RowKey, AttributeType = ScalarAttributeType.N } - }).ConfigureAwait(false); - - var (results, lastEvaluatedKey) = await storage.QueryAsync(this.options.TableName, - new Dictionary - { - { PartitionKeyAlias, new AttributeValue(StartRowPartitionKey) }, - { RowKeyAlias, StartRowRowKey } - }, - $"{PartitionKey} = {PartitionKeyAlias} AND {RowKey} <= {RowKeyAlias}", - (fields) => - { - return new StartRow(AttributeToLong(fields[AllocatedTransactionIdsKey])); - }).ConfigureAwait(false); - - if (results.Count == 0) - { - // This is a fresh deployment, the StartRecord isn't created yet. - // Create it here. - await storage.PutEntryAsync(this.options.TableName, - new Dictionary - { - { PartitionKey, new AttributeValue(StartRowPartitionKey) }, - { RowKey, StartRowRowKey }, - { AllocatedTransactionIdsKey, new AttributeValue { N = "0" } } - }).ConfigureAwait(false); - - startRecordValue = 0; - } - else - { - startRecordValue = results[0].AllocatedTransactionIds; - } - } - - public static Factory> Create(IServiceProvider serviceProvider) - { - return async () => - { - DynamoDBTransactionLogStorage logStorage = ActivatorUtilities.CreateInstance(serviceProvider, new object[0]); - await logStorage.Initialize(); - return logStorage; - }; - } - - public async Task GetFirstCommitRecord() - { - currentLastEvaluatedKey = null; - - await ReadRowsFromTable(0); - - if (currentQueryResult.Count == 0) - { - // The log has no log entries - currentQueryResult = null; - - nextLogSequenceNumber = 1; - - return null; - } - - currentRowTransactions = DeserializeCommitRecords(currentQueryResult[0].Transactions); - - // TODO: Assert not empty? - - nextLogSequenceNumber = currentRowTransactions[currentRowTransactionsIndex].LSN + 1; - - return currentRowTransactions[currentRowTransactionsIndex++]; - } - - public async Task GetNextCommitRecord() - { - // Based on the current implementation logic in TransactionManager, this must be not be null, since - // GetFirstCommitRecord sets a query or if no start record, the TransactionManager exits its loop. - if (currentQueryResult == null) - { - throw new InvalidOperationException("GetNextCommitRecord called but currentQueryResult is null."); - } - - if (currentRowTransactionsIndex == currentRowTransactions.Count) - { - currentQueryResultIndex++; - currentRowTransactionsIndex = 0; - currentRowTransactions = null; - } - - if (currentQueryResultIndex == currentQueryResult.Count) - { - // No more rows in our current segment, retrieve the next segment from the Table. - if (currentLastEvaluatedKey == null || currentLastEvaluatedKey.Count == 0) - { - currentQueryResult = null; - return null; - } - - await ReadRowsFromTable(0); - } - - if (currentRowTransactions == null) - { - // TODO: assert currentRowTransactionsIndex = 0? - currentRowTransactions = DeserializeCommitRecords(currentQueryResult[currentQueryResultIndex].Transactions); - } - - var currentTransaction = currentRowTransactions[currentRowTransactionsIndex++]; - - nextLogSequenceNumber = currentTransaction.LSN + 1; - - return currentTransaction; - } - - public Task GetStartRecord() - { - return Task.FromResult(startRecordValue); - } - - public async Task UpdateStartRecord(long transactionId) - { - await storage.UpsertEntryAsync(this.options.TableName, - new Dictionary - { - { PartitionKey, new AttributeValue(StartRowPartitionKey) }, - { RowKey, StartRowRowKey } - }, - new Dictionary - { - { AllocatedTransactionIdsKey, LongToAttribute(transactionId) } - }).ConfigureAwait(false); - - startRecordValue = transactionId; - } - - public async Task Append(IEnumerable commitRecords) - { - var batchOperation = new List>(); - - // TODO modify this to be able to use IEnumerable, fixed size array for serialization in the size of CommitRecordsPerRow, list is temporary - var transactionList = new List(commitRecords); - - for (int nextRecord = 0; nextRecord < transactionList.Count; nextRecord += CommitRecordsPerRow) - { - var recordCount = Math.Min(transactionList.Count - nextRecord, CommitRecordsPerRow); - var transactionSegment = transactionList.GetRange(nextRecord, recordCount); - var commitRow = new CommitRow(nextLogSequenceNumber); - - foreach (var transaction in transactionSegment) - { - transaction.LSN = nextLogSequenceNumber++; - } - - commitRow.Transactions = SerializeCommitRecords(transactionSegment); - - batchOperation.Add( - new Dictionary - { - { PartitionKey, new AttributeValue(CommitRecordPartitionKey) }, - { RowKey, commitRow.FirstLSNAttribute }, - { TransactionsKey, new AttributeValue { B = new MemoryStream(commitRow.Transactions.Value.Array) } } - }); - - if (batchOperation.Count == BatchOperationLimit) - { - await storage.PutEntriesAsync(this.options.TableName, batchOperation).ConfigureAwait(false); - - batchOperation = new List>(); - } - } - - if (batchOperation.Count > 0) - { - await storage.PutEntriesAsync(this.options.TableName, batchOperation).ConfigureAwait(false); - } - } - - public async Task TruncateLog(long lsn) - { - var keyValues = new Dictionary - { - { PartitionKeyAlias, new AttributeValue(CommitRecordPartitionKey) }, - { RowKeyAlias, LongToAttribute(lsn) } - }; - string query = $"{PartitionKey} = {PartitionKeyAlias} AND {RowKey} <= {RowKeyAlias}"; - Dictionary lastEvaluatedKey = null; - var batchOperation = new List>(); - - do - { - var result = await storage.QueryAsync(this.options.TableName, keyValues, query, - CommitRowResolver, lastEvaluatedKey: lastEvaluatedKey).ConfigureAwait(false); - lastEvaluatedKey = result.lastEvaluatedKey; - - foreach (var row in result.results) - { - var transactions = DeserializeCommitRecords(row.Transactions); - - if (transactions.Count > 0 && transactions[transactions.Count - 1].LSN <= lsn) - { - batchOperation.Add( - new Dictionary - { - { PartitionKey, new AttributeValue(CommitRecordPartitionKey) }, - { RowKey, row.FirstLSNAttribute } - }); - - if (batchOperation.Count == BatchOperationLimit) - { - await storage.DeleteEntriesAsync(this.options.TableName, batchOperation).ConfigureAwait(false); - - batchOperation = new List>(); - } - } - else - { - break; - } - } - - } while (lastEvaluatedKey.Count != 0); - - if (batchOperation.Count > 0) - { - await storage.DeleteEntriesAsync(this.options.TableName, batchOperation).ConfigureAwait(false); - } - } - - private async Task ReadRowsFromTable(long keyLowerBound) - { - var (results, lastEvaluatedKey) = await storage.QueryAsync(this.options.TableName, - new Dictionary - { - { PartitionKeyAlias, new AttributeValue(CommitRecordPartitionKey) }, - { RowKeyAlias, LongToAttribute(keyLowerBound) } - }, - $"{PartitionKey} = {PartitionKeyAlias} AND {RowKey} >= {RowKeyAlias}", - CommitRowResolver, - lastEvaluatedKey: currentLastEvaluatedKey).ConfigureAwait(false); - currentQueryResult = results; - - // Reset the indexes - currentQueryResultIndex = 0; - currentRowTransactionsIndex = 0; - currentLastEvaluatedKey = lastEvaluatedKey; - } - - private ArraySegment SerializeCommitRecords(List commitRecords) - { - var serializableList = new List>>(commitRecords.Count); - - foreach (var commitRecord in commitRecords) - { - serializableList.Add(new Tuple>(commitRecord.LSN, commitRecord.TransactionId, commitRecord.Resources)); - } - - var streamWriter = new BinaryTokenStreamWriter(); - - serializationManager.Serialize(serializableList, streamWriter); - - return new ArraySegment(streamWriter.ToByteArray()); - } - - private List DeserializeCommitRecords(ArraySegment? serializerCommitRecords) - { - if (!serializerCommitRecords.HasValue) - { - return new List(); - } - - var streamReader = new BinaryTokenStreamReader(serializerCommitRecords.Value); - - var deserializedList = serializationManager.Deserialize>>>(streamReader); - - var commitRecords = new List(deserializedList.Count); - - foreach (var item in deserializedList) - { - commitRecords.Add(new CommitRecord { LSN = item.Item1, TransactionId = item.Item2, Resources = item.Item3 }); - } - - return commitRecords; - } - - private static AttributeValue LongToAttribute(long value) - { - return new AttributeValue { N = value.ToString("d", CultureInfo.InvariantCulture) }; - } - - private static long AttributeToLong(AttributeValue value) - { - return long.Parse(value.N, CultureInfo.InvariantCulture); - } - - private static Func, CommitRow> CommitRowResolver => (fields) => - { - var commitRow = new CommitRow(AttributeToLong(fields[RowKey])); - var stream = fields[TransactionsKey].B; - if (stream.TryGetBuffer(out ArraySegment buffer)) - { - commitRow.Transactions = buffer; - } - else - { - commitRow.Transactions = new ArraySegment(stream.ToArray()); - } - return commitRow; - }; - - private class CommitRow - { - public CommitRow(long firstLSN) - { - FirstLSN = firstLSN; - } - - public ArraySegment? Transactions { get; set; } - - public long FirstLSN { get; set; } - - public AttributeValue FirstLSNAttribute - { - get - { - return LongToAttribute(FirstLSN); - } - } - } - - private class StartRow - { - public StartRow(long transactionId) - { - AllocatedTransactionIds = transactionId; - } - - public long AllocatedTransactionIds { get; set; } - } - } -} diff --git a/src/Azure/Orleans.Transactions.AzureStorage/Hosting/AzureTableTransactionsSiloBuilderExtensions.cs b/src/Azure/Orleans.Transactions.AzureStorage/Hosting/AzureTableTransactionsSiloBuilderExtensions.cs index 7f358eb40d..d7f6f50564 100644 --- a/src/Azure/Orleans.Transactions.AzureStorage/Hosting/AzureTableTransactionsSiloBuilderExtensions.cs +++ b/src/Azure/Orleans.Transactions.AzureStorage/Hosting/AzureTableTransactionsSiloBuilderExtensions.cs @@ -3,9 +3,9 @@ using Orleans.Runtime; using Orleans.Configuration; using Orleans.Transactions.Abstractions; -using Orleans.Transactions.AzureStorage.TransactionalState; using Microsoft.Extensions.DependencyInjection.Extensions; using Orleans.Providers; +using Orleans.Transactions.AzureStorage; namespace Orleans.Hosting { @@ -48,17 +48,10 @@ private static IServiceCollection AddAzureTableTransactionalStateStorage(this IS { configureOptions?.Invoke(services.AddOptions(name)); - // single TM - services.ConfigureNamedOptionForLogging(name); services.TryAddSingleton(sp => sp.GetServiceByName(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME)); services.AddSingletonNamedService(name, AzureTableTransactionalStateStorageFactory.Create); services.AddSingletonNamedService>(name, (s, n) => (ILifecycleParticipant)s.GetRequiredServiceByName(n)); - // distributed TM - services.TryAddSingleton(sp => sp.GetServiceByName(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME)); - services.AddSingletonNamedService(name, Orleans.Transactions.DistributedTM.AzureStorage.AzureTableTransactionalStateStorageFactory.Create); - services.AddSingletonNamedService>(name, (s, n) => (ILifecycleParticipant)s.GetRequiredServiceByName(n)); - return services; } diff --git a/src/Azure/Orleans.Transactions.AzureStorage/Hosting/SiloBuilderExtensions.cs b/src/Azure/Orleans.Transactions.AzureStorage/Hosting/SiloBuilderExtensions.cs deleted file mode 100644 index 223dc7c196..0000000000 --- a/src/Azure/Orleans.Transactions.AzureStorage/Hosting/SiloBuilderExtensions.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System; -using Microsoft.Extensions.DependencyInjection; -using Orleans.Configuration; -using Orleans.Transactions.AzureStorage; - -namespace Orleans.Hosting -{ - public static class SiloBuilderExtensions - { - /// - /// Configure cluster to use azure transaction log using configure action. - /// - public static ISiloHostBuilder UseAzureTransactionLog(this ISiloHostBuilder builder, Action configureOptions) - { - return builder.UseAzureTransactionLog(ob => ob.Configure(configureOptions)); - } - - /// - /// Configure cluster to use azure transaction log using configuration builder. - /// - public static ISiloHostBuilder UseAzureTransactionLog(this ISiloHostBuilder builder, Action> configureOptions) - { - return builder.ConfigureServices(services => - { - configureOptions?.Invoke(services.AddOptions()); - services.AddTransient(); - services.AddTransient(AzureTransactionLogStorage.Create); - }); - } - } -} \ No newline at end of file diff --git a/src/Azure/Orleans.Transactions.AzureStorage/Storage/AzureTransactionArchiveLogOptions.cs b/src/Azure/Orleans.Transactions.AzureStorage/Storage/AzureTransactionArchiveLogOptions.cs deleted file mode 100644 index 53f4f74eae..0000000000 --- a/src/Azure/Orleans.Transactions.AzureStorage/Storage/AzureTransactionArchiveLogOptions.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Orleans.Configuration.Development -{ - /// - /// Option class to configure Azure transaction log archive behavior - /// - public class AzureTransactionArchiveLogOptions - { - public const bool DEFAULT_ARCHIVE_LOG = false; - //whether to archive commited transaction log or not. turned off by default - public bool ArchiveLog { get; set; } = DEFAULT_ARCHIVE_LOG; - } -} diff --git a/src/Azure/Orleans.Transactions.AzureStorage/Storage/AzureTransactionLogConfiguration.cs b/src/Azure/Orleans.Transactions.AzureStorage/Storage/AzureTransactionLogConfiguration.cs deleted file mode 100644 index 604678014a..0000000000 --- a/src/Azure/Orleans.Transactions.AzureStorage/Storage/AzureTransactionLogConfiguration.cs +++ /dev/null @@ -1,17 +0,0 @@ - -namespace Orleans.Transactions.AzureStorage -{ - public class AzureTransactionLogConfiguration - { - public string ConnectionString { get; set; } - - public string TableName { get; set; } = "TransactionLog"; - - internal void Copy(AzureTransactionLogConfiguration other) - { - if (other == null) Copy(new AzureTransactionLogConfiguration()); - this.ConnectionString = other.ConnectionString; - this.TableName = other.TableName; - } - } -} diff --git a/src/Azure/Orleans.Transactions.AzureStorage/Storage/AzureTransactionLogOptions.cs b/src/Azure/Orleans.Transactions.AzureStorage/Storage/AzureTransactionLogOptions.cs deleted file mode 100644 index 25e5b1fcb6..0000000000 --- a/src/Azure/Orleans.Transactions.AzureStorage/Storage/AzureTransactionLogOptions.cs +++ /dev/null @@ -1,36 +0,0 @@ - -using Microsoft.Extensions.Options; -using Orleans.Runtime; - -namespace Orleans.Configuration -{ - public class AzureTransactionLogOptions - { - [RedactConnectionString] - public string ConnectionString { get; set; } - - public string TableName { get; set; } = "TransactionLog"; - } - - public class AzureTransactionLogOptionsValidator : IConfigurationValidator - { - private readonly AzureTransactionLogOptions options; - - public AzureTransactionLogOptionsValidator(IOptions configurationOptions) - { - this.options = configurationOptions.Value; - } - - public void ValidateConfiguration() - { - if (string.IsNullOrWhiteSpace(this.options.ConnectionString)) - { - throw new OrleansConfigurationException($"Invalid AzureTransactionLogOptions. ConnectionString is required."); - } - if (string.IsNullOrWhiteSpace(this.options.TableName)) - { - throw new OrleansConfigurationException($"Invalid AzureTransactionLogOptions. TableName is required."); - } - } - } -} diff --git a/src/Azure/Orleans.Transactions.AzureStorage/Storage/AzureTransactionLogStorage.cs b/src/Azure/Orleans.Transactions.AzureStorage/Storage/AzureTransactionLogStorage.cs deleted file mode 100644 index bd8fab0a68..0000000000 --- a/src/Azure/Orleans.Transactions.AzureStorage/Storage/AzureTransactionLogStorage.cs +++ /dev/null @@ -1,449 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Table; -using Microsoft.WindowsAzure.Storage.RetryPolicies; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.DependencyInjection; -using Orleans.Configuration; -using Orleans.Configuration.Development; -using Orleans.Serialization; -using Orleans.Transactions.Abstractions; - -namespace Orleans.Transactions.AzureStorage -{ - /// - /// TransactionLog ported from research. Placeholder, is being rewritten. - /// - public class AzureTransactionLogStorage : ITransactionLogStorage - { - private const string RowKey = "RowKey"; - private const string PartitionKey = "PartitionKey"; - - private const int BatchOperationLimit = 100; - private const int CommitRecordsPerRow = 10; - - //TODO: jbragg - Do not use serializationManager for persistent data!! - private readonly SerializationManager serializationManager; - private readonly AzureTransactionLogOptions options; - - // Azure Tables objects for persistent storage - private CloudTable table; - - private long startRecordValue; - private long nextLogSequenceNumber; - private readonly string commitRecordPartitionKey; - - // Log iteration indexes, reused between operations - private TableContinuationToken currentContinuationToken; - private TableQuerySegment currentQueryResult; - private int currentQueryResultIndex; - private List currentRowTransactions; - private int currentRowTransactionsIndex; - private readonly ClusterOptions clusterOptions; - private readonly AzureTransactionArchiveLogOptions archiveLogOptions; - public AzureTransactionLogStorage(SerializationManager serializationManager, IOptions configurationOptions, - IOptions archiveOptions, IOptions clusterOptions) - { - this.serializationManager = serializationManager; - this.options = configurationOptions.Value; - this.clusterOptions = clusterOptions.Value; - this.archiveLogOptions = archiveOptions.Value; - this.commitRecordPartitionKey = ArchivalRow.MakePartitionKey(this.clusterOptions.ServiceId); - } - - public async Task Initialize() - { - if (string.IsNullOrWhiteSpace(this.options.ConnectionString)) - { - throw new ArgumentNullException(nameof(this.options.ConnectionString)); - } - - // Retrieve the storage account from the connection string. - var storageAccount = CloudStorageAccount.Parse(this.options.ConnectionString); - - // Create the table if not exists. - CloudTableClient creationClient = storageAccount.CreateCloudTableClient(); - // TODO - do not hard code DefaultRequestOptions in rewrite. - creationClient.DefaultRequestOptions.RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(1), 60); - creationClient.DefaultRequestOptions.ServerTimeout = TimeSpan.FromMinutes(3); - creationClient.DefaultRequestOptions.PayloadFormat = TablePayloadFormat.JsonNoMetadata; - CloudTable creationTable = creationClient.GetTableReference(this.options.TableName); - await creationTable.CreateIfNotExistsAsync().ConfigureAwait(false); - - // get table for operations - CloudTableClient operationClient = storageAccount.CreateCloudTableClient(); - // TODO - do not hard code DefaultRequestOptions in rewrite. - operationClient.DefaultRequestOptions.RetryPolicy = new LinearRetry(TimeSpan.FromMilliseconds(100), 5); - operationClient.DefaultRequestOptions.ServerTimeout = TimeSpan.FromSeconds(3); - operationClient.DefaultRequestOptions.PayloadFormat = TablePayloadFormat.JsonNoMetadata; - this.table = operationClient.GetTableReference(this.options.TableName); - - var query = new TableQuery().Where(TableQuery.CombineFilters( - TableQuery.GenerateFilterCondition(PartitionKey, QueryComparisons.Equal, StartRow.MakePartitionkey(this.clusterOptions.ServiceId)), - TableOperators.And, - TableQuery.GenerateFilterCondition(RowKey, QueryComparisons.LessThanOrEqual, StartRow.StartRowRowKey))); - var queryResult = await table.ExecuteQuerySegmentedAsync(query, null).ConfigureAwait(false); - - if (queryResult.Results.Count == 0) - { - // This is a fresh deployment, the StartRecord isn't created yet. - // Create it here. - var row = new StartRow(this.clusterOptions, 0); - var operation = TableOperation.Insert(row); - - await table.ExecuteAsync(operation).ConfigureAwait(false); - - startRecordValue = 0; - } - else - { - startRecordValue = queryResult.Results[0].AllocatedTransactionIds; - } - } - - public async Task GetFirstCommitRecord() - { - currentContinuationToken = null; - - await ReadRowsFromTable(0); - - if (currentQueryResult.Results.Count == 0) - { - // The log has no log entries - currentQueryResult = null; - - nextLogSequenceNumber = 1; - - return null; - } - - currentRowTransactions = DeserializeCommitRecords(currentQueryResult.Results[0].Transactions); - - // TODO: Assert not empty? - - nextLogSequenceNumber = currentRowTransactions[currentRowTransactionsIndex].LSN + 1; - - return currentRowTransactions[currentRowTransactionsIndex++]; - } - - public async Task GetNextCommitRecord() - { - // Based on the current implementation logic in TransactionManager, this must be not be null, since - // GetFirstCommitRecord sets a query or if no start record, the TransactionManager exits its loop. - if (currentQueryResult == null) - { - throw new InvalidOperationException("GetNextCommitRecord called but currentQueryResult is null."); - } - - if (currentRowTransactionsIndex == currentRowTransactions.Count) - { - currentQueryResultIndex++; - currentRowTransactionsIndex = 0; - currentRowTransactions = null; - } - - if (currentQueryResultIndex == currentQueryResult.Results.Count) - { - // No more rows in our current segment, retrieve the next segment from the Table. - if (currentContinuationToken == null) - { - currentQueryResult = null; - return null; - } - - await ReadRowsFromTable(0); - } - - if (currentRowTransactions == null) - { - // TODO: assert currentRowTransactionsIndex = 0? - currentRowTransactions = DeserializeCommitRecords(currentQueryResult.Results[currentQueryResultIndex].Transactions); - } - - var currentTransaction = currentRowTransactions[currentRowTransactionsIndex++]; - - nextLogSequenceNumber = currentTransaction.LSN + 1; - - return currentTransaction; - } - - public Task GetStartRecord() - { - return Task.FromResult(startRecordValue); - } - - public async Task UpdateStartRecord(long transactionId) - { - var tableOperation = TableOperation.Replace(new StartRow(this.clusterOptions, transactionId)); - - await table.ExecuteAsync(tableOperation).ConfigureAwait(false); - - startRecordValue = transactionId; - } - - public async Task Append(IEnumerable transactions) - { - var batchOperation = new TableBatchOperation(); - - // TODO modify this to be able to use IEnumerable, fixed size array for serialization in the size of CommitRecordsPerRow, list is temporary - var transactionList = new List(transactions); - - for (int nextRecord = 0; nextRecord < transactionList.Count; nextRecord += CommitRecordsPerRow) - { - var recordCount = Math.Min(transactionList.Count - nextRecord, CommitRecordsPerRow); - var transactionSegment = transactionList.GetRange(nextRecord, recordCount); - var commitRow = new CommitRow(this.clusterOptions, nextLogSequenceNumber); - - foreach (var transaction in transactionSegment) - { - transaction.LSN = nextLogSequenceNumber++; - } - - commitRow.Transactions = SerializeCommitRecords(transactionSegment); - - batchOperation.Insert(commitRow); - - if (batchOperation.Count == BatchOperationLimit) - { - await table.ExecuteBatchAsync(batchOperation).ConfigureAwait(false); - - batchOperation = new TableBatchOperation(); - } - } - - if (batchOperation.Count > 0) - { - await table.ExecuteBatchAsync(batchOperation).ConfigureAwait(false); - } - } - - public async Task> QueryArchivalRecords(TableQuery query) - { - var continuationToken = default(TableContinuationToken); - var deserializedResults = new List(); - do - { - var queryResult = await table.ExecuteQuerySegmentedAsync(query, continuationToken).ConfigureAwait(false); - continuationToken = queryResult.ContinuationToken; - - if (queryResult.Results.Count > 0) - { - foreach (var row in queryResult) - { - var transactions = DeserializeCommitRecords(row.Transactions); - deserializedResults.AddRange(transactions); - } - } - } while (continuationToken != default(TableContinuationToken)); - - return deserializedResults; - } - - public async Task TruncateLog(long lsn) - { - var continuationToken = default(TableContinuationToken); - var query = new TableQuery().Where(TableQuery.CombineFilters( - TableQuery.GenerateFilterCondition(PartitionKey, QueryComparisons.Equal, commitRecordPartitionKey), - TableOperators.And, - TableQuery.CombineFilters( - TableQuery.GenerateFilterCondition(RowKey, QueryComparisons.LessThanOrEqual, CommitRow.MakeRowKey(lsn)), - TableOperators.And, - TableQuery.GenerateFilterCondition(RowKey, QueryComparisons.GreaterThanOrEqual, CommitRow.MinRowKey)))); - var batchOperation = new TableBatchOperation(); - do - { - var queryResult = await table.ExecuteQuerySegmentedAsync(query, continuationToken).ConfigureAwait(false); - - continuationToken = queryResult.ContinuationToken; - - if (queryResult.Results.Count > 0) - { - - foreach (var row in queryResult) - { - var transactions = DeserializeCommitRecords(row.Transactions); - - if (transactions.Count > 0 && transactions[transactions.Count - 1].LSN <= lsn) - { - batchOperation.Delete(row); - if (this.archiveLogOptions.ArchiveLog) - { - var archiveRow = new ArchivalRow(this.clusterOptions, row.Transactions, transactions.Select(tx => tx.TransactionId).Min(), transactions.Select(tx => tx.LSN).Min()); - batchOperation.Insert(archiveRow); - } - - if (batchOperation.Count == BatchOperationLimit) - { - await table.ExecuteBatchAsync(batchOperation).ConfigureAwait(false); - batchOperation = new TableBatchOperation(); - } - } - else - { - break; - } - } - } - - } while (continuationToken != default(TableContinuationToken)); - - if (batchOperation.Count > 0) - { - await table.ExecuteBatchAsync(batchOperation).ConfigureAwait(false); - } - } - - public static Factory> Create(IServiceProvider serviceProvider) - { - return async () => - { - AzureTransactionLogStorage storage = ActivatorUtilities.CreateInstance(serviceProvider, new object[0]); - await storage.Initialize(); - return storage; - }; - } - - private async Task ReadRowsFromTable(long keyLowerBound) - { - var query = new TableQuery().Where(TableQuery.CombineFilters( - TableQuery.GenerateFilterCondition(PartitionKey, QueryComparisons.Equal, commitRecordPartitionKey), - TableOperators.And, - TableQuery.CombineFilters( - TableQuery.GenerateFilterCondition(RowKey, QueryComparisons.GreaterThanOrEqual, CommitRow.MakeRowKey(keyLowerBound)), - TableOperators.And, - TableQuery.GenerateFilterCondition(RowKey, QueryComparisons.LessThanOrEqual, CommitRow.MaxRowKey)))); - - currentQueryResult = await table.ExecuteQuerySegmentedAsync(query, currentContinuationToken).ConfigureAwait(false); - - // Reset the indexes - currentQueryResultIndex = 0; - currentRowTransactionsIndex = 0; - currentContinuationToken = currentQueryResult.ContinuationToken; - } - - private byte[] SerializeCommitRecords(List commitRecords) - { - var serializableList = new List>>(commitRecords.Count); - - foreach (var commitRecord in commitRecords) - { - serializableList.Add(new Tuple>(commitRecord.LSN, commitRecord.TransactionId, commitRecord.Resources)); - } - - var streamWriter = new BinaryTokenStreamWriter(); - - serializationManager.Serialize(serializableList, streamWriter); - - return streamWriter.ToByteArray(); - } - - private List DeserializeCommitRecords(byte[] serializerCommitRecords) - { - if (serializerCommitRecords == null) - { - return new List(); - } - - var streamReader = new BinaryTokenStreamReader(serializerCommitRecords); - - var deserializedList = serializationManager.Deserialize>>>(streamReader); - - var commitRecords = new List(deserializedList.Count); - - foreach (var item in deserializedList) - { - commitRecords.Add(new CommitRecord { LSN = item.Item1, TransactionId = item.Item2, Resources = item.Item3 }); - } - - return commitRecords; - } - - public class ArchivalRow : TableEntity - { - public const string MinRowKey = "arch_"; - public const string MaxRowKey = "arch_~"; - - public static string MakeRowKey(long firstTransactionId) - { - return $"arch_{firstTransactionId:x16}"; - } - - //CommitRow and ArchivalRow should share the same partitionKey making. So they - //the same method - public static string MakePartitionKey(string serviceId) - { - return $"tlpk_{serviceId}"; - } - - public ArchivalRow() - { - } - - public ArchivalRow(ClusterOptions clusterOptions, byte[] transactions, long firstTransactionId, long firstLSN) - { - this.Transactions = transactions; - this.ClusterId = clusterOptions.ClusterId; - this.FirstLSN = firstLSN; - this.FirstTransactionId = firstTransactionId; - PartitionKey = MakePartitionKey(clusterOptions.ServiceId); - RowKey = MakeRowKey(firstTransactionId); - } - - public string ClusterId { get; set; } - public long FirstLSN { get; set; } - public long FirstTransactionId { get; set; } - public byte[] Transactions { get; set; } - } - - public class CommitRow : TableEntity - { - public const string MaxRowKey = "crrk_~"; - public const string MinRowKey = "crrk_"; - public CommitRow(ClusterOptions clusterOptions, long firstLSN) - { - // All entities are in the same partition for atomic read/writes. - PartitionKey = ArchivalRow.MakePartitionKey(clusterOptions.ServiceId); - RowKey = MakeRowKey(firstLSN); - } - - public CommitRow() - { - } - - public byte[] Transactions { get; set; } - - internal static string MakeRowKey(long lsn) - { - return $"crrk_{lsn:x16}"; - } - } - - private class StartRow : TableEntity - { - internal const string StartRowRowKey = "srrk_"; - - internal static string MakePartitionkey(string serviceId) - { - return $"srpk_{serviceId}"; - } - - public StartRow(ClusterOptions options, long transactionId) - { - // only row in the table with this partition key - PartitionKey = MakePartitionkey(options.ServiceId); - RowKey = StartRowRowKey; - ETag = "*"; - AllocatedTransactionIds = transactionId; - } - - public StartRow() - { - } - - public long AllocatedTransactionIds { get; set; } - } - } -} diff --git a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/AzureTableTransactionalStateStorage.cs b/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/AzureTableTransactionalStateStorage.cs index 5fce5a1f6d..0a072ba424 100644 --- a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/AzureTableTransactionalStateStorage.cs +++ b/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/AzureTableTransactionalStateStorage.cs @@ -14,16 +14,18 @@ public class AzureTableTransactionalStateStorage : ITransactionalStateSt { private readonly CloudTable table; private readonly string partition; + private readonly string stateName; private readonly JsonSerializerSettings jsonSettings; private readonly ILogger logger; private KeyEntity key; - private List states; + private List> states; - public AzureTableTransactionalStateStorage(CloudTable table, string partition, JsonSerializerSettings JsonSettings, ILogger> logger) + public AzureTableTransactionalStateStorage(CloudTable table, string partition, string stateName, JsonSerializerSettings JsonSettings, ILogger> logger) { this.table = table; this.partition = partition; + this.stateName = stateName; this.jsonSettings = JsonSettings; this.logger = logger; } @@ -32,98 +34,187 @@ public async Task> Load() { try { - Task keyTask = ReadKey(); - Task> statesTask = ReadStates(); - this.key = await keyTask.ConfigureAwait(false); - this.states = await statesTask.ConfigureAwait(false); - if (string.IsNullOrEmpty(this.key.ETag)) + var keyTask = ReadKey(); + var statesTask = ReadStates(); + key = await keyTask.ConfigureAwait(false); + states = await statesTask.ConfigureAwait(false); + + if (string.IsNullOrEmpty(key.ETag)) { + if (logger.IsEnabled(LogLevel.Debug)) + logger.LogDebug($"{partition} Loaded v0, fresh"); + + // first time load return new TransactionalStorageLoadResponse(); } - TState commitedState = (!string.IsNullOrEmpty(this.key.CommittedTransactionId)) ? FindState(this.key.CommittedTransactionId) : new TState(); - if (commitedState == null) + else { - this.logger.LogCritical("Transactional state non-recoverable error. Commited state for transaction {TransactionId} not found.", this.key.CommittedTransactionId); - throw new InvalidOperationException($"Transactional state non-recoverable error. Commited state for transaction {this.key.CommittedTransactionId} not found."); + if (!FindState(this.key.CommittedSequenceId, out var pos)) + { + var error = $"Storage state corrupted: no record for committed state"; + logger.LogCritical(error); + throw new InvalidOperationException(error); + } + var committedState = states[pos].Value.GetState(this.jsonSettings); + + var PrepareRecordsToRecover = new List>(); + for (int i = 0; i < states.Count; i++) + { + var kvp = states[i]; + + // pending states for already committed transactions can be ignored + if (kvp.Key <= key.CommittedSequenceId) + continue; + + // upon recovery, local non-committed transactions are considered aborted + if (kvp.Value.TransactionManager == null) + break; + + PrepareRecordsToRecover.Add(new PendingTransactionState() + { + SequenceId = kvp.Key, + State = kvp.Value.GetState(this.jsonSettings), + TimeStamp = kvp.Value.TransactionTimestamp, + TransactionId = kvp.Value.TransactionId, + TransactionManager = kvp.Value.TransactionManager + }); + } + + // clear the state strings... no longer needed, ok to GC now + for (int i = 0; i < states.Count; i++) + { + states[i].Value.StateJson = null; + } + + if (logger.IsEnabled(LogLevel.Debug)) + logger.LogDebug($"{partition} Loaded v{this.key.CommittedSequenceId} rows={string.Join(",", states.Select(s => s.Key.ToString("x16")))}"); + + return new TransactionalStorageLoadResponse(this.key.ETag, committedState, this.key.CommittedSequenceId, this.key.Metadata, PrepareRecordsToRecover); } - var pendingStates = states.Select(s => new PendingTransactionState(s.TransactionId, s.SequenceId, s.GetState(this.jsonSettings))).ToList(); - return new TransactionalStorageLoadResponse(this.key.ETag, commitedState, this.key.Metadata, pendingStates); - } catch(Exception ex) + } + catch (Exception ex) { this.logger.LogError("Transactional state load failed {Exception}.", ex); throw; } } - public async Task Persist(string expectedETag, string metadata, List> statesToPrepare) + + public async Task Store(string expectedETag, string metadata, List> statesToPrepare, long? commitUpTo, long? abortAfter) { - try - { - var batchOperation = new TableBatchOperation(); + if (this.key.ETag != expectedETag) + throw new ArgumentException(nameof(expectedETag), "Etag does not match"); - this.key.ETag = expectedETag; - this.key.Metadata = metadata; - if (string.IsNullOrEmpty(this.key.ETag)) - batchOperation.Insert(this.key); - else - batchOperation.Replace(this.key); + // assemble all storage operations into a single batch + // these operations must commit in sequence, but not necessarily atomically + // so we can split this up if needed + var batchOperation = new BatchOperation(logger, key, table); - // add new states - List> stored = this.states.Select(s => Tuple.Create(s.TransactionId, s.SequenceId)).ToList(); - List newStates = new List(); - foreach (PendingTransactionState pendingState in statesToPrepare.Where(p => !stored.Contains(Tuple.Create(p.TransactionId, p.SequenceId)))) + // first, clean up aborted records + if (abortAfter.HasValue && states.Count != 0) + { + while (states.Count > 0 && states[states.Count - 1].Key > abortAfter) { - var newState = StateEntity.Create(this.jsonSettings, this.partition, pendingState); - newStates.Add(newState); - batchOperation.Insert(newState); - } + var entity = states[states.Count - 1].Value; + await batchOperation.Add(TableOperation.Delete(entity)).ConfigureAwait(false); + states.RemoveAt(states.Count - 1); - if (batchOperation.Count > AzureTableConstants.MaxBatchSize) - { - this.logger.LogError("Too many pending states. PendingStateCount {PendingStateCount}.", batchOperation.Count); - throw new InvalidOperationException($"Too many pending states. PendingStateCount {batchOperation.Count}"); + if (logger.IsEnabled(LogLevel.Trace)) + logger.LogTrace($"{partition}.{states[states.Count - 1].Key:x16} Delete {entity.TransactionId}"); } - - await table.ExecuteBatchAsync(batchOperation).ConfigureAwait(false); - this.states.AddRange(newStates); - return this.key.ETag; } - catch (Exception ex) + + // second, persist non-obsolete prepare records + var obsoleteBefore = commitUpTo.HasValue ? commitUpTo.Value : key.CommittedSequenceId; + if (statesToPrepare != null) + foreach (var s in statesToPrepare) + if (s.SequenceId >= obsoleteBefore) + { + if (FindState(s.SequenceId, out var pos)) + { + // overwrite with new pending state + var existing = states[pos].Value; + existing.TransactionId = s.TransactionId; + existing.TransactionTimestamp = s.TimeStamp; + existing.TransactionManager = s.TransactionManager; + existing.SetState(s.State, this.jsonSettings); + await batchOperation.Add(TableOperation.Replace(existing)).ConfigureAwait(false); + states.RemoveAt(pos); + + if (logger.IsEnabled(LogLevel.Trace)) + logger.LogTrace($"{partition}.{existing.SequenceId:x16} Update {existing.TransactionId}"); + } + else + { + var entity = StateEntity.Create(this.jsonSettings, this.partition, s); + await batchOperation.Add(TableOperation.Insert(entity)).ConfigureAwait(false); + states.Insert(pos, new KeyValuePair(s.SequenceId, entity)); + + if (logger.IsEnabled(LogLevel.Trace)) + logger.LogTrace($"{partition}.{s.SequenceId:x16} Insert {entity.TransactionId}"); + } + } + + // third, persist metadata and commit position + key.Metadata = metadata; + if (commitUpTo.HasValue && commitUpTo.Value > key.CommittedSequenceId) { - this.logger.LogError("Transactional state persist failed {Exception}.", ex); - throw; + key.CommittedSequenceId = commitUpTo.Value; } - } + if (string.IsNullOrEmpty(this.key.ETag)) + { + await batchOperation.Add(TableOperation.Insert(this.key)).ConfigureAwait(false); - public async Task Confirm(string expectedETag, string metadata, string transactionIdToCommit) - { - try + if (logger.IsEnabled(LogLevel.Trace)) + logger.LogTrace($"{partition}.k Insert"); + } + else { - // only update storage if transaction id is greater then previously committed - if (string.Compare(transactionIdToCommit, this.key.CommittedTransactionId) <= 0) - return this.key.ETag; + await batchOperation.Add(TableOperation.Replace(this.key)).ConfigureAwait(false); + + if (logger.IsEnabled(LogLevel.Trace)) + logger.LogTrace($"{partition}.k Update"); + } - TState state = FindState(transactionIdToCommit); - if (state == null) + // fourth, remove obsolete records + if (states.Count > 0 && states[0].Key < obsoleteBefore) + { + FindState(obsoleteBefore, out var pos); + for (int i = 0; i < pos; i++) { - this.logger.LogCritical("Transactional state non-recoverable error. Attempting to confirm a transaction {TransactionId} for which no state exists.", transactionIdToCommit); - throw new InvalidOperationException($"Transactional state non-recoverable error. Attempting to confirm a transaction {transactionIdToCommit} for which no state exists."); - } + await batchOperation.Add(TableOperation.Delete(states[i].Value)).ConfigureAwait(false); - this.key.ETag = expectedETag; - this.key.Metadata = metadata; - this.key.CommittedTransactionId = transactionIdToCommit; - await WriteKey().ConfigureAwait(false); - var dead = this.states.Where(p => string.Compare(p.TransactionId, transactionIdToCommit) <0).ToList(); - this.states = this.states.Where(p => string.Compare(p.TransactionId, transactionIdToCommit) >= 0).ToList(); - Cleanup(dead).Ignore(); - return this.key.ETag; + if (logger.IsEnabled(LogLevel.Trace)) + logger.LogTrace($"{partition}.{states[i].Key:x16} Delete {states[i].Value.TransactionId}"); + } + states.RemoveRange(0, pos); } - catch (Exception ex) + + await batchOperation.Flush().ConfigureAwait(false); + + if (logger.IsEnabled(LogLevel.Debug)) + logger.LogDebug($"{partition} Stored v{this.key.CommittedSequenceId} eTag={key.ETag}"); + + return key.ETag; + } + + private bool FindState(long sequenceId, out int pos) + { + pos = 0; + while (pos < states.Count) { - this.logger.LogError("Transactional state confirm failed {Exception}.", ex); - throw; + switch (states[pos].Key.CompareTo(sequenceId)) + { + case 0: + return true; + case -1: + pos++; + continue; + case 1: + break; + } } + return false; } private async Task ReadKey() @@ -136,82 +227,100 @@ private async Task ReadKey() : queryResult.Results[0]; } - private async Task WriteKey() - { - Task write = (string.IsNullOrEmpty(this.key.ETag)) - ? this.table.ExecuteAsync(TableOperation.Insert(this.key)) - : this.table.ExecuteAsync(TableOperation.Replace(this.key)); - await write.ConfigureAwait(false); - } - - private async Task> ReadStates() + private async Task>> ReadStates() { var query = new TableQuery() - .Where(AzureStorageUtils.RangeQuery(this.partition, StateEntity.RKMin, StateEntity.RKMax)); + .Where(AzureStorageUtils.RangeQuery(this.partition, StateEntity.RK_MIN, StateEntity.RK_MAX)); TableContinuationToken continuationToken = null; - List results = new List(); + var results = new List>(); do { TableQuerySegment queryResult = await table.ExecuteQuerySegmentedAsync(query, continuationToken).ConfigureAwait(false); - results.AddRange(queryResult.Results); + foreach (var x in queryResult.Results) + { + results.Add(new KeyValuePair(x.SequenceId, x)); + }; continuationToken = queryResult.ContinuationToken; } while (continuationToken != null); return results; } - private TState FindState(string transactionId) + private class BatchOperation { - StateEntity entity = this.states.FirstOrDefault(s => s.TransactionId == transactionId); - return entity?.GetState(this.jsonSettings); - } + private readonly TableBatchOperation batchOperation; + private readonly ILogger logger; + private readonly KeyEntity key; + private readonly CloudTable table; - private async Task Cleanup(List deadStates) - { - var batchOperation = new TableBatchOperation(); - var pendingTasks = new List(); - const int MaxInFlight = 3; - foreach (StateEntity deadState in deadStates) - { - batchOperation.Delete(deadState); - // if batch is full, execute and make new batch - if (batchOperation.Count == AzureTableConstants.MaxBatchSize) - { - pendingTasks.Add(table.ExecuteBatchAsync(batchOperation)); - // if we've more than MaxInFlight storage calls in flight, wait for those to execute before continuing and clear pending tasks - if (pendingTasks.Count == MaxInFlight) - { - try - { - await Task.WhenAll(pendingTasks).ConfigureAwait(false); - } - catch (Exception ex) - { - this.logger.LogInformation("Error cleaning up transactional states {Exception}. Ignoring", ex); - } - pendingTasks.Clear(); - } - batchOperation = new TableBatchOperation(); - } - } + private bool batchContainsKey; - if (batchOperation.Count != 0) + public BatchOperation(ILogger logger, KeyEntity key, CloudTable table) { - pendingTasks.Add(table.ExecuteBatchAsync(batchOperation)); - batchOperation = new TableBatchOperation(); + this.batchOperation = new TableBatchOperation(); + this.logger = logger; + this.key = key; + this.table = table; } - if (pendingTasks.Count != 0) + public async Task Add(TableOperation operation) { - try + batchOperation.Add(operation); + + if (operation.Entity == key) { - await Task.WhenAll(pendingTasks).ConfigureAwait(false); + batchContainsKey = true; } - catch (Exception ex) + + if (batchOperation.Count == AzureTableConstants.MaxBatchSize - (batchContainsKey ? 0 : 1)) { - this.logger.LogInformation("Error cleaning up transactional states {Exception}. Ignoring", ex); + // the key serves as a synchronizer, to prevent modification by multiple grains under edge conditions, + // like duplicate activations or deployments.Every batch write needs to include the key, + // even if the key values don't change. + + if (!batchContainsKey) + { + if (string.IsNullOrEmpty(key.ETag)) + batchOperation.Insert(key); + else + batchOperation.Replace(key); + } + + await Flush().ConfigureAwait(false); + + batchOperation.Clear(); + batchContainsKey = false; } - pendingTasks.Clear(); + } + + public async Task Flush() + { + if (batchOperation.Count > 0) + try + { + await table.ExecuteBatchAsync(batchOperation).ConfigureAwait(false); + + batchOperation.Clear(); + batchContainsKey = false; + + if (logger.IsEnabled(LogLevel.Trace)) + { + for (int i = 0; i < batchOperation.Count; i++) + logger.LogTrace($"batch-op ok {i} PK={batchOperation[i].Entity.PartitionKey} RK={batchOperation[i].Entity.RowKey}"); + } + } + catch (Exception ex) + { + if (logger.IsEnabled(LogLevel.Trace)) + { + for (int i = 0; i < batchOperation.Count; i++) + logger.LogTrace($"batch-op failed {i} PK={batchOperation[i].Entity.PartitionKey} RK={batchOperation[i].Entity.RowKey}"); + } + + this.logger.LogError("Transactional state store failed {Exception}.", ex); + throw; + } } } + } -} +} \ No newline at end of file diff --git a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/AzureTableTransactionalStateStorageFactory.cs b/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/AzureTableTransactionalStateStorageFactory.cs index 32a9fd8f7f..91976480a7 100644 --- a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/AzureTableTransactionalStateStorageFactory.cs +++ b/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/AzureTableTransactionalStateStorageFactory.cs @@ -11,7 +11,7 @@ using Orleans.Serialization; using Orleans.Transactions.Abstractions; -namespace Orleans.Transactions.AzureStorage.TransactionalState +namespace Orleans.Transactions.AzureStorage { public class AzureTableTransactionalStateStorageFactory : ITransactionalStateStorageFactory, ILifecycleParticipant { @@ -40,7 +40,7 @@ public AzureTableTransactionalStateStorageFactory(string name, AzureTableTransac public ITransactionalStateStorage Create(string stateName, IGrainActivationContext context) where TState : class, new() { string partitionKey = MakePartitionKey(context, stateName); - return ActivatorUtilities.CreateInstance>(context.ActivationServices, this.table, partitionKey, this.jsonSettings); + return ActivatorUtilities.CreateInstance>(context.ActivationServices, this.table, partitionKey, stateName, this.jsonSettings); } public void Participate(ISiloLifecycle lifecycle) diff --git a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/DistributedTM/AzureTableTransactionalStateStorage.cs b/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/DistributedTM/AzureTableTransactionalStateStorage.cs deleted file mode 100644 index 764535fd07..0000000000 --- a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/DistributedTM/AzureTableTransactionalStateStorage.cs +++ /dev/null @@ -1,327 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage.Table; -using Newtonsoft.Json; -using Orleans.Transactions.Abstractions; -using Orleans.Transactions.AzureStorage; - -namespace Orleans.Transactions.DistributedTM.AzureStorage -{ - public class AzureTableTransactionalStateStorage : ITransactionalStateStorage - where TState : class, new() - { - private readonly CloudTable table; - private readonly string partition; - private readonly string stateName; - private readonly JsonSerializerSettings jsonSettings; - private readonly ILogger logger; - - private KeyEntity key; - private List> states; - - public AzureTableTransactionalStateStorage(CloudTable table, string partition, string stateName, JsonSerializerSettings JsonSettings, ILogger> logger) - { - this.table = table; - this.partition = partition; - this.stateName = stateName; - this.jsonSettings = JsonSettings; - this.logger = logger; - } - - public async Task> Load() - { - try - { - var keyTask = ReadKey(); - var statesTask = ReadStates(); - key = await keyTask.ConfigureAwait(false); - states = await statesTask.ConfigureAwait(false); - - if (string.IsNullOrEmpty(key.ETag)) - { - if (logger.IsEnabled(LogLevel.Debug)) - logger.LogDebug($"{partition} Loaded v0, fresh"); - - // first time load - return new TransactionalStorageLoadResponse(); - } - else - { - if (!FindState(this.key.CommittedSequenceId, out var pos)) - { - var error = $"Storage state corrupted: no record for committed state"; - logger.LogCritical(error); - throw new InvalidOperationException(error); - } - var committedState = states[pos].Value.GetState(this.jsonSettings); - - var PrepareRecordsToRecover = new List>(); - for (int i = 0; i < states.Count; i++) - { - var kvp = states[i]; - - // pending states for already committed transactions can be ignored - if (kvp.Key <= key.CommittedSequenceId) - continue; - - // upon recovery, local non-committed transactions are considered aborted - if (kvp.Value.TransactionManager == null) - break; - - PrepareRecordsToRecover.Add(new PendingTransactionState() - { - SequenceId = kvp.Key, - State = kvp.Value.GetState(this.jsonSettings), - TimeStamp = kvp.Value.TransactionTimestamp, - TransactionId = kvp.Value.TransactionId, - TransactionManager = kvp.Value.TransactionManager - }); - } - - // clear the state strings... no longer needed, ok to GC now - for (int i = 0; i < states.Count; i++) - { - states[i].Value.StateJson = null; - } - - if (logger.IsEnabled(LogLevel.Debug)) - logger.LogDebug($"{partition} Loaded v{this.key.CommittedSequenceId} rows={string.Join(",", states.Select(s => s.Key.ToString("x16")))}"); - - return new TransactionalStorageLoadResponse(this.key.ETag, committedState, this.key.CommittedSequenceId, this.key.Metadata, PrepareRecordsToRecover); - } - } - catch (Exception ex) - { - this.logger.LogError("Transactional state load failed {Exception}.", ex); - throw; - } - } - - - public async Task Store(string expectedETag, string metadata, List> statesToPrepare, long? commitUpTo, long? abortAfter) - { - if (this.key.ETag != expectedETag) - throw new ArgumentException(nameof(expectedETag), "Etag does not match"); - - // assemble all storage operations into a single batch - // these operations must commit in sequence, but not necessarily atomically - // so we can split this up if needed - var batchOperation = new BatchOperation(logger, key, table); - - // first, clean up aborted records - if (abortAfter.HasValue && states.Count != 0) - { - while (states.Count > 0 && states[states.Count - 1].Key > abortAfter) - { - var entity = states[states.Count - 1].Value; - await batchOperation.Add(TableOperation.Delete(entity)).ConfigureAwait(false); - states.RemoveAt(states.Count - 1); - - if (logger.IsEnabled(LogLevel.Trace)) - logger.LogTrace($"{partition}.{states[states.Count - 1].Key:x16} Delete {entity.TransactionId}"); - } - } - - // second, persist non-obsolete prepare records - var obsoleteBefore = commitUpTo.HasValue ? commitUpTo.Value : key.CommittedSequenceId; - if (statesToPrepare != null) - foreach (var s in statesToPrepare) - if (s.SequenceId >= obsoleteBefore) - { - if (FindState(s.SequenceId, out var pos)) - { - // overwrite with new pending state - var existing = states[pos].Value; - existing.TransactionId = s.TransactionId; - existing.TransactionTimestamp = s.TimeStamp; - existing.TransactionManager = s.TransactionManager; - existing.SetState(s.State, this.jsonSettings); - await batchOperation.Add(TableOperation.Replace(existing)).ConfigureAwait(false); - states.RemoveAt(pos); - - if (logger.IsEnabled(LogLevel.Trace)) - logger.LogTrace($"{partition}.{existing.SequenceId:x16} Update {existing.TransactionId}"); - } - else - { - var entity = StateEntity.Create(this.jsonSettings, this.partition, s); - await batchOperation.Add(TableOperation.Insert(entity)).ConfigureAwait(false); - states.Insert(pos, new KeyValuePair(s.SequenceId, entity)); - - if (logger.IsEnabled(LogLevel.Trace)) - logger.LogTrace($"{partition}.{s.SequenceId:x16} Insert {entity.TransactionId}"); - } - } - - // third, persist metadata and commit position - key.Metadata = metadata; - if (commitUpTo.HasValue && commitUpTo.Value > key.CommittedSequenceId) - { - key.CommittedSequenceId = commitUpTo.Value; - } - if (string.IsNullOrEmpty(this.key.ETag)) - { - await batchOperation.Add(TableOperation.Insert(this.key)).ConfigureAwait(false); - - if (logger.IsEnabled(LogLevel.Trace)) - logger.LogTrace($"{partition}.k Insert"); - } - else - { - await batchOperation.Add(TableOperation.Replace(this.key)).ConfigureAwait(false); - - if (logger.IsEnabled(LogLevel.Trace)) - logger.LogTrace($"{partition}.k Update"); - } - - // fourth, remove obsolete records - if (states.Count > 0 && states[0].Key < obsoleteBefore) - { - FindState(obsoleteBefore, out var pos); - for (int i = 0; i < pos; i++) - { - await batchOperation.Add(TableOperation.Delete(states[i].Value)).ConfigureAwait(false); - - if (logger.IsEnabled(LogLevel.Trace)) - logger.LogTrace($"{partition}.{states[i].Key:x16} Delete {states[i].Value.TransactionId}"); - } - states.RemoveRange(0, pos); - } - - await batchOperation.Flush().ConfigureAwait(false); - - if (logger.IsEnabled(LogLevel.Debug)) - logger.LogDebug($"{partition} Stored v{this.key.CommittedSequenceId} eTag={key.ETag}"); - - return key.ETag; - } - - private bool FindState(long sequenceId, out int pos) - { - pos = 0; - while (pos < states.Count) - { - switch (states[pos].Key.CompareTo(sequenceId)) - { - case 0: - return true; - case -1: - pos++; - continue; - case 1: - break; - } - } - return false; - } - - private async Task ReadKey() - { - var query = new TableQuery() - .Where(AzureStorageUtils.PointQuery(this.partition, KeyEntity.RK)); - TableQuerySegment queryResult = await table.ExecuteQuerySegmentedAsync(query, null).ConfigureAwait(false); - return queryResult.Results.Count == 0 - ? new KeyEntity() { PartitionKey = this.partition } - : queryResult.Results[0]; - } - - private async Task>> ReadStates() - { - var query = new TableQuery() - .Where(AzureStorageUtils.RangeQuery(this.partition, StateEntity.RK_MIN, StateEntity.RK_MAX)); - TableContinuationToken continuationToken = null; - var results = new List>(); - do - { - TableQuerySegment queryResult = await table.ExecuteQuerySegmentedAsync(query, continuationToken).ConfigureAwait(false); - foreach (var x in queryResult.Results) - { - results.Add(new KeyValuePair(x.SequenceId, x)); - }; - continuationToken = queryResult.ContinuationToken; - } while (continuationToken != null); - return results; - } - - private class BatchOperation - { - private readonly TableBatchOperation batchOperation; - private readonly ILogger logger; - private readonly KeyEntity key; - private readonly CloudTable table; - - private bool batchContainsKey; - - public BatchOperation(ILogger logger, KeyEntity key, CloudTable table) - { - this.batchOperation = new TableBatchOperation(); - this.logger = logger; - this.key = key; - this.table = table; - } - - public async Task Add(TableOperation operation) - { - batchOperation.Add(operation); - - if (operation.Entity == key) - { - batchContainsKey = true; - } - - if (batchOperation.Count == AzureTableConstants.MaxBatchSize - (batchContainsKey ? 0 : 1)) - { - // the key serves as a synchronizer, to prevent modification by multiple grains under edge conditions, - // like duplicate activations or deployments.Every batch write needs to include the key, - // even if the key values don't change. - - if (!batchContainsKey) - { - if (string.IsNullOrEmpty(key.ETag)) - batchOperation.Insert(key); - else - batchOperation.Replace(key); - } - - await Flush().ConfigureAwait(false); - - batchOperation.Clear(); - batchContainsKey = false; - } - } - - public async Task Flush() - { - if (batchOperation.Count > 0) - try - { - await table.ExecuteBatchAsync(batchOperation).ConfigureAwait(false); - - batchOperation.Clear(); - batchContainsKey = false; - - if (logger.IsEnabled(LogLevel.Trace)) - { - for (int i = 0; i < batchOperation.Count; i++) - logger.LogTrace($"batch-op ok {i} PK={batchOperation[i].Entity.PartitionKey} RK={batchOperation[i].Entity.RowKey}"); - } - } - catch (Exception ex) - { - if (logger.IsEnabled(LogLevel.Trace)) - { - for (int i = 0; i < batchOperation.Count; i++) - logger.LogTrace($"batch-op failed {i} PK={batchOperation[i].Entity.PartitionKey} RK={batchOperation[i].Entity.RowKey}"); - } - - this.logger.LogError("Transactional state store failed {Exception}.", ex); - throw; - } - } - } - - } -} \ No newline at end of file diff --git a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/DistributedTM/AzureTableTransactionalStateStorageFactory.cs b/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/DistributedTM/AzureTableTransactionalStateStorageFactory.cs deleted file mode 100644 index 91890f6c3a..0000000000 --- a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/DistributedTM/AzureTableTransactionalStateStorageFactory.cs +++ /dev/null @@ -1,71 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.WindowsAzure.Storage.Table; -using Newtonsoft.Json; -using Orleans.Configuration; -using Orleans.Runtime; -using Orleans.Serialization; -using Orleans.Transactions.Abstractions; -using Orleans.Transactions.AzureStorage; - -namespace Orleans.Transactions.DistributedTM.AzureStorage -{ - public class AzureTableTransactionalStateStorageFactory : ITransactionalStateStorageFactory, ILifecycleParticipant - { - private readonly string name; - private readonly AzureTableTransactionalStateOptions options; - private readonly ClusterOptions clusterOptions; - private readonly JsonSerializerSettings jsonSettings; - private readonly ILoggerFactory loggerFactory; - private CloudTable table; - - public static ITransactionalStateStorageFactory Create(IServiceProvider services, string name) - { - IOptionsSnapshot optionsSnapshot = services.GetRequiredService>(); - return ActivatorUtilities.CreateInstance(services, name, optionsSnapshot.Get(name)); - } - - public AzureTableTransactionalStateStorageFactory(string name, AzureTableTransactionalStateOptions options, IOptions clusterOptions, ITypeResolver typeResolver, IGrainFactory grainFactory, ILoggerFactory loggerFactory) - { - this.name = name; - this.options = options; - this.clusterOptions = clusterOptions.Value; - this.jsonSettings = OrleansJsonSerializer.GetDefaultSerializerSettings(typeResolver, grainFactory); - this.loggerFactory = loggerFactory; - } - - public ITransactionalStateStorage Create(string stateName, IGrainActivationContext context) where TState : class, new() - { - string partitionKey = MakePartitionKey(context, stateName); - return ActivatorUtilities.CreateInstance>(context.ActivationServices, this.table, partitionKey, stateName, this.jsonSettings); - } - - public void Participate(ISiloLifecycle lifecycle) - { - lifecycle.Subscribe(OptionFormattingUtilities.Name(this.name), this.options.InitStage, Init); - } - - private string MakePartitionKey(IGrainActivationContext context, string stateName) - { - string grainKey = context.GrainInstance.GrainReference.ToKeyString(); - var key = $"ts_{this.clusterOptions.ServiceId}_{grainKey}_{stateName}"; - return AzureStorageUtils.SanitizeTableProperty(key); - } - - private async Task CreateTable() - { - var tableManager = new AzureTableDataManager(this.options.TableName, this.options.ConnectionString, this.loggerFactory); - await tableManager.InitTableAsync().ConfigureAwait(false); - this.table = tableManager.Table; - } - - private Task Init(CancellationToken cancellationToken) - { - return CreateTable(); - } - } -} diff --git a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/DistributedTM/KeyEntity.cs b/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/DistributedTM/KeyEntity.cs deleted file mode 100644 index e8d1f63fc0..0000000000 --- a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/DistributedTM/KeyEntity.cs +++ /dev/null @@ -1,17 +0,0 @@ -using Microsoft.WindowsAzure.Storage.Table; - -namespace Orleans.Transactions.DistributedTM.AzureStorage -{ - internal class KeyEntity : TableEntity - { - public const string RK = "k"; - - public KeyEntity() - { - this.RowKey = RK; - } - - public long CommittedSequenceId { get; set; } - public string Metadata { get; set; } - } -} diff --git a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/DistributedTM/StateEntity.cs b/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/DistributedTM/StateEntity.cs deleted file mode 100644 index 0a7034c9ba..0000000000 --- a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/DistributedTM/StateEntity.cs +++ /dev/null @@ -1,54 +0,0 @@ -using Microsoft.WindowsAzure.Storage.Table; -using Newtonsoft.Json; -using Orleans.Transactions.Abstractions; -using System; - -namespace Orleans.Transactions.DistributedTM.AzureStorage -{ - internal class StateEntity : TableEntity - { - public static string MakeRowKey(long sequenceId) - { - return $"{RK_PREFIX}{sequenceId.ToString("x16")}"; - } - - public long SequenceId => long.Parse(RowKey.Substring(RK_PREFIX.Length)); - - // row keys range from s0000000000000001 to s7fffffffffffffff - public const string RK_PREFIX = "s"; - public const string RK_MIN = RK_PREFIX; - public const string RK_MAX = RK_PREFIX + "~"; - - public string TransactionId { get; set; } - - public DateTime TransactionTimestamp { get; set; } - - public string TransactionManager { get; set; } - - public string StateJson { get; set; } - - public static StateEntity Create(JsonSerializerSettings JsonSettings, - string partitionKey, PendingTransactionState pendingState) - where T : class, new() - { - return new StateEntity - { - PartitionKey = partitionKey, - RowKey = MakeRowKey(pendingState.SequenceId), - TransactionId = pendingState.TransactionId, - TransactionTimestamp = pendingState.TimeStamp, - TransactionManager = pendingState.TransactionManager, - StateJson = JsonConvert.SerializeObject(pendingState.State, JsonSettings) - }; - } - - public T GetState(JsonSerializerSettings JsonSettings) - { - return JsonConvert.DeserializeObject(this.StateJson, JsonSettings); - } - public void SetState(T state, JsonSerializerSettings JsonSettings) - { - StateJson = JsonConvert.SerializeObject(state, JsonSettings); - } - } -} diff --git a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/KeyEntity.cs b/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/KeyEntity.cs index 3b8e17b8fc..e1d158e80e 100644 --- a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/KeyEntity.cs +++ b/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/KeyEntity.cs @@ -4,14 +4,14 @@ namespace Orleans.Transactions.AzureStorage { internal class KeyEntity : TableEntity { - public const string RK = "tsk"; + public const string RK = "k"; public KeyEntity() { this.RowKey = RK; } - public string CommittedTransactionId { get; set; } + public long CommittedSequenceId { get; set; } public string Metadata { get; set; } } } diff --git a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/StateEntity.cs b/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/StateEntity.cs index 344b14c9cb..c7c090ff60 100644 --- a/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/StateEntity.cs +++ b/src/Azure/Orleans.Transactions.AzureStorage/TransactionalState/StateEntity.cs @@ -1,4 +1,5 @@ -using Microsoft.WindowsAzure.Storage.Table; +using System; +using Microsoft.WindowsAzure.Storage.Table; using Newtonsoft.Json; using Orleans.Transactions.Abstractions; @@ -6,11 +7,24 @@ namespace Orleans.Transactions.AzureStorage { internal class StateEntity : TableEntity { - public const string RKMin = "ts_"; - public const string RKMax = "ts_~"; + public static string MakeRowKey(long sequenceId) + { + return $"{RK_PREFIX}{sequenceId.ToString("x16")}"; + } + + public long SequenceId => long.Parse(RowKey.Substring(RK_PREFIX.Length)); + + // row keys range from s0000000000000001 to s7fffffffffffffff + public const string RK_PREFIX = "s_"; + public const string RK_MIN = RK_PREFIX; + public const string RK_MAX = RK_PREFIX + "~"; public string TransactionId { get; set; } - public long SequenceId { get; set; } + + public DateTime TransactionTimestamp { get; set; } + + public string TransactionManager { get; set; } + public string StateJson { get; set; } public static StateEntity Create(JsonSerializerSettings JsonSettings, @@ -20,21 +34,21 @@ public static StateEntity Create(JsonSerializerSettings JsonSettings, return new StateEntity { PartitionKey = partitionKey, - RowKey = MakeRowKey(pendingState.TransactionId, pendingState.SequenceId), + RowKey = MakeRowKey(pendingState.SequenceId), TransactionId = pendingState.TransactionId, - SequenceId = pendingState.SequenceId, + TransactionTimestamp = pendingState.TimeStamp, + TransactionManager = pendingState.TransactionManager, StateJson = JsonConvert.SerializeObject(pendingState.State, JsonSettings) }; } - public static string MakeRowKey(string TransactionId, long sequenceId) - { - return AzureStorageUtils.SanitizeTableProperty($"{RKMin}{TransactionId}_{sequenceId.ToString("x16")}"); - } - public T GetState(JsonSerializerSettings JsonSettings) { return JsonConvert.DeserializeObject(this.StateJson, JsonSettings); } + public void SetState(T state, JsonSerializerSettings JsonSettings) + { + StateJson = JsonConvert.SerializeObject(state, JsonSettings); + } } } diff --git a/src/Orleans.Core/Properties/AssemblyInfo.cs b/src/Orleans.Core/Properties/AssemblyInfo.cs index fb287ec22e..3287fae157 100644 --- a/src/Orleans.Core/Properties/AssemblyInfo.cs +++ b/src/Orleans.Core/Properties/AssemblyInfo.cs @@ -20,7 +20,6 @@ [assembly: InternalsVisibleTo("Orleans.TestingHost")] [assembly: InternalsVisibleTo("Orleans.TestingHost.AppDomain")] [assembly: InternalsVisibleTo("Orleans.TestingHost.Legacy")] -[assembly: InternalsVisibleTo("Orleans.Transactions.DynamoDB")] [assembly: InternalsVisibleTo("OrleansCounterControl")] [assembly: InternalsVisibleTo("OrleansManager")] [assembly: InternalsVisibleTo("OrleansProviders")] diff --git a/src/Orleans.Core/Transactions/ITransactionAgent.cs b/src/Orleans.Core/Transactions/ITransactionAgent.cs index f949e1e233..455c068faa 100644 --- a/src/Orleans.Core/Transactions/ITransactionAgent.cs +++ b/src/Orleans.Core/Transactions/ITransactionAgent.cs @@ -49,19 +49,5 @@ public interface ITransactionAgent /// None. /// This method is exception-free void Abort(ITransactionInfo transactionInfo, OrleansTransactionAbortedException reason); - - /// - /// Check if a transaction is known to have aborted. - /// - /// the id of the transaction - /// true if the transaction is known to have aborted, false otherwise - /// - /// Note that the transaction could have aborted but this still returns false, if the agent - /// did not learn about the outcome yet. - /// This method is exception-free. - /// - bool IsAborted(long transactionId); - - long ReadOnlyTransactionId { get; } } } diff --git a/src/Orleans.Core/Transactions/ITransactionInfo.cs b/src/Orleans.Core/Transactions/ITransactionInfo.cs index 4d5f687628..e0e73b0e79 100644 --- a/src/Orleans.Core/Transactions/ITransactionInfo.cs +++ b/src/Orleans.Core/Transactions/ITransactionInfo.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; - + namespace Orleans.Transactions { /// @@ -38,6 +35,4 @@ public interface ITransactionInfo /// true if there are no orphans, false otherwise bool ReconcilePending(out int numberOrphans); } - - } diff --git a/src/Orleans.Core/Transactions/ITransactionalResource.cs b/src/Orleans.Core/Transactions/ITransactionalResource.cs index 230dcd15c5..a7f59e829e 100644 --- a/src/Orleans.Core/Transactions/ITransactionalResource.cs +++ b/src/Orleans.Core/Transactions/ITransactionalResource.cs @@ -1,7 +1,7 @@  -using Orleans.Concurrency; using System; using System.Threading.Tasks; +using Orleans.Concurrency; namespace Orleans.Transactions { diff --git a/src/Orleans.Core/Transactions/TransactionContext.cs b/src/Orleans.Core/Transactions/TransactionContext.cs index 509e716af2..300b1f321d 100644 --- a/src/Orleans.Core/Transactions/TransactionContext.cs +++ b/src/Orleans.Core/Transactions/TransactionContext.cs @@ -1,7 +1,5 @@  -using System; using System.Collections.Generic; -using System.Linq; using Orleans.Runtime; namespace Orleans.Transactions diff --git a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs index 0ba717b4f7..684448865b 100644 --- a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs +++ b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs @@ -228,11 +228,10 @@ internal static void AddDefaultServices(HostBuilderContext context, IServiceColl services.AddFromExisting(); services.AddSingleton(); services.AddFromExisting(); - + // Transactions - services.TryAddSingleton(); + services.TryAddSingleton(); services.TryAddSingleton>(sp => () => sp.GetRequiredService()); - services.TryAddSingleton(); // Application Parts var applicationPartManager = context.GetApplicationPartManager(); diff --git a/src/Orleans.Runtime/Transactions/DisabledTransactionAgent.cs b/src/Orleans.Runtime/Transactions/DisabledTransactionAgent.cs new file mode 100644 index 0000000000..da81d20c3b --- /dev/null +++ b/src/Orleans.Runtime/Transactions/DisabledTransactionAgent.cs @@ -0,0 +1,31 @@ +using Orleans.Transactions; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +namespace Orleans.Transactions +{ + internal class DisabledTransactionAgent : ITransactionAgent + { + public void Abort(ITransactionInfo transactionInfo, OrleansTransactionAbortedException reason) + { + throw new OrleansTransactionsDisabledException(); + } + + public Task Commit(ITransactionInfo transactionInfo) + { + throw new OrleansTransactionsDisabledException(); + } + + public Task Start() + { + return Task.CompletedTask; + } + + public Task StartTransaction(bool readOnly, TimeSpan timeout) + { + throw new OrleansStartTransactionFailedException(new OrleansTransactionsDisabledException()); + } + } +} diff --git a/src/Orleans.Runtime/Transactions/DisabledTransactionManagerService.cs b/src/Orleans.Runtime/Transactions/DisabledTransactionManagerService.cs deleted file mode 100644 index 575186542f..0000000000 --- a/src/Orleans.Runtime/Transactions/DisabledTransactionManagerService.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Orleans.Transactions; - -namespace Orleans.Transactions -{ - internal class DisabledTransactionManagerService : ITransactionManagerService - { - public Task AbortTransaction(long transactionId, OrleansTransactionAbortedException reason) - { - throw new OrleansTransactionsDisabledException(); - } - - public Task CommitTransactions(List transactions, HashSet queries) - { - throw new OrleansTransactionsDisabledException(); - } - - public Task StartTransactions(List timeouts) - { - throw new OrleansTransactionsDisabledException(); - } - } -} diff --git a/src/Orleans.Runtime/Transactions/ITransactionManagerService.cs b/src/Orleans.Runtime/Transactions/ITransactionManagerService.cs deleted file mode 100644 index b89612522f..0000000000 --- a/src/Orleans.Runtime/Transactions/ITransactionManagerService.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace Orleans.Transactions -{ - public interface ITransactionManagerService - { - Task StartTransactions(List timeouts); - Task CommitTransactions(List transactions, HashSet queries); - Task AbortTransaction(long transactionId, OrleansTransactionAbortedException reason); - } - - [Serializable] - public struct CommitResult - { - public bool Success { get; set; } - - public OrleansTransactionAbortedException AbortingException { get; set; } - } - - [Serializable] - public class CommitTransactionsResponse - { - public long ReadOnlyTransactionId { get; set; } - public long AbortLowerBound { get; set; } - public Dictionary CommitResult { get; set; } - } - - [Serializable] - public class StartTransactionsResponse - { - public long ReadOnlyTransactionId { get; set; } - public long AbortLowerBound { get; set; } - public List TransactionId { get; set; } - } -} diff --git a/src/Orleans.Runtime/Transactions/TransactionAgent.cs b/src/Orleans.Runtime/Transactions/TransactionAgent.cs deleted file mode 100644 index ddfc8d41bf..0000000000 --- a/src/Orleans.Runtime/Transactions/TransactionAgent.cs +++ /dev/null @@ -1,476 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Orleans.Runtime; -using Orleans.Concurrency; -using DateTime = System.DateTime; -using Microsoft.Extensions.Options; -using Orleans.Configuration; - -namespace Orleans.Transactions -{ - internal class TransactionAgentMetrics - { - //TPS in current monitor window - private const string BatchStartTransanctionsTPS = "TransactionAgent.BatchStartTransactions.TPS"; - //avg latency in current monitor window - private const string AvgBatchStartTransactionsLatency = "TransactionAgent.BatchStartTransactions.AvgLatency"; - private const string AvgBatchStartTransactionsBatchSize = "TransactionAgent.BatchStartTransactions.AvgBatchSize"; - private const string BatchCommitTransactionsTPS = "TransactionAgent.BatchCommitTransactions.TPS"; - private const string AvgBatchCommitTransactionsLatency = "TransactionAgent.BatchCommitTransactions.AvgLatency"; - private const string AvgBatchCommitTransactionsSize = "TransactionAgent.BatchCommitTransactions.AvgBatchSize"; - internal int BatchStartTransactionsRequestCounter { get; set; } - - internal int BatchCommitTransactionsRequestsCounter { get; set; } - internal TimeSpan BatchStartTransactionsRequestLatencyCounter { get; set; } = TimeSpan.Zero; - - internal TimeSpan BatchCommitTransactionsRequestLatencyCounter { get; set; } = TimeSpan.Zero; - internal int BatchStartTransactionsRequestSizeCounter { get; set; } - internal int BatchCommitTransactionsRequestSizeCounter { get; set; } - private DateTime lastReportTime = DateTime.UtcNow; - private ITelemetryProducer telemetryProducer; - private PeriodicAction periodicMonitor; - - public TransactionAgentMetrics(ITelemetryProducer producer, TimeSpan interval) - { - this.telemetryProducer = producer; - this.periodicMonitor = new PeriodicAction(interval, this.ReportMetrics); - } - - public void TryReportMetrics() - { - this.periodicMonitor.TryAction(DateTime.UtcNow); - } - - private void ResetCounters(DateTime lastReportTimeStamp) - { - //record last report time stamp - lastReportTime = lastReportTimeStamp; - this.BatchStartTransactionsRequestCounter = 0; - this.BatchCommitTransactionsRequestsCounter = 0; - this.BatchStartTransactionsRequestLatencyCounter = TimeSpan.Zero; - this.BatchCommitTransactionsRequestLatencyCounter = TimeSpan.Zero; - this.BatchCommitTransactionsRequestSizeCounter = 0; - this.BatchStartTransactionsRequestSizeCounter = 0; - } - - private void ReportMetrics() - { - if (this.telemetryProducer == null) - return; - var now = DateTime.UtcNow; - var timeSinceLastReportInSeconds = Math.Max(1, (now - this.lastReportTime).TotalSeconds); - //batch start metrics - var batchStartTransactionTPS = BatchStartTransactionsRequestCounter / timeSinceLastReportInSeconds; - this.telemetryProducer.TrackMetric(BatchStartTransanctionsTPS, batchStartTransactionTPS); - if (BatchStartTransactionsRequestCounter > 0) - { - var avgBatchStartTransactionLatency = BatchStartTransactionsRequestLatencyCounter.Divide(BatchStartTransactionsRequestCounter); - this.telemetryProducer.TrackMetric(AvgBatchStartTransactionsLatency, - avgBatchStartTransactionLatency); - var avgBatchStartSize = BatchStartTransactionsRequestSizeCounter / BatchStartTransactionsRequestCounter; - this.telemetryProducer.TrackMetric(AvgBatchStartTransactionsBatchSize, avgBatchStartSize); - } - - //batch commit metrics - var batchCommitTransactionTPS = BatchCommitTransactionsRequestsCounter / timeSinceLastReportInSeconds; - this.telemetryProducer.TrackMetric(BatchCommitTransactionsTPS, batchCommitTransactionTPS); - - if (BatchCommitTransactionsRequestsCounter > 0) - { - var avgBatchCommitTransactionLatency = - BatchCommitTransactionsRequestLatencyCounter.Divide(BatchCommitTransactionsRequestsCounter); - this.telemetryProducer.TrackMetric(AvgBatchCommitTransactionsLatency, - avgBatchCommitTransactionLatency); - var avgBatchCommitSzie = - BatchCommitTransactionsRequestSizeCounter / BatchCommitTransactionsRequestsCounter; - this.telemetryProducer.TrackMetric(AvgBatchCommitTransactionsSize, avgBatchCommitSzie); - } - - this.ResetCounters(now); - } - } - - [Reentrant] - internal class TransactionAgent : SystemTarget, ITransactionAgent, ITransactionAgentSystemTarget - { - private readonly ITransactionManagerService tmService; - - //private long abortSequenceNumber; - private long abortLowerBound; - private readonly ConcurrentDictionary abortedTransactions; - - private readonly ConcurrentQueue>> transactionStartQueue; - private readonly ConcurrentQueue transactionCommitQueue; - private readonly ConcurrentDictionary> commitCompletions; - private readonly HashSet outstandingCommits; - - private readonly ILogger logger; - private readonly ILoggerFactory loggerFactory; - private IGrainTimer requestProcessor; - private Task startTransactionsTask = Task.CompletedTask; - private Task commitTransactionsTask = Task.CompletedTask; - - public long ReadOnlyTransactionId { get; private set; } - - //metrics related - private TransactionAgentMetrics metrics; - public TransactionAgent( - ILocalSiloDetails siloDetails, - ITransactionManagerService tmService, - ILoggerFactory loggerFactory, - ITelemetryProducer telemetryProducer, - IOptions options) - : base(Constants.TransactionAgentSystemTargetId, siloDetails.SiloAddress, loggerFactory) - { - logger = loggerFactory.CreateLogger(); - this.tmService = tmService; - ReadOnlyTransactionId = 0; - //abortSequenceNumber = 0; - abortLowerBound = 0; - this.loggerFactory = loggerFactory; - - abortedTransactions = new ConcurrentDictionary(); - transactionStartQueue = new ConcurrentQueue>>(); - transactionCommitQueue = new ConcurrentQueue(); - commitCompletions = new ConcurrentDictionary>(); - outstandingCommits = new HashSet(); - this.metrics = new TransactionAgentMetrics(telemetryProducer, options.Value.MetricsWritePeriod); - } - - #region ITransactionAgent - - public async Task StartTransaction(bool readOnly, TimeSpan timeout) - { - if (readOnly) - { - return new TransactionInfo(ReadOnlyTransactionId, true); - } - - TransactionsStatisticsGroup.OnTransactionStartRequest(); - var completion = new TaskCompletionSource(); - transactionStartQueue.Enqueue(new Tuple>(timeout, completion)); - - long id = await completion.Task; - return new TransactionInfo(id); - } - - public async Task Commit(ITransactionInfo info) - { - var transactionInfo = (TransactionInfo)info; - - TransactionsStatisticsGroup.OnTransactionCommitRequest(); - - if (transactionInfo.IsReadOnly) - { - return; - } - - var completion = new TaskCompletionSource(); - bool canCommit = true; - - List> prepareTasks = new List>(transactionInfo.WriteSet.Count); - foreach (var g in transactionInfo.WriteSet.Keys) - { - TransactionalResourceVersion write = TransactionalResourceVersion.Create(transactionInfo.TransactionId, transactionInfo.WriteSet[g]); - TransactionalResourceVersion? read = null; - if (transactionInfo.ReadSet.ContainsKey(g)) - { - read = transactionInfo.ReadSet[g]; - transactionInfo.ReadSet.Remove(g); - } - prepareTasks.Add(g.Prepare(transactionInfo.TransactionId, write, read)); - } - - foreach (var g in transactionInfo.ReadSet.Keys) - { - TransactionalResourceVersion read = transactionInfo.ReadSet[g]; - prepareTasks.Add(g.Prepare(transactionInfo.TransactionId, null, read)); - } - - await Task.WhenAll(prepareTasks); - foreach (var t in prepareTasks) - { - if (!t.Result) - { - canCommit = false; - } - } - - if (!canCommit) - { - TransactionsStatisticsGroup.OnTransactionAborted(); - abortedTransactions.TryAdd(transactionInfo.TransactionId, 0); - throw new OrleansPrepareFailedException(transactionInfo.TransactionId.ToString()); - } - commitCompletions.TryAdd(transactionInfo.TransactionId, completion); - transactionCommitQueue.Enqueue(transactionInfo); - await completion.Task; - } - - public void Abort(ITransactionInfo info, OrleansTransactionAbortedException reason) - { - var transactionInfo = (TransactionInfo)info; - - abortedTransactions.TryAdd(transactionInfo.TransactionId, 0); - foreach (var g in transactionInfo.WriteSet.Keys) - { - g.Abort(transactionInfo.TransactionId).Ignore(); - } - - // TODO: should we wait for the abort tasks to complete before returning? - // If so, how do we handle exceptions? - - // There is no guarantee that the WriteSet is complete and has all the grains. - // Notify the TM of the abort as well. - this.tmService.AbortTransaction(transactionInfo.TransactionId, reason).Ignore(); - } - - public bool IsAborted(long transactionId) - { - if (transactionId <= abortLowerBound) - { - return true; - } - - return abortedTransactions.ContainsKey(transactionId) || transactionId < this.abortLowerBound; - } - - #endregion - - private async Task ProcessRequests(object args) - { - // NOTE: This code is a bit complicated because we want to issue both start and commit requests, - // but wait for each one separately in its own continuation. This can be significantly simplified - // if we can register a separate timer for start and commit. - - List committingTransactions = new List(); - List startingTransactions = new List(); - List> startCompletions = new List>(); - - while (transactionCommitQueue.Count > 0 || transactionStartQueue.Count > 0 || outstandingCommits.Count > 0) - { - this.metrics.TryReportMetrics(); - var initialAbortLowerBound = this.abortLowerBound; - - await Task.Yield(); - await WaitForWork(); - - int startCount = transactionStartQueue.Count; - while (startCount > 0 && startTransactionsTask.IsCompleted) - { - Tuple> elem; - transactionStartQueue.TryDequeue(out elem); - startingTransactions.Add(elem.Item1); - startCompletions.Add(elem.Item2); - - startCount--; - } - - int commitCount = transactionCommitQueue.Count; - while (commitCount > 0 && commitTransactionsTask.IsCompleted) - { - TransactionInfo elem; - transactionCommitQueue.TryDequeue(out elem); - committingTransactions.Add(elem); - outstandingCommits.Add(elem.TransactionId); - - commitCount--; - } - - - if (startingTransactions.Count > 0 && startTransactionsTask.IsCompleted) - { - logger.Debug(ErrorCode.Transactions_SendingTMRequest, "Calling TM to start {0} transactions", startingTransactions.Count); - - startTransactionsTask = this.StartTransactions(startingTransactions, startCompletions); - } - - if ((committingTransactions.Count > 0 || outstandingCommits.Count > 0) && commitTransactionsTask.IsCompleted) - { - logger.Debug(ErrorCode.Transactions_SendingTMRequest, "Calling TM to commit {0} transactions", committingTransactions.Count); - - commitTransactionsTask = this.CommitTransactions(committingTransactions, outstandingCommits); - - // Removed transactions below the abort lower bound. - if (this.abortLowerBound != initialAbortLowerBound) - { - foreach (var aborted in this.abortedTransactions) - { - if (aborted.Key < this.abortLowerBound) - { - long ignored; - this.abortedTransactions.TryRemove(aborted.Key, out ignored); - } - } - } - } - } - this.metrics.TryReportMetrics(); - - } - - private async Task CommitTransactions(List committingTransactions, - HashSet outstandingCommits) - { - var stopWatch = Stopwatch.StartNew(); - try - { - metrics.BatchCommitTransactionsRequestsCounter++; - metrics.BatchCommitTransactionsRequestSizeCounter += committingTransactions.Count; - CommitTransactionsResponse commitResponse; - try - { - commitResponse = await this.tmService.CommitTransactions(committingTransactions, outstandingCommits); - } - finally - { - stopWatch.Stop(); - metrics.BatchCommitTransactionsRequestLatencyCounter += stopWatch.Elapsed; - } - - var commitResults = commitResponse.CommitResult; - - // reply to clients with the outcomes we received from the TM. - foreach (var completedId in commitResults.Keys) - { - outstandingCommits.Remove(completedId); - - TaskCompletionSource completion; - if (commitCompletions.TryRemove(completedId, out completion)) - { - if (commitResults[completedId].Success) - { - TransactionsStatisticsGroup.OnTransactionCommitted(); - completion.SetResult(true); - } - else - { - if (commitResults[completedId].AbortingException != null) - { - TransactionsStatisticsGroup.OnTransactionAborted(); - completion.SetException(commitResults[completedId].AbortingException); - } - else - { - TransactionsStatisticsGroup.OnTransactionInDoubt(); - completion.SetException(new OrleansTransactionInDoubtException(completedId.ToString())); - } - } - } - } - - // Refresh cached values using new values from TM. - this.ReadOnlyTransactionId = Math.Max(this.ReadOnlyTransactionId, - commitResponse.ReadOnlyTransactionId); - this.abortLowerBound = Math.Max(this.abortLowerBound, commitResponse.AbortLowerBound); - logger.Debug(ErrorCode.Transactions_ReceivedTMResponse, - "{0} transactions committed. readOnlyTransactionId {1}, abortLowerBound {2}", - committingTransactions.Count, ReadOnlyTransactionId, abortLowerBound); - } - catch (Exception e) - { - logger.Error(ErrorCode.Transactions_TMError, "TM Error", e); - // Propagate the exception to every transaction in the request. - foreach (var tx in committingTransactions) - { - TransactionsStatisticsGroup.OnTransactionInDoubt(); - - TaskCompletionSource completion; - if (commitCompletions.TryRemove(tx.TransactionId, out completion)) - { - outstandingCommits.Remove(tx.TransactionId); - completion.SetException(new OrleansTransactionInDoubtException(tx.TransactionId.ToString())); - } - } - } - - committingTransactions.Clear(); - - } - - private async Task StartTransactions(List startingTransactions, List> startCompletions) - { - var stopWatch = Stopwatch.StartNew(); - try - { - metrics.BatchStartTransactionsRequestCounter++; - metrics.BatchStartTransactionsRequestSizeCounter += startingTransactions.Count; - StartTransactionsResponse startResponse; - try - { - startResponse = await this.tmService.StartTransactions(startingTransactions); - } - finally - { - stopWatch.Stop(); - metrics.BatchStartTransactionsRequestLatencyCounter += stopWatch.Elapsed; - } - List startedIds = startResponse.TransactionId; - - // reply to clients with results - for (int i = 0; i < startCompletions.Count; i++) - { - TransactionsStatisticsGroup.OnTransactionStarted(); - startCompletions[i].SetResult(startedIds[i]); - } - - // Refresh cached values using new values from TM. - this.ReadOnlyTransactionId = Math.Max(this.ReadOnlyTransactionId, startResponse.ReadOnlyTransactionId); - this.abortLowerBound = Math.Max(this.abortLowerBound, startResponse.AbortLowerBound); - logger.Debug(ErrorCode.Transactions_ReceivedTMResponse, - "{0} Transactions started. readOnlyTransactionId {1}, abortLowerBound {2}", - startingTransactions.Count, ReadOnlyTransactionId, abortLowerBound); - } - catch (Exception e) - { - logger.Error(ErrorCode.Transactions_TMError, "Transaction manager failed to start transactions.", e); - - foreach (var completion in startCompletions) - { - TransactionsStatisticsGroup.OnTransactionStartFailed(); - completion.SetException(new OrleansStartTransactionFailedException(e)); - } - } - - startingTransactions.Clear(); - startCompletions.Clear(); - } - - private Task WaitForWork() - { - // Returns a task that can be waited on until the RequestProcessor has - // actionable work. The purpose is to avoid looping indefinitely while waiting - // for the outstanding start or commit requests to complete. - List toWait = new List(); - - if (transactionStartQueue.Count > 0) - { - toWait.Add(startTransactionsTask); - } - - if (transactionCommitQueue.Count > 0) - { - toWait.Add(commitTransactionsTask); - } - - if (toWait.Count == 0) - { - return Task.CompletedTask; - } - - return Task.WhenAny(toWait); - } - - public Task Start() - { - requestProcessor = GrainTimer.FromTaskCallback(this.RuntimeClient.Scheduler, this.loggerFactory.CreateLogger(), ProcessRequests, null, TimeSpan.FromMilliseconds(0), TimeSpan.FromMilliseconds(10), "TransactionAgent"); - requestProcessor.Start(); - return Task.CompletedTask; - } - - } -} diff --git a/src/Orleans.Runtime/Transactions/TransactionInfo.cs b/src/Orleans.Runtime/Transactions/TransactionInfo.cs deleted file mode 100644 index 9dea80ab24..0000000000 --- a/src/Orleans.Runtime/Transactions/TransactionInfo.cs +++ /dev/null @@ -1,199 +0,0 @@ -using Orleans.Concurrency; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace Orleans.Transactions -{ - [Serializable] - public class TransactionInfo : ITransactionInfo - { - public TransactionInfo() - { - this.joined = new ConcurrentQueue(); - } - - public TransactionInfo(long id, bool readOnly = false) - : this() - { - TransactionId = id; - IsReadOnly = readOnly; - IsAborted = false; - PendingCalls = 0; - ReadSet = new Dictionary(); - WriteSet = new Dictionary(); - DependentTransactions = new HashSet(); - } - - /// - /// Constructor used when TransactionInfo is transferred to a request - /// - /// - public TransactionInfo(TransactionInfo other) - : this() - { - TransactionId = other.TransactionId; - IsReadOnly = other.IsReadOnly; - IsAborted = other.IsAborted; - PendingCalls = 0; - ReadSet = new Dictionary(); - WriteSet = new Dictionary(); - DependentTransactions = new HashSet(); - } - - public string Id => TransactionId.ToString(); - - public long TransactionId { get; } - - public bool IsReadOnly { get; } - - public bool IsAborted { get; set; } - - public Dictionary ReadSet { get; } - public Dictionary WriteSet { get; } - public HashSet DependentTransactions { get; } - - [NonSerialized] - public int PendingCalls; - - [NonSerialized] - private readonly ConcurrentQueue joined; - - public ITransactionInfo Fork() - { - PendingCalls++; - return new TransactionInfo(this); - } - - public void Join(ITransactionInfo other) - { - this.joined.Enqueue((TransactionInfo)other); - } - - /// - /// Reconciles all pending calls that have join the transaction. - /// - /// true if there are no orphans, false otherwise - public bool ReconcilePending(out int numberOrphans) - { - TransactionInfo trasactionInfo; - while (this.joined.TryDequeue(out trasactionInfo)) - { - Union(trasactionInfo); - PendingCalls--; - } - numberOrphans = PendingCalls; - return numberOrphans == 0; - } - - private void Union(TransactionInfo other) - { - if (TransactionId != other.TransactionId) - { - IsAborted = true; - string error = $"Attempting to perform union between different Transactions. Attempted union between Transactions {TransactionId} and {other.TransactionId}"; - throw new InvalidOperationException(error); - } - - if (other.IsAborted) - { - IsAborted = true; - } - - // Take a union of the ReadSets. - foreach (var grain in other.ReadSet.Keys) - { - if (ReadSet.ContainsKey(grain)) - { - if (ReadSet[grain] != other.ReadSet[grain]) - { - // Conflict! Transaction must abort - IsAborted = true; - } - } - else - { - ReadSet.Add(grain, other.ReadSet[grain]); - } - } - - // Take a union of the WriteSets. - foreach (var grain in other.WriteSet.Keys) - { - if (!WriteSet.ContainsKey(grain)) - { - WriteSet[grain] = 0; - } - - WriteSet[grain] += other.WriteSet[grain]; - } - - DependentTransactions.UnionWith(other.DependentTransactions); - } - - - public void RecordRead(ITransactionalResource transactionalResource, TransactionalResourceVersion readVersion, long stableVersion) - { - if (readVersion.TransactionId == TransactionId) - { - // Just reading our own write here. - // Sanity check to see if there's a lost write. - int resourceWriteNumber; - if (WriteSet.TryGetValue(transactionalResource, out resourceWriteNumber) - && resourceWriteNumber > readVersion.WriteNumber) - { - // Context has record of more writes than we have, some writes must be lost. - throw new OrleansTransactionAbortedException(TransactionId.ToString(), "Lost Write"); - } - } - else - { - TransactionalResourceVersion resourceReadVersion; - if (ReadSet.TryGetValue(transactionalResource, out resourceReadVersion) - && resourceReadVersion != readVersion) - { - // Uh-oh. Read two different versions of the grain. - throw new OrleansValidationFailedException(TransactionId.ToString()); - } - - ReadSet[transactionalResource] = readVersion; - - if (readVersion.TransactionId != TransactionId && - readVersion.TransactionId > stableVersion) - { - DependentTransactions.Add(readVersion.TransactionId); - } - } - } - - public void RecordWrite(ITransactionalResource transactionalResource, TransactionalResourceVersion latestVersion, long stableVersion) - { - int writeNumber; - WriteSet.TryGetValue(transactionalResource, out writeNumber); - WriteSet[transactionalResource] = writeNumber + 1; - - if (latestVersion.TransactionId != TransactionId && latestVersion.TransactionId > stableVersion) - { - DependentTransactions.Add(latestVersion.TransactionId); - } - } - - /// - /// For verbose tracing and debugging. - /// - public override string ToString() - { - return string.Join("", - TransactionId, - (IsReadOnly ? " RO" : ""), - (IsAborted ? " Aborted" : ""), - $" R{{{string.Join(",", ReadSet.Select(kvp => $"{kvp.Key.ToShortString()}.{kvp.Value}"))}}}", - $" W{{{string.Join(",", WriteSet.Select(kvp => $"{kvp.Key.ToShortString()}.{TransactionId}#{kvp.Value}"))}}}", - $" D{{{string.Join(",", DependentTransactions)}}}" - ); - } - } - -} diff --git a/src/Orleans.Runtime/Transactions/TransactionsOptions.cs b/src/Orleans.Runtime/Transactions/TransactionsOptions.cs deleted file mode 100644 index 0221ed5d03..0000000000 --- a/src/Orleans.Runtime/Transactions/TransactionsOptions.cs +++ /dev/null @@ -1,44 +0,0 @@ -using System; - -namespace Orleans.Configuration -{ - public class TransactionsOptions - { - public TransactionsOptions() - { - UseDefaults(); - } - - /// - /// The number of new Transaction Ids allocated on every write to the log. - /// To avoid writing to log on every transaction start, transaction Ids are allocated in batches. - /// - public int TransactionIdAllocationBatchSize { get; set; } - public const int DefaultTransactionIdAllocationBatchSize = 50000; - - /// - /// A new batch of transaction Ids will be automatically allocated if the available ids drop below - /// this threshold. - /// - public int AvailableTransactionIdThreshold { get; set; } - public const int DefaultAvailableTransactionIdThreshold = 20000; - - /// - /// How long to preserve a transaction record in the TM memory after the transaction has completed. - /// This is used to answer queries about the outcome of the transaction. - /// - public TimeSpan TransactionRecordPreservationDuration { get; set; } - public static readonly TimeSpan DefaultTransactionRecordPreservationDuration = TimeSpan.FromMinutes(1); - - public TimeSpan MetricsWritePeriod { get; set; } - public static readonly TimeSpan DefaultMetricsWritePeriod = TimeSpan.FromSeconds(30); - - private void UseDefaults() - { - this.TransactionIdAllocationBatchSize = DefaultTransactionIdAllocationBatchSize; - this.AvailableTransactionIdThreshold = DefaultAvailableTransactionIdThreshold; - this.TransactionRecordPreservationDuration = DefaultTransactionRecordPreservationDuration; - this.MetricsWritePeriod = DefaultMetricsWritePeriod; - } - } -} diff --git a/src/Orleans.Transactions/Abstractions/CommitRecord.cs b/src/Orleans.Transactions/Abstractions/CommitRecord.cs deleted file mode 100644 index 935960dc88..0000000000 --- a/src/Orleans.Transactions/Abstractions/CommitRecord.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace Orleans.Transactions.Abstractions -{ - [Serializable] - public class CommitRecord - { - public CommitRecord() - { - Resources = new HashSet(); - } - - public long TransactionId { get; set; } - public long LSN { get; set; } - - public HashSet Resources { get; set; } - } -} diff --git a/src/Orleans.Transactions/DistributedTM/Protocol/TransactionParticipantExtensionExtensions.cs b/src/Orleans.Transactions/Abstractions/Extensions/TransactionParticipantExtensionExtensions.cs similarity index 99% rename from src/Orleans.Transactions/DistributedTM/Protocol/TransactionParticipantExtensionExtensions.cs rename to src/Orleans.Transactions/Abstractions/Extensions/TransactionParticipantExtensionExtensions.cs index ac5916c617..ed1971b886 100644 --- a/src/Orleans.Transactions/DistributedTM/Protocol/TransactionParticipantExtensionExtensions.cs +++ b/src/Orleans.Transactions/Abstractions/Extensions/TransactionParticipantExtensionExtensions.cs @@ -7,7 +7,7 @@ using Orleans.Runtime; using Orleans.Serialization; -namespace Orleans.Transactions.DistributedTM +namespace Orleans.Transactions.Abstractions.Extensions { public static class TransactionParticipantExtensionExtensions { diff --git a/src/Orleans.Transactions/Abstractions/TransactionalExtensionExtensions.cs b/src/Orleans.Transactions/Abstractions/Extensions/TransactionalExtensionExtensions.cs similarity index 97% rename from src/Orleans.Transactions/Abstractions/TransactionalExtensionExtensions.cs rename to src/Orleans.Transactions/Abstractions/Extensions/TransactionalExtensionExtensions.cs index 2bfeb72352..d58ce5bff7 100644 --- a/src/Orleans.Transactions/Abstractions/TransactionalExtensionExtensions.cs +++ b/src/Orleans.Transactions/Abstractions/Extensions/TransactionalExtensionExtensions.cs @@ -2,7 +2,7 @@ using System.Threading.Tasks; using Orleans.Concurrency; -namespace Orleans.Transactions.Abstractions +namespace Orleans.Transactions.Abstractions.Extensions { public static class TransactionalExtensionExtensions { diff --git a/src/Orleans.Transactions/Abstractions/INamedTransactionalStateStorageFactory.cs b/src/Orleans.Transactions/Abstractions/INamedTransactionalStateStorageFactory.cs index 09f4bf0a08..174f959896 100644 --- a/src/Orleans.Transactions/Abstractions/INamedTransactionalStateStorageFactory.cs +++ b/src/Orleans.Transactions/Abstractions/INamedTransactionalStateStorageFactory.cs @@ -1,6 +1,4 @@  -using System; - namespace Orleans.Transactions.Abstractions { /// diff --git a/src/Orleans.Transactions/Abstractions/ITransactionLogStorage.cs b/src/Orleans.Transactions/Abstractions/ITransactionLogStorage.cs deleted file mode 100644 index 825580cf78..0000000000 --- a/src/Orleans.Transactions/Abstractions/ITransactionLogStorage.cs +++ /dev/null @@ -1,60 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace Orleans.Transactions.Abstractions -{ - /// - /// This interface provides the abstraction for various durable transaction log storages. - /// - public interface ITransactionLogStorage - { - ///// - ///// Gets the first CommitRecord in the log. - ///// - ///// - ///// The CommitRecord with the lowest LSN in the log, or null if there is none. - ///// - Task GetFirstCommitRecord(); - - /// - /// Returns the CommitRecord with LSN following the LSN of record returned by the last - /// GetFirstcommitRecord() or GetNextCommitRecord() call. - /// - /// - /// The next CommitRecord, or null if there is none. - /// - Task GetNextCommitRecord(); - - /// - /// Returns the first available transaction id for new transactions. - /// - /// - /// This method helps to ensure that a given transaction id is never issued more than once. - /// - Task GetStartRecord(); - - /// - /// Update the start record with the value. - /// - /// Id of the transaction to update the start record with. - /// - Task UpdateStartRecord(long transactionId); - - /// - /// Append the given records to the log in order - /// - /// Commit Records - /// - /// If an exception is thrown it is possible that a prefix of the records are persisted - /// to the log. - /// - Task Append(IEnumerable commitRecords); - - /// - /// Truncates the transaction log from the start until the given LSN provided in the parameter. - /// - /// Last LSN until the log should be truncated, this value is inclusive. - /// - Task TruncateLog(long lsn); - } -} diff --git a/src/Orleans.Transactions/DistributedTM/Protocol/ITransactionParticipant.cs b/src/Orleans.Transactions/Abstractions/ITransactionParticipant.cs similarity index 99% rename from src/Orleans.Transactions/DistributedTM/Protocol/ITransactionParticipant.cs rename to src/Orleans.Transactions/Abstractions/ITransactionParticipant.cs index b74e8d2abe..73b8c31b23 100644 --- a/src/Orleans.Transactions/DistributedTM/Protocol/ITransactionParticipant.cs +++ b/src/Orleans.Transactions/Abstractions/ITransactionParticipant.cs @@ -1,10 +1,9 @@  -using Orleans.Concurrency; using System; using System.Collections.Generic; using System.Threading.Tasks; -namespace Orleans.Transactions +namespace Orleans.Transactions.Abstractions { /// /// Interface that allows a component to be a transaction participant. diff --git a/src/Orleans.Transactions/DistributedTM/Protocol/ITransactionParticipantExtension.cs b/src/Orleans.Transactions/Abstractions/ITransactionParticipantExtension.cs similarity index 97% rename from src/Orleans.Transactions/DistributedTM/Protocol/ITransactionParticipantExtension.cs rename to src/Orleans.Transactions/Abstractions/ITransactionParticipantExtension.cs index 56e5c48199..773e45fd10 100644 --- a/src/Orleans.Transactions/DistributedTM/Protocol/ITransactionParticipantExtension.cs +++ b/src/Orleans.Transactions/Abstractions/ITransactionParticipantExtension.cs @@ -4,7 +4,7 @@ using Orleans.Concurrency; using Orleans.Runtime; -namespace Orleans.Transactions.DistributedTM +namespace Orleans.Transactions.Abstractions { /// /// This is a grain extension interface that allows a grain to be a participant in a transaction. diff --git a/src/Orleans.Transactions/Abstractions/ITransactionalExtension.cs b/src/Orleans.Transactions/Abstractions/ITransactionalExtension.cs index 2be7bf00c8..323d49bb3a 100644 --- a/src/Orleans.Transactions/Abstractions/ITransactionalExtension.cs +++ b/src/Orleans.Transactions/Abstractions/ITransactionalExtension.cs @@ -2,7 +2,7 @@ using Orleans.Concurrency; using Orleans.Runtime; -namespace Orleans.Transactions +namespace Orleans.Transactions.Abstractions { /// /// This is a grain extension interface that allows a grain to take part in transaction orchestration. diff --git a/src/Orleans.Transactions/Abstractions/ITransactionalState.cs b/src/Orleans.Transactions/Abstractions/ITransactionalState.cs index 84d27c9fd2..f0f761f2f9 100644 --- a/src/Orleans.Transactions/Abstractions/ITransactionalState.cs +++ b/src/Orleans.Transactions/Abstractions/ITransactionalState.cs @@ -1,13 +1,41 @@  +using System; +using System.Threading.Tasks; + namespace Orleans.Transactions.Abstractions { /// - /// State that respects Orleans transaction semantics + /// State that respects Orleans transaction semantics, and allows + /// read/write locking /// - public interface ITransactionalState + /// The type of the state + public interface ITransactionalState where TState : class, new() { - TState State { get; } - void Save(); + /// + /// Performs a read operation and returns the result, without modifying the state. + /// + /// The type of the return value + /// A function that reads the state and returns the result. MUST NOT modify the state. + Task PerformRead(Func readFunction); + + /// + /// Performs an update operation, without returning any result. + /// + /// An action that updates the state. + Task PerformUpdate(Action updateAction); + + /// + /// Performs an update operation and returns the result. + /// + /// The type of the return value + /// A function that can read and update the state, and return a result + Task PerformUpdate(Func updateFunction); + + /// + /// An identifier of the current transaction. Can be used for tracing and debugging. + /// + string CurrentTransactionId { get; } } + } diff --git a/src/Orleans.Transactions/Abstractions/ITransactionalStateFactory.cs b/src/Orleans.Transactions/Abstractions/ITransactionalStateFactory.cs index bf7de1c85b..90b1db489e 100644 --- a/src/Orleans.Transactions/Abstractions/ITransactionalStateFactory.cs +++ b/src/Orleans.Transactions/Abstractions/ITransactionalStateFactory.cs @@ -3,6 +3,6 @@ namespace Orleans.Transactions.Abstractions { public interface ITransactionalStateFactory { - ITransactionalState Create(ITransactionalStateConfiguration config) where TState : class, new(); + ITransactionalState Create(Abstractions.ITransactionalStateConfiguration config) where TState : class, new(); } } diff --git a/src/Orleans.Transactions/Abstractions/ITransactionalStateStorage.cs b/src/Orleans.Transactions/Abstractions/ITransactionalStateStorage.cs index b19d543acb..c44877c53b 100644 --- a/src/Orleans.Transactions/Abstractions/ITransactionalStateStorage.cs +++ b/src/Orleans.Transactions/Abstractions/ITransactionalStateStorage.cs @@ -5,20 +5,29 @@ namespace Orleans.Transactions.Abstractions { + /// + /// Storage interface for transactional state + /// + /// the type of the state public interface ITransactionalStateStorage where TState : class, new() { Task> Load(); - Task Persist( - string expectedETag, - string metadata, - List> statesToPrepare); + Task Store( - Task Confirm( string expectedETag, string metadata, - string transactionIdToCommit); + + // a list of transactions to prepare. + List> statesToPrepare, + + // if non-null, commit all pending transaction up to and including this sequence number. + long? commitUpTo, + + // if non-null, abort all pending transactions with sequence numbers strictly larger than this one. + long? abortAfter + ); } [Serializable] @@ -26,18 +35,35 @@ Task Confirm( public class PendingTransactionState where TState : class, new() { - public PendingTransactionState(string transactionId, long sequenceId, TState state) - { - this.TransactionId = transactionId; - this.SequenceId = sequenceId; - this.State = state; - } + /// + /// Transactions are given dense local sequence numbers 1,2,3,4... + /// If a new transaction is prepared with the same sequence number as a + /// previously prepared transaction, it replaces it. + /// + public long SequenceId { get; set; } - public string TransactionId { get; } + /// + /// A globally unique identifier of the transaction. + /// + public string TransactionId { get; set; } - public long SequenceId { get; } + /// + /// The logical timestamp of the transaction. + /// Timestamps are guaranteed to be monotonically increasing. + /// + public DateTime TimeStamp { get; set; } - public TState State { get; } + /// + /// The transaction manager that knows about the status of this prepared transaction, + /// or null if this is the transaction manager. + /// Used during recovery to inquire about the fate of the transaction. + /// + public string TransactionManager { get; set; } + + /// + /// A snapshot of the state after this transaction executed + /// + public TState State { get; set; } } [Serializable] @@ -45,22 +71,34 @@ public PendingTransactionState(string transactionId, long sequenceId, TState sta public class TransactionalStorageLoadResponse where TState : class, new() { - public TransactionalStorageLoadResponse() : this(null, new TState(), null, Array.Empty>()){} + public TransactionalStorageLoadResponse() : this(null, new TState(), 0, null, Array.Empty>()) { } - public TransactionalStorageLoadResponse(string etag, TState committedState, string metadata, IReadOnlyList> pendingStates) + public TransactionalStorageLoadResponse(string etag, TState committedState, long committedSequenceId, string metadata, IReadOnlyList> pendingStates) { this.ETag = etag; this.CommittedState = committedState; + this.CommittedSequenceId = committedSequenceId; this.Metadata = metadata; this.PendingStates = pendingStates; } - public string ETag { get; } + public string ETag { get; set; } + + public TState CommittedState { get; set; } - public TState CommittedState { get; } + /// + /// The local sequence id of the last committed transaction, or zero if none + /// + public long CommittedSequenceId { get; set; } - public string Metadata { get; } + /// + /// Additional state maintained by the transaction algorithm, such as commit records + /// + public string Metadata { get; set; } - public IReadOnlyList> PendingStates { get; } + /// + /// List of pending states, ordered by sequence id + /// + public IReadOnlyList> PendingStates { get; set; } } } diff --git a/src/Orleans.Transactions/DistributedTM/Protocol/TransactionInfo.cs b/src/Orleans.Transactions/Abstractions/TransactionInfo.cs similarity index 97% rename from src/Orleans.Transactions/DistributedTM/Protocol/TransactionInfo.cs rename to src/Orleans.Transactions/Abstractions/TransactionInfo.cs index ae9a34435a..4921ddcf84 100644 --- a/src/Orleans.Transactions/DistributedTM/Protocol/TransactionInfo.cs +++ b/src/Orleans.Transactions/Abstractions/TransactionInfo.cs @@ -1,14 +1,11 @@  using System; using System.Collections.Generic; -using System.Diagnostics; -using System.Runtime.CompilerServices; -using Orleans.Concurrency; -using Orleans.Serialization; using System.Linq; using System.Collections.Concurrent; +using Orleans.Transactions.Abstractions.Extensions; -namespace Orleans.Transactions.DistributedTM +namespace Orleans.Transactions.Abstractions { [Serializable] public class TransactionInfo : ITransactionInfo diff --git a/src/Orleans.Transactions/Development/DevelopmentSiloBuilderExtensions.cs b/src/Orleans.Transactions/Development/DevelopmentSiloBuilderExtensions.cs deleted file mode 100644 index 7806f9d59c..0000000000 --- a/src/Orleans.Transactions/Development/DevelopmentSiloBuilderExtensions.cs +++ /dev/null @@ -1,26 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Orleans.Transactions.Development; - -namespace Orleans.Hosting.Development -{ - public static class DevelopmentSiloBuilderExtensions - { - /// - /// Configure cluster to use an in-memory transaction log. - /// For development and test purposes only - /// - public static ISiloHostBuilder UseInMemoryTransactionLog(this ISiloHostBuilder builder) - { - return builder.ConfigureServices(services => services.UseInMemoryTransactionLog()); - } - - /// - /// Configure cluster to use an in-memory transaction log. - /// For development and test purposes only - /// - public static IServiceCollection UseInMemoryTransactionLog(this IServiceCollection services) - { - return services.AddTransient(InMemoryTransactionLogStorage.Create); - } - } -} diff --git a/src/Orleans.Transactions/Development/InMemoryTransactionLogStorage.cs b/src/Orleans.Transactions/Development/InMemoryTransactionLogStorage.cs deleted file mode 100644 index 7eeb1e5511..0000000000 --- a/src/Orleans.Transactions/Development/InMemoryTransactionLogStorage.cs +++ /dev/null @@ -1,118 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Orleans.Transactions.Abstractions; - -namespace Orleans.Transactions.Development -{ - /// - /// In memory transaction log - for test and development purposes. - /// NOT FOR PRODUCTION USE. - /// - public class InMemoryTransactionLogStorage : ITransactionLogStorage - { - private static readonly Task NullCommitRecordTask = Task.FromResult(null); - - private long startRecordValue; - - private readonly List log; - - private int lastLogRecordIndex; - - private long nextLogSequenceNumber; - - public InMemoryTransactionLogStorage() - { - log = new List(); - - startRecordValue = 0; - lastLogRecordIndex = 0; - } - - public Task GetFirstCommitRecord() - { - if (log.Count == 0) - { - // - // Initialize LSN here, to be semantically correct with other providers. - // - - nextLogSequenceNumber = 1; - - return NullCommitRecordTask; - } - - // - // If the log has records, then this method should not get called for the in memory provider. - // - - throw new InvalidOperationException($"GetFirstCommitRecord was called while the log already has {log.Count} records."); - } - - public Task GetNextCommitRecord() - { - if (log.Count <= lastLogRecordIndex) - { - return NullCommitRecordTask; - } - - nextLogSequenceNumber++; - - return Task.FromResult(log[lastLogRecordIndex++]); - } - - public Task GetStartRecord() - { - startRecordValue = 50000; - - return Task.FromResult(startRecordValue); - } - - public Task UpdateStartRecord(long transactionId) - { - startRecordValue = transactionId; - - return Task.CompletedTask; - } - - public Task Append(IEnumerable commitRecords) - { - lock (this) - { - foreach (var commitRecord in commitRecords) - { - commitRecord.LSN = nextLogSequenceNumber++; - } - - log.AddRange(commitRecords); - } - - return Task.CompletedTask; - } - - public Task TruncateLog(long lsn) - { - lock (this) - { - var itemsToRemove = 0; - - for (itemsToRemove = 0; itemsToRemove < log.Count; itemsToRemove++) - { - if (log[itemsToRemove].LSN > lsn) - { - break; - } - } - - log.RemoveRange(0, itemsToRemove); - } - - return Task.CompletedTask; - } - - public static Factory> Create(IServiceProvider sp) - { - return () => Task.FromResult(new InMemoryTransactionLogStorage()); - } - } -} diff --git a/src/Orleans.Transactions/DistributedTM/Factories/INamedTransactionalStateStorageFactory.cs b/src/Orleans.Transactions/DistributedTM/Factories/INamedTransactionalStateStorageFactory.cs deleted file mode 100644 index 941ce7a5de..0000000000 --- a/src/Orleans.Transactions/DistributedTM/Factories/INamedTransactionalStateStorageFactory.cs +++ /dev/null @@ -1,18 +0,0 @@ - -namespace Orleans.Transactions.DistributedTM -{ - /// - /// Factory which creates an ITransactionalStateStorage by name. - /// - public interface INamedTransactionalStateStorageFactory - { - /// - /// Create an ITransactionalStateStorage by name. - /// - /// - /// Name of transaction state storage to create. - /// Name of transaction state. - /// ITransactionalStateStorage, null if not found. - ITransactionalStateStorage Create(string storageName, string stateName) where TState : class, new(); - } -} diff --git a/src/Orleans.Transactions/DistributedTM/Factories/ITransactionalStateFactory.cs b/src/Orleans.Transactions/DistributedTM/Factories/ITransactionalStateFactory.cs deleted file mode 100644 index 41697b59f5..0000000000 --- a/src/Orleans.Transactions/DistributedTM/Factories/ITransactionalStateFactory.cs +++ /dev/null @@ -1,8 +0,0 @@ - -namespace Orleans.Transactions.DistributedTM -{ - public interface ITransactionalStateFactory - { - ITransactionalState Create(Abstractions.ITransactionalStateConfiguration config) where TState : class, new(); - } -} diff --git a/src/Orleans.Transactions/DistributedTM/Factories/ITransactionalStateStorageFactory.cs b/src/Orleans.Transactions/DistributedTM/Factories/ITransactionalStateStorageFactory.cs deleted file mode 100644 index 0070e65c58..0000000000 --- a/src/Orleans.Transactions/DistributedTM/Factories/ITransactionalStateStorageFactory.cs +++ /dev/null @@ -1,10 +0,0 @@ - -using Orleans.Runtime; - -namespace Orleans.Transactions.DistributedTM -{ - public interface ITransactionalStateStorageFactory - { - ITransactionalStateStorage Create(string stateName, IGrainActivationContext context) where TState : class, new(); - } -} diff --git a/src/Orleans.Transactions/DistributedTM/Factories/NamedTransactionalStateStorageFactory.cs b/src/Orleans.Transactions/DistributedTM/Factories/NamedTransactionalStateStorageFactory.cs deleted file mode 100644 index 309b450558..0000000000 --- a/src/Orleans.Transactions/DistributedTM/Factories/NamedTransactionalStateStorageFactory.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System; -using Microsoft.Extensions.DependencyInjection; -using Orleans.Runtime; -using Orleans.Transactions.Abstractions; -using Orleans.Storage; -using Microsoft.Extensions.Logging; - -namespace Orleans.Transactions.DistributedTM -{ - public class NamedTransactionalStateStorageFactory : INamedTransactionalStateStorageFactory - { - private readonly IGrainActivationContext context; - private readonly ILoggerFactory loggerFactory; - - public NamedTransactionalStateStorageFactory(IGrainActivationContext context, ILoggerFactory loggerFactory) - { - this.context = context; - this.loggerFactory = loggerFactory; - } - - public ITransactionalStateStorage Create(string storageName, string stateName) - where TState : class, new() - { - // Try to get ITransactionalStateStorage from factory - ITransactionalStateStorageFactory factory = string.IsNullOrEmpty(storageName) - ? this.context.ActivationServices.GetService() - : this.context.ActivationServices.GetServiceByName(storageName); - if (factory != null) return factory.Create(stateName, context); - - // Else try to get storage provider and wrap it - IGrainStorage grainStorage = string.IsNullOrEmpty(storageName) - ? this.context.ActivationServices.GetService() - : this.context.ActivationServices.GetServiceByName(storageName); - if (grainStorage != null) return new TransactionalStateStorageProviderWrapper(grainStorage, stateName, context, this.loggerFactory); - throw (string.IsNullOrEmpty(storageName)) - ? new InvalidOperationException($"No default {nameof(ITransactionalStateStorageFactory)} nor {nameof(IGrainStorage)} was found while attempting to create transactional state storage.") - : new InvalidOperationException($"No {nameof(ITransactionalStateStorageFactory)} nor {nameof(IGrainStorage)} with the name {storageName} was found while attempting to create transactional state storage."); - } - } -} diff --git a/src/Orleans.Transactions/DistributedTM/Factories/TransactionalStateAttributeMapper.cs b/src/Orleans.Transactions/DistributedTM/Factories/TransactionalStateAttributeMapper.cs deleted file mode 100644 index 846138bdff..0000000000 --- a/src/Orleans.Transactions/DistributedTM/Factories/TransactionalStateAttributeMapper.cs +++ /dev/null @@ -1,27 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Orleans.Runtime; -using Orleans.Transactions.Abstractions; -using System.Reflection; - -namespace Orleans.Transactions.DistributedTM -{ - internal class TransactionalStateAttributeMapper : IAttributeToFactoryMapper - { - private static readonly MethodInfo create = typeof(ITransactionalStateFactory).GetMethod("Create"); - - public Factory GetFactory(ParameterInfo parameter, TransactionalStateAttribute attribute) - { - ITransactionalStateConfiguration config = attribute; - // use generic type args to define collection type. - MethodInfo genericCreate = create.MakeGenericMethod(parameter.ParameterType.GetGenericArguments()); - object[] args = new object[] { config }; - return context => Create(context, genericCreate, args); - } - - private object Create(IGrainActivationContext context, MethodInfo genericCreate, object[] args) - { - ITransactionalStateFactory factory = context.ActivationServices.GetRequiredService(); - return genericCreate.Invoke(factory, args); - } - } -} diff --git a/src/Orleans.Transactions/DistributedTM/Factories/TransactionalStateFactory.cs b/src/Orleans.Transactions/DistributedTM/Factories/TransactionalStateFactory.cs deleted file mode 100644 index de38edd30a..0000000000 --- a/src/Orleans.Transactions/DistributedTM/Factories/TransactionalStateFactory.cs +++ /dev/null @@ -1,23 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Orleans.Runtime; -using Orleans.Transactions.Abstractions; - -namespace Orleans.Transactions.DistributedTM -{ - public class TransactionalStateFactory : ITransactionalStateFactory - { - private IGrainActivationContext context; - - public TransactionalStateFactory(IGrainActivationContext context) - { - this.context = context; - } - - public ITransactionalState Create(ITransactionalStateConfiguration config) where TState : class, new() - { - TransactionalState transactionalState = ActivatorUtilities.CreateInstance>(this.context.ActivationServices, config, this.context); - transactionalState.Participate(context.ObservableLifecycle); - return transactionalState; - } - } -} diff --git a/src/Orleans.Transactions/DistributedTM/ITransactionalState.cs b/src/Orleans.Transactions/DistributedTM/ITransactionalState.cs deleted file mode 100644 index a2ee16e12e..0000000000 --- a/src/Orleans.Transactions/DistributedTM/ITransactionalState.cs +++ /dev/null @@ -1,41 +0,0 @@ - -using System; -using System.Threading.Tasks; - -namespace Orleans.Transactions.DistributedTM -{ - /// - /// State that respects Orleans transaction semantics, and allows - /// read/write locking - /// - /// The type of the state - public interface ITransactionalState - where TState : class, new() - { - /// - /// Performs a read operation and returns the result, without modifying the state. - /// - /// The type of the return value - /// A function that reads the state and returns the result. MUST NOT modify the state. - Task PerformRead(Func readFunction); - - /// - /// Performs an update operation, without returning any result. - /// - /// An action that updates the state. - Task PerformUpdate(Action updateAction); - - /// - /// Performs an update operation and returns the result. - /// - /// The type of the return value - /// A function that can read and update the state, and return a result - Task PerformUpdate(Func updateFunction); - - /// - /// An identifier of the current transaction. Can be used for tracing and debugging. - /// - string CurrentTransactionId { get; } - } - -} diff --git a/src/Orleans.Transactions/DistributedTM/ITransactionalStateStorage.cs b/src/Orleans.Transactions/DistributedTM/ITransactionalStateStorage.cs deleted file mode 100644 index f65c0c8dfd..0000000000 --- a/src/Orleans.Transactions/DistributedTM/ITransactionalStateStorage.cs +++ /dev/null @@ -1,105 +0,0 @@ -using Orleans.Concurrency; -using Orleans.Transactions; -using System; -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace Orleans.Transactions.DistributedTM -{ - /// - /// Storage interface for transactional state - /// - /// the type of the state - public interface ITransactionalStateStorage - where TState : class, new() - { - Task> Load(); - - Task Store( - - string expectedETag, - string metadata, - - // a list of transactions to prepare. - List> statesToPrepare, - - // if non-null, commit all pending transaction up to and including this sequence number. - long? commitUpTo, - - // if non-null, abort all pending transactions with sequence numbers strictly larger than this one. - long? abortAfter - ); - } - - [Serializable] - [Immutable] - public class PendingTransactionState - where TState : class, new() - { - /// - /// Transactions are given dense local sequence numbers 1,2,3,4... - /// If a new transaction is prepared with the same sequence number as a - /// previously prepared transaction, it replaces it. - /// - public long SequenceId { get; set; } - - /// - /// A globally unique identifier of the transaction. - /// - public string TransactionId { get; set; } - - /// - /// The logical timestamp of the transaction. - /// Timestamps are guaranteed to be monotonically increasing. - /// - public DateTime TimeStamp { get; set; } - - /// - /// The transaction manager that knows about the status of this prepared transaction, - /// or null if this is the transaction manager. - /// Used during recovery to inquire about the fate of the transaction. - /// - public string TransactionManager { get; set; } - - /// - /// A snapshot of the state after this transaction executed - /// - public TState State { get; set; } - } - - [Serializable] - [Immutable] - public class TransactionalStorageLoadResponse - where TState : class, new() - { - public TransactionalStorageLoadResponse() : this(null, new TState(), 0, null, Array.Empty>()) { } - - public TransactionalStorageLoadResponse(string etag, TState committedState, long committedSequenceId, string metadata, IReadOnlyList> pendingStates) - { - this.ETag = etag; - this.CommittedState = committedState; - this.CommittedSequenceId = committedSequenceId; - this.Metadata = metadata; - this.PendingStates = pendingStates; - } - - public string ETag { get; set; } - - public TState CommittedState { get; set; } - - /// - /// The local sequence id of the last committed transaction, or zero if none - /// - public long CommittedSequenceId { get; set; } - - /// - /// Additional state maintained by the transaction algorithm, such as commit records - /// - public string Metadata { get; set; } - - /// - /// List of pending states, ordered by sequence id - /// - public IReadOnlyList> PendingStates { get; set; } - } -} diff --git a/src/Orleans.Transactions/DistributedTM/Implementation/TransactionalState.cs b/src/Orleans.Transactions/DistributedTM/Implementation/TransactionalState.cs deleted file mode 100644 index 422222ac70..0000000000 --- a/src/Orleans.Transactions/DistributedTM/Implementation/TransactionalState.cs +++ /dev/null @@ -1,264 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Orleans; -using Orleans.Providers; -using Orleans.Runtime; -using Orleans.Transactions.Abstractions; -using Newtonsoft.Json; -using Orleans.Transactions; -using Orleans.Serialization; - -namespace Orleans.Transactions.DistributedTM -{ - /// - /// Stateful facet that respects Orleans transaction semantics - /// - public partial class TransactionalState : ITransactionalState, ITransactionParticipant, ILifecycleParticipant - where TState : class, new() - { - private readonly ITransactionalStateConfiguration config; - private readonly IGrainActivationContext context; - private readonly ITransactionDataCopier copier; - private readonly ITransactionAgent transactionAgent; - private readonly IProviderRuntime runtime; - private readonly ILoggerFactory loggerFactory; - - private ILogger logger; - - private ITransactionParticipant thisParticipant; - - // storage - private ITransactionalStateStorage storage; - - private string stateName; - private string StateName => stateName ?? (stateName = StoredName()); - - //private TimeSpan DebuggerAllowance = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromTicks(0); - private TimeSpan DebuggerAllowance = TimeSpan.FromTicks(0); - - // max time the TM will wait for prepare phase to complete - private TimeSpan PrepareTimeout => TimeSpan.FromSeconds(20) + DebuggerAllowance; - - // max time a group can occupy the lock - private TimeSpan LockTimeout => TimeSpan.FromSeconds(8) + DebuggerAllowance; - - // max time a transaction will wait for the lock to become available - private TimeSpan LockAcquireTimeout => TimeSpan.FromSeconds(10) + DebuggerAllowance; - - - private TimeSpan RemoteTransactionPingFrequency => TimeSpan.FromSeconds(60); - private static TimeSpan ConfirmationRetryDelay => TimeSpan.FromSeconds(30); - private static int ConfirmationRetryLimit => 3; - - - private TState stableState; - private long stableSequenceNumber; - - // the queues handling the various stages - private CommitQueue commitQueue; - private StorageBatch storageBatch; - - private Dictionary> _confirmationTasks; - private Dictionary> confirmationTasks - { - get - { - if (_confirmationTasks == null) - { - _confirmationTasks = new Dictionary>(); - } - return _confirmationTasks; - } - } - - private TransactionalStatus problemFlag; - private int failCounter; - - // moves transactions into and out of the lock stage - private BatchWorker lockWorker; - - // processes storage and post-storage queues, moves transactions out of the commit stage - private BatchWorker storageWorker; - - // processes confirmation tasks - private BatchWorker confirmationWorker; - - private DateTime clock; - - // collection tasks - private Dictionary unprocessedPreparedMessages; - private class PMessages - { - public int Count; - public TransactionalStatus Status; - } - - // clock read and merge - private DateTime MergeAndReadClock(DateTime other) - { - return clock = new DateTime(Math.Max(Math.Max(clock.Ticks + 1, other.Ticks + 1), DateTime.UtcNow.Ticks)); - } - private void MergeClock(DateTime other) - { - if (other > clock) - clock = other; - } - - public TransactionalState( - ITransactionalStateConfiguration transactionalStateConfiguration, - IGrainActivationContext context, - ITransactionDataCopier copier, - ITransactionAgent transactionAgent, - IProviderRuntime runtime, - ILoggerFactory loggerFactory, - ITypeResolver typeResolver, - IGrainFactory grainFactory) - { - this.config = transactionalStateConfiguration; - this.context = context; - this.copier = copier; - this.transactionAgent = transactionAgent; - this.runtime = runtime; - this.loggerFactory = loggerFactory; - - lockWorker = new BatchWorkerFromDelegate(LockWork); - storageWorker = new BatchWorkerFromDelegate(StorageWork); - confirmationWorker = new BatchWorkerFromDelegate(ConfirmationWork); - - if (MetaData.SerializerSettings == null) - { - MetaData.SerializerSettings = TransactionParticipantExtensionExtensions.GetJsonSerializerSettings(typeResolver, grainFactory); - } - } - - #region lifecycle - - public void Participate(IGrainLifecycle lifecycle) - { - lifecycle.Subscribe>(GrainLifecycleStage.SetupState, OnSetupState); - } - - private async Task OnSetupState(CancellationToken ct) - { - if (ct.IsCancellationRequested) return; - - var boundExtension = await this.runtime.BindExtension(() => new TransactionParticipantExtension()); - boundExtension.Item1.Register(this.config.StateName, this); - this.thisParticipant = boundExtension.Item2.AsTransactionParticipant(this.config.StateName); - - this.logger = loggerFactory.CreateLogger($"{context.GrainType.Name}.{this.config.StateName}.{this.thisParticipant.ToShortString()}"); - - var storageFactory = this.context.ActivationServices.GetRequiredService(); - this.storage = storageFactory.Create(this.config.StorageName, this.config.StateName); - - // recover state - await Restore(); - - storageWorker.Notify(); - } - - #endregion lifecycle - - private string StoredName() - { - return $"{this.context.GrainInstance.GetType().FullName}-{this.config.StateName}"; - } - - public bool Equals(ITransactionParticipant other) - { - return thisParticipant.Equals(other); - } - - public override string ToString() - { - return $"{this.context.GrainInstance}.{this.config.StateName}"; - } - - /// - /// called on activation, and when recovering from storage conflicts or other exceptions. - /// - private async Task Restore() - { - // start the load - var loadtask = this.storage.Load(); - - // abort active transactions, without waking up waiters just yet - AbortExecutingTransactions("due to restore"); - - // abort all entries in the commit queue - foreach (var entry in commitQueue.Elements) - { - NotifyOfAbort(entry, problemFlag); - } - commitQueue.Clear(); - - var loadresponse = await loadtask; - - storageBatch = new StorageBatch(loadresponse); - - stableState = loadresponse.CommittedState; - stableSequenceNumber = loadresponse.CommittedSequenceId; - - if (logger.IsEnabled(LogLevel.Debug)) - logger.Debug($"Load v{stableSequenceNumber} {loadresponse.PendingStates.Count}p {storageBatch.MetaData.CommitRecords.Count}c"); - - // ensure clock is consistent with loaded state - MergeClock(storageBatch.MetaData.TimeStamp); - - // resume prepared transactions (not TM) - foreach (var pr in loadresponse.PendingStates.OrderBy(ps => ps.TimeStamp)) - { - if (pr.SequenceId > stableSequenceNumber && pr.TransactionManager != null) - { - if (logger.IsEnabled(LogLevel.Debug)) - logger.Debug($"recover two-phase-commit {pr.TransactionId}"); - - var tm = (pr.TransactionManager == null) ? null : - (ITransactionParticipant) JsonConvert.DeserializeObject(pr.TransactionManager, MetaData.SerializerSettings); - - commitQueue.Add(new TransactionRecord() - { - Role = CommitRole.RemoteCommit, - TransactionId = Guid.Parse(pr.TransactionId), - Timestamp = pr.TimeStamp, - State = pr.State, - TransactionManager = tm, - PrepareIsPersisted = true, - LastSent = default(DateTime), - ConfirmationResponsePromise = null - }); - } - } - - // resume committed transactions (on TM) - foreach (var kvp in storageBatch.MetaData.CommitRecords) - { - if (logger.IsEnabled(LogLevel.Debug)) - logger.Debug($"recover commit confirmation {kvp.Key}"); - - confirmationTasks.Add(kvp.Key, new TransactionRecord() - { - Role = CommitRole.LocalCommit, - TransactionId = kvp.Key, - Timestamp = kvp.Value.Timestamp, - WriteParticipants = kvp.Value.WriteParticipants - }); - } - - // clear the problem flag - problemFlag = TransactionalStatus.Ok; - - // check for work - confirmationWorker.Notify(); - storageWorker.Notify(); - lockWorker.Notify(); - } - - - } -} diff --git a/src/Orleans.Transactions/DistributedTM/Implementation/TransactionalStateStorageProviderWrapper.cs b/src/Orleans.Transactions/DistributedTM/Implementation/TransactionalStateStorageProviderWrapper.cs deleted file mode 100644 index 5845c26726..0000000000 --- a/src/Orleans.Transactions/DistributedTM/Implementation/TransactionalStateStorageProviderWrapper.cs +++ /dev/null @@ -1,115 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Collections.Concurrent; -using System.Linq; -using System.Threading.Tasks; -using Orleans.Core; -using Orleans.Runtime; -using Orleans.Storage; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.DependencyInjection; -using Orleans.Utilities; -using System.Diagnostics; - -namespace Orleans.Transactions.DistributedTM -{ - internal class TransactionalStateStorageProviderWrapper : ITransactionalStateStorage - where TState : class, new() - { - private readonly IGrainStorage grainStorage; - private readonly IGrainActivationContext context; - private readonly ILoggerFactory loggerFactory; - private readonly string stateName; - - private IStorage> stateStorage; - private IStorage> StateStorage => stateStorage ?? (stateStorage = GetStateStorage()); - - - public TransactionalStateStorageProviderWrapper(IGrainStorage grainStorage, string stateName, IGrainActivationContext context, ILoggerFactory loggerFactory) - { - this.grainStorage = grainStorage; - this.context = context; - this.loggerFactory = loggerFactory; - this.stateName = stateName; - } - - public async Task> Load() - { - await this.StateStorage.ReadStateAsync(); - return new TransactionalStorageLoadResponse(stateStorage.Etag, stateStorage.State.CommittedState, stateStorage.State.CommittedSequenceId, stateStorage.State.Metadata, stateStorage.State.PendingStates); - } - - public async Task Store(string expectedETag, string metadata, List> statesToPrepare, long? commitUpTo, long? abortAfter) - { - if (this.StateStorage.Etag != expectedETag) - throw new ArgumentException(nameof(expectedETag), "Etag does not match"); - stateStorage.State.Metadata = metadata; - - var pendinglist = stateStorage.State.PendingStates; - - // abort - if (abortAfter.HasValue && pendinglist.Count != 0) - { - var pos = pendinglist.FindIndex(t => t.SequenceId > abortAfter.Value); - if (pos != -1) - { - pendinglist.RemoveRange(pos, pendinglist.Count - pos); - } - } - - // prepare - if (statesToPrepare?.Count > 0) - { - if (pendinglist.Count != 0) - { - // remove prepare records that are being overwritten - while (pendinglist[pendinglist.Count - 1].SequenceId >= statesToPrepare[0].SequenceId) - { - pendinglist.RemoveAt(pendinglist.Count - 1); - } - } - pendinglist.AddRange(statesToPrepare); - } - - // commit - if (commitUpTo.HasValue && commitUpTo.Value > stateStorage.State.CommittedSequenceId) - { - var pos = pendinglist.FindIndex(t => t.SequenceId == commitUpTo.Value); - if (pos != -1) - { - var committedState = pendinglist[pos]; - stateStorage.State.CommittedSequenceId = committedState.SequenceId; - stateStorage.State.CommittedState = committedState.State; - pendinglist.RemoveRange(0, pos + 1); - } - else - { - throw new InvalidOperationException($"Transactional state corrupted. Missing prepare record (SequenceId={commitUpTo.Value}) for committed transaction."); - } - } - - await stateStorage.WriteStateAsync(); - return stateStorage.Etag; - } - - private IStorage> GetStateStorage() - { - string formattedTypeName = RuntimeTypeNameFormatter.Format(this.context.GrainInstance.GetType()); - string fullStateName = $"{formattedTypeName}-{this.stateName}"; - return new StateStorageBridge>(fullStateName, this.context.GrainInstance.GrainReference, grainStorage, this.loggerFactory); - } - } - - [Serializable] - public class TransactionalStateRecord - where TState : class, new() - { - public TState CommittedState { get; set; } = new TState(); - - public long CommittedSequenceId { get; set; } - - public string Metadata { get; set; } - - public List> PendingStates { get; set; } = new List>(); - } -} \ No newline at end of file diff --git a/src/Orleans.Transactions/DistributedTM/Implementation/TransactionAgent.cs b/src/Orleans.Transactions/DistributedTM/TransactionAgent.cs similarity index 94% rename from src/Orleans.Transactions/DistributedTM/Implementation/TransactionAgent.cs rename to src/Orleans.Transactions/DistributedTM/TransactionAgent.cs index d0615554d2..161a2eeaef 100644 --- a/src/Orleans.Transactions/DistributedTM/Implementation/TransactionAgent.cs +++ b/src/Orleans.Transactions/DistributedTM/TransactionAgent.cs @@ -1,16 +1,14 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Threading.Tasks; using Orleans.Runtime; using Orleans.Concurrency; -using System.Threading; using System.Linq; -using Orleans.Runtime.Configuration; using Microsoft.Extensions.Logging; +using Orleans.Transactions.Abstractions; -namespace Orleans.Transactions.DistributedTM +namespace Orleans.Transactions { [Reentrant] internal class TransactionAgent : ITransactionAgent @@ -221,18 +219,5 @@ public void Abort(ITransactionInfo info, OrleansTransactionAbortedException reas } } } - - #region vestigial - - // these methods are not used by the distributed-TM implementation - - public long ReadOnlyTransactionId => throw new NotImplementedException(); - - public bool IsAborted(long transactionId) - { - throw new NotImplementedException(); - } - - #endregion } } diff --git a/src/Orleans.Transactions/DistributedTM/Implementation/TransactionRecord.cs b/src/Orleans.Transactions/DistributedTM/TransactionRecord.cs similarity index 97% rename from src/Orleans.Transactions/DistributedTM/Implementation/TransactionRecord.cs rename to src/Orleans.Transactions/DistributedTM/TransactionRecord.cs index 628ec29e31..169fa9c95f 100644 --- a/src/Orleans.Transactions/DistributedTM/Implementation/TransactionRecord.cs +++ b/src/Orleans.Transactions/DistributedTM/TransactionRecord.cs @@ -1,14 +1,10 @@ -using Orleans.Transactions; +using Orleans.Transactions.Abstractions; using System; using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Threading.Tasks; -namespace Orleans.Transactions.DistributedTM +namespace Orleans.Transactions { - - /// /// Each participant plays a particular role in the commit protocol /// diff --git a/src/Orleans.Transactions/DistributedTM/Implementation/TransactionalStatusExtensions.cs b/src/Orleans.Transactions/DistributedTM/TransactionalStatusExtensions.cs similarity index 91% rename from src/Orleans.Transactions/DistributedTM/Implementation/TransactionalStatusExtensions.cs rename to src/Orleans.Transactions/DistributedTM/TransactionalStatusExtensions.cs index 0e2a0308cf..3d48a6d982 100644 --- a/src/Orleans.Transactions/DistributedTM/Implementation/TransactionalStatusExtensions.cs +++ b/src/Orleans.Transactions/DistributedTM/TransactionalStatusExtensions.cs @@ -1,9 +1,7 @@ -using System; -using System.Collections.Generic; -using System.Runtime.Serialization; -using System.Text; + +using Orleans.Transactions.Abstractions; -namespace Orleans.Transactions.DistributedTM +namespace Orleans.Transactions { internal static class TransactionalStatusExtensions { diff --git a/src/Orleans.Transactions/Hosting/SiloBuilderExtensions.cs b/src/Orleans.Transactions/Hosting/SiloBuilderExtensions.cs index 99100ad70f..5efdad9fbd 100644 --- a/src/Orleans.Transactions/Hosting/SiloBuilderExtensions.cs +++ b/src/Orleans.Transactions/Hosting/SiloBuilderExtensions.cs @@ -10,65 +10,6 @@ namespace Orleans.Hosting { public static class SiloBuilderExtensions { - /// - /// Configure cluster to use an in-cluster transaction manager using a configure action. - /// - public static ISiloHostBuilder UseInClusterTransactionManager(this ISiloHostBuilder builder, Action configureOptions) - { - return builder.ConfigureServices(services => services.UseInClusterTransactionManager(configureOptions)); - } - - /// - /// Configure cluster to use an in-cluster transaction manager using a configuration builder. - /// - public static ISiloHostBuilder UseInClusterTransactionManager(this ISiloHostBuilder builder, Action> configureOptions = null) - { - return builder.ConfigureServices(services => services.UseInClusterTransactionManager(configureOptions)); - } - - /// - /// Configure cluster services to use an in-cluster transaction manager using a configure action. - /// - public static IServiceCollection UseInClusterTransactionManager(this IServiceCollection services, Action configureOptions) - { - return services.UseInClusterTransactionManager(ob => ob.Configure(configureOptions)); - } - - /// - /// Configure cluster services to use an in-cluster transaction manager using a configuration builder. - /// - public static IServiceCollection UseInClusterTransactionManager(this IServiceCollection services, - Action> configureOptions = null) - { - configureOptions?.Invoke(services.AddOptions()); - return services.AddTransient() - .AddTransient() - .AddSingleton() - .AddSingleton(sp => sp.GetRequiredService().CreateTransactionManagerService()); - } - - /// - /// Configure cluster to support the use of transactional state. - /// - public static ISiloHostBuilder UseTransactionalState(this ISiloHostBuilder builder) - { - return builder.ConfigureServices(services => services.UseTransactionalState()); - } - - /// - /// Configure cluster to support the use of transactional state. - /// - public static IServiceCollection UseTransactionalState(this IServiceCollection services) - { - services.TryAddSingleton(typeof(ITransactionDataCopier<>), typeof(DefaultTransactionDataCopier<>)); - services.AddSingleton, TransactionalStateAttributeMapper>(); - services.TryAddTransient(); - services.TryAddTransient(); - services.AddTransient(typeof(ITransactionalState<>), typeof(TransactionalState<>)); - return services; - } - - /// /// Configure cluster to use the distributed TM algorithm /// @@ -82,11 +23,11 @@ public static ISiloHostBuilder UseDistributedTM(this ISiloHostBuilder builder) /// public static IServiceCollection UseDistributedTM(this IServiceCollection services) { - services.TryAddSingleton(); + services.AddSingleton(); services.TryAddSingleton(typeof(ITransactionDataCopier<>), typeof(DefaultTransactionDataCopier<>)); - services.AddSingleton, Orleans.Transactions.DistributedTM.TransactionalStateAttributeMapper>(); - services.TryAddTransient(); - services.TryAddTransient(); + services.AddSingleton, TransactionalStateAttributeMapper>(); + services.TryAddTransient(); + services.TryAddTransient(); services.AddTransient(typeof(ITransactionalState<>), typeof(TransactionalState<>)); return services; } diff --git a/src/Orleans.Transactions/InClusterTM/ActiveTransactionsTracker.cs b/src/Orleans.Transactions/InClusterTM/ActiveTransactionsTracker.cs deleted file mode 100644 index c9045ae760..0000000000 --- a/src/Orleans.Transactions/InClusterTM/ActiveTransactionsTracker.cs +++ /dev/null @@ -1,140 +0,0 @@ -using System; -using System.Threading; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Logging; -using Orleans.Configuration; - -namespace Orleans.Transactions -{ - internal class ActiveTransactionsTracker : IDisposable - { - private readonly TransactionsOptions options; - private readonly TransactionLog transactionLog; - private readonly ILogger logger; - private readonly object lockObj; - private readonly Thread allocationThread; - private readonly AutoResetEvent allocationEvent; - - private long smallestActiveTransactionId; - private long highestActiveTransactionId; - - private long maxAllocatedTransactionId; - private volatile bool disposed; - - public ActiveTransactionsTracker(IOptions configOption, TransactionLog transactionLog, ILoggerFactory loggerFactory) - { - this.options = configOption.Value; - this.transactionLog = transactionLog; - this.logger = loggerFactory.CreateLogger(nameof(ActiveTransactionsTracker)); - lockObj = new object(); - - allocationEvent = new AutoResetEvent(true); - allocationThread = new Thread(AllocateTransactionId) - { - IsBackground = true, - Name = nameof(ActiveTransactionsTracker) - }; - } - - public void Start(long initialTransactionId) - { - smallestActiveTransactionId = initialTransactionId + 1; - highestActiveTransactionId = initialTransactionId; - maxAllocatedTransactionId = initialTransactionId; - - allocationThread.Start(); - } - - public long GetNewTransactionId() - { - var id = Interlocked.Increment(ref highestActiveTransactionId); - - if (maxAllocatedTransactionId - highestActiveTransactionId <= options.AvailableTransactionIdThreshold) - { - // Signal the allocation thread to allocate more Ids - allocationEvent.Set(); - } - - while (id > maxAllocatedTransactionId) - { - // Wait until the allocation thread catches up before returning. - // This should never happen if we are pre-allocating fast enough. - allocationEvent.Set(); - lock (lockObj) - { - } - } - - return id; - } - - public long GetSmallestActiveTransactionId() - { - // NOTE: this result is not strictly correct if there are NO active transactions - // but for all purposes in which this is used it is still valid. - // TODO: consider renaming this or handling the no active transactions case. - return Interlocked.Read(ref smallestActiveTransactionId); - } - - public long GetHighestActiveTransactionId() - { - // NOTE: this result is not strictly correct if there are NO active transactions - // but for all purposes in which this is used it is still valid. - // TODO: consider renaming this or handling the no active transactions case. - lock (lockObj) - { - return Math.Min(highestActiveTransactionId, maxAllocatedTransactionId); - } - } - - - public void PopSmallestActiveTransactionId() - { - Interlocked.Increment(ref smallestActiveTransactionId); - } - - private void AllocateTransactionId(object args) - { - while (!this.disposed) - { - try - { - allocationEvent.WaitOne(); - if (this.disposed) return; - - lock (lockObj) - { - if (maxAllocatedTransactionId - highestActiveTransactionId <= options.AvailableTransactionIdThreshold) - { - var batchSize = options.TransactionIdAllocationBatchSize; - transactionLog.UpdateStartRecord(maxAllocatedTransactionId + batchSize).GetAwaiter().GetResult(); - - maxAllocatedTransactionId += batchSize; - } - } - } - catch (ThreadAbortException) - { - throw; - } - catch (Exception exception) - { - this.logger.Warn( - OrleansTransactionsErrorCode.Transactions_IdAllocationFailed, - "Ignoring exception in " + nameof(this.AllocateTransactionId), - exception); - } - } - } - - public void Dispose() - { - if (!this.disposed) - { - this.disposed = true; - this.allocationEvent.Set(); - this.allocationEvent.Dispose(); - } - } - } -} diff --git a/src/Orleans.Transactions/InClusterTM/Transaction.cs b/src/Orleans.Transactions/InClusterTM/Transaction.cs deleted file mode 100644 index 161e211635..0000000000 --- a/src/Orleans.Transactions/InClusterTM/Transaction.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace Orleans.Transactions -{ - internal enum TransactionState - { - Started = 0, - PendingDependency, - Validated, - Committed, - Checkpointed, - Aborted, - Unknown - }; - - internal class Transaction - { - public long TransactionId { get; set; } - - public TransactionState State { get; set; } - - // Sequence of the transaction in the log. - // LSN is valid only if State is Committed. - public long LSN { get; set; } - - // Time to abort the transaction if it was not completed. - public long ExpirationTime { get; set; } - - public TransactionInfo Info { get; set; } - - // Transactions waiting on the result of this transaction. - public HashSet WaitingTransactions { get; private set; } - - // Number of transactions this transaction is waiting for an outcome of. - public int PendingCount { get; set; } - - public long HighestActiveTransactionIdAtCheckpoint { get; set; } - - // Time the transaction was completed (i.e. either committed or aborted) - public DateTime CompletionTimeUtc { get; set; } - - public OrleansTransactionAbortedException AbortingException { get; set; } - - public Transaction(long transactionId) - { - TransactionId = transactionId; - WaitingTransactions = new HashSet(); - PendingCount = 0; - LSN = 0; - HighestActiveTransactionIdAtCheckpoint = 0; - } - } -} diff --git a/src/Orleans.Transactions/InClusterTM/TransactionLog.cs b/src/Orleans.Transactions/InClusterTM/TransactionLog.cs deleted file mode 100644 index c37bba8c96..0000000000 --- a/src/Orleans.Transactions/InClusterTM/TransactionLog.cs +++ /dev/null @@ -1,150 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Orleans.Transactions.Abstractions; - -namespace Orleans.Transactions -{ - /// - /// This class represents the durable Transaction Log. - /// Orleans Transaction Log has 2 types of entries: - /// 1- StartRecord: There is exactly 1 entry of this type in the log. It logs the - /// number of started transactions so far. - /// 2- CommitRecord: An entry is appended to the log when a transaction commits. - /// - /// Usage: - /// The log can be in 2 modes. - /// 1- When first initialized the log is in Recovery Mode. In this mode the client calls - /// GetFirstCommitRecord followed by a sequence of GetNextCommitRecord() calls to - /// retrieve the log entries. Finally the client calls EndRecovery(). - /// 2- The log becomes in Append Mode after the call to EndRecovery(). - /// This is the normal mode of operation in which the caller can modify the log by - /// appending entries and removing entries that are no longer necessary. - /// - public class TransactionLog - { - private readonly Factory> storageFactory; - private ITransactionLogStorage transactionLogStorage; - - private TransactionLogOperatingMode currentLogMode; - - private long lastStartRecordValue; - - public TransactionLog(Factory> storageFactory) - { - this.storageFactory = storageFactory; - - currentLogMode = TransactionLogOperatingMode.Uninitialized; - } - - /// - /// Initialize the log (in Recovery Mode). This method must be called before any other method - /// is called on the log. - /// - /// - public async Task Initialize() - { - currentLogMode = TransactionLogOperatingMode.RecoveryMode; - - transactionLogStorage = await storageFactory(); - } - - /// - /// Gets the first CommitRecord in the log. - /// - /// - /// The CommitRecord with the lowest LSN in the log, or null if there is none. - /// - public Task GetFirstCommitRecord() - { - ThrowIfNotInMode(TransactionLogOperatingMode.RecoveryMode); - - return transactionLogStorage.GetFirstCommitRecord(); - } - - /// - /// Returns the CommitRecord with LSN following the LSN of record returned by the last - /// GetFirstcommitRecord() or GetNextCommitRecord() call. - /// - /// - /// The next CommitRecord, or null if there is none. - /// - public Task GetNextCommitRecord() - { - ThrowIfNotInMode(TransactionLogOperatingMode.RecoveryMode); - - return transactionLogStorage.GetNextCommitRecord(); - } - - /// - /// Exit recovery and enter Append Mode. - /// - public Task EndRecovery() - { - ThrowIfNotInMode(TransactionLogOperatingMode.RecoveryMode); - - currentLogMode = TransactionLogOperatingMode.AppendMode; - - return Task.CompletedTask; - } - - public async Task GetStartRecord() - { - ThrowIfNotInMode(TransactionLogOperatingMode.AppendMode); - - lastStartRecordValue = await transactionLogStorage.GetStartRecord(); - - return lastStartRecordValue; - } - - public Task UpdateStartRecord(long transactionId) - { - ThrowIfNotInMode(TransactionLogOperatingMode.AppendMode); - - if (transactionId > lastStartRecordValue) - { - return transactionLogStorage.UpdateStartRecord(transactionId); - } - - throw new InvalidOperationException($"UpdateStartRecord was called in an invalid state. TransactionId: {transactionId}, lastStartRecordValue: {lastStartRecordValue}."); - } - - /// - /// Append the given records to the log in order - /// - /// Commit Records - /// - /// If an exception is thrown it is possible that a prefix of the records are persisted - /// to the log. - /// - public Task Append(IEnumerable commitRecords) - { - ThrowIfNotInMode(TransactionLogOperatingMode.AppendMode); - - - return transactionLogStorage.Append(commitRecords); - } - - public Task TruncateLog(long lsn) - { - ThrowIfNotInMode(TransactionLogOperatingMode.AppendMode); - - return transactionLogStorage.TruncateLog(lsn); - } - - private void ThrowIfNotInMode(TransactionLogOperatingMode expectedLogMode) - { - if (currentLogMode != expectedLogMode) - { - new InvalidOperationException($"Log has to be in {expectedLogMode} mode, but it is in {currentLogMode} mode."); - } - } - - private enum TransactionLogOperatingMode - { - Uninitialized = 0, - RecoveryMode, - AppendMode - }; - } -} diff --git a/src/Orleans.Transactions/InClusterTM/TransactionManager.cs b/src/Orleans.Transactions/InClusterTM/TransactionManager.cs deleted file mode 100644 index bf7c28c9e0..0000000000 --- a/src/Orleans.Transactions/InClusterTM/TransactionManager.cs +++ /dev/null @@ -1,709 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using System.Collections.Concurrent; -using System.Threading; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Logging; -using Orleans.Configuration; -using Orleans.Runtime; -using Orleans.Transactions.Abstractions; - -namespace Orleans.Transactions -{ - public class TransactionManager : ITransactionManager - { - private const int MaxCheckpointBatchSize = 200; - private static readonly TimeSpan DefaultLogMaintenanceInterval = TimeSpan.FromSeconds(1); - - private readonly TransactionsOptions options; - private readonly TransactionLog transactionLog; - private readonly ActiveTransactionsTracker activeTransactionsTracker; - private readonly TimeSpan logMaintenanceInterval; - - // Index of transactions by transactionId. - private readonly ConcurrentDictionary transactionsTable; - - // Transactions that have validated dependencies. - private readonly ConcurrentQueue dependencyQueue; - - // Transactions that are waiting on group commit. - private readonly ConcurrentQueue> groupCommitQueue; - - // Queue of committed transactions in commit order. - private readonly ConcurrentQueue checkpointQueue; - private readonly Queue checkpointRetryQueue = new Queue(MaxCheckpointBatchSize); - - private readonly InterlockedExchangeLock dependencyLock; - private readonly InterlockedExchangeLock commitLock; - private readonly InterlockedExchangeLock checkpointLock; - private readonly Dictionary resources; - private readonly List transactions; - - private long checkpointedLSN; - - protected readonly ILogger logger; - private bool IsRunning; - private Task transactionLogMaintenanceTask; - private TransactionManagerMetrics metrics; - public TransactionManager( - TransactionLog transactionLog, - IOptions configOption, - ILoggerFactory loggerFactory, - ITelemetryProducer telemetryProducer, - IOptions statisticsOptions, - TimeSpan? logMaintenanceInterval = null) - { - this.transactionLog = transactionLog; - this.options = configOption.Value; - this.logger = loggerFactory.CreateLogger(); - this.logMaintenanceInterval = logMaintenanceInterval ?? DefaultLogMaintenanceInterval; - - activeTransactionsTracker = new ActiveTransactionsTracker(configOption, this.transactionLog, loggerFactory); - - transactionsTable = new ConcurrentDictionary(2, 1000000); - - dependencyQueue = new ConcurrentQueue(); - groupCommitQueue = new ConcurrentQueue>(); - checkpointQueue = new ConcurrentQueue(); - - this.dependencyLock = new InterlockedExchangeLock(); - this.commitLock = new InterlockedExchangeLock(); - this.checkpointLock = new InterlockedExchangeLock(); - this.resources = new Dictionary(); - this.transactions = new List(); - this.metrics = - new TransactionManagerMetrics(telemetryProducer, configOption.Value.MetricsWritePeriod); - this.checkpointedLSN = 0; - this.IsRunning = false; - } - - #region ITransactionManager - - public async Task StartAsync() - { - await transactionLog.Initialize(); - CommitRecord record = await transactionLog.GetFirstCommitRecord(); - long prevLSN = 0; - while (record != null) - { - Transaction tx = new Transaction(record.TransactionId) - { - State = TransactionState.Committed, - LSN = record.LSN, - Info = new TransactionInfo(record.TransactionId) - }; - - if (prevLSN == 0) - { - checkpointedLSN = record.LSN - 1; - } - prevLSN = record.LSN; - - foreach (var resource in record.Resources) - { - tx.Info.WriteSet.Add(resource, 1); - } - - transactionsTable[record.TransactionId] = tx; - checkpointQueue.Enqueue(tx); - this.SignalCheckpointEnqueued(); - - record = await transactionLog.GetNextCommitRecord(); - } - - await transactionLog.EndRecovery(); - var maxAllocatedTransactionId = await transactionLog.GetStartRecord(); - - activeTransactionsTracker.Start(maxAllocatedTransactionId); - - this.BeginDependencyCompletionLoop(); - this.BeginGroupCommitLoop(); - this.BeginCheckpointLoop(); - - this.IsRunning = true; - this.transactionLogMaintenanceTask = MaintainTransactionLog(); - this.transactionLogMaintenanceTask.Ignore(); // protect agains unhandled exception in unexpected cases. - } - - public async Task StopAsync() - { - this.IsRunning = false; - if (this.transactionLogMaintenanceTask != null) - { - await this.transactionLogMaintenanceTask; - } - this.activeTransactionsTracker.Dispose(); - } - - public long StartTransaction(TimeSpan timeout) - { - this.metrics.StartTransactionRequestCounter++; - var transactionId = activeTransactionsTracker.GetNewTransactionId(); - Transaction tx = new Transaction(transactionId) - { - State = TransactionState.Started, - ExpirationTime = DateTime.UtcNow.Ticks + timeout.Ticks, - }; - - transactionsTable[transactionId] = tx; - - return tx.TransactionId; - } - - public void AbortTransaction(long transactionId, OrleansTransactionAbortedException reason) - { - this.metrics.AbortTransactionRequestCounter++; - this.metrics.AbortedTransactionCounter++; - if(this.logger.IsEnabled(LogLevel.Debug)) this.logger.LogDebug($"Abort transaction {transactionId} due to reason {reason}"); - if (transactionsTable.TryGetValue(transactionId, out Transaction tx)) - { - bool justAborted = false; - - lock (tx) - { - if (tx.State == TransactionState.Started || - tx.State == TransactionState.PendingDependency) - { - tx.State = TransactionState.Aborted; - justAborted = true; - } - } - - if (justAborted) - { - foreach (var waiting in tx.WaitingTransactions) - { - var cascading = new OrleansCascadingAbortException(waiting.Info.TransactionId.ToString(), tx.TransactionId.ToString()); - AbortTransaction(waiting.Info.TransactionId, cascading); - } - - tx.CompletionTimeUtc = DateTime.UtcNow; - tx.AbortingException = reason; - } - } - } - - public void CommitTransaction(TransactionInfo transactionInfo) - { - this.metrics.CommitTransactionRequestCounter++; - if (transactionsTable.TryGetValue(transactionInfo.TransactionId, out Transaction tx)) - { - bool abort = false; - long cascadingDependentId = 0; - - bool pending = false; - bool signal = false; - lock (tx) - { - if (tx.State == TransactionState.Started) - { - tx.Info = transactionInfo; - - // Check our dependent transactions. - // - If all dependent transactions committed, put transaction in validating queue - // (dependencyQueue) - // - If at least one dependent transaction aborted, abort. - // - If at least one dependent transaction is still pending, put in pending queue - // (dependentTx.WaitingTransactions) - foreach (var dependentId in tx.Info.DependentTransactions) - { - // Transaction does not exist in the transaction table; - // therefore, presumed abort. - if (!transactionsTable.TryGetValue(dependentId, out Transaction dependentTx)) - { - abort = true; - if(this.logger.IsEnabled(LogLevel.Debug)) this.logger.LogDebug($"Will abort transaction {transactionInfo.TransactionId} because it doesn't exist in the transaction table"); - cascadingDependentId = dependentId; - this.metrics.AbortedTransactionDueToMissingInfoInTransactionTableCounter++; - break; - } - - // NOTE: our deadlock prevention mechanism ensures that we are acquiring - // the locks in proper order and there is no risk of deadlock. - lock (dependentTx) - { - // Dependent transactions has aborted; therefore, abort. - if (dependentTx.State == TransactionState.Aborted) - { - abort = true; - if(this.logger.IsEnabled(LogLevel.Debug)) this.logger.LogDebug($"Will abort transaction {transactionInfo.TransactionId} because one of its dependent transaction {dependentTx.TransactionId} has aborted"); - cascadingDependentId = dependentId; - this.metrics.AbortedTransactionDueToDependencyCounter++; - break; - } - - // Dependent transaction is still executing or has a pending dependency. - if (dependentTx.State == TransactionState.Started || - dependentTx.State == TransactionState.PendingDependency) - { - pending = true; - dependentTx.WaitingTransactions.Add(tx); - tx.PendingCount++; - } - } - } - - if (abort) - { - AbortTransaction(transactionInfo.TransactionId, new OrleansCascadingAbortException(transactionInfo.TransactionId.ToString(), cascadingDependentId.ToString())); - } - else if (pending) - { - tx.State = TransactionState.PendingDependency; - } - else - { - tx.State = TransactionState.Validated; - dependencyQueue.Enqueue(tx); - signal = true; - } - } - - } - if (signal) - { - this.SignalDependencyEnqueued(); - } - } - else - { - // Don't have a record of the transaction any more so presumably it's aborted. - throw new OrleansTransactionAbortedException(transactionInfo.TransactionId.ToString(), "Transaction presumed to be aborted"); - } - } - - public TransactionStatus GetTransactionStatus(long transactionId, out OrleansTransactionAbortedException abortingException) - { - abortingException = null; - if (transactionsTable.TryGetValue(transactionId, out Transaction tx)) - { - if (tx.State == TransactionState.Aborted) - { - lock (tx) - { - abortingException = tx.AbortingException; - } - return TransactionStatus.Aborted; - } - else if (tx.State == TransactionState.Committed || tx.State == TransactionState.Checkpointed) - { - return TransactionStatus.Committed; - } - else - { - return TransactionStatus.InProgress; - } - } - return TransactionStatus.Unknown; - } - - public long GetReadOnlyTransactionId() - { - long readId = activeTransactionsTracker.GetSmallestActiveTransactionId(); - if (readId > 0) - { - readId--; - } - return readId; - } - - #endregion - - private void BeginDependencyCompletionLoop() - { - BeginDependencyCompletionLoopAsync().Ignore(); - } - - private void BeginGroupCommitLoop() - { - BeginGroupCommitLoopAsync().Ignore(); - } - - private void BeginCheckpointLoop() - { - BeginCheckpointLoopAsync().Ignore(); - } - - private void SignalDependencyEnqueued() - { - BeginDependencyCompletionLoop(); - } - - private void SignalGroupCommitEnqueued() - { - BeginGroupCommitLoop(); - } - - private void SignalCheckpointEnqueued() - { - BeginCheckpointLoop(); - } - - private async Task BeginDependencyCompletionLoopAsync() - { - bool gotLock = false; - try - { - if (!(gotLock = dependencyLock.TryGetLock())) - { - return; - } - - while (this.CheckDependenciesCompleted()) - { - // force yield thread - await Task.Delay(TimeSpan.FromTicks(1)); - } - } - finally - { - if (gotLock) - dependencyLock.ReleaseLock(); - } - } - - private async Task BeginGroupCommitLoopAsync() - { - bool gotLock = false; - try - { - if (!(gotLock = commitLock.TryGetLock())) - { - return; - } - - while (await this.GroupCommit()) - { - // force yield thread - await Task.Delay(TimeSpan.FromTicks(1)); - } - } - finally - { - if (gotLock) - commitLock.ReleaseLock(); - } - } - - private async Task BeginCheckpointLoopAsync() - { - bool gotLock = false; - try - { - if (!(gotLock = checkpointLock.TryGetLock())) - { - return; - } - - while (await this.Checkpoint(resources, transactions)) - { - } - } - finally - { - if (gotLock) - checkpointLock.ReleaseLock(); - } - } - - private bool CheckDependenciesCompleted() - { - bool processed = false; - while (dependencyQueue.TryDequeue(out Transaction tx)) - { - processed = true; - CommitRecord commitRecord = new CommitRecord - { - TransactionId = tx.TransactionId - }; - - foreach (var resource in tx.Info.WriteSet.Keys) - { - commitRecord.Resources.Add(resource); - } - groupCommitQueue.Enqueue(new Tuple(commitRecord, tx)); - this.SignalGroupCommitEnqueued(); - - // We don't need to hold the transaction lock any more to access - // the WaitingTransactions set, since nothing can be added to it - // after this point. - - // If a transaction is waiting on us, decrement their waiting count; - // - if they are no longer waiting, mark as validated. - // TODO: Can't we clear WaitingTransactions here and no longer track it? - foreach (var waiting in tx.WaitingTransactions) - { - bool signal = false; - lock (waiting) - { - if (waiting.State != TransactionState.Aborted) - { - waiting.PendingCount--; - - if (waiting.PendingCount == 0) - { - waiting.State = TransactionState.Validated; - dependencyQueue.Enqueue(waiting); - signal = true; - } - } - } - if (signal) - { - this.SignalDependencyEnqueued(); - } - } - } - - return processed; - } - - private async Task GroupCommit() - { - bool processed = false; - int batchSize = groupCommitQueue.Count; - List records = new List(batchSize); - List transactions = new List(batchSize); - while (batchSize > 0) - { - processed = true; - if (groupCommitQueue.TryDequeue(out Tuple t)) - { - records.Add(t.Item1); - transactions.Add(t.Item2); - batchSize--; - } - else - { - break; - } - - } - - try - { - await transactionLog.Append(records); - } - catch (Exception e) - { - this.logger.Error(OrleansTransactionsErrorCode.TransactionManager_GroupCommitError, "Group Commit error", e); - // Failure to get an acknowledgment of the commits from the log (e.g. timeout exception) - // will put the transactions in doubt. We crash and let this be handled in recovery. - // TODO: handle other exceptions more gracefuly - throw; - - } - - for (int i = 0; i < transactions.Count; i++) - { - var transaction = transactions[i]; - lock (transaction) - { - transaction.State = TransactionState.Committed; - transaction.LSN = records[i].LSN; - transaction.CompletionTimeUtc = DateTime.UtcNow; - } - checkpointQueue.Enqueue(transaction); - this.SignalCheckpointEnqueued(); - } - - return processed; - } - - private async Task Checkpoint(Dictionary resources, List transactions) - { - // Rather than continue processing forever, only process the number of transactions which were waiting at invocation time. - var total = this.checkpointRetryQueue.Count + checkpointQueue.Count; - - // The maximum number of transactions checkpointed in each batch. - int batchSize = Math.Min(total, MaxCheckpointBatchSize); - long lsn = 0; - - try - { - var processed = 0; - while (processed < total && (this.checkpointRetryQueue.Count > 0 || !checkpointQueue.IsEmpty)) - { - resources.Clear(); - transactions.Clear(); - - // Take a batch of transactions to checkpoint. - var currentBatchSize = 0; - while (currentBatchSize < batchSize) - { - currentBatchSize++; - Transaction tx; - - // If some previous operation had failed, retry it before proceeding with new work. - if (this.checkpointRetryQueue.Count > 0) tx = this.checkpointRetryQueue.Dequeue(); - else if (!checkpointQueue.TryDequeue(out tx)) break; - - foreach (var resource in tx.Info.WriteSet.Keys) - { - resources[resource] = tx.Info.TransactionId; - } - - lsn = Math.Max(lsn, tx.LSN); - transactions.Add(tx); - } - - processed += currentBatchSize; - - // If the transaction involved writes, send a commit notification to each resource which performed - // a write and wait for acknowledgement. - if (resources.Count > 0) - { - // Send commit notifications to all of the resources involved. - var completion = new MultiCompletionSource(resources.Count); - foreach (var resource in resources) - { - NotifyResourceOfCommit(resource.Key, resource.Value, completion); - } - - // Wait for the commit notifications to be acknowledged by the resources. - await completion.Task; - } - - // Mark the transactions as checkpointed. - foreach (var tx in transactions) - { - lock (tx) - { - tx.State = TransactionState.Checkpointed; - tx.HighestActiveTransactionIdAtCheckpoint = activeTransactionsTracker.GetHighestActiveTransactionId(); - } - } - - // Allow the transaction log to be truncated. - this.checkpointedLSN = lsn; - } - } - catch (Exception e) - { - // Retry all failed checkpoint operations. - foreach (var tx in transactions) this.checkpointRetryQueue.Enqueue(tx); - - this.logger.Error(OrleansTransactionsErrorCode.TransactionManager_CheckpointError, "Failure during checkpoint", e); - throw; - } - - return total > 0; - } - - private static void NotifyResourceOfCommit(ITransactionalResource resource, long transaction, MultiCompletionSource completionSource) - { - resource.Commit(transaction) - .ContinueWith( - (result, state) => - { - var completion = (MultiCompletionSource)state; - if (result.Exception != null) completion.SetException(result.Exception); - else completion.SetOneResult(); - }, - completionSource, - CancellationToken.None, - TaskContinuationOptions.ExecuteSynchronously, - TaskScheduler.Default); - } - - private async Task MaintainTransactionLog() - { - while(this.IsRunning) - { - try - { - await TransactionLogMaintenance(); - } catch(Exception ex) - { - this.logger.Error(OrleansTransactionsErrorCode.TransactionManager_TransactionLogMaintenanceError, $"Error while maintaining transaction log.", ex); - } - this.metrics.TryReportMetrics(); - await Task.Delay(this.logMaintenanceInterval); - } - } - - private async Task TransactionLogMaintenance() - { - // - // Truncate log - // - if (checkpointedLSN > 0) - { - try - { - await transactionLog.TruncateLog(checkpointedLSN - 1); - } - catch (Exception e) - { - this.logger.Error(OrleansTransactionsErrorCode.TransactionManager_TransactionLogTruncationError, $"Failed to truncate log. LSN: {checkpointedLSN}", e); - } - } - - // - // Timeout expired transactions - // - long now = DateTime.UtcNow.Ticks; - foreach (var txRecord in transactionsTable) - { - if (txRecord.Value.State == TransactionState.Started && - txRecord.Value.ExpirationTime < now) - { - AbortTransaction(txRecord.Key, new OrleansTransactionTimeoutException(txRecord.Key.ToString())); - } - } - - // - // Find the oldest active transaction - // - long lowestActiveId = activeTransactionsTracker.GetSmallestActiveTransactionId(); - long highestActiveId = activeTransactionsTracker.GetHighestActiveTransactionId(); - while (lowestActiveId <= highestActiveId) - { - - if (transactionsTable.TryGetValue(lowestActiveId, out Transaction tx)) - { - if (tx.State != TransactionState.Aborted && - tx.State != TransactionState.Checkpointed) - { - break; - } - } - - lowestActiveId++; - activeTransactionsTracker.PopSmallestActiveTransactionId(); - } - - // - // Remove transactions that we no longer need to keep a record of from transactions table. - // a transaction is presumed to be aborted if we try to look it up and it does not exist in the - // table. - // - foreach (var txRecord in transactionsTable) - { - if (txRecord.Value.State == TransactionState.Aborted && - txRecord.Value.CompletionTimeUtc + this.options.TransactionRecordPreservationDuration < DateTime.UtcNow) - { - transactionsTable.TryRemove(txRecord.Key, out Transaction temp); - } - else if (txRecord.Value.State == TransactionState.Checkpointed) - { - lock (txRecord.Value) - { - if (txRecord.Value.HighestActiveTransactionIdAtCheckpoint < activeTransactionsTracker.GetSmallestActiveTransactionId() && - txRecord.Value.CompletionTimeUtc + this.options.TransactionRecordPreservationDuration < DateTime.UtcNow) - { - // The oldest active transaction started after this transaction was checkpointed - // so no in progress transaction is going to take a dependency on this transaction - // which means we can safely forget about it. - - // When receiving an arbitrarily delayed message, a transaction that may have committed - // will appear to have aborted, causing the delayed transaction to abort. - transactionsTable.TryRemove(txRecord.Key, out Transaction temp); - } - } - } - } - } - } -} diff --git a/src/Orleans.Transactions/InClusterTM/TransactionManagerGrain.cs b/src/Orleans.Transactions/InClusterTM/TransactionManagerGrain.cs deleted file mode 100644 index ad58d90206..0000000000 --- a/src/Orleans.Transactions/InClusterTM/TransactionManagerGrain.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Orleans.Concurrency; -using Orleans.Transactions.Abstractions; - -namespace Orleans.Transactions -{ - public interface ITransactionManagerGrain : ITransactionManagerService, IGrainWithIntegerKey - { - } - - [Reentrant] - public class TransactionManagerGrain : Grain, ITransactionManagerGrain - { - //for test use only - public static bool IsActive = false; - private readonly ITransactionManager transactionManager; - private readonly ITransactionManagerService transactionManagerService; - - public TransactionManagerGrain(ITransactionManager transactionManager) - { - this.transactionManager = transactionManager; - this.transactionManagerService = new TransactionManagerService(transactionManager); - } - - public override async Task OnActivateAsync() - { - IsActive = true; - await transactionManager.StartAsync(); - } - - public override async Task OnDeactivateAsync() - { - IsActive = false; - await transactionManager.StopAsync(); - } - - public Task StartTransactions(List timeouts) - { - return this.transactionManagerService.StartTransactions(timeouts); - } - - public Task CommitTransactions(List transactions, HashSet queries) - { - return this.transactionManagerService.CommitTransactions(transactions, queries); - } - - public Task AbortTransaction(long transactionId, OrleansTransactionAbortedException reason) - { - return this.transactionManagerService.AbortTransaction(transactionId, reason); - } - } -} diff --git a/src/Orleans.Transactions/InClusterTM/TransactionManagerMetrics.cs b/src/Orleans.Transactions/InClusterTM/TransactionManagerMetrics.cs deleted file mode 100644 index 8bc12201f4..0000000000 --- a/src/Orleans.Transactions/InClusterTM/TransactionManagerMetrics.cs +++ /dev/null @@ -1,94 +0,0 @@ -using Orleans.Runtime; -using System; -using System.Collections.Generic; -using System.Text; - -namespace Orleans.Transactions -{ - internal class TransactionManagerMetrics - { - private const string StartTransactionRequestTPS = "TransactionManager.StartTransaction.PerSecond"; - private const string AbortTransactionRequestTPS = "TransactionManager.AbortTransaction.PerSecond"; - private const string CommitTransactionRequestTPS = "TransactionManager.CommitTransaction.PerSecond"; - - private const string AbortedTransactionDueToDependencyTPS = "Transaction.AbortedDueToDependency.PerSecond"; - - private const string AbortedTransactionDueToMissingInfoInTransactionTableTPS = - "Transaction.AbortedDueToMissingInfoInTransactionTable.PerSecond"; - private const string AbortedTransactionTPS = "Transaction.Aborted.PerSecond"; - //pending transaction produced TPS in the monitor window - private const string PendingTransactionTPS = "Transaction.Pending.PerSecond"; - //TPS for validated transaction in the monitor window - private const string ValidatedTransactionTPS = "Transaction.Validated.PerSecond"; - - internal long StartTransactionRequestCounter { get; set; } - internal long AbortTransactionRequestCounter { get; set; } - internal long CommitTransactionRequestCounter { get; set; } - internal long AbortedTransactionCounter { get; set; } - internal long AbortedTransactionDueToDependencyCounter { get; set; } - internal long AbortedTransactionDueToMissingInfoInTransactionTableCounter { get; set; } - internal long PendingTransactionCounter { get; set; } - internal long ValidatedTransactionCounter { get; set; } - - private DateTime lastReportTime = DateTime.Now; - private ITelemetryProducer telemetryProducer; - private PeriodicAction monitor; - public TransactionManagerMetrics(ITelemetryProducer telemetryProducer, TimeSpan metricReportInterval) - { - this.telemetryProducer = telemetryProducer; - this.monitor = new PeriodicAction(metricReportInterval, this.ReportMetrics); - } - - public void TryReportMetrics() - { - this.monitor.TryAction(DateTime.Now); - } - - private void ResetCounters(DateTime lastReportTime) - { - this.lastReportTime = lastReportTime; - this.StartTransactionRequestCounter = 0; - this.AbortTransactionRequestCounter = 0; - this.CommitTransactionRequestCounter = 0; - - this.AbortedTransactionDueToDependencyCounter = 0; - this.AbortedTransactionDueToMissingInfoInTransactionTableCounter = 0; - this.PendingTransactionCounter = 0; - this.ValidatedTransactionCounter = 0; - this.AbortedTransactionCounter = 0; - } - - private void ReportMetrics() - { - if (this.telemetryProducer == null) - return; - var now = DateTime.Now; - var timeSinceLastReportInSeconds = Math.Max(1, (now - this.lastReportTime).TotalSeconds); - var startTransactionTps = StartTransactionRequestCounter / timeSinceLastReportInSeconds; - this.telemetryProducer.TrackMetric(StartTransactionRequestTPS, startTransactionTps); - - var abortTransactionTPS = AbortTransactionRequestCounter / timeSinceLastReportInSeconds; - this.telemetryProducer.TrackMetric(AbortTransactionRequestTPS, abortTransactionTPS); - - var commitTransactionTPS = CommitTransactionRequestCounter / timeSinceLastReportInSeconds; - this.telemetryProducer.TrackMetric(CommitTransactionRequestTPS, commitTransactionTPS); - - var abortedTransactionDueToDependentTPS = AbortedTransactionDueToDependencyCounter / timeSinceLastReportInSeconds; - this.telemetryProducer.TrackMetric(AbortedTransactionDueToDependencyTPS, abortedTransactionDueToDependentTPS); - - var abortedTransactionDueToMissingInfoInTransactionTableTPS = AbortedTransactionDueToMissingInfoInTransactionTableCounter / timeSinceLastReportInSeconds; - this.telemetryProducer.TrackMetric(AbortedTransactionDueToMissingInfoInTransactionTableTPS, abortedTransactionDueToMissingInfoInTransactionTableTPS); - - var pendingTransactionTPS = PendingTransactionCounter / timeSinceLastReportInSeconds; - this.telemetryProducer.TrackMetric(PendingTransactionTPS, pendingTransactionTPS); - - var validatedTransactionTPS = ValidatedTransactionCounter / timeSinceLastReportInSeconds; - this.telemetryProducer.TrackMetric(ValidatedTransactionTPS, validatedTransactionTPS); - - var abortedTransactionTPS = AbortedTransactionCounter / timeSinceLastReportInSeconds; - this.telemetryProducer.TrackMetric(AbortedTransactionTPS, abortedTransactionTPS); - - this.ResetCounters(now); - } - } -} diff --git a/src/Orleans.Transactions/InClusterTM/TransactionManagerService.cs b/src/Orleans.Transactions/InClusterTM/TransactionManagerService.cs deleted file mode 100644 index a0a6322c70..0000000000 --- a/src/Orleans.Transactions/InClusterTM/TransactionManagerService.cs +++ /dev/null @@ -1,97 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Orleans.Transactions.Abstractions; - -namespace Orleans.Transactions -{ - public class TransactionManagerService : ITransactionManagerService - { - private readonly ITransactionManager tm; - - public TransactionManagerService(ITransactionManager tm) - { - this.tm = tm; - } - - public Task StartTransactions(List timeouts) - { - var result = new StartTransactionsResponse { TransactionId = new List() }; - - foreach (var timeout in timeouts) - { - result.TransactionId.Add(tm.StartTransaction(timeout)); - } - - result.ReadOnlyTransactionId = tm.GetReadOnlyTransactionId(); - result.AbortLowerBound = result.ReadOnlyTransactionId; - - return Task.FromResult(result); - } - - public Task CommitTransactions(List transactions, HashSet queries) - { - List tasks = new List(); - - var result = new CommitTransactionsResponse { CommitResult = new Dictionary() }; - - foreach (var ti in transactions) - { - try - { - tm.CommitTransaction(ti); - } - catch (OrleansTransactionAbortedException e) - { - var cr = new CommitResult() - { - Success = false, - AbortingException = e, - }; - result.CommitResult[ti.TransactionId] = cr; - } - } - - foreach (var q in queries) - { - OrleansTransactionAbortedException abortingException; - var status = tm.GetTransactionStatus(q, out abortingException); - if (status == TransactionStatus.InProgress) - { - continue; - } - - var cr = new CommitResult(); - if (status == TransactionStatus.Aborted) - { - cr.Success = false; - cr.AbortingException = abortingException; - } - else if (status == TransactionStatus.Committed) - { - cr.Success = true; - } - else if (status == TransactionStatus.Unknown) - { - // Note that the way we communicate an unknown transaction is a false Success - // and a null aborting exception. - // TODO: make this more explicit? - cr.Success = false; - } - - result.CommitResult[q] = cr; - } - - result.ReadOnlyTransactionId = tm.GetReadOnlyTransactionId(); - result.AbortLowerBound = result.ReadOnlyTransactionId; - - return Task.FromResult(result); - } - - public Task AbortTransaction(long transactionId, OrleansTransactionAbortedException reason) - { - tm.AbortTransaction(transactionId, reason); - return Task.CompletedTask; - } - } -} diff --git a/src/Orleans.Transactions/InClusterTM/TransactionServiceGrainFactory.cs b/src/Orleans.Transactions/InClusterTM/TransactionServiceGrainFactory.cs deleted file mode 100644 index f5874e5120..0000000000 --- a/src/Orleans.Transactions/InClusterTM/TransactionServiceGrainFactory.cs +++ /dev/null @@ -1,18 +0,0 @@ - -namespace Orleans.Transactions -{ - internal class TransactionServiceGrainFactory - { - private readonly IGrainFactory grainFactory; - - public TransactionServiceGrainFactory(IGrainFactory grainFactory) - { - this.grainFactory = grainFactory; - } - - public ITransactionManagerService CreateTransactionManagerService() - { - return this.grainFactory.GetGrain(0); - } - } -} diff --git a/src/Orleans.Transactions/Properties/AssemblyInfo.cs b/src/Orleans.Transactions/Properties/AssemblyInfo.cs index 038923fbdf..4e6b6330a0 100644 --- a/src/Orleans.Transactions/Properties/AssemblyInfo.cs +++ b/src/Orleans.Transactions/Properties/AssemblyInfo.cs @@ -1,9 +1,9 @@ using Orleans.CodeGeneration; using Orleans.Transactions.Abstractions; using Orleans.Transactions; +using Orleans.Transactions.Abstractions.Extensions; [assembly: GenerateSerializer(typeof(TransactionalExtensionExtensions.TransactionalResourceExtensionWrapper))] -[assembly: GenerateSerializer(typeof(CommitRecord))] [assembly: GenerateSerializer(typeof(TransactionalStateRecord<>))] [assembly: GenerateSerializer(typeof(PendingTransactionState<>))] [assembly: GenerateSerializer(typeof(TransactionalStorageLoadResponse<>))] diff --git a/src/Orleans.Transactions/State/NamedTransactionalStateStorageFactory.cs b/src/Orleans.Transactions/State/NamedTransactionalStateStorageFactory.cs index 6399b27cfe..96ae686e68 100644 --- a/src/Orleans.Transactions/State/NamedTransactionalStateStorageFactory.cs +++ b/src/Orleans.Transactions/State/NamedTransactionalStateStorageFactory.cs @@ -1,9 +1,9 @@ -using System; +using System; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; using Orleans.Runtime; using Orleans.Transactions.Abstractions; using Orleans.Storage; +using Microsoft.Extensions.Logging; namespace Orleans.Transactions { diff --git a/src/Orleans.Transactions/DistributedTM/Implementation/ReaderWriterLock.cs b/src/Orleans.Transactions/State/ReaderWriterLock.cs similarity index 99% rename from src/Orleans.Transactions/DistributedTM/Implementation/ReaderWriterLock.cs rename to src/Orleans.Transactions/State/ReaderWriterLock.cs index e4ef5f45d5..bde7dce633 100644 --- a/src/Orleans.Transactions/DistributedTM/Implementation/ReaderWriterLock.cs +++ b/src/Orleans.Transactions/State/ReaderWriterLock.cs @@ -1,13 +1,13 @@ -using Microsoft.Extensions.Logging; -using Orleans.Runtime; -using Orleans.Transactions; -using System; +using System; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.Runtime; +using Orleans.Transactions; +using Orleans.Transactions.Abstractions; -namespace Orleans.Transactions.DistributedTM +namespace Orleans.Transactions { public partial class TransactionalState : ITransactionalState, ITransactionParticipant, ILifecycleParticipant where TState : class, new() diff --git a/src/Orleans.Transactions/DistributedTM/Implementation/StorageBatch.cs b/src/Orleans.Transactions/State/StorageBatch.cs similarity index 96% rename from src/Orleans.Transactions/DistributedTM/Implementation/StorageBatch.cs rename to src/Orleans.Transactions/State/StorageBatch.cs index e7b44c33a0..2abc2502fa 100644 --- a/src/Orleans.Transactions/DistributedTM/Implementation/StorageBatch.cs +++ b/src/Orleans.Transactions/State/StorageBatch.cs @@ -1,14 +1,12 @@ -using Newtonsoft.Json; -using Orleans.Concurrency; -using Orleans.Serialization; -using Orleans.Transactions; -using System; +using System; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading.Tasks; +using Newtonsoft.Json; +using Orleans.Concurrency; +using Orleans.Transactions.Abstractions; -namespace Orleans.Transactions.DistributedTM +namespace Orleans.Transactions { /// /// Events streamed to storage. @@ -180,13 +178,13 @@ public void Prepare(long sequenceNumber, Guid transactionId, DateTime timestamp, var tmstring = (transactionManager == null) ? null : JsonConvert.SerializeObject(transactionManager, MetaData.SerializerSettings); - prepares[sequenceNumber] = new PendingTransactionState() + prepares[sequenceNumber] = new PendingTransactionState { SequenceId = sequenceNumber, - State = state, - TimeStamp = timestamp, TransactionId = transactionId.ToString(), - TransactionManager = tmstring + TimeStamp = timestamp, + TransactionManager = tmstring, + State = state }; if (cancelAbove < sequenceNumber) diff --git a/src/Orleans.Transactions/DistributedTM/Implementation/TransactionParticipant.cs b/src/Orleans.Transactions/State/TransactionParticipant.cs similarity index 99% rename from src/Orleans.Transactions/DistributedTM/Implementation/TransactionParticipant.cs rename to src/Orleans.Transactions/State/TransactionParticipant.cs index ae15ebfc7c..8213e14330 100644 --- a/src/Orleans.Transactions/DistributedTM/Implementation/TransactionParticipant.cs +++ b/src/Orleans.Transactions/State/TransactionParticipant.cs @@ -3,17 +3,14 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Orleans; -using Orleans.Providers; using Orleans.Runtime; using Orleans.Transactions.Abstractions; -using Newtonsoft.Json; using Orleans.Transactions; using Orleans.Storage; -namespace Orleans.Transactions.DistributedTM +namespace Orleans.Transactions { /// /// Stateful facet that respects Orleans transaction semantics diff --git a/src/Orleans.Transactions/DistributedTM/Protocol/TransactionParticipantExtension.cs b/src/Orleans.Transactions/State/TransactionParticipantExtension.cs similarity index 97% rename from src/Orleans.Transactions/DistributedTM/Protocol/TransactionParticipantExtension.cs rename to src/Orleans.Transactions/State/TransactionParticipantExtension.cs index ae8c10508c..842ff7c98b 100644 --- a/src/Orleans.Transactions/DistributedTM/Protocol/TransactionParticipantExtension.cs +++ b/src/Orleans.Transactions/State/TransactionParticipantExtension.cs @@ -1,9 +1,10 @@  +using Orleans.Transactions.Abstractions; using System; using System.Collections.Generic; using System.Threading.Tasks; -namespace Orleans.Transactions.DistributedTM +namespace Orleans.Transactions { public class TransactionParticipantExtension : ITransactionParticipantExtension { diff --git a/src/Orleans.Transactions/State/TransactionalState.cs b/src/Orleans.Transactions/State/TransactionalState.cs index ba3818364a..0e26dc2d6e 100644 --- a/src/Orleans.Transactions/State/TransactionalState.cs +++ b/src/Orleans.Transactions/State/TransactionalState.cs @@ -5,20 +5,19 @@ using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using Orleans; using Orleans.Providers; using Orleans.Runtime; using Orleans.Transactions.Abstractions; using Newtonsoft.Json; using Orleans.Transactions; +using Orleans.Transactions.Abstractions.Extensions; namespace Orleans.Transactions { /// /// Stateful facet that respects Orleans transaction semantics /// - /// - public class TransactionalState : ITransactionalState, ITransactionalResource, ILifecycleParticipant + public partial class TransactionalState : ITransactionalState, ITransactionParticipant, ILifecycleParticipant where TState : class, new() { private readonly ITransactionalStateConfiguration config; @@ -26,581 +25,237 @@ public class TransactionalState : ITransactionalState, ITransact private readonly ITransactionDataCopier copier; private readonly ITransactionAgent transactionAgent; private readonly IProviderRuntime runtime; - private readonly ILogger logger; + private readonly ILoggerFactory loggerFactory; - private readonly Dictionary transactionCopy; - private readonly AsyncSerialExecutor storageExecutor; + private ILogger logger; - private ITransactionalResource transactionalResource; + private ITransactionParticipant thisParticipant; // storage private ITransactionalStateStorage storage; - // only to be modified at save/load time - private Metadata metadata; - private string eTag; - private bool validState; - private TState commitedState; + private string stateName; + private string StateName => stateName ?? (stateName = StoredName()); - // In-memory version of the persistent state. - private TState value; - private readonly SortedDictionary> log; - private TransactionalResourceVersion version; - private long highestReadTransactionId; - private long highCommitTransactionId; + //private TimeSpan DebuggerAllowance = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromTicks(0); + private TimeSpan DebuggerAllowance = TimeSpan.FromTicks(0); - public TState State => GetState(); + // max time the TM will wait for prepare phase to complete + private TimeSpan PrepareTimeout => TimeSpan.FromSeconds(20) + DebuggerAllowance; - public TransactionalState(ITransactionalStateConfiguration transactionalStateConfiguration, IGrainActivationContext context, ITransactionDataCopier copier, ITransactionAgent transactionAgent, IProviderRuntime runtime, ILoggerFactory loggerFactory) - { - this.config = transactionalStateConfiguration; - this.context = context; - this.copier = copier; - this.transactionAgent = transactionAgent; - this.runtime = runtime; - this.logger = loggerFactory.CreateLogger($"{this.GetType().FullName}.{this.context.GrainIdentity}.{this.config.StateName}"); - this.transactionCopy = new Dictionary(); - this.storageExecutor = new AsyncSerialExecutor(); - this.log = new SortedDictionary>(); - } - - /// - /// Transactional Write procedure. - /// - public void Save() - { - var info = TransactionContext.GetRequiredTransactionInfo(); - - if(this.logger.IsEnabled(LogLevel.Debug)) this.logger.Debug("Write {0}", info); + // max time a group can occupy the lock + private TimeSpan LockTimeout => TimeSpan.FromSeconds(8) + DebuggerAllowance; - if (info.IsReadOnly) - { - // For obvious reasons... - throw new OrleansReadOnlyViolatedException(info.TransactionId.ToString()); - } - - Rollback(); - - var copiedValue = this.transactionCopy[info.TransactionId]; - - // - // Validation - // - - if (this.version.TransactionId > info.TransactionId || this.highestReadTransactionId >= info.TransactionId) - { - // Prevent cycles. Wait-die - throw new OrleansTransactionWaitDieException(info.TransactionId.ToString()); - } - - TransactionalResourceVersion nextVersion = TransactionalResourceVersion.Create(info.TransactionId, - this.version.TransactionId == info.TransactionId ? this.version.WriteNumber + 1 : 1); + // max time a transaction will wait for the lock to become available + private TimeSpan LockAcquireTimeout => TimeSpan.FromSeconds(10) + DebuggerAllowance; - // - // Update Transaction Context - // - info.RecordWrite(transactionalResource, this.version, this.metadata.StableVersion.TransactionId); - // - // Modify the State - // - if (!this.log.ContainsKey(info.TransactionId)) - { - LogRecord r = new LogRecord(); - this.log[info.TransactionId] = r; - } + private TimeSpan RemoteTransactionPingFrequency => TimeSpan.FromSeconds(60); + private static TimeSpan ConfirmationRetryDelay => TimeSpan.FromSeconds(30); + private static int ConfirmationRetryLimit => 3; - LogRecord logRecord = this.log[info.TransactionId]; - logRecord.NewVal = copiedValue; - logRecord.Version = nextVersion; - this.value = copiedValue; - this.version = nextVersion; - this.transactionCopy.Remove(info.TransactionId); - } + private TState stableState; + private long stableSequenceNumber; - #region lifecycle - public void Participate(IGrainLifecycle lifecycle) - { - lifecycle.Subscribe>(GrainLifecycleStage.SetupState, OnSetupState); - } - #endregion lifecycle + // the queues handling the various stages + private CommitQueue commitQueue; + private StorageBatch storageBatch; - #region ITransactionalResource - async Task ITransactionalResource.Prepare(long transactionId, TransactionalResourceVersion? writeVersion, - TransactionalResourceVersion? readVersion) + private Dictionary> _confirmationTasks; + private Dictionary> confirmationTasks { - if (this.logger.IsEnabled(LogLevel.Debug)) this.logger.Debug("Prepare {0}", transactionId); - - this.transactionCopy.Remove(transactionId); - - long wlb = 0; - if (readVersion.HasValue) + get { - this.highestReadTransactionId = Math.Max(this.highestReadTransactionId, readVersion.Value.TransactionId - 1); - wlb = this.highestReadTransactionId; + if (_confirmationTasks == null) + { + _confirmationTasks = new Dictionary>(); + } + return _confirmationTasks; } + } - if (!ValidateWrite(writeVersion)) - { - return false; - } + private TransactionalStatus problemFlag; + private int failCounter; - if (!ValidateRead(transactionId, readVersion)) - { - return false; - } + // moves transactions into and out of the lock stage + private BatchWorker lockWorker; - try - { - // Note that the checks above will need to be done again - // after we aquire the lock because things could change in the meantime. - return await this.storageExecutor.AddNext(() => GuardState(() => PersistPrepare(wlb, transactionId, writeVersion, readVersion))); - } - catch (Exception ex) - { - // On error, queue up a recovery action. Will do nothing if state recovers before this is processed - this.storageExecutor.AddNext(() => GuardState(() => Task.FromResult(true))).Ignore(); - this.logger.Error(OrleansTransactionsErrorCode.Transactions_PrepareFailed, $"Prepare of transaction {transactionId} failed.", ex); - await ((ITransactionalResource)this).Abort(transactionId); - return false; - } - } + // processes storage and post-storage queues, moves transactions out of the commit stage + private BatchWorker storageWorker; - /// - /// Implementation of ITransactionalGrain Abort method. See interface documentation for more details. - /// - Task ITransactionalResource.Abort(long transactionId) - { - if (this.logger.IsEnabled(LogLevel.Debug)) this.logger.Debug("Abort {0}", transactionId); + // processes confirmation tasks + private BatchWorker confirmationWorker; - // Rollback t if it has changed the grain - if (this.log.ContainsKey(transactionId)) - { - Rollback(transactionId); - } - return Task.CompletedTask; - } + private DateTime clock; - /// - /// Implementation of ITransactionalGrain Commit method. See interface documentation for more details. - /// - async Task ITransactionalResource.Commit(long transactionId) + // collection tasks + private Dictionary unprocessedPreparedMessages; + private class PMessages { - if (this.logger.IsEnabled(LogLevel.Debug)) this.logger.Debug("Commit {0}", transactionId); - - // Learning that t is committed implies that all pending transactions before t also committed - if (transactionId > this.metadata.StableVersion.TransactionId) - { - this.highCommitTransactionId = Math.Max(this.highCommitTransactionId, transactionId); - try - { - bool success = await this.storageExecutor.AddNext(() => GuardState(() => PersistCommit(transactionId))); - } catch(Exception ex) - { - this.logger.Info(OrleansTransactionsErrorCode.Transactions_CommitFailed, "Commit {0} failed. Recovering. Exception: {1}", transactionId, ex); - // On error, queue up a recovery action. Will do nothing if state recovers before this is processed - this.storageExecutor.AddNext(() => GuardState(() => Task.FromResult(true))).Ignore(); - throw; - } - } + public int Count; + public TransactionalStatus Status; } - private async Task GuardState(Func> action) + // clock read and merge + private DateTime MergeAndReadClock(DateTime other) { - if (!this.validState) - { - this.logger.Debug("Invalid state found. Recovering."); - await DoLoad(); - } - this.validState = false; - bool results = await action(); - this.validState = true; - return results; + return clock = new DateTime(Math.Max(Math.Max(clock.Ticks + 1, other.Ticks + 1), DateTime.UtcNow.Ticks)); } - - private async Task PersistPrepare(long wlb, long transactionId, TransactionalResourceVersion? writeVersion, TransactionalResourceVersion? readVersion) + private void MergeClock(DateTime other) { - if (this.logger.IsEnabled(LogLevel.Debug)) this.logger.Debug("PersistPrepare. WLB: {0}, transactionId: {1}", wlb, transactionId); - - if (!ValidateWrite(writeVersion)) - { - return false; - } - - if (!ValidateRead(transactionId, readVersion)) - { - return false; - } - - // check if we need to do a log write - if (this.metadata.StableVersion.TransactionId >= transactionId && this.metadata.HighestReadTransactionId >= wlb) - { - // Logs already persisted, nothing to do here - if (this.logger.IsEnabled(LogLevel.Debug)) this.logger.Debug("PersistPrepare TransactionId {1} ignored. Already persisted", transactionId); - return true; - } - - List> pending = this.log.Select(kvp => new PendingTransactionState(kvp.Value.Version.ToString(), kvp.Key, kvp.Value.NewVal)).ToList(); - Metadata metadata = new Metadata() - { - StableVersion = this.metadata.StableVersion, - HighestVersion = this.version, - HighestReadTransactionId = wlb - }; - this.eTag = await this.storage.Persist(this.eTag, metadata.ToString(), pending); - this.metadata = metadata; - - if (this.logger.IsEnabled(LogLevel.Debug)) this.logger.Debug("PersistPrepare TransactionId {1} succeded", transactionId); - return true; + if (other > clock) + clock = other; } - private async Task PersistCommit(long transactionId) + public TransactionalState( + ITransactionalStateConfiguration transactionalStateConfiguration, + IGrainActivationContext context, + ITransactionDataCopier copier, + ITransactionAgent transactionAgent, + IProviderRuntime runtime, + ILoggerFactory loggerFactory, + ITypeResolver typeResolver, + IGrainFactory grainFactory) { - transactionId = Math.Max(this.highCommitTransactionId, transactionId); - if (transactionId <= this.metadata.StableVersion.TransactionId) - { - // Transaction commit already persisted. - return true; - } - - // find version related to this transaction - LogRecord stableRecord = this.log.First(kvp => kvp.Key <= transactionId).Value; - TransactionalResourceVersion stableversion = stableRecord.Version; - TState stableState = stableRecord.NewVal; - - // Trim the logs to remove old versions. - // Note that we try to keep the highest version that is below or equal to the ReadOnlyTransactionId - // so that we can use it to serve read only transactions. - long highestKey = transactionId; - foreach (var key in this.log.Keys) - { - if (key > this.transactionAgent.ReadOnlyTransactionId) - { - break; - } + this.config = transactionalStateConfiguration; + this.context = context; + this.copier = copier; + this.transactionAgent = transactionAgent; + this.runtime = runtime; + this.loggerFactory = loggerFactory; - highestKey = key; - } + lockWorker = new BatchWorkerFromDelegate(LockWork); + storageWorker = new BatchWorkerFromDelegate(StorageWork); + confirmationWorker = new BatchWorkerFromDelegate(ConfirmationWork); - if (this.log.Count != 0) + if (MetaData.SerializerSettings == null) { - List>> records = this.log.TakeWhile(kvp => kvp.Key < highestKey).ToList(); - if (this.logger.IsEnabled(LogLevel.Debug)) - records.ForEach(kvp => this.logger.Debug("Removing committed transaction from log: transactionId: {1}", kvp.Key)); - records.ForEach(kvp => this.log.Remove(kvp.Key)); + MetaData.SerializerSettings = TransactionParticipantExtensionExtensions.GetJsonSerializerSettings(typeResolver, grainFactory); } - - Metadata newMetadata = new Metadata() - { - StableVersion = stableversion, - HighestVersion = this.version, - HighestReadTransactionId = this.highestReadTransactionId, - }; - this.eTag = await this.storage.Confirm(this.eTag, newMetadata.ToString(), stableversion.ToString()); - this.metadata = newMetadata; - this.commitedState = stableState; - UpdateActiveState(); - - return true; } - #endregion ITransactionalResource + #region lifecycle - /// - /// Find the appropriate version of the state to serve for this transaction. - /// We enforce reads in transaction id order, hence we find the version written by the highest - /// transaction less than or equal to this one - /// - private bool TryGetVersion(long transactionId, out TState readState, out TransactionalResourceVersion readVersion) + public void Participate(IGrainLifecycle lifecycle) { - readState = this.value; - readVersion = this.version; - bool versionAvailable = this.version.TransactionId <= transactionId; - - LogRecord logRecord = null; - foreach (KeyValuePair> kvp in this.log) - { - if (kvp.Key > transactionId) - { - break; - } - logRecord = kvp.Value; - } - - if (logRecord == null) return versionAvailable; - - readState = logRecord.NewVal; - readVersion = logRecord.Version; - - return true; + lifecycle.Subscribe>(GrainLifecycleStage.SetupState, OnSetupState); } - /// - /// Transactional Read procedure. - /// - private TState GetState() + private async Task OnSetupState(CancellationToken ct) { + if (ct.IsCancellationRequested) return; - var info = TransactionContext.GetRequiredTransactionInfo(); - - Rollback(); - - if (this.transactionCopy.TryGetValue(info.TransactionId, out TState state)) - { - return state; - } - - if (this.logger.IsEnabled(LogLevel.Debug)) this.logger.Debug("GetState {0}", info); - - if (!TryGetVersion(info.TransactionId, out TState readState, out TransactionalResourceVersion readVersion)) - { - // This can only happen if old versions are gone due to checkpointing. - throw new OrleansTransactionVersionDeletedException(info.TransactionId.ToString()); - } - - if (info.IsReadOnly && readVersion.TransactionId > this.metadata.StableVersion.TransactionId) - { - throw new OrleansTransactionUnstableVersionException(info.TransactionId.ToString()); - } - - info.RecordRead(transactionalResource, readVersion, this.metadata.StableVersion.TransactionId); + var boundExtension = await this.runtime.BindExtension(() => new TransactionParticipantExtension()); + boundExtension.Item1.Register(this.config.StateName, this); + this.thisParticipant = boundExtension.Item2.AsTransactionParticipant(this.config.StateName); - this.highestReadTransactionId = Math.Max(this.highestReadTransactionId, info.TransactionId - 1); + this.logger = loggerFactory.CreateLogger($"{context.GrainType.Name}.{this.config.StateName}.{this.thisParticipant.ToShortString()}"); - TState copy = this.copier.DeepCopy(readState); + var storageFactory = this.context.ActivationServices.GetRequiredService(); + this.storage = storageFactory.Create(this.config.StorageName, this.config.StateName); - if (!info.IsReadOnly) - { - this.transactionCopy[info.TransactionId] = copy; - } + // recover state + await Restore(); - return copy; + storageWorker.Notify(); } - /// - /// Undo writes to restore state to pre transaction value. - /// - private void Rollback(long transactionId) + #endregion lifecycle + + private string StoredName() { - List>> records = this.log.SkipWhile(kvp => kvp.Key < transactionId).ToList(); - foreach (KeyValuePair> kvp in records) - { - if (this.logger.IsEnabled(LogLevel.Debug)) this.logger.Debug("Removing transaction {0} in rollback", kvp.Key); - this.log.Remove(kvp.Key); - this.transactionCopy.Remove(kvp.Key); - } - - if (this.log.Count > 0) - { - LogRecord lastLogRecord = this.log.Values.Last(); - if (this.logger.IsEnabled(LogLevel.Debug) && this.version != lastLogRecord.Version) - this.logger.Debug("Rolling back from {0} to {1}", this.version, lastLogRecord.Version); - this.version = lastLogRecord.Version; - this.value = lastLogRecord.NewVal; - } - else - { - if (this.logger.IsEnabled(LogLevel.Debug) && this.version != this.metadata.StableVersion) - this.logger.Debug("Rolling back to stable version, from {0} to {1}", this.version, this.metadata.StableVersion); - this.version = this.metadata.StableVersion; - this.value = this.commitedState; - } + return $"{this.context.GrainInstance.GetType().FullName}-{this.config.StateName}"; } - /// - /// Check with the transaction agent and rollback any aborted transaction. - /// - private void Rollback() + public bool Equals(ITransactionParticipant other) { - foreach (var transactionId in this.log.Keys) - { - if (transactionId > this.metadata.StableVersion.TransactionId && transactionAgent.IsAborted(transactionId)) - { - Rollback(transactionId); - return; - } - } + return thisParticipant.Equals(other); } - private bool ValidateWrite(TransactionalResourceVersion? writeVersion) + public override string ToString() { - if (!writeVersion.HasValue) - return true; - - // Validate that we still have all of the transaction's writes. - var validate = this.log.TryGetValue(writeVersion.Value.TransactionId, out LogRecord logRecord) && logRecord.Version == writeVersion.Value; - if(!validate && this.logger.IsEnabled(LogLevel.Debug)) - if(logRecord != null) - this.logger.Debug($"ValidateWrite failed, because version is not the same as recorded in state log record of the same transaction, write version in the log record is {logRecord.Version}, version to be validated is {writeVersion.Value}"); - else - this.logger.Debug($"ValidateWrite failed, because no log record was found for transaction {writeVersion.Value.TransactionId} of write version to be validated {writeVersion.Value}"); - return validate; + return $"{this.context.GrainInstance}.{this.config.StateName}"; } - private bool ValidateRead(long transactionId, TransactionalResourceVersion? readVersion) + /// + /// called on activation, and when recovering from storage conflicts or other exceptions. + /// + private async Task Restore() { - if (!readVersion.HasValue) - return true; + // start the load + var loadtask = this.storage.Load(); - foreach (var key in this.log.Keys) - { - if (key >= transactionId) - { - break; - } + // abort active transactions, without waking up waiters just yet + AbortExecutingTransactions("due to restore"); - if (key > readVersion.Value.TransactionId && key < transactionId) - { - if(this.logger.IsEnabled(LogLevel.Debug)) - this.logger.Debug($"ValidateRead failed, due to one of the log record has larger transaction Id than the one in read version {readVersion.Value.TransactionId}, and smaller than transaction {transactionId}"); - return false; - } - } - - if (readVersion.Value.TransactionId == this.metadata.StableVersion.TransactionId) - { - if (readVersion.Value.WriteNumber != this.metadata.StableVersion.WriteNumber && this.logger.IsEnabled(LogLevel.Debug)) - this.logger.Debug("ValidateRead failed, due to invalid write count. TransactionId: {0}, ReadVersion: {1}, StableVersion {3}", transactionId, readVersion.Value, this.metadata.StableVersion); - return readVersion.Value.WriteNumber == this.metadata.StableVersion.WriteNumber; - } - if (readVersion.Value.TransactionId < this.metadata.StableVersion.TransactionId) return true; - // If version read by the transaction is lost, return false. - if (!this.log.TryGetValue(readVersion.Value.TransactionId, out LogRecord logRecord)) - { - if(this.logger.IsEnabled(LogLevel.Debug)) - this.logger.Debug("ValidateRead failed, due to version read by the transaction lost. TransactionId: {0}, ReadVersion: {1}", transactionId, readVersion.Value); - return false; - } - - - // If version is not same it was overridden by the same transaction that originally wrote it. - if (logRecord.Version != readVersion.Value) - { - if(this.logger.IsEnabled(LogLevel.Debug)) - this.logger.Debug("ValidateRead failed, because version is not same as recorded by state log record of the same transaction. TransactionId: {0}, ReadVersion: {1}, logVersion {2}", transactionId, readVersion.Value, logRecord.Version); - return false; - } - - return true; - } - - private async Task DoLoad() - { - this.logger.Debug("DoLoad"); - // load inital state - TransactionalStorageLoadResponse loadResponse = await this.storage.Load(); - this.eTag = loadResponse.ETag; - this.metadata = Metadata.FromString(loadResponse.Metadata); - this.version = this.metadata.HighestVersion; - this.highestReadTransactionId = this.metadata.HighestReadTransactionId; - this.commitedState = loadResponse.CommittedState; - foreach (PendingTransactionState pendingState in loadResponse.PendingStates) + // abort all entries in the commit queue + foreach (var entry in commitQueue.Elements) { - if (this.logger.IsEnabled(LogLevel.Debug)) - this.logger.Debug("Rebuilding log from storage for {0}", pendingState.SequenceId); - this.log[pendingState.SequenceId] = new LogRecord - { - NewVal = pendingState.State, - Version = (TransactionalResourceVersion.TryParse(pendingState.TransactionId, out TransactionalResourceVersion version)) ? version : default(TransactionalResourceVersion) - }; + NotifyOfAbort(entry, problemFlag); } + commitQueue.Clear(); - UpdateActiveState(); - } + var loadresponse = await loadtask; - private void UpdateActiveState() - { - this.logger.Debug("UpdateActiveState"); - if(this.metadata.HighestVersion > this.version) - this.version = this.metadata.HighestVersion; - this.value = this.log.TryGetValue(this.version.TransactionId, out LogRecord record) - ? record.NewVal - : this.commitedState; - this.log.Clear(); - - // Rollback any known aborted transactions - Rollback(); - } + storageBatch = new StorageBatch(loadresponse); - private async Task OnSetupState(CancellationToken ct) - { - if (ct.IsCancellationRequested) return; + stableState = loadresponse.CommittedState; + stableSequenceNumber = loadresponse.CommittedSequenceId; - Tuple boundExtension = await this.runtime.BindExtension(() => new TransactionalExtension()); - boundExtension.Item1.Register(this.config.StateName, this); - this.transactionalResource = boundExtension.Item2.AsTransactionalResource(this.config.StateName); + if (logger.IsEnabled(LogLevel.Debug)) + logger.Debug($"Load v{stableSequenceNumber} {loadresponse.PendingStates.Count}p {storageBatch.MetaData.CommitRecords.Count}c"); - INamedTransactionalStateStorageFactory storageFactory = this.context.ActivationServices.GetRequiredService(); - this.storage = storageFactory.Create(this.config.StorageName, this.config.StateName); + // ensure clock is consistent with loaded state + MergeClock(storageBatch.MetaData.TimeStamp); - // recover state - await DoLoad(); - - this.validState = true; - } - - public bool Equals(ITransactionalResource other) - { - return transactionalResource.Equals(other); - } - - public override string ToString() - { - return $"{this.context.GrainInstance}.{this.config.StateName}"; - } - - private class Metadata - { - private static readonly JsonSerializerSettings settings = new JsonSerializerSettings + // resume prepared transactions (not TM) + foreach (var pr in loadresponse.PendingStates.OrderBy(ps => ps.TimeStamp)) { - TypeNameHandling = TypeNameHandling.All, - PreserveReferencesHandling = PreserveReferencesHandling.Objects, - DateFormatHandling = DateFormatHandling.IsoDateFormat, - DefaultValueHandling = DefaultValueHandling.Ignore, - MissingMemberHandling = MissingMemberHandling.Ignore, - NullValueHandling = NullValueHandling.Ignore, - ConstructorHandling = ConstructorHandling.AllowNonPublicDefaultConstructor, - TypeNameAssemblyFormatHandling = TypeNameAssemblyFormatHandling.Simple, - Formatting = Formatting.None - }; - - [JsonIgnore] - public TransactionalResourceVersion HighestVersion { get; set; } - public string HighestVersionString - { - get { return this.HighestVersion.ToString(); } - set { this.HighestVersion = (TransactionalResourceVersion.TryParse(value, out TransactionalResourceVersion version)) ? version : default(TransactionalResourceVersion); } + if (pr.SequenceId > stableSequenceNumber && pr.TransactionManager != null) + { + if (logger.IsEnabled(LogLevel.Debug)) + logger.Debug($"recover two-phase-commit {pr.TransactionId}"); + + var tm = (pr.TransactionManager == null) ? null : + (ITransactionParticipant) JsonConvert.DeserializeObject(pr.TransactionManager, MetaData.SerializerSettings); + + commitQueue.Add(new TransactionRecord() + { + Role = CommitRole.RemoteCommit, + TransactionId = Guid.Parse(pr.TransactionId), + Timestamp = pr.TimeStamp, + State = pr.State, + TransactionManager = tm, + PrepareIsPersisted = true, + LastSent = default(DateTime), + ConfirmationResponsePromise = null + }); + } } - [JsonIgnore] - public TransactionalResourceVersion StableVersion { get; set; } - public string StableVersionString + // resume committed transactions (on TM) + foreach (var kvp in storageBatch.MetaData.CommitRecords) { - get { return this.StableVersion.ToString(); } - set { this.StableVersion = (TransactionalResourceVersion.TryParse(value, out TransactionalResourceVersion version)) ? version : default(TransactionalResourceVersion); } - } - - public long HighestReadTransactionId { get; set; } + if (logger.IsEnabled(LogLevel.Debug)) + logger.Debug($"recover commit confirmation {kvp.Key}"); - // TODO consider passing metadata type to storage, and letting storage handle serialization - #region Serialization - public override string ToString() - { - return JsonConvert.SerializeObject(this, settings); + confirmationTasks.Add(kvp.Key, new TransactionRecord() + { + Role = CommitRole.LocalCommit, + TransactionId = kvp.Key, + Timestamp = kvp.Value.Timestamp, + WriteParticipants = kvp.Value.WriteParticipants + }); } - public static Metadata FromString(string metadataString) - { - return (!string.IsNullOrEmpty(metadataString)) ? JsonConvert.DeserializeObject(metadataString, settings) : new Metadata(); - } - #endregion - } + // clear the problem flag + problemFlag = TransactionalStatus.Ok; - private class LogRecord - { - public T NewVal { get; set; } - public TransactionalResourceVersion Version { get; set; } + // check for work + confirmationWorker.Notify(); + storageWorker.Notify(); + lockWorker.Notify(); } } } diff --git a/src/Orleans.Transactions/State/TransactionalStateStorageProviderWrapper.cs b/src/Orleans.Transactions/State/TransactionalStateStorageProviderWrapper.cs index 8f7cf0e788..46ba78e827 100644 --- a/src/Orleans.Transactions/State/TransactionalStateStorageProviderWrapper.cs +++ b/src/Orleans.Transactions/State/TransactionalStateStorageProviderWrapper.cs @@ -1,13 +1,12 @@ -using System; +using System; using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Orleans.Transactions.Abstractions; using Orleans.Core; using Orleans.Runtime; using Orleans.Storage; +using Microsoft.Extensions.Logging; using Orleans.Utilities; +using Orleans.Transactions.Abstractions; namespace Orleans.Transactions { @@ -22,6 +21,7 @@ internal class TransactionalStateStorageProviderWrapper : ITransactional private IStorage> stateStorage; private IStorage> StateStorage => stateStorage ?? (stateStorage = GetStateStorage()); + public TransactionalStateStorageProviderWrapper(IGrainStorage grainStorage, string stateName, IGrainActivationContext context, ILoggerFactory loggerFactory) { this.grainStorage = grainStorage; @@ -33,36 +33,60 @@ public TransactionalStateStorageProviderWrapper(IGrainStorage grainStorage, stri public async Task> Load() { await this.StateStorage.ReadStateAsync(); - return new TransactionalStorageLoadResponse(this.StateStorage.Etag, this.StateStorage.State.CommittedState, this.StateStorage.State.Metadata, this.StateStorage.State.PendingStates); + return new TransactionalStorageLoadResponse(stateStorage.Etag, stateStorage.State.CommittedState, stateStorage.State.CommittedSequenceId, stateStorage.State.Metadata, stateStorage.State.PendingStates); } - public async Task Persist(string expectedETag, string metadata, List> statesToPrepare) + public async Task Store(string expectedETag, string metadata, List> statesToPrepare, long? commitUpTo, long? abortAfter) { if (this.StateStorage.Etag != expectedETag) throw new ArgumentException(nameof(expectedETag), "Etag does not match"); - this.StateStorage.State.Metadata = metadata; - foreach(PendingTransactionState pendingState in statesToPrepare.Where(s => !this.StateStorage.State.PendingStates.Contains(s))) + stateStorage.State.Metadata = metadata; + + var pendinglist = stateStorage.State.PendingStates; + + // abort + if (abortAfter.HasValue && pendinglist.Count != 0) { - this.StateStorage.State.PendingStates.Add(pendingState); + var pos = pendinglist.FindIndex(t => t.SequenceId > abortAfter.Value); + if (pos != -1) + { + pendinglist.RemoveRange(pos, pendinglist.Count - pos); + } } - await this.StateStorage.WriteStateAsync(); - return this.StateStorage.Etag; - } - public async Task Confirm(string expectedETag, string metadata, string transactionIdToCommit) - { - if (this.StateStorage.Etag != expectedETag) - throw new ArgumentException(nameof(expectedETag), "Etag does not match"); - this.StateStorage.State.Metadata = metadata; - PendingTransactionState committedState = this.StateStorage.State.PendingStates.FirstOrDefault(pending => transactionIdToCommit == pending.TransactionId); - if (committedState != null) + // prepare + if (statesToPrepare?.Count > 0) { - this.StateStorage.State.CommittedTransactionId = committedState.TransactionId; - this.StateStorage.State.CommittedState = committedState.State; - this.StateStorage.State.PendingStates = StateStorage.State.PendingStates.Where(pending => pending.SequenceId > committedState.SequenceId).ToList(); + if (pendinglist.Count != 0) + { + // remove prepare records that are being overwritten + while (pendinglist[pendinglist.Count - 1].SequenceId >= statesToPrepare[0].SequenceId) + { + pendinglist.RemoveAt(pendinglist.Count - 1); + } + } + pendinglist.AddRange(statesToPrepare); } - await this.StateStorage.WriteStateAsync(); - return this.StateStorage.Etag; + + // commit + if (commitUpTo.HasValue && commitUpTo.Value > stateStorage.State.CommittedSequenceId) + { + var pos = pendinglist.FindIndex(t => t.SequenceId == commitUpTo.Value); + if (pos != -1) + { + var committedState = pendinglist[pos]; + stateStorage.State.CommittedSequenceId = committedState.SequenceId; + stateStorage.State.CommittedState = committedState.State; + pendinglist.RemoveRange(0, pos + 1); + } + else + { + throw new InvalidOperationException($"Transactional state corrupted. Missing prepare record (SequenceId={commitUpTo.Value}) for committed transaction."); + } + } + + await stateStorage.WriteStateAsync(); + return stateStorage.Etag; } private IStorage> GetStateStorage() @@ -79,7 +103,7 @@ public class TransactionalStateRecord { public TState CommittedState { get; set; } = new TState(); - public string CommittedTransactionId { get; set; } + public long CommittedSequenceId { get; set; } public string Metadata { get; set; } diff --git a/src/Orleans.Transactions/TransactionalExtension.cs b/src/Orleans.Transactions/TransactionalExtension.cs index 924bbb9ca6..c355fb2e5f 100644 --- a/src/Orleans.Transactions/TransactionalExtension.cs +++ b/src/Orleans.Transactions/TransactionalExtension.cs @@ -1,4 +1,5 @@  +using Orleans.Transactions.Abstractions; using System.Collections.Generic; using System.Threading.Tasks; diff --git a/src/Orleans.Transactions/DistributedTM/Implementation/CommitQueue.cs b/src/Orleans.Transactions/Utilities/CommitQueue.cs similarity index 99% rename from src/Orleans.Transactions/DistributedTM/Implementation/CommitQueue.cs rename to src/Orleans.Transactions/Utilities/CommitQueue.cs index 182d7eb969..acb078fae6 100644 --- a/src/Orleans.Transactions/DistributedTM/Implementation/CommitQueue.cs +++ b/src/Orleans.Transactions/Utilities/CommitQueue.cs @@ -6,7 +6,7 @@ using System.Text; using System.Threading.Tasks; -namespace Orleans.Transactions.DistributedTM +namespace Orleans.Transactions { /// /// A deque data structure that stores transaction records in a circular buffer, sorted by timestamps. diff --git a/test/Benchmarks/Program.cs b/test/Benchmarks/Program.cs index 4f81ead31e..ffa6721223 100644 --- a/test/Benchmarks/Program.cs +++ b/test/Benchmarks/Program.cs @@ -5,7 +5,6 @@ using BenchmarkDotNet.Running; using Benchmarks.MapReduce; using Benchmarks.Serialization; -using Benchmarks.TransactionManager; using Benchmarks.Ping; using Benchmarks.Transactions; @@ -32,28 +31,6 @@ class Program { BenchmarkRunner.Run(); }, - ["TransactionManager.Azure"] = () => - { - RunBenchmark( - "Running Azure TransactionManager benchmark", - () => - { - return new TransactionManagerBenchmarks(); - }, - benchmark => benchmark.RunAgainstAzure(), - benchmark => { }); - }, - ["TransactionManager.Memory"] = () => - { - RunBenchmark( - "Running Azure TransactionManager benchmark", - () => - { - return new TransactionManagerBenchmarks(); - }, - benchmark => benchmark.RunAgainstMemory(), - benchmark => { }); - }, ["Transactions"] = () => { RunBenchmark( diff --git a/test/Benchmarks/TransactionManager/TransactionManagerBentchmarks.cs b/test/Benchmarks/TransactionManager/TransactionManagerBentchmarks.cs deleted file mode 100644 index f724c769aa..0000000000 --- a/test/Benchmarks/TransactionManager/TransactionManagerBentchmarks.cs +++ /dev/null @@ -1,172 +0,0 @@ -using System; -using System.Linq; -using System.Threading.Tasks; -using System.Collections.Generic; -using System.Diagnostics; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Logging.Abstractions; -using Orleans; -using Orleans.Configuration; -using Orleans.Configuration.Development; -using Orleans.Runtime.Configuration; -using Orleans.Serialization; -using Orleans.Transactions.Abstractions; -using Orleans.Transactions; -using Orleans.Transactions.Development; -using Orleans.Transactions.AzureStorage; -using Orleans.TestingHost.Utils; - -namespace Benchmarks.TransactionManager -{ - public class TransactionManagerBenchmarks - { - private const int TransactionsPerRun = 500000; - private const int ConcurrentTransactionsTransactions = TransactionsPerRun/10; - private static readonly TimeSpan LogMaintenanceInterval = TimeSpan.FromMilliseconds(10); - private static readonly TimeSpan TransactionTimeout = TimeSpan.FromSeconds(10); - List transactionBatchTimeouts = Enumerable.Range(0, ConcurrentTransactionsTransactions) - .Select(i => TransactionTimeout) - .ToList(); - - public void RunAgainstMemory() - { - Factory> storageFactory = () => Task.FromResult(new InMemoryTransactionLogStorage()); - Run(storageFactory).GetAwaiter().GetResult(); - } - - public void RunAgainstAzure() - { - Run(AzureStorageFactory).GetAwaiter().GetResult(); - } - - private async Task Run(Factory> storageFactory) - { - ITransactionManager tm = new Orleans.Transactions.TransactionManager(new TransactionLog(storageFactory), Options.Create(new TransactionsOptions()), NullLoggerFactory.Instance, NullTelemetryProducer.Instance, Options.Create(new StatisticsOptions()), LogMaintenanceInterval); - await tm.StartAsync(); - ITransactionManagerService tms = new TransactionManagerService(tm); - Stopwatch sw; - int success = 0; - int failed = 0; - - sw = Stopwatch.StartNew(); - HashSet transactionsInFlight = new HashSet(); - int generatedTransactions = 0; - while (generatedTransactions < TransactionsPerRun) - { - int generateCount = Math.Min(TransactionsPerRun - generatedTransactions, ConcurrentTransactionsTransactions - transactionsInFlight.Count); - StartTransactionsResponse startResponse = await tms.StartTransactions(this.transactionBatchTimeouts.Take(generateCount).ToList()); - List newTransactions = startResponse.TransactionId - .Select(this.MakeTransactionInfo) - .ToList(); - generatedTransactions += newTransactions.Count; - Console.WriteLine($"Generated {newTransactions.Count} Transactions."); - transactionsInFlight.UnionWith(newTransactions.Select(ti => ti.TransactionId)); - do - { - Tuple results = await CommitAndCount(tms, newTransactions, transactionsInFlight); - success += results.Item1; - failed += results.Item2; - } - while (transactionsInFlight.Count == ConcurrentTransactionsTransactions); - } - Console.WriteLine($"Generation Complete."); - List empty = new List(); - while (transactionsInFlight.Count != 0) - { - Tuple results = await CommitAndCount(tms, empty, transactionsInFlight); - success += results.Item1; - failed += results.Item2; - } - sw.Stop(); - Console.WriteLine($"{generatedTransactions} Transactions performed in {sw.ElapsedMilliseconds}ms. Succeeded: {success}, Failed: {failed}."); - Console.WriteLine($"{success * 1000 / sw.ElapsedMilliseconds} Transactions/sec."); - await tm.StopAsync(); - } - - private async Task> CommitAndCount(ITransactionManagerService tms, List newTransactions, HashSet transactionsInFlight) - { - int success = 0; - int failed = 0; - CommitTransactionsResponse commitResponse = await tms.CommitTransactions(newTransactions, transactionsInFlight); - if (commitResponse.CommitResult.Count != 0) - { - Console.WriteLine($"Commited {commitResponse.CommitResult.Count} Transactions."); - } - else await Task.Delay(10); - foreach (KeyValuePair kvp in commitResponse.CommitResult) - { - bool removed = transactionsInFlight.Remove(kvp.Key); - if (!removed) - { - Console.WriteLine($"Unrequested result: {kvp.Key}."); - continue; - } - if (kvp.Value.Success) - { - success++; - } - else - { - failed++; - } - } - return Tuple.Create(success, failed); - } - - private TransactionInfo MakeTransactionInfo(long transactionId) - { - var transactionInfo = new TransactionInfo(transactionId); - transactionInfo.WriteSet[NoOpTransactionalResource.Instance] = 1; - return transactionInfo; - } - - [Serializable] - private class NoOpTransactionalResource : ITransactionalResource - { - public static ITransactionalResource Instance => new NoOpTransactionalResource(); - - public Task Abort(long transactionId) - { - return Task.CompletedTask; - } - - public Task Commit(long transactionId) - { - return Task.CompletedTask; - } - - public bool Equals(ITransactionalResource other) - { - return Object.Equals(this,other); - } - - public Task Prepare(long transactionId, TransactionalResourceVersion? writeVersion, TransactionalResourceVersion? readVersion) - { - return Task.FromResult(true); - } - } - - private static async Task AzureStorageFactory() - { - var serviceId = Guid.NewGuid().ToString(); - var clusterId = Guid.NewGuid().ToString(); - var client = new ClientBuilder().Configure(o => - { - o.ServiceId = serviceId; - o.ClusterId = clusterId; - }).Build(); - var azureConfig = Options.Create(new AzureTransactionLogOptions() - { - // TODO: Find better way for test isolation. - TableName = "TransactionLog", - ConnectionString = "UseDevelopmentStorage=true" - }); - AzureTransactionLogStorage storage = new AzureTransactionLogStorage(client.ServiceProvider.GetRequiredService(), azureConfig, - Options.Create(new AzureTransactionArchiveLogOptions()), Options.Create(new ClusterOptions(){ClusterId = clusterId, ServiceId = serviceId })); - await storage.Initialize(); - return storage; - } - } - -} \ No newline at end of file diff --git a/test/Benchmarks/Transactions/TransactionBenchmark.cs b/test/Benchmarks/Transactions/TransactionBenchmark.cs index b1d39dd3e5..92eeaa79b4 100644 --- a/test/Benchmarks/Transactions/TransactionBenchmark.cs +++ b/test/Benchmarks/Transactions/TransactionBenchmark.cs @@ -1,9 +1,7 @@ using System; using System.Threading.Tasks; using System.Linq; -using Orleans.Runtime.Configuration; using Orleans.Hosting; -using Orleans.Hosting.Development; using Orleans.TestingHost; using BenchmarkGrainInterfaces.Transaction; @@ -80,10 +78,7 @@ public sealed class SiloBuilderConfigurator : ISiloBuilderConfigurator { public void Configure(ISiloHostBuilder hostBuilder) { - hostBuilder - .UseInClusterTransactionManager() - .UseInMemoryTransactionLog() - .UseTransactionalState(); + hostBuilder.UseDistributedTM(); } } } diff --git a/test/Grains/BenchmarkGrains/Transaction/TransactionGrain.cs b/test/Grains/BenchmarkGrains/Transaction/TransactionGrain.cs index 036dd0344b..c6acca34c4 100644 --- a/test/Grains/BenchmarkGrains/Transaction/TransactionGrain.cs +++ b/test/Grains/BenchmarkGrains/Transaction/TransactionGrain.cs @@ -27,8 +27,7 @@ public TransactionGrain( public Task Run() { - this.info.State.Value += 1; - this.info.Save(); + this.info.PerformUpdate(s => s.Value += 1); return Task.CompletedTask; } } diff --git a/test/Transactions/Directory.Build.props b/test/Transactions/Directory.Build.props index 0b6ae78c7d..b9c97a99ac 100644 --- a/test/Transactions/Directory.Build.props +++ b/test/Transactions/Directory.Build.props @@ -110,17 +110,4 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/test/Transactions/Orleans.Transactions.Azure.Test/AzureTransactionLogTests.cs b/test/Transactions/Orleans.Transactions.Azure.Test/AzureTransactionLogTests.cs deleted file mode 100644 index e9194ad872..0000000000 --- a/test/Transactions/Orleans.Transactions.Azure.Test/AzureTransactionLogTests.cs +++ /dev/null @@ -1,87 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using Microsoft.WindowsAzure.Storage.Table; -using Orleans.Configuration; -using Orleans.Configuration.Development; -using Orleans.Runtime.Configuration; -using Orleans.Transactions.Abstractions; -using TestExtensions; -using Xunit; -using Orleans.Persistence.AzureStorage; - -namespace Orleans.Transactions.AzureStorage.Tests -{ - [TestCategory("Azure"), TestCategory("Transactions"), TestCategory("BVT")] - public class AzureTransactionLogTests - { - private static string ClusterServiceId = Guid.NewGuid().ToString(); - public AzureTransactionLogTests() - { - TestFixture.CheckForAzureStorage(TestDefaultConfiguration.DataConnectionString); - } - - [SkippableFact] - public async Task TransactionLogCanArchiveAndQuery() - { - var azureOptions = new AzureTransactionLogOptions() - { - ConnectionString = TestDefaultConfiguration.DataConnectionString, - TableName = "TransactionLog" - }; - var archiveOptions = new AzureTransactionArchiveLogOptions() - { - ArchiveLog = true - }; - - var logStorage = await StorageFactory(azureOptions, archiveOptions); - var recordsNum = 20; - var allTransactions = new List(recordsNum); - for (int i = 0; i< recordsNum; i++) - { - allTransactions.Add(new CommitRecord(){LSN=i, TransactionId = i}); - } - await logStorage.Append(allTransactions); - //all transactions will be archived - var maxSLN = recordsNum - 11; - await logStorage.TruncateLog(maxSLN); - //get all archived records - var archQuery = new TableQuery().Where(TableQuery.CombineFilters( - TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, AzureTransactionLogStorage.ArchivalRow.MakePartitionKey(ClusterServiceId)), - TableOperators.And, - TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, AzureTransactionLogStorage.ArchivalRow.MaxRowKey), - TableOperators.And, - TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, AzureTransactionLogStorage.ArchivalRow.MinRowKey)))); - var archivalTransactions = await logStorage.QueryArchivalRecords(archQuery); - //query remaining commited records - var cmQuery = new TableQuery().Where(TableQuery.CombineFilters( - TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, AzureTransactionLogStorage.ArchivalRow.MakePartitionKey(ClusterServiceId)), - TableOperators.And, - TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, AzureTransactionLogStorage.CommitRow.MaxRowKey), - TableOperators.And, - TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, AzureTransactionLogStorage.CommitRow.MinRowKey)))); - var commitTransactions = await logStorage.QueryArchivalRecords(cmQuery); - //assert a subset of transaction has been archived - Assert.Equal(allTransactions.FindAll(r=>r.LSN <= maxSLN).Select(r => r.LSN).Distinct(), archivalTransactions.Select(tx=>tx.LSN).Distinct()); - //assert the remaining transactions still isn't archived - Assert.Equal(allTransactions.FindAll(r => r.LSN > maxSLN).Select(r => r.LSN).Distinct(), commitTransactions.Select(tx => tx.LSN).Distinct()); - } - - private static async Task StorageFactory(AzureTransactionLogOptions azureOptions, AzureTransactionArchiveLogOptions archiveOptions) - { - var config = new ClientConfiguration(); - var environment = SerializationTestEnvironment.InitializeWithDefaults(config); - var azureConfig = Options.Create(azureOptions); - AzureTransactionLogStorage storage = new AzureTransactionLogStorage(environment.SerializationManager, azureConfig, - Options.Create(archiveOptions), Options.Create(new ClusterOptions(){ClusterId = Guid.NewGuid().ToString(), - ServiceId = ClusterServiceId})); - await storage.Initialize(); - return storage; - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Azure.Test/DistributedTM/GoldenPathTests.cs b/test/Transactions/Orleans.Transactions.Azure.Test/DistributedTM/GoldenPathTests.cs deleted file mode 100644 index 5f7f1328c8..0000000000 --- a/test/Transactions/Orleans.Transactions.Azure.Test/DistributedTM/GoldenPathTests.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Xunit; -using Xunit.Abstractions; -using Orleans.Transactions.Tests; - -namespace Orleans.Transactions.AzureStorage.Tests.DistributedTM -{ - [TestCategory("Azure"), TestCategory("Transactions"), TestCategory("Functional")] - public class GoldenPathTests : GoldenPathTransactionTestRunner, IClassFixture - { - public GoldenPathTests(TestFixture fixture, ITestOutputHelper output) - : base(fixture.GrainFactory, output, true) - { - fixture.EnsurePreconditionsMet(); - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Azure.Test/DistributedTM/GrainFaultTests.cs b/test/Transactions/Orleans.Transactions.Azure.Test/DistributedTM/GrainFaultTests.cs deleted file mode 100644 index a93ed534d9..0000000000 --- a/test/Transactions/Orleans.Transactions.Azure.Test/DistributedTM/GrainFaultTests.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Xunit; -using Xunit.Abstractions; -using Orleans.Transactions.Tests; - -namespace Orleans.Transactions.AzureStorage.Tests.DistributedTM -{ - [TestCategory("Azure"), TestCategory("Transactions"), TestCategory("Functional")] - public class GrainFaultTests : GrainFaultTransactionTestRunner, IClassFixture - { - public GrainFaultTests(TestFixture fixture, ITestOutputHelper output) - : base(fixture.GrainFactory, output, true) - { - fixture.EnsurePreconditionsMet(); - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Azure.Test/DistributedTM/TestFixture.cs b/test/Transactions/Orleans.Transactions.Azure.Test/DistributedTM/TestFixture.cs deleted file mode 100644 index 101fd62389..0000000000 --- a/test/Transactions/Orleans.Transactions.Azure.Test/DistributedTM/TestFixture.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System; -using Xunit; -using Orleans.Hosting; -using Orleans.TestingHost; -using Orleans.Transactions.Tests; -using Orleans.TestingHost.Utils; -using TestExtensions; -using Microsoft.Extensions.Logging; - -namespace Orleans.Transactions.AzureStorage.Tests.DistributedTM -{ - public class TestFixture : BaseTestClusterFixture - { - protected override void CheckPreconditionsOrThrow() - { - base.CheckPreconditionsOrThrow(); - CheckForAzureStorage(TestDefaultConfiguration.DataConnectionString); - } - - protected override void ConfigureTestCluster(TestClusterBuilder builder) - { - builder.AddSiloBuilderConfigurator(); - } - - public class SiloBuilderConfigurator : ISiloBuilderConfigurator - { - public void Configure(ISiloHostBuilder hostBuilder) - { - var id = (uint)Guid.NewGuid().GetHashCode() % 100000; - hostBuilder - .ConfigureLogging(builder => builder.AddFilter("SingleStateTransactionalGrain.data", LogLevel.Trace)) - .ConfigureLogging(builder => builder.AddFilter("DoubleStateTransactionalGrain.data", LogLevel.Trace)) - .ConfigureLogging(builder => builder.AddFilter("MaxStateTransactionalGrain.data", LogLevel.Trace)) - .ConfigureLogging(builder => builder.AddFilter("TransactionAgent", LogLevel.Trace)) - .ConfigureLogging(builder => builder.AddFilter("Orleans.Transactions.DistributedTM.AzureStorage.AzureTableTransactionalStateStorage", LogLevel.Trace)) - .AddAzureTableTransactionalStateStorage(TransactionTestConstants.TransactionStore, options => - { - options.ConnectionString = TestDefaultConfiguration.DataConnectionString; - }) - .UseDistributedTM(); - } - } - - - public static void CheckForAzureStorage(string dataConnectionString) - { - if (string.IsNullOrWhiteSpace(dataConnectionString)) - { - throw new SkipException("No connection string found. Skipping"); - } - - bool usingLocalWAS = string.Equals(dataConnectionString, "UseDevelopmentStorage=true", StringComparison.OrdinalIgnoreCase); - - if (!usingLocalWAS) - { - // Tests are using Azure Cloud Storage, not local WAS emulator. - return; - } - - //Starts the storage emulator if not started already and it exists (i.e. is installed). - if (!StorageEmulator.TryStart()) - { - throw new SkipException("Azure Storage Emulator could not be started."); - } - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Azure.Test/DistributedTM/TransactionRecoveryTests.cs b/test/Transactions/Orleans.Transactions.Azure.Test/DistributedTM/TransactionRecoveryTests.cs deleted file mode 100644 index bdd5132722..0000000000 --- a/test/Transactions/Orleans.Transactions.Azure.Test/DistributedTM/TransactionRecoveryTests.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Orleans.TestingHost; -using Orleans.Transactions.Tests; -using TestExtensions; -using Xunit; -using Xunit.Abstractions; - -namespace Orleans.Transactions.AzureStorage.Tests.DistributedTM -{ - [TestCategory("Transactions"), TestCategory("Functional")] - public class TransactionRecoveryTests : TestClusterPerTest - { - private readonly TransactionRecoveryTestsRunner testRunner; - - public TransactionRecoveryTests(ITestOutputHelper helper) - { - this.testRunner = new TransactionRecoveryTestsRunner(this.HostedCluster, helper, true); - } - - protected override void ConfigureTestCluster(TestClusterBuilder builder) - { - TestFixture.CheckForAzureStorage(TestDefaultConfiguration.DataConnectionString); - builder.Options.InitialSilosCount = 5; - builder.AddSiloBuilderConfigurator(); - } - - [SkippableTheory(Skip = "Intermittent failure, investigating...")] - [InlineData(TransactionTestConstants.TransactionGrainStates.SingleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.DoubleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.MaxStateTransaction)] - public Task TransactionWillRecoverAfterRandomSiloFailure(TransactionTestConstants.TransactionGrainStates transactionGrainStates) - { - return this.testRunner.TransactionWillRecoverAfterRandomSiloFailure(transactionGrainStates); - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Azure.Test/GoldenPathTransactionManagerTest.cs b/test/Transactions/Orleans.Transactions.Azure.Test/GoldenPathTransactionManagerTest.cs deleted file mode 100644 index 0005b51e45..0000000000 --- a/test/Transactions/Orleans.Transactions.Azure.Test/GoldenPathTransactionManagerTest.cs +++ /dev/null @@ -1,55 +0,0 @@ -using System; -using System.Threading.Tasks; -using Xunit.Abstractions; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using Orleans.Configuration; -using Orleans.Configuration.Development; -using Orleans.Runtime.Configuration; -using Orleans.Transactions.Abstractions; -using Orleans.Transactions.Tests; -using TestExtensions; -using Orleans.TestingHost.Utils; - -namespace Orleans.Transactions.AzureStorage.Tests -{ - [TestCategory("Azure"), TestCategory("Transactions"), TestCategory("Functional")] - public class GoldenPathTransactionManagerTest : GoldenPathTransactionManagerTestRunner - { - private static readonly TimeSpan LogMaintenanceInterval = TimeSpan.FromMilliseconds(10); - private static readonly TimeSpan StorageDelay = TimeSpan.FromSeconds(30); - public GoldenPathTransactionManagerTest(ITestOutputHelper output) - : base(MakeTransactionManager(), LogMaintenanceInterval, StorageDelay, output) - { - } - - private static ITransactionManager MakeTransactionManager() - { - TestFixture.CheckForAzureStorage(TestDefaultConfiguration.DataConnectionString); - ITransactionManager tm = new TransactionManager(new TransactionLog(StorageFactory), Options.Create(new TransactionsOptions()), NullLoggerFactory.Instance, NullTelemetryProducer.Instance, Options.Create(new StatisticsOptions()), LogMaintenanceInterval); - tm.StartAsync().GetAwaiter().GetResult(); - return tm; - } - - private static async Task StorageFactory() - { - TestFixture.CheckForAzureStorage(TestDefaultConfiguration.DataConnectionString); - var config = new ClientConfiguration(); - var environment = SerializationTestEnvironment.InitializeWithDefaults(config); - var azureConfig = Options.Create(new AzureTransactionLogOptions() - { - // TODO: Find better way for test isolation. - TableName = "TransactionLog", - ConnectionString = TestDefaultConfiguration.DataConnectionString - }); - AzureTransactionLogStorage storage = new AzureTransactionLogStorage(environment.SerializationManager, azureConfig, - Options.Create(new AzureTransactionArchiveLogOptions()), Options.Create(new ClusterOptions() - { - ClusterId = Guid.NewGuid().ToString(), - ServiceId = Guid.NewGuid().ToString() - })); - await storage.Initialize(); - return storage; - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Azure.Test/OrchestrationTests.cs b/test/Transactions/Orleans.Transactions.Azure.Test/OrchestrationTests.cs deleted file mode 100644 index f743b11b58..0000000000 --- a/test/Transactions/Orleans.Transactions.Azure.Test/OrchestrationTests.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Xunit; -using Xunit.Abstractions; -using Orleans.Transactions.Tests; - -namespace Orleans.Transactions.AzureStorage.Tests -{ - [TestCategory("Azure"), TestCategory("Transactions"), TestCategory("Functional")] - public class OrchestrationTests : OrchestrationsTransactionsTestRunner, IClassFixture - { - public OrchestrationTests(TestFixture fixture, ITestOutputHelper output) - : base(fixture.GrainFactory, output) - { - fixture.EnsurePreconditionsMet(); - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Azure.Test/TestFixture.cs b/test/Transactions/Orleans.Transactions.Azure.Test/TestFixture.cs index 49950aae1d..2940149e30 100644 --- a/test/Transactions/Orleans.Transactions.Azure.Test/TestFixture.cs +++ b/test/Transactions/Orleans.Transactions.Azure.Test/TestFixture.cs @@ -5,8 +5,6 @@ using Orleans.Transactions.Tests; using Orleans.TestingHost.Utils; using TestExtensions; -using Microsoft.Extensions.Options; -using Orleans.Configuration; using Microsoft.Extensions.Logging; namespace Orleans.Transactions.AzureStorage.Tests @@ -28,21 +26,17 @@ public class SiloBuilderConfigurator : ISiloBuilderConfigurator { public void Configure(ISiloHostBuilder hostBuilder) { - var id = (uint) Guid.NewGuid().GetHashCode() % 100000; hostBuilder - .ConfigureLogging(builder => builder.AddFilter("Orleans.Transactions.TransactionalState", LogLevel.Debug)) - .UseInClusterTransactionManager() - .UseAzureTransactionLog(options => - { - // TODO: Find better way for test isolation. Possibly different partition keys. - options.TableName = $"TransactionLog{id:X}"; - options.ConnectionString = TestDefaultConfiguration.DataConnectionString; - }) - .UseTransactionalState() + .ConfigureLogging(builder => builder.AddFilter("SingleStateTransactionalGrain.data", LogLevel.Trace)) + .ConfigureLogging(builder => builder.AddFilter("DoubleStateTransactionalGrain.data", LogLevel.Trace)) + .ConfigureLogging(builder => builder.AddFilter("MaxStateTransactionalGrain.data", LogLevel.Trace)) + .ConfigureLogging(builder => builder.AddFilter("TransactionAgent", LogLevel.Trace)) + .ConfigureLogging(builder => builder.AddFilter("Orleans.Transactions.AzureStorage.AzureTableTransactionalStateStorage", LogLevel.Trace)) .AddAzureTableTransactionalStateStorage(TransactionTestConstants.TransactionStore, options => { options.ConnectionString = TestDefaultConfiguration.DataConnectionString; - }); + }) + .UseDistributedTM(); } } diff --git a/test/Transactions/Orleans.Transactions.Azure.Test/TransactionRecoveryAzureTests.cs b/test/Transactions/Orleans.Transactions.Azure.Test/TransactionRecoveryAzureTests.cs index 39a07c6e3e..e0d6688c27 100644 --- a/test/Transactions/Orleans.Transactions.Azure.Test/TransactionRecoveryAzureTests.cs +++ b/test/Transactions/Orleans.Transactions.Azure.Test/TransactionRecoveryAzureTests.cs @@ -1,8 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using System.Threading.Tasks; using Orleans.TestingHost; using Orleans.Transactions.AzureStorage.Tests; using Orleans.Transactions.Tests; @@ -30,17 +26,12 @@ protected override void ConfigureTestCluster(TestClusterBuilder builder) } [SkippableTheory] - [InlineData(TransactionTestConstants.SingleStateTransactionalGrain, true)] - [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain, true)] - [InlineData(TransactionTestConstants.MaxStateTransactionalGrain, true)] - [InlineData(TransactionTestConstants.SingleStateTransactionalGrain, false)] - [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain, false)] - [InlineData(TransactionTestConstants.MaxStateTransactionalGrain, false)] - public Task TransactionWillRecoverAfterRandomSiloFailure(string transactionTestGrainClassName, - bool killSiloWhichRunsTm) + [InlineData(TransactionTestConstants.SingleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.MaxStateTransactionalGrain)] + public Task TransactionWillRecoverAfterRandomSiloFailure(string transactionTestGrainClassName) { - return this.testRunner.TransactionWillRecoverAfterRandomSiloFailure(transactionTestGrainClassName, - killSiloWhichRunsTm); + return this.testRunner.TransactionWillRecoverAfterRandomSiloFailure(transactionTestGrainClassName); } } } diff --git a/test/Transactions/Orleans.Transactions.DynamoDB.Test/GoldenPathTests.cs b/test/Transactions/Orleans.Transactions.DynamoDB.Test/GoldenPathTests.cs deleted file mode 100644 index 62631beb95..0000000000 --- a/test/Transactions/Orleans.Transactions.DynamoDB.Test/GoldenPathTests.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Xunit; -using Xunit.Abstractions; -using Orleans.Transactions.Tests; - -namespace Orleans.Transactions.DynamoDB.Tests -{ - [TestCategory("DynamoDb"), TestCategory("Transactions"), TestCategory("Functional")] - public class GoldenPathTests : GoldenPathTransactionTestRunner, IClassFixture - { - public GoldenPathTests(TestFixture fixture, ITestOutputHelper output) - : base(fixture.GrainFactory, output) - { - fixture.EnsurePreconditionsMet(); - } - } -} diff --git a/test/Transactions/Orleans.Transactions.DynamoDB.Test/GoldenPathTransactionManagerTest.cs b/test/Transactions/Orleans.Transactions.DynamoDB.Test/GoldenPathTransactionManagerTest.cs deleted file mode 100644 index a276dc8b32..0000000000 --- a/test/Transactions/Orleans.Transactions.DynamoDB.Test/GoldenPathTransactionManagerTest.cs +++ /dev/null @@ -1,53 +0,0 @@ -using System; -using System.Threading.Tasks; -using Xunit.Abstractions; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using Orleans.Configuration; -using Orleans.Runtime.Configuration; -using Orleans.Transactions.Abstractions; -using Orleans.Transactions.Tests; -using TestExtensions; -using Orleans.TestingHost.Utils; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.DependencyInjection; -using AWSUtils.Tests.StorageTests; - -namespace Orleans.Transactions.DynamoDB.Tests -{ - [TestCategory("DynamoDb"), TestCategory("Transactions"), TestCategory("Functional")] - public class GoldenPathTransactionManagerTest : GoldenPathTransactionManagerTestRunner - { - private static readonly TimeSpan LogMaintenanceInterval = TimeSpan.FromMilliseconds(10); - private static readonly TimeSpan StorageDelay = TimeSpan.FromSeconds(30); - - public GoldenPathTransactionManagerTest(ITestOutputHelper output) - : base(MakeTransactionManager(), LogMaintenanceInterval, StorageDelay, output) - { - } - - private static ITransactionManager MakeTransactionManager() - { - TestFixture.CheckForDynamoDBStorage(); - ITransactionManager tm = new TransactionManager(new TransactionLog(StorageFactory), Options.Create(new TransactionsOptions()), NullLoggerFactory.Instance, NullTelemetryProducer.Instance, Options.Create(new StatisticsOptions()), LogMaintenanceInterval); - tm.StartAsync().GetAwaiter().GetResult(); - return tm; - } - - private static async Task StorageFactory() - { - TestFixture.CheckForDynamoDBStorage(); - var config = new ClientConfiguration(); - var environment = SerializationTestEnvironment.InitializeWithDefaults(config); - var dynamoConfig = Options.Create(new DynamoDBTransactionLogOptions() - { - // TODO: Find better way for test isolation. - TableName = $"TransactionLog{((uint)Guid.NewGuid().GetHashCode()) % 100000}", - Service = AWSTestConstants.Service - }); - DynamoDBTransactionLogStorage storage = new DynamoDBTransactionLogStorage(environment.SerializationManager, dynamoConfig, environment.Client.ServiceProvider.GetRequiredService()); - await storage.Initialize(); - return storage; - } - } -} diff --git a/test/Transactions/Orleans.Transactions.DynamoDB.Test/GrainFaultTests.cs b/test/Transactions/Orleans.Transactions.DynamoDB.Test/GrainFaultTests.cs deleted file mode 100644 index 9c4df0bb88..0000000000 --- a/test/Transactions/Orleans.Transactions.DynamoDB.Test/GrainFaultTests.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Xunit; -using Xunit.Abstractions; -using Orleans.Transactions.Tests; - -namespace Orleans.Transactions.DynamoDB.Tests -{ - [TestCategory("DynamoDb"), TestCategory("Transactions"), TestCategory("Functional")] - public class GrainFaultTests : GrainFaultTransactionTestRunner, IClassFixture - { - public GrainFaultTests(TestFixture fixture, ITestOutputHelper output) - : base(fixture.GrainFactory, output) - { - fixture.EnsurePreconditionsMet(); - } - } -} diff --git a/test/Transactions/Orleans.Transactions.DynamoDB.Test/OrchestrationTests.cs b/test/Transactions/Orleans.Transactions.DynamoDB.Test/OrchestrationTests.cs deleted file mode 100644 index 76db752007..0000000000 --- a/test/Transactions/Orleans.Transactions.DynamoDB.Test/OrchestrationTests.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Xunit; -using Xunit.Abstractions; -using Orleans.Transactions.Tests; - -namespace Orleans.Transactions.DynamoDB.Tests -{ - [TestCategory("DynamoDb"), TestCategory("Transactions"), TestCategory("Functional")] - public class OrchestrationTests : OrchestrationsTransactionsTestRunner, IClassFixture - { - public OrchestrationTests(TestFixture fixture, ITestOutputHelper output) - : base(fixture.GrainFactory, output) - { - fixture.EnsurePreconditionsMet(); - } - } -} diff --git a/test/Transactions/Orleans.Transactions.DynamoDB.Test/Orleans.Transactions.DynamoDB.Test.csproj b/test/Transactions/Orleans.Transactions.DynamoDB.Test/Orleans.Transactions.DynamoDB.Test.csproj deleted file mode 100644 index e438a6145d..0000000000 --- a/test/Transactions/Orleans.Transactions.DynamoDB.Test/Orleans.Transactions.DynamoDB.Test.csproj +++ /dev/null @@ -1,20 +0,0 @@ - - - Orleans.Transactions.DynamoDB.Tests - Orleans.Transactions.DynamoDB.Tests - true - - - - - - - - - - - - - - - diff --git a/test/Transactions/Orleans.Transactions.DynamoDB.Test/TestFixture.cs b/test/Transactions/Orleans.Transactions.DynamoDB.Test/TestFixture.cs deleted file mode 100644 index c878ce5a91..0000000000 --- a/test/Transactions/Orleans.Transactions.DynamoDB.Test/TestFixture.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System; -using Xunit; -using Orleans.Hosting; -using Orleans.TestingHost; -using TestExtensions; -using AWSUtils.Tests.StorageTests; -using Orleans.Transactions.Tests; - -namespace Orleans.Transactions.DynamoDB.Tests -{ - public class TestFixture : BaseTestClusterFixture - { - protected override void CheckPreconditionsOrThrow() - { - base.CheckPreconditionsOrThrow(); - CheckForDynamoDBStorage(); - } - - protected override void ConfigureTestCluster(TestClusterBuilder builder) - { - builder.AddSiloBuilderConfigurator(); - } - - private class SiloBuilderConfigurator : ISiloBuilderConfigurator - { - public void Configure(ISiloHostBuilder hostBuilder) - { - var id = (uint) Guid.NewGuid().GetHashCode() % 100000; - hostBuilder - .UseInClusterTransactionManager() - .AddDynamoDBGrainStorage(TransactionTestConstants.TransactionStore, options => - { - options.Service = AWSTestConstants.Service; - }) - .UseDynamoDBTransactionLog(options => - { - // TODO: Find better way for test isolation. Possibly different partition keys. - options.TableName = $"TransactionLog{id:X}"; - options.Service = AWSTestConstants.Service; - }) - .UseTransactionalState(); - } - } - - public static void CheckForDynamoDBStorage() - { - if (!AWSTestConstants.IsDynamoDbAvailable) - throw new SkipException("Unable to connect to AWS DynamoDB simulator"); - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Tests/DistributedTM/GoldenPathTransactionMemoryTests.cs b/test/Transactions/Orleans.Transactions.Tests/DistributedTM/GoldenPathTransactionMemoryTests.cs deleted file mode 100644 index ff523f3d35..0000000000 --- a/test/Transactions/Orleans.Transactions.Tests/DistributedTM/GoldenPathTransactionMemoryTests.cs +++ /dev/null @@ -1,15 +0,0 @@ -using Xunit.Abstractions; -using Xunit; -using System; - -namespace Orleans.Transactions.Tests.DistributedTM -{ - [TestCategory("BVT"), TestCategory("Transactions")] - public class GoldenPathTransactionMemoryTests : GoldenPathTransactionTestRunner, IClassFixture - { - public GoldenPathTransactionMemoryTests(MemoryTransactionsFixture fixture, ITestOutputHelper output) - : base(fixture.GrainFactory, output, true) - { - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Tests/DistributedTM/GrainFaultTransactionMemoryTests.cs b/test/Transactions/Orleans.Transactions.Tests/DistributedTM/GrainFaultTransactionMemoryTests.cs deleted file mode 100644 index 3aa1ae9e62..0000000000 --- a/test/Transactions/Orleans.Transactions.Tests/DistributedTM/GrainFaultTransactionMemoryTests.cs +++ /dev/null @@ -1,15 +0,0 @@ -using Xunit.Abstractions; -using Xunit; -using System; - -namespace Orleans.Transactions.Tests.DistributedTM -{ - [TestCategory("BVT"), TestCategory("Transactions")] - public class GrainFaultTransactionMemoryTests : GrainFaultTransactionTestRunner, IClassFixture - { - public GrainFaultTransactionMemoryTests(MemoryTransactionsFixture fixture, ITestOutputHelper output) - : base(fixture.GrainFactory, output, true) - { - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Tests/DistributedTM/MemoryTransactionsFixture.cs b/test/Transactions/Orleans.Transactions.Tests/DistributedTM/MemoryTransactionsFixture.cs deleted file mode 100644 index 0936fd687e..0000000000 --- a/test/Transactions/Orleans.Transactions.Tests/DistributedTM/MemoryTransactionsFixture.cs +++ /dev/null @@ -1,27 +0,0 @@ -using Orleans.TestingHost; -using Orleans.Hosting; -using TestExtensions; -using Microsoft.Extensions.Logging; - -namespace Orleans.Transactions.Tests.DistributedTM -{ - public class MemoryTransactionsFixture : BaseTestClusterFixture - { - protected override void ConfigureTestCluster(TestClusterBuilder builder) - { - builder.AddSiloBuilderConfigurator(); - } - - public class SiloBuilderConfigurator : ISiloBuilderConfigurator - { - public void Configure(ISiloHostBuilder hostBuilder) - { - hostBuilder - .ConfigureLogging(builder => builder.AddFilter("SingleStateTransactionalGrain.data", LogLevel.Trace)) - .ConfigureLogging(builder => builder.AddFilter("TransactionAgent", LogLevel.Trace)) - .AddMemoryGrainStorage(TransactionTestConstants.TransactionStore) - .UseDistributedTM(); - } - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Tests/DistributedTM/TransactionRecoveryMemoryTests.cs b/test/Transactions/Orleans.Transactions.Tests/DistributedTM/TransactionRecoveryMemoryTests.cs deleted file mode 100644 index 504d672b60..0000000000 --- a/test/Transactions/Orleans.Transactions.Tests/DistributedTM/TransactionRecoveryMemoryTests.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Orleans.TestingHost; -using TestExtensions; -using Xunit; -using Xunit.Abstractions; - -namespace Orleans.Transactions.Tests.DistributedTM -{ - [TestCategory("Transactions"), TestCategory("Functional")] - public class TransactionRecoveryMemoryTests : TestClusterPerTest - { - private readonly TransactionRecoveryTestsRunner testRunner; - - public TransactionRecoveryMemoryTests(ITestOutputHelper helper) - { - this.testRunner = new TransactionRecoveryTestsRunner(this.HostedCluster, helper, true); - } - - protected override void ConfigureTestCluster(TestClusterBuilder builder) - { - builder.Options.InitialSilosCount = 5; - builder.AddSiloBuilderConfigurator(); - } - - [SkippableTheory(Skip = "Intermittent failure, investigating...")] - [InlineData(TransactionTestConstants.TransactionGrainStates.SingleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.DoubleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.MaxStateTransaction)] - public Task TransactionWillRecoverAfterRandomSiloFailure(TransactionTestConstants.TransactionGrainStates transactionGrainStates) - { - return this.testRunner.TransactionWillRecoverAfterRandomSiloFailure(transactionGrainStates); - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Tests/Grains/ITransactionOrchestrationResultGrain.cs b/test/Transactions/Orleans.Transactions.Tests/Grains/ITransactionOrchestrationResultGrain.cs deleted file mode 100644 index 33a02c2b68..0000000000 --- a/test/Transactions/Orleans.Transactions.Tests/Grains/ITransactionOrchestrationResultGrain.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace Orleans.Transactions.Tests -{ - public class TransactionOrchestrationResult - { - public HashSet Prepared { get; } = new HashSet(); - public HashSet Aborted { get; } = new HashSet(); - public HashSet Committed { get; } = new HashSet(); - } - - public interface ITransactionOrchestrationResultGrain : IGrainWithGuidKey - { - Task RecordPrepare(long transactionId); - Task RecordAbort(long transactionId); - Task RecordCommit(long transactionId); - Task GetResults(); - } -} diff --git a/test/Transactions/Orleans.Transactions.Tests/DistributedTM/MultiStateTransactionalGrain.cs b/test/Transactions/Orleans.Transactions.Tests/Grains/MultiStateTransactionalGrain.cs similarity index 79% rename from test/Transactions/Orleans.Transactions.Tests/DistributedTM/MultiStateTransactionalGrain.cs rename to test/Transactions/Orleans.Transactions.Tests/Grains/MultiStateTransactionalGrain.cs index 83c9eb1c8e..1b29319537 100644 --- a/test/Transactions/Orleans.Transactions.Tests/DistributedTM/MultiStateTransactionalGrain.cs +++ b/test/Transactions/Orleans.Transactions.Tests/Grains/MultiStateTransactionalGrain.cs @@ -1,17 +1,20 @@ using Microsoft.Extensions.Logging; using Orleans.Transactions.Abstractions; using System; -using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading.Tasks; -namespace Orleans.Transactions.Tests.DistributedTM +namespace Orleans.Transactions.Tests { + [Serializable] + public class GrainData + { + public int Value { get; set; } + } + public class MaxStateTransactionalGrain : MultiStateTransactionalGrainBaseClass { - public MaxStateTransactionalGrain( - Orleans.Transactions.DistributedTM.ITransactionalStateFactory stateFactory, + public MaxStateTransactionalGrain(ITransactionalStateFactory stateFactory, ILoggerFactory loggerFactory) : base(Enumerable.Range(0, TransactionTestConstants.MaxCoordinatedTransactions) .Select(i => stateFactory.Create(new TransactionalStateAttribute($"data{i}", TransactionTestConstants.TransactionStore))) @@ -25,11 +28,11 @@ public class DoubleStateTransactionalGrain : MultiStateTransactionalGrainBaseCla { public DoubleStateTransactionalGrain( [TransactionalState("data1", TransactionTestConstants.TransactionStore)] - Orleans.Transactions.DistributedTM.ITransactionalState data1, + ITransactionalState data1, [TransactionalState("data2", TransactionTestConstants.TransactionStore)] - Orleans.Transactions.DistributedTM.ITransactionalState data2, + ITransactionalState data2, ILoggerFactory loggerFactory) - : base(new Orleans.Transactions.DistributedTM.ITransactionalState[2] { data1, data2 }, loggerFactory) + : base(new ITransactionalState[2] { data1, data2 }, loggerFactory) { } } @@ -38,22 +41,30 @@ public class SingleStateTransactionalGrain : MultiStateTransactionalGrainBaseCla { public SingleStateTransactionalGrain( [TransactionalState("data", TransactionTestConstants.TransactionStore)] - Orleans.Transactions.DistributedTM.ITransactionalState data, + ITransactionalState data, ILoggerFactory loggerFactory) - : base(new Orleans.Transactions.DistributedTM.ITransactionalState[1] { data }, loggerFactory) + : base(new ITransactionalState[1] { data }, loggerFactory) { } } + public class NoStateTransactionalGrain : MultiStateTransactionalGrainBaseClass + { + public NoStateTransactionalGrain( + ILoggerFactory loggerFactory) + : base(Array.Empty>(), loggerFactory) + { + } + } public class MultiStateTransactionalGrainBaseClass : Grain, ITransactionTestGrain { - protected Orleans.Transactions.DistributedTM.ITransactionalState[] dataArray; + protected ITransactionalState[] dataArray; private readonly ILoggerFactory loggerFactory; protected ILogger logger; public MultiStateTransactionalGrainBaseClass( - Orleans.Transactions.DistributedTM.ITransactionalState[] dataArray, + ITransactionalState[] dataArray, ILoggerFactory loggerFactory) { this.dataArray = dataArray; diff --git a/test/Transactions/Orleans.Transactions.Tests/Grains/MultiStateTransactionalGrains.cs b/test/Transactions/Orleans.Transactions.Tests/Grains/MultiStateTransactionalGrains.cs deleted file mode 100644 index 57d05b5e23..0000000000 --- a/test/Transactions/Orleans.Transactions.Tests/Grains/MultiStateTransactionalGrains.cs +++ /dev/null @@ -1,128 +0,0 @@ -using System; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Orleans.Transactions.Abstractions; - -namespace Orleans.Transactions.Tests -{ - [Serializable] - public class GrainData - { - public int Value { get; set; } - } - - public class MaxStateTransactionalGrain : MultiStateTransactionalGrainBaseClass - { - public MaxStateTransactionalGrain( - ITransactionalStateFactory stateFactory, - ILoggerFactory loggerFactory) - : base(Enumerable.Range(0, TransactionTestConstants.MaxCoordinatedTransactions) - .Select(i => stateFactory.Create(new TransactionalStateAttribute($"data{i}", TransactionTestConstants.TransactionStore))) - .ToArray(), - loggerFactory) - { - } - } - - public class DoubleStateTransactionalGrain : MultiStateTransactionalGrainBaseClass - { - public DoubleStateTransactionalGrain( - [TransactionalState("data1", TransactionTestConstants.TransactionStore)] - ITransactionalState data1, - [TransactionalState("data2", TransactionTestConstants.TransactionStore)] - ITransactionalState data2, - ILoggerFactory loggerFactory) - : base(new ITransactionalState[2] { data1, data2 }, loggerFactory) - { - } - } - - public class SingleStateTransactionalGrain : MultiStateTransactionalGrainBaseClass - { - public SingleStateTransactionalGrain( - [TransactionalState("data", TransactionTestConstants.TransactionStore)] ITransactionalState data, - ILoggerFactory loggerFactory) - :base(new ITransactionalState[1]{data}, loggerFactory) - { - } - } - - public class MultiStateTransactionalGrainBaseClass: Grain, ITransactionTestGrain - { - protected ITransactionalState[] dataArray; - private readonly ILoggerFactory loggerFactory; - protected ILogger logger; - - public MultiStateTransactionalGrainBaseClass( - ITransactionalState[] dataArray, - ILoggerFactory loggerFactory) - { - this.dataArray = dataArray; - this.loggerFactory = loggerFactory; - } - - public override Task OnActivateAsync() - { - this.logger = this.loggerFactory.CreateLogger(this.GetGrainIdentity().ToString()); - return base.OnActivateAsync(); - } - - public Task Set(int newValue) - { - foreach (var data in this.dataArray) - { - Set(data, newValue, this.logger); - } - return Task.CompletedTask; - } - - public Task Add(int numberToAdd) - { - return Task.FromResult(this.dataArray.Select(data => Add(data, numberToAdd, this.logger)).ToArray()); - } - - public Task Get() - { - return Task.FromResult(this.dataArray.Select(data => Get(data, this.logger)).ToArray()); - } - - public Task AddAndThrow(int numberToAdd) - { - foreach (var data in dataArray) - { - Add(data, numberToAdd, this.logger); - } - throw new Exception($"{GetType().Name} test exception"); - } - - public Task Deactivate() - { - DeactivateOnIdle(); - return Task.CompletedTask; - } - - private static void Set(ITransactionalState data, int newValue, ILogger logger) - { - logger.LogInformation($"Setting from {data.State.Value} to {newValue}."); - data.State.Value = newValue; - data.Save(); - logger.LogInformation($"Set to {data.State.Value}."); - } - - private static int Get(ITransactionalState data, ILogger logger) - { - logger.LogInformation($"Get {data.State.Value}."); - return data.State.Value; - } - - private static int Add(ITransactionalState data, int numberToAdd, ILogger logger) - { - logger.LogInformation($"Adding {numberToAdd} to {data.State.Value}."); - data.State.Value += numberToAdd; - data.Save(); - logger.LogInformation($"Value after Adding {numberToAdd} is {data.State.Value}."); - return data.State.Value; - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Tests/Grains/TransactionOrchestrationGrain.cs b/test/Transactions/Orleans.Transactions.Tests/Grains/TransactionOrchestrationGrain.cs deleted file mode 100644 index f43d83c26e..0000000000 --- a/test/Transactions/Orleans.Transactions.Tests/Grains/TransactionOrchestrationGrain.cs +++ /dev/null @@ -1,153 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Orleans.Runtime; -using Orleans.Providers; -using Orleans.Transactions.Abstractions; -using Orleans.Transactions; - -namespace Orleans.Transactions.Tests -{ - public class TransactionOrchestrationGrain : Grain, ITransactionTestGrain - { - private readonly TransactionalResource resource = new TransactionalResource(); - - public override Task OnActivateAsync() - { - var resultGrain = this.GrainFactory.GetGrain(this.GetPrimaryKey()); - return this.resource.BindAsync(this, this.ServiceProvider, resultGrain); - } - - public Task Set(int newValue) - { - this.resource.JoinTransaction(); - return Task.CompletedTask; - } - - public Task Add(int numberToAdd) - { - this.resource.JoinTransaction(); - return Task.FromResult(new int[1]{0}); - } - - public Task Get() - { - this.resource.JoinTransaction(); - return Task.FromResult(new int[1]{0}); - } - - public Task AddAndThrow(int numberToAdd) - { - throw new NotImplementedException(nameof(AddAndThrow)); - } - - public Task Deactivate() - { - DeactivateOnIdle(); - return Task.CompletedTask; - } - - private class TransactionalResource : ITransactionalResource - { - private ITransactionOrchestrationResultGrain resultGrain; - private Grain grain; - - private ILogger logger; - private ITransactionalResource transactionalResource; - - private readonly List transactions= new List(); - - private TransactionalResourceVersion version; - private long stableVersion; - - private long writeLowerBound; - - public async Task Prepare(long transactionId, TransactionalResourceVersion? writeVersion, - TransactionalResourceVersion? readVersion) - { - await this.resultGrain.RecordPrepare(transactionId); - return true; - } - - public Task Abort(long transactionId) - { - logger.Info($"Transaction {transactionId} was aborted for grain {grain}."); - return this.resultGrain.RecordAbort(transactionId); - } - - public Task Commit(long transactionId) - { - logger.Info($"Transaction {transactionId} was committed for grain {grain}."); - this.stableVersion = transactionId; - return this.resultGrain.RecordCommit(transactionId); - } - - public void JoinTransaction() - { - TransactionInfo info = TransactionContext.GetRequiredTransactionInfo(); - logger.Info($"Grain {grain} is joining transaction {info.TransactionId}."); - - // are we already part of the transaction? - if (this.transactions.Contains(info.TransactionId)) - { - return; - } - - TransactionalResourceVersion readVersion; - if (!TryGetVersion(info.TransactionId, out readVersion)) - { - throw new OrleansTransactionVersionDeletedException(info.TransactionId.ToString()); - } - - if (info.IsReadOnly && readVersion.TransactionId > this.stableVersion) - { - throw new OrleansTransactionUnstableVersionException(info.TransactionId.ToString()); - } - - info.RecordRead(transactionalResource, readVersion, this.stableVersion); - - writeLowerBound = Math.Max(writeLowerBound, info.TransactionId - 1); - - if (this.version.TransactionId > info.TransactionId || this.writeLowerBound >= info.TransactionId) - { - throw new OrleansTransactionWaitDieException(info.TransactionId.ToString()); - } - - TransactionalResourceVersion nextVersion = TransactionalResourceVersion.Create(info.TransactionId, - this.version.TransactionId == info.TransactionId ? this.version.WriteNumber + 1 : 1); - - info.RecordWrite(transactionalResource, this.version, this.stableVersion); - - this.version = nextVersion; - - this.transactions.Remove(info.TransactionId); - } - - public async Task BindAsync(Grain containerGrain, IServiceProvider services, ITransactionOrchestrationResultGrain resultGrain) - { - this.grain = containerGrain; - this.resultGrain = resultGrain; - this.logger = services.GetService>(); - - // bind extension to grain - IProviderRuntime runtime = services.GetRequiredService(); - Tuple boundExtension = await runtime.BindExtension(() => new TransactionalExtension()); - boundExtension.Item1.Register(nameof(TransactionalResource), this); - this.transactionalResource = boundExtension.Item2.AsTransactionalResource(nameof(TransactionalResource)); - } - - public bool Equals(ITransactionalResource other) - { - return transactionalResource.Equals(other); - } - - private bool TryGetVersion(long transactionId, out TransactionalResourceVersion readVersion) - { - readVersion = this.version; - return this.version.TransactionId <= transactionId; - } - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Tests/Grains/TransactionOrchestrationResultGrain.cs b/test/Transactions/Orleans.Transactions.Tests/Grains/TransactionOrchestrationResultGrain.cs deleted file mode 100644 index e3913a4e82..0000000000 --- a/test/Transactions/Orleans.Transactions.Tests/Grains/TransactionOrchestrationResultGrain.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Orleans.Runtime; - -namespace Orleans.Transactions.Tests -{ - public class TransactionOrchestrationResultGrain : Grain, ITransactionOrchestrationResultGrain - { - private readonly TransactionOrchestrationResult orchestrationResult = new TransactionOrchestrationResult(); - private ILogger logger; - - public override Task OnActivateAsync() - { - this.logger = this.ServiceProvider.GetRequiredService>(); - return Task.CompletedTask; - } - - public Task RecordPrepare(long transactionId) - { - this.logger.Info($"Grain {this.GetPrimaryKey()} prepared transaction {transactionId}."); - this.orchestrationResult.Prepared.Add(transactionId); - return Task.CompletedTask; - } - - public Task RecordAbort(long transactionId) - { - this.logger.Info($"Grain {this.GetPrimaryKey()} aborted transaction {transactionId}."); - this.orchestrationResult.Aborted.Add(transactionId); - return Task.CompletedTask; - } - - public Task RecordCommit(long transactionId) - { - this.logger.Info($"Grain {this.GetPrimaryKey()} committed transaction {transactionId}."); - this.orchestrationResult.Committed.Add(transactionId); - return Task.CompletedTask; - } - - public Task GetResults() - { - this.logger.Info($"Reporting transaction orchestration results for Grain {this.GetPrimaryKey()}."); - return Task.FromResult(this.orchestrationResult); - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Tests/Memory/GoldenPathTransactionManagerMemoryTests.cs b/test/Transactions/Orleans.Transactions.Tests/Memory/GoldenPathTransactionManagerMemoryTests.cs deleted file mode 100644 index 51dd89409c..0000000000 --- a/test/Transactions/Orleans.Transactions.Tests/Memory/GoldenPathTransactionManagerMemoryTests.cs +++ /dev/null @@ -1,33 +0,0 @@ - -using System; -using System.Threading.Tasks; -using Xunit.Abstractions; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Logging.Abstractions; -using Orleans.Configuration; -using Orleans.Transactions.Abstractions; -using Orleans.Transactions.Development; -using Orleans.TestingHost.Utils; - -namespace Orleans.Transactions.Tests -{ - [TestCategory("BVT"), TestCategory("Transactions")] - public class GoldenPathTransactionManagerMemoryTests : GoldenPathTransactionManagerTestRunner - { - private static readonly TimeSpan LogMaintenanceInterval = TimeSpan.FromMilliseconds(10); - private static readonly TimeSpan StorageDelay = TimeSpan.FromMilliseconds(30); - - public GoldenPathTransactionManagerMemoryTests(ITestOutputHelper output) - : base(MakeTransactionManager(), LogMaintenanceInterval, StorageDelay, output) - { - } - - private static ITransactionManager MakeTransactionManager() - { - Factory> storageFactory = () => Task.FromResult(new InMemoryTransactionLogStorage()); - ITransactionManager tm = new TransactionManager(new TransactionLog(storageFactory), Options.Create(new TransactionsOptions()), NullLoggerFactory.Instance, NullTelemetryProducer.Instance, Options.Create(new StatisticsOptions()), LogMaintenanceInterval); - tm.StartAsync().GetAwaiter().GetResult(); - return tm; - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Tests/Memory/MemoryTransactionsFixture.cs b/test/Transactions/Orleans.Transactions.Tests/Memory/MemoryTransactionsFixture.cs index c87d7e76cc..c161b225e0 100644 --- a/test/Transactions/Orleans.Transactions.Tests/Memory/MemoryTransactionsFixture.cs +++ b/test/Transactions/Orleans.Transactions.Tests/Memory/MemoryTransactionsFixture.cs @@ -1,8 +1,5 @@ -using Orleans.Runtime.Configuration; using Orleans.TestingHost; using Orleans.Hosting; -using Orleans.Hosting.Development; -using Orleans.Logging; using TestExtensions; namespace Orleans.Transactions.Tests @@ -20,9 +17,7 @@ public void Configure(ISiloHostBuilder hostBuilder) { hostBuilder .AddMemoryGrainStorage(TransactionTestConstants.TransactionStore) - .UseInClusterTransactionManager() - .UseInMemoryTransactionLog() - .UseTransactionalState(); + .UseDistributedTM(); } } } diff --git a/test/Transactions/Orleans.Transactions.Tests/Memory/OrchestrationsTransactionsMemoryTests.cs b/test/Transactions/Orleans.Transactions.Tests/Memory/OrchestrationsTransactionsMemoryTests.cs deleted file mode 100644 index 431e3f1ce5..0000000000 --- a/test/Transactions/Orleans.Transactions.Tests/Memory/OrchestrationsTransactionsMemoryTests.cs +++ /dev/null @@ -1,14 +0,0 @@ -using Xunit.Abstractions; -using Xunit; - -namespace Orleans.Transactions.Tests -{ - [TestCategory("BVT"), TestCategory("Transactions")] - public class OrchestrationsTransactionsMemoryTests : OrchestrationsTransactionsTestRunner, IClassFixture - { - public OrchestrationsTransactionsMemoryTests(MemoryTransactionsFixture fixture, ITestOutputHelper output) - : base(fixture.GrainFactory, output) - { - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Tests/Memory/TransactionRecoveryMemoryTests.cs b/test/Transactions/Orleans.Transactions.Tests/Memory/TransactionRecoveryMemoryTests.cs index 2993851d6a..83a1dc6e83 100644 --- a/test/Transactions/Orleans.Transactions.Tests/Memory/TransactionRecoveryMemoryTests.cs +++ b/test/Transactions/Orleans.Transactions.Tests/Memory/TransactionRecoveryMemoryTests.cs @@ -28,17 +28,12 @@ protected override void ConfigureTestCluster(TestClusterBuilder builder) } [SkippableTheory] - [InlineData(TransactionTestConstants.SingleStateTransactionalGrain, true)] - [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain, true)] - [InlineData(TransactionTestConstants.MaxStateTransactionalGrain, true)] - [InlineData(TransactionTestConstants.SingleStateTransactionalGrain, false)] - [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain, false)] - [InlineData(TransactionTestConstants.MaxStateTransactionalGrain, false)] - public Task TransactionWillRecoverAfterRandomSiloFailure(string transactionTestGrainClassName, - bool killSiloWhichRunsTm) + [InlineData(TransactionTestConstants.SingleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.MaxStateTransactionalGrain)] + public Task TransactionWillRecoverAfterRandomSiloFailure(string transactionTestGrainClassName) { - return this.testRunner.TransactionWillRecoverAfterRandomSiloFailure(transactionTestGrainClassName, - killSiloWhichRunsTm); + return this.testRunner.TransactionWillRecoverAfterRandomSiloFailure(transactionTestGrainClassName); } } } diff --git a/test/Transactions/Orleans.Transactions.Tests/Runners/DisabledTransactionsTestRunner.cs b/test/Transactions/Orleans.Transactions.Tests/Runners/DisabledTransactionsTestRunner.cs index 851b3fa623..0ba9d00d93 100644 --- a/test/Transactions/Orleans.Transactions.Tests/Runners/DisabledTransactionsTestRunner.cs +++ b/test/Transactions/Orleans.Transactions.Tests/Runners/DisabledTransactionsTestRunner.cs @@ -13,7 +13,7 @@ protected DisabledTransactionsTestRunner(IGrainFactory grainFactory, ITestOutput : base(grainFactory, output) { } [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionOrchestrationGrain)] + [InlineData(TransactionTestConstants.NoStateTransactionalGrain)] public virtual async Task TransactionGrainsThrowWhenTransactions(string transactionTestGrainClassName) { const int delta = 5; @@ -23,7 +23,7 @@ public virtual async Task TransactionGrainsThrowWhenTransactions(string transact } [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionOrchestrationGrain)] + [InlineData(TransactionTestConstants.NoStateTransactionalGrain)] public virtual async Task MultiTransactionGrainsThrowWhenTransactions(string transactionTestGrainClassName) { const int delta = 5; diff --git a/test/Transactions/Orleans.Transactions.Tests/Runners/GoldenPathTransactionManagerTestRunner.cs b/test/Transactions/Orleans.Transactions.Tests/Runners/GoldenPathTransactionManagerTestRunner.cs deleted file mode 100644 index 36dbabac37..0000000000 --- a/test/Transactions/Orleans.Transactions.Tests/Runners/GoldenPathTransactionManagerTestRunner.cs +++ /dev/null @@ -1,145 +0,0 @@ -using System; -using System.Threading.Tasks; -using System.Diagnostics; -using Xunit; -using Xunit.Abstractions; -using Orleans.Transactions.Abstractions; - -namespace Orleans.Transactions.Tests -{ - public class GoldenPathTransactionManagerTestRunner : IDisposable - { - private readonly TimeSpan logMaintenanceInterval; - private readonly TimeSpan storageDelay; - private readonly ITestOutputHelper output; - private readonly ITransactionManager transactionManager; - - protected GoldenPathTransactionManagerTestRunner(ITransactionManager transactionManager, TimeSpan logMaintenanceInterval, TimeSpan storageDelay, ITestOutputHelper output) - { - this.transactionManager = transactionManager; - this.logMaintenanceInterval = logMaintenanceInterval; - this.storageDelay = storageDelay; - this.output = output; - } - - [SkippableFact] - public async Task StartCommitTransaction() - { - long id = this.transactionManager.StartTransaction(TimeSpan.FromTicks(this.storageDelay.Ticks*2)); - var info = new TransactionInfo(id); - this.transactionManager.CommitTransaction(info); - await WaitForTransactionCommit(id, this.logMaintenanceInterval + this.storageDelay); - } - - [SkippableFact(Skip = "Intermittent failure, jbragg investigating")] - public async Task TransactionTimeout() - { - long id = this.transactionManager.StartTransaction(TimeSpan.FromTicks(this.logMaintenanceInterval.Ticks / 2)); - await Task.Delay(logMaintenanceInterval); - await Assert.ThrowsAsync(() => WaitForTransactionCommit(id, this.logMaintenanceInterval + this.storageDelay)); - } - - [SkippableFact(Skip = "Intermittent failure, jbragg investigating")] - public async Task DependentTransaction() - { - long id1 = this.transactionManager.StartTransaction(TimeSpan.FromTicks(this.storageDelay.Ticks * 2)); - long id2 = this.transactionManager.StartTransaction(TimeSpan.FromTicks(this.storageDelay.Ticks * 4)); - - // commit transaction - var info = new TransactionInfo(id1); - Stopwatch sw = Stopwatch.StartNew(); - this.transactionManager.CommitTransaction(info); - sw.Stop(); - this.output.WriteLine($"Transaction {info} took {sw.ElapsedMilliseconds}ms to commit."); - - // resolve transaction - sw = Stopwatch.StartNew(); - await WaitForTransactionCommit(id1, this.logMaintenanceInterval + this.storageDelay); - sw.Stop(); - this.output.WriteLine($"Transaction {id1} took {sw.ElapsedMilliseconds}ms to resolve."); - - // commit dependent transaction - var info2 = new TransactionInfo(id2); - info2.DependentTransactions.Add(id1); - sw = Stopwatch.StartNew(); - this.transactionManager.CommitTransaction(info2); - sw.Stop(); - this.output.WriteLine($"Transaction {info2} took {sw.ElapsedMilliseconds}ms to commit."); - - // resolve dependent transaction - sw = Stopwatch.StartNew(); - await WaitForTransactionCommit(id2, this.logMaintenanceInterval + this.storageDelay); - sw.Stop(); - this.output.WriteLine($"Transaction {id2} took {sw.ElapsedMilliseconds}ms to resolve."); - } - - [SkippableFact] - public async Task OutOfOrderCommitTransaction() - { - long id1 = this.transactionManager.StartTransaction(TimeSpan.FromTicks(this.storageDelay.Ticks * 2)); - long id2 = this.transactionManager.StartTransaction(TimeSpan.FromTicks(this.storageDelay.Ticks * 4)); - - var info2 = new TransactionInfo(id2); - info2.DependentTransactions.Add(id1); - - this.transactionManager.CommitTransaction(info2); - OrleansTransactionAbortedException e; - Assert.True(this.transactionManager.GetTransactionStatus(id2, out e) == TransactionStatus.InProgress); - - var info = new TransactionInfo(id1); - this.transactionManager.CommitTransaction(info); - - await WaitForTransactionCommit(id2, this.logMaintenanceInterval + this.storageDelay); - } - - [SkippableFact] - public async Task CascadingAbortTransaction() - { - long id1 = this.transactionManager.StartTransaction(TimeSpan.FromTicks(this.storageDelay.Ticks * 2)); - long id2 = this.transactionManager.StartTransaction(TimeSpan.FromTicks(this.storageDelay.Ticks * 4)); - - var info2 = new TransactionInfo(id2); - info2.DependentTransactions.Add(id1); - - this.transactionManager.CommitTransaction(info2); - OrleansTransactionAbortedException abort; - Assert.True(this.transactionManager.GetTransactionStatus(id2, out abort) == TransactionStatus.InProgress); - - this.transactionManager.AbortTransaction(id1, new OrleansTransactionAbortedException(id1.ToString())); - - var e = await Assert.ThrowsAsync(() => WaitForTransactionCommit(id2, this.logMaintenanceInterval + this.storageDelay)); - Assert.True(e.TransactionId == id2.ToString()); - Assert.True(e.DependentTransactionId == id1.ToString()); - } - - private async Task WaitForTransactionCommit(long transactionId, TimeSpan timeout) - { - var endTime = DateTime.UtcNow + timeout; - while (DateTime.UtcNow < endTime) - { - OrleansTransactionAbortedException e; - var result = this.transactionManager.GetTransactionStatus(transactionId, out e); - switch (result) - { - case TransactionStatus.Committed: - return; - case TransactionStatus.Aborted: - throw e; - case TransactionStatus.Unknown: - throw new OrleansTransactionInDoubtException(transactionId.ToString()); - default: - Assert.True(result == TransactionStatus.InProgress); - await Task.Delay(logMaintenanceInterval); - break; - } - } - - throw new TimeoutException("Timed out waiting for the transaction to complete"); - } - - public void Dispose() - { - (this.transactionManager as IDisposable)?.Dispose(); - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Tests/Runners/GoldenPathTransactionTestRunner.cs b/test/Transactions/Orleans.Transactions.Tests/Runners/GoldenPathTransactionTestRunner.cs index d16a7842f3..f344907f1d 100644 --- a/test/Transactions/Orleans.Transactions.Tests/Runners/GoldenPathTransactionTestRunner.cs +++ b/test/Transactions/Orleans.Transactions.Tests/Runners/GoldenPathTransactionTestRunner.cs @@ -9,14 +9,14 @@ namespace Orleans.Transactions.Tests { public abstract class GoldenPathTransactionTestRunner : TransactionTestRunnerBase { - protected GoldenPathTransactionTestRunner(IGrainFactory grainFactory, ITestOutputHelper output, bool distributedTm = false) - : base(grainFactory, output, distributedTm) { } + protected GoldenPathTransactionTestRunner(IGrainFactory grainFactory, ITestOutputHelper output) + : base(grainFactory, output) { } [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionGrainStates.SingleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.DoubleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.MaxStateTransaction)] - public virtual async Task SingleGrainReadTransaction(TransactionTestConstants.TransactionGrainStates grainStates) + [InlineData(TransactionTestConstants.SingleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.MaxStateTransactionalGrain)] + public virtual async Task SingleGrainReadTransaction(string grainStates) { const int expected = 0; @@ -30,10 +30,10 @@ public virtual async Task SingleGrainReadTransaction(TransactionTestConstants.Tr } [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionGrainStates.SingleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.DoubleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.MaxStateTransaction)] - public virtual async Task SingleGrainWriteTransaction(TransactionTestConstants.TransactionGrainStates grainStates) + [InlineData(TransactionTestConstants.SingleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.MaxStateTransactionalGrain)] + public virtual async Task SingleGrainWriteTransaction(string grainStates) { const int delta = 5; ITransactionTestGrain grain = RandomTestGrain(grainStates); @@ -45,10 +45,10 @@ public virtual async Task SingleGrainWriteTransaction(TransactionTestConstants.T } [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionGrainStates.SingleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.DoubleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.MaxStateTransaction)] - public virtual async Task MultiGrainWriteTransaction(TransactionTestConstants.TransactionGrainStates grainStates) + [InlineData(TransactionTestConstants.SingleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.MaxStateTransactionalGrain)] + public virtual async Task MultiGrainWriteTransaction(string grainStates) { const int expected = 5; const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions; @@ -73,10 +73,10 @@ public virtual async Task MultiGrainWriteTransaction(TransactionTestConstants.Tr } [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionGrainStates.SingleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.DoubleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.MaxStateTransaction)] - public virtual async Task MultiGrainReadWriteTransaction(TransactionTestConstants.TransactionGrainStates grainStates) + [InlineData(TransactionTestConstants.SingleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.MaxStateTransactionalGrain)] + public virtual async Task MultiGrainReadWriteTransaction(string grainStates) { const int delta = 5; const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions; @@ -104,10 +104,10 @@ public virtual async Task MultiGrainReadWriteTransaction(TransactionTestConstant } [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionGrainStates.SingleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.DoubleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.MaxStateTransaction)] - public virtual async Task RepeatGrainReadWriteTransaction(TransactionTestConstants.TransactionGrainStates grainStates) + [InlineData(TransactionTestConstants.SingleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.MaxStateTransactionalGrain)] + public virtual async Task RepeatGrainReadWriteTransaction(string grainStates) { const int repeat = 10; const int delta = 5; @@ -142,10 +142,10 @@ public virtual async Task RepeatGrainReadWriteTransaction(TransactionTestConstan } [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionGrainStates.SingleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.DoubleStateTransaction)] - [InlineData(TransactionTestConstants.TransactionGrainStates.MaxStateTransaction)] - public virtual async Task MultiWriteToSingleGrainTransaction(TransactionTestConstants.TransactionGrainStates grainStates) + [InlineData(TransactionTestConstants.SingleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.MaxStateTransactionalGrain)] + public virtual async Task MultiWriteToSingleGrainTransaction(string grainStates) { const int delta = 5; const int concurrentWrites = 3; diff --git a/test/Transactions/Orleans.Transactions.Tests/Runners/GrainFaultTransactionTestRunner.cs b/test/Transactions/Orleans.Transactions.Tests/Runners/GrainFaultTransactionTestRunner.cs index a4755959e3..23dc8feaf2 100644 --- a/test/Transactions/Orleans.Transactions.Tests/Runners/GrainFaultTransactionTestRunner.cs +++ b/test/Transactions/Orleans.Transactions.Tests/Runners/GrainFaultTransactionTestRunner.cs @@ -9,13 +9,15 @@ namespace Orleans.Transactions.Tests { public abstract class GrainFaultTransactionTestRunner : TransactionTestRunnerBase { - public GrainFaultTransactionTestRunner(IGrainFactory grainFactory, ITestOutputHelper output, bool distributedTm = false) - : base(grainFactory, output, distributedTm) + public GrainFaultTransactionTestRunner(IGrainFactory grainFactory, ITestOutputHelper output) + : base(grainFactory, output) { } [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionGrainStates.SingleStateTransaction)] - public async Task AbortTransactionOnExceptions(TransactionTestConstants.TransactionGrainStates grainStates) + [InlineData(TransactionTestConstants.SingleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.MaxStateTransactionalGrain)] + public async Task AbortTransactionOnExceptions(string grainStates) { const int expected = 5; @@ -36,8 +38,10 @@ await TestAfterDustSettles(async () => } [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionGrainStates.SingleStateTransaction)] - public async Task MultiGrainAbortTransactionOnExceptions(TransactionTestConstants.TransactionGrainStates grainStates) + [InlineData(TransactionTestConstants.SingleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.MaxStateTransactionalGrain)] + public async Task MultiGrainAbortTransactionOnExceptions(string grainStates) { const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions - 1; const int expected = 5; @@ -69,8 +73,10 @@ await TestAfterDustSettles(async () => } [SkippableTheory(Skip = "Intermittent failure, jbragg investigating")] - [InlineData(TransactionTestConstants.TransactionGrainStates.SingleStateTransaction)] - public async Task AbortTransactionOnOrphanCalls(TransactionTestConstants.TransactionGrainStates grainStates) + [InlineData(TransactionTestConstants.SingleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.DoubleStateTransactionalGrain)] + [InlineData(TransactionTestConstants.MaxStateTransactionalGrain)] + public async Task AbortTransactionOnOrphanCalls(string grainStates) { const int expected = 5; diff --git a/test/Transactions/Orleans.Transactions.Tests/Runners/OrchestrationsTransactionsTestRunner.cs b/test/Transactions/Orleans.Transactions.Tests/Runners/OrchestrationsTransactionsTestRunner.cs deleted file mode 100644 index 6c740b57ef..0000000000 --- a/test/Transactions/Orleans.Transactions.Tests/Runners/OrchestrationsTransactionsTestRunner.cs +++ /dev/null @@ -1,152 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Xunit; -using Xunit.Abstractions; - -namespace Orleans.Transactions.Tests -{ - public abstract class OrchestrationsTransactionsTestRunner : TransactionTestRunnerBase - { - private readonly TimeSpan DefaultWaitTime = TimeSpan.FromSeconds(30); - private readonly TimeSpan waitTime; - protected OrchestrationsTransactionsTestRunner(IGrainFactory grainFactory, ITestOutputHelper output, TimeSpan? waitTime = null) - : base(grainFactory, output) - { - this.waitTime = waitTime ?? DefaultWaitTime; - } - - [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionOrchestrationGrain)] - public virtual async Task SingleGrainReadTransaction(string transactionTestGrainClassName) - { - Guid grainId = Guid.NewGuid(); - ITransactionTestGrain grain = TestGrain(transactionTestGrainClassName, grainId); - await grain.Get(); - await CheckReport(grainId, 1, 0); - } - - [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionOrchestrationGrain)] - public virtual async Task SingleGrainWriteTransaction(string transactionTestGrainClassName) - { - const int delta = 5; - Guid grainId = Guid.NewGuid(); - ITransactionTestGrain grain = TestGrain(transactionTestGrainClassName, grainId); - await grain.Get(); - await grain.Add(delta); - await grain.Get(); - await CheckReport(grainId, 3, 0); - } - - [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionOrchestrationGrain)] - public virtual async Task MultiGrainWriteTransaction(string transactionTestGrainClassName) - { - const int expected = 5; - const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions; - - List grainIds = Enumerable.Range(0, grainCount) - .Select(i => Guid.NewGuid()) - .ToList(); - List grains = grainIds - .Select(id => TestGrain(transactionTestGrainClassName, id)) - .ToList(); - - ITransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain(Guid.NewGuid()); - - await coordinator.MultiGrainAdd(grains, expected); - - foreach (var grain in grains) - { - await grain.Get(); - } - foreach (var grainId in grainIds) - { - await CheckReport(grainId, 2, 0); - } - } - - [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionOrchestrationGrain)] - public virtual async Task MultiGrainReadWriteTransaction(string transactionTestGrainClassName) - { - const int delta = 5; - const int grainCount = TransactionTestConstants.MaxCoordinatedTransactions; - - List grainIds = Enumerable.Range(0, grainCount) - .Select(i => Guid.NewGuid()) - .ToList(); - List grains = grainIds - .Select(id => TestGrain(transactionTestGrainClassName, id)) - .ToList(); - - ITransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain(Guid.NewGuid()); - - await coordinator.MultiGrainSet(grains, delta); - await coordinator.MultiGrainDouble(grains); - - foreach (var grain in grains) - { - await grain.Get(); - } - foreach (var grainId in grainIds) - { - await CheckReport(grainId, 3, 0); - } - } - - [SkippableTheory] - [InlineData(TransactionTestConstants.TransactionOrchestrationGrain)] - public virtual async Task MultiWriteToSingleGrainTransaction(string transactionTestGrainClassName) - { - const int delta = 5; - const int concurrentWrites = TransactionTestConstants.MaxCoordinatedTransactions; - - Guid grainId = Guid.NewGuid(); - ITransactionTestGrain grain = TestGrain(transactionTestGrainClassName, grainId); - List grains = Enumerable.Repeat(grain, concurrentWrites).ToList(); - - ITransactionCoordinatorGrain coordinator = this.grainFactory.GetGrain(Guid.NewGuid()); - - await coordinator.MultiGrainAdd(grains, delta); - - await grains[0].Get(); - await CheckReport(grainId, 2, 0); - } - - private async Task CheckReport(Guid grainId, int perpareCount, int abortCount) - { - var endTime = DateTime.UtcNow + this.waitTime; - while (DateTime.UtcNow < endTime) - { - bool passed = await CheckReport(grainId, perpareCount, abortCount, false); - if (passed) return; - await Task.Delay(this.waitTime.Milliseconds / 10); - } - await CheckReport(grainId, perpareCount, abortCount, true); - } - - private async Task CheckReport(Guid grainId, int perpareCount, int abortCount, bool assert) - { - var resultGrain = this.grainFactory.GetGrain(grainId); - - TransactionOrchestrationResult results = await resultGrain.GetResults(); - - if(assert) - { - Assert.Equal(perpareCount, results.Prepared.Count); - Assert.Equal(abortCount, results.Aborted.Count); - Assert.Equal(results.Prepared.Max(), results.Committed.Aggregate((long)0, (t1, t2) => Math.Max(t1, t2))); - } - else if(perpareCount != results.Prepared.Count || - abortCount != results.Aborted.Count || - results.Prepared.Max() != results.Committed.Aggregate((long)0, (t1, t2) => Math.Max(t1, t2))) - { - return false; - } - return true; - } - } -} diff --git a/test/Transactions/Orleans.Transactions.Tests/Runners/TransactionRecoveryTestsRunner.cs b/test/Transactions/Orleans.Transactions.Tests/Runners/TransactionRecoveryTestsRunner.cs index ebd1cec3f9..caf8a39d28 100644 --- a/test/Transactions/Orleans.Transactions.Tests/Runners/TransactionRecoveryTestsRunner.cs +++ b/test/Transactions/Orleans.Transactions.Tests/Runners/TransactionRecoveryTestsRunner.cs @@ -1,10 +1,6 @@ using System; -using System.Collections; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; -using System.Text; -using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Extensions.Logging; @@ -25,8 +21,8 @@ public class TransactionRecoveryTestsRunner : TransactionTestRunnerBase private readonly TestCluster testCluster; private readonly ILogger logger; private static TimeSpan RecoveryTimeout = TimeSpan.FromSeconds(60); - public TransactionRecoveryTestsRunner(TestCluster testCluster, ITestOutputHelper output, bool distributedTM = false) - : base(testCluster.GrainFactory, output, distributedTM) + public TransactionRecoveryTestsRunner(TestCluster testCluster, ITestOutputHelper output) + : base(testCluster.GrainFactory, output) { this.testCluster = testCluster; this.logger = this.testCluster.ServiceProvider.GetService>(); @@ -42,8 +38,7 @@ public void Configure(ISiloHostBuilder hostBuilder) } } - //For distributedTM - public virtual async Task TransactionWillRecoverAfterRandomSiloFailure(TransactionTestConstants.TransactionGrainStates transactionTestGrainClassName) + public virtual async Task TransactionWillRecoverAfterRandomSiloFailure(string transactionTestGrainClassName) { const int grainCount = 100; var txGrains = Enumerable.Range(0, grainCount) @@ -56,39 +51,20 @@ public virtual async Task TransactionWillRecoverAfterRandomSiloFailure(Transacti await TestingUtils.WaitUntilAsync(lastTry => CheckTxResult(txGrains, lastTry), RecoveryTimeout); } - //For singleTM , only in singleTM we have the need of test two scenarios, whic is kill silo which runs the TM and ramdom silo which doesn't - public virtual async Task TransactionWillRecoverAfterRandomSiloFailure(string transactionTestGrainClassName, bool killSiloWhichRunsTm) - { - const int grainCount = 100; - var txGrains = Enumerable.Range(0, grainCount) - .Select(i => RandomTestGrain(transactionTestGrainClassName)) - .ToList(); - var txSucceedBeforeInterruption = await AllTxSucceed(txGrains); - this.logger.LogInformation($"Tx succeed before interruption : {txSucceedBeforeInterruption}"); - if(killSiloWhichRunsTm) - await KillSilo(location => location.ActiveInCurrentSilo); - else - { - await KillSilo(location => !location.ActiveInCurrentSilo); - } - await TestingUtils.WaitUntilAsync(lastTry => CheckTxResult(txGrains, lastTry), RecoveryTimeout); - } - //given a predicate, whether to kill a silo - private async Task KillSilo(Func predicate) + private async Task KillRandomSilo() { var mgmt = this.testCluster.GrainFactory.GetGrain(0); object[] results = await mgmt.SendControlCommandToProvider(typeof(TMGrainLocator).FullName, typeof(TMGrainLocator).Name, 1); - this.logger.LogInformation($"Current TMGrainLocator list : {String.Join(";", results.Select(re => re.As().ToString()))}"); - var murderCandidates = results.Where(re => predicate(re as TMGrainLocator.TMGrainLocation)); - if (murderCandidates.Count() == 0) + this.logger.LogInformation($"Current TMGrainLocator list : {String.Join(";", results.Select(re => re.As().ToString()))}"); + if (results.Length == 0) throw new Exception("No silo fits the predicate, potential test configuration issues"); - var murderTarget = murderCandidates.ElementAt(this.seed.Next(murderCandidates.Count())) as TMGrainLocator.TMGrainLocation; + var murderTarget = results.ElementAt(this.seed.Next(results.Length)) as SiloAddress; this.logger.LogInformation($"Current hard kill target is {murderTarget}"); foreach (var siloHanle in this.testCluster.Silos) { - if(siloHanle.SiloAddress.Equals(murderTarget.CurrentSiloAddress)) + if(siloHanle.SiloAddress.Equals(murderTarget)) siloHanle.StopSilo(false); } } @@ -103,22 +79,7 @@ public TMGrainLocator(ILocalSiloDetails siloDetails) public Task ExecuteCommand(int command, object arg) { - return Task.FromResult(new TMGrainLocation() - { - ActiveInCurrentSilo = TransactionManagerGrain.IsActive, - CurrentSiloAddress = this.siloDetails.SiloAddress - }); - } - - public class TMGrainLocation - { - public bool ActiveInCurrentSilo { get; set; } - public SiloAddress CurrentSiloAddress { get; set; } - - public override string ToString() - { - return $"{nameof(ActiveInCurrentSilo)} : {this.ActiveInCurrentSilo}, {nameof(CurrentSiloAddress)} : {this.CurrentSiloAddress}"; - } + return Task.FromResult(this.siloDetails.SiloAddress); } } @@ -148,6 +109,7 @@ private async Task AllTxSucceed(IEnumerable txGrain } catch (Exception) { + base.output.WriteLine($"Some transactions failed. {tasks.Count(t => t.IsFaulted)} out of {tasks.Count} failed"); return false; } diff --git a/test/Transactions/Orleans.Transactions.Tests/Runners/TransactionTestRunnerBase.cs b/test/Transactions/Orleans.Transactions.Tests/Runners/TransactionTestRunnerBase.cs index ac8a99b11b..0dde3ddecd 100644 --- a/test/Transactions/Orleans.Transactions.Tests/Runners/TransactionTestRunnerBase.cs +++ b/test/Transactions/Orleans.Transactions.Tests/Runners/TransactionTestRunnerBase.cs @@ -1,5 +1,4 @@ using System; -using Xunit; using Xunit.Abstractions; namespace Orleans.Transactions.Tests @@ -8,23 +7,11 @@ public class TransactionTestRunnerBase { protected readonly IGrainFactory grainFactory; protected readonly ITestOutputHelper output; - private bool distributedTm; - protected TransactionTestRunnerBase(IGrainFactory grainFactory, ITestOutputHelper output, bool distributedTm = false) + protected TransactionTestRunnerBase(IGrainFactory grainFactory, ITestOutputHelper output) { this.output = output; this.grainFactory = grainFactory; - this.distributedTm = distributedTm; - } - - protected ITransactionTestGrain RandomTestGrain(TransactionTestConstants.TransactionGrainStates grainStates) - { - return TestGrain(grainStates, Guid.NewGuid()); - } - - protected virtual ITransactionTestGrain TestGrain(TransactionTestConstants.TransactionGrainStates grainStates, Guid id) - { - return TestGrain(GetTestGrainClassName(grainStates), id); } protected ITransactionTestGrain RandomTestGrain(string transactionTestGrainClassNames) @@ -36,23 +23,5 @@ protected virtual ITransactionTestGrain TestGrain(string transactionTestGrainCla { return grainFactory.GetGrain(id, transactionTestGrainClassName); } - - private string GetTestGrainClassName(TransactionTestConstants.TransactionGrainStates grainStates) - { - if(this.distributedTm) - { - if (grainStates == TransactionTestConstants.TransactionGrainStates.SingleStateTransaction) - return TransactionTestConstants.SingleStateTransactionalGrainDistributedTM; - if (grainStates == TransactionTestConstants.TransactionGrainStates.DoubleStateTransaction) - return TransactionTestConstants.DoubleStateTransactionalGrainDistributedTM; - if (grainStates == TransactionTestConstants.TransactionGrainStates.MaxStateTransaction) - return TransactionTestConstants.MaxStateTransactionalGrainDistributedTM; - throw new SkipException($"{grainStates} not supported when using distributed transaction manager."); - } - - if (grainStates == TransactionTestConstants.TransactionGrainStates.SingleStateTransaction) - return TransactionTestConstants.SingleStateTransactionalGrain; - throw new SkipException($"{grainStates} not supported when using distributed transaction manager."); - } } } diff --git a/test/Transactions/Orleans.Transactions.Tests/TransactionTestConstants.cs b/test/Transactions/Orleans.Transactions.Tests/TransactionTestConstants.cs index 505e8ae470..dc93d940c8 100644 --- a/test/Transactions/Orleans.Transactions.Tests/TransactionTestConstants.cs +++ b/test/Transactions/Orleans.Transactions.Tests/TransactionTestConstants.cs @@ -11,25 +11,10 @@ public static class TransactionTestConstants // storage providers public const string TransactionStore = "TransactionStore"; - // Transaction orchestration grains - public const string TransactionOrchestrationGrain = "Orleans.Transactions.Tests.TransactionOrchestrationGrain"; - - public enum TransactionGrainStates - { - SingleStateTransaction, - DoubleStateTransaction, - MaxStateTransaction - } - // grain implementations singleton TM + public const string NoStateTransactionalGrain = "Orleans.Transactions.Tests.NoStateTransactionalGrain"; public const string SingleStateTransactionalGrain = "Orleans.Transactions.Tests.SingleStateTransactionalGrain"; public const string DoubleStateTransactionalGrain = "Orleans.Transactions.Tests.DoubleStateTransactionalGrain"; public const string MaxStateTransactionalGrain = "Orleans.Transactions.Tests.MaxStateTransactionalGrain"; - - // grain implementations using distributed TM - public const string SingleStateTransactionalGrainDistributedTM = "Orleans.Transactions.Tests.DistributedTM.SingleStateTransactionalGrain"; - public const string DoubleStateTransactionalGrainDistributedTM = "Orleans.Transactions.Tests.DistributedTM.DoubleStateTransactionalGrain"; - public const string MaxStateTransactionalGrainDistributedTM = "Orleans.Transactions.Tests.DistributedTM.MaxStateTransactionalGrain"; - } }