From c11c9f9211826ea5c41d0b4b7d916b7af0ef9260 Mon Sep 17 00:00:00 2001 From: Mohit Chakraborty <8271806+Mohit-Chakraborty@users.noreply.github.com> Date: Sun, 7 Feb 2021 10:54:02 -0800 Subject: [PATCH] Rename a few properties in SearchIndexingBufferedSenderOptions (#18285) * Rename a few properties in SearchIndexingBufferedSenderOptions * Update samples and API using scripts * Update Batching.Publisher.BatchActionCount when handling a 413 --- .../Azure.Search.Documents.netstandard2.0.cs | 7 ++++--- .../samples/Sample05_IndexingDocuments.md | 2 +- .../src/Batching/Publisher.cs | 4 ++-- .../Batching/SearchIndexingBufferedSender.cs | 20 ++++++++++++------- .../SearchIndexingBufferedSenderOptions.cs | 6 +++--- .../src/Batching/SearchIndexingPublisher.cs | 8 +++++--- .../tests/Batching/BatchingTests.cs | 12 +++++------ .../Samples/Sample05_IndexingDocuments.cs | 2 +- 8 files changed, 35 insertions(+), 26 deletions(-) diff --git a/sdk/search/Azure.Search.Documents/api/Azure.Search.Documents.netstandard2.0.cs b/sdk/search/Azure.Search.Documents/api/Azure.Search.Documents.netstandard2.0.cs index c5a2d505b30e7..1122464ff8af6 100644 --- a/sdk/search/Azure.Search.Documents/api/Azure.Search.Documents.netstandard2.0.cs +++ b/sdk/search/Azure.Search.Documents/api/Azure.Search.Documents.netstandard2.0.cs @@ -82,13 +82,14 @@ public SearchIndexingBufferedSenderOptions() { } public System.Threading.CancellationToken FlushCancellationToken { get { throw null; } set { } } public int? InitialBatchActionCount { get { throw null; } set { } } public System.Func KeyFieldAccessor { get { throw null; } set { } } - public int MaxRetries { get { throw null; } set { } } - public System.TimeSpan MaxRetryDelay { get { throw null; } set { } } - public System.TimeSpan RetryDelay { get { throw null; } set { } } + public int MaxRetriesPerIndexAction { get { throw null; } set { } } + public System.TimeSpan MaxThrottlingDelay { get { throw null; } set { } } + public System.TimeSpan ThrottlingDelay { get { throw null; } set { } } } public partial class SearchIndexingBufferedSender : System.IAsyncDisposable, System.IDisposable { protected SearchIndexingBufferedSender() { } + public SearchIndexingBufferedSender(Azure.Search.Documents.SearchClient searchClient, Azure.Search.Documents.SearchIndexingBufferedSenderOptions options = null) { } public virtual System.Uri Endpoint { get { throw null; } } public virtual string IndexName { get { throw null; } } public virtual string ServiceName { get { throw null; } } diff --git a/sdk/search/Azure.Search.Documents/samples/Sample05_IndexingDocuments.md b/sdk/search/Azure.Search.Documents/samples/Sample05_IndexingDocuments.md index e4fb18700d927..32105b673b049 100644 --- a/sdk/search/Azure.Search.Documents/samples/Sample05_IndexingDocuments.md +++ b/sdk/search/Azure.Search.Documents/samples/Sample05_IndexingDocuments.md @@ -126,7 +126,7 @@ again. ```C# Snippet:Azure_Search_Documents_Tests_Samples_Sample05_IndexingDocuments_BufferedSender1 await using SearchIndexingBufferedSender indexer = - searchClient.CreateIndexingBufferedSender(); + new SearchIndexingBufferedSender(searchClient); await indexer.UploadDocumentsAsync(GenerateCatalog(count: 100000)); ``` diff --git a/sdk/search/Azure.Search.Documents/src/Batching/Publisher.cs b/sdk/search/Azure.Search.Documents/src/Batching/Publisher.cs index 52267ec5f6d22..3d347f90da329 100644 --- a/sdk/search/Azure.Search.Documents/src/Batching/Publisher.cs +++ b/sdk/search/Azure.Search.Documents/src/Batching/Publisher.cs @@ -101,10 +101,10 @@ public int IndexingActionsCount protected CancellationToken PublisherCancellationToken { get; } /// - /// Gets a value indicating the number of actions to group into a batch + /// Gets or sets a value indicating the number of actions to group into a batch /// when tuning the behavior of the publisher. /// - protected int BatchActionCount { get; } // TODO: Not automatically tuning yet + protected int BatchActionCount { get; set; } /// /// Gets a value indicating the number of bytes to use when tuning the diff --git a/sdk/search/Azure.Search.Documents/src/Batching/SearchIndexingBufferedSender.cs b/sdk/search/Azure.Search.Documents/src/Batching/SearchIndexingBufferedSender.cs index 36851bb0bce8b..cffac85504e06 100644 --- a/sdk/search/Azure.Search.Documents/src/Batching/SearchIndexingBufferedSender.cs +++ b/sdk/search/Azure.Search.Documents/src/Batching/SearchIndexingBufferedSender.cs @@ -36,7 +36,7 @@ public class SearchIndexingBufferedSender : IDisposable, IAsyncDisposable /// The single publisher responsible for submitting requests. /// #pragma warning disable CA2213 // Member should be disposed. Disposed in DisposeAsync - private SearchIndexingPublisher _publisher; + private readonly SearchIndexingPublisher _publisher; #pragma warning restore CA2213 // Member should be disposed. Disposed in DisposeAsync /// @@ -103,15 +103,21 @@ public class SearchIndexingBufferedSender : IDisposable, IAsyncDisposable protected SearchIndexingBufferedSender() { } /// - /// Creates a new instance of the SearchIndexingBufferedSender. + /// Creates a new instance of that + /// can be used to index search documents with intelligent batching, + /// automatic flushing, and retries for failed indexing actions. /// /// /// The SearchClient used to send requests to the service. /// /// - /// Provides the configuration options for the sender. + /// The to + /// customize the sender's behavior. /// - internal SearchIndexingBufferedSender( + /// + /// Thrown when the is null. + /// + public SearchIndexingBufferedSender( SearchClient searchClient, SearchIndexingBufferedSenderOptions options = null) { @@ -126,9 +132,9 @@ internal SearchIndexingBufferedSender( options.AutoFlushInterval, options.InitialBatchActionCount, options.BatchPayloadSize, - options.MaxRetries, - options.RetryDelay, - options.MaxRetryDelay, + options.MaxRetriesPerIndexAction, + options.ThrottlingDelay, + options.MaxThrottlingDelay, options.FlushCancellationToken); } diff --git a/sdk/search/Azure.Search.Documents/src/Batching/SearchIndexingBufferedSenderOptions.cs b/sdk/search/Azure.Search.Documents/src/Batching/SearchIndexingBufferedSenderOptions.cs index fb6699d2d1074..e526f32fc9e67 100644 --- a/sdk/search/Azure.Search.Documents/src/Batching/SearchIndexingBufferedSenderOptions.cs +++ b/sdk/search/Azure.Search.Documents/src/Batching/SearchIndexingBufferedSenderOptions.cs @@ -63,7 +63,7 @@ public class SearchIndexingBufferedSenderOptions /// to control the number of attempts we will make to submit an indexing /// action. /// - public int MaxRetries { get; set; } = 3; + public int MaxRetriesPerIndexAction { get; set; } = 3; /// /// The initial retry delay. The delay will increase exponentially with @@ -73,7 +73,7 @@ public class SearchIndexingBufferedSenderOptions /// is used to add delay between additional batch submissions when our /// requests are being throttled by the service. /// - public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(0.8); + public TimeSpan ThrottlingDelay { get; set; } = TimeSpan.FromSeconds(0.8); /// /// The maximum permissible delay between retry attempts. Note that @@ -82,7 +82,7 @@ public class SearchIndexingBufferedSenderOptions /// property is used to add delay between additional batch /// submissions when our requests are being throttled by the service. /// - public TimeSpan MaxRetryDelay { get; set; } = TimeSpan.FromMinutes(1); + public TimeSpan MaxThrottlingDelay { get; set; } = TimeSpan.FromMinutes(1); /// /// Gets or sets a function that can be used to access the index key diff --git a/sdk/search/Azure.Search.Documents/src/Batching/SearchIndexingPublisher.cs b/sdk/search/Azure.Search.Documents/src/Batching/SearchIndexingPublisher.cs index ca435dcab33dd..cf97ef8a47d50 100644 --- a/sdk/search/Azure.Search.Documents/src/Batching/SearchIndexingPublisher.cs +++ b/sdk/search/Azure.Search.Documents/src/Batching/SearchIndexingPublisher.cs @@ -137,9 +137,11 @@ protected override async Task OnSubmitBatchAsync(IList>>(batch.Take(half)); - foreach (PublisherAction> action in batch.Skip(half)) + // Update 'BatchActionCount' so future submissions can avoid this error. + BatchActionCount = (int)Math.Floor((double)batch.Count / 2.0); + + var smaller = new List>>(batch.Take(BatchActionCount)); + foreach (PublisherAction> action in batch.Skip(BatchActionCount)) { // Add the second half to the retry queue without // counting this as a retry attempt diff --git a/sdk/search/Azure.Search.Documents/tests/Batching/BatchingTests.cs b/sdk/search/Azure.Search.Documents/tests/Batching/BatchingTests.cs index 7ca17cbec6d1b..9ea7f05335530 100644 --- a/sdk/search/Azure.Search.Documents/tests/Batching/BatchingTests.cs +++ b/sdk/search/Azure.Search.Documents/tests/Batching/BatchingTests.cs @@ -1184,8 +1184,8 @@ public async Task Behavior_MaxRetries() client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions() { - MaxRetries = 5, - MaxRetryDelay = TimeSpan.FromSeconds(1) + MaxRetriesPerIndexAction = 5, + MaxThrottlingDelay = TimeSpan.FromSeconds(1) }); // Keep 503ing to count the retries @@ -1213,8 +1213,8 @@ public async Task Behavior_RetryDelay() client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions() { - MaxRetries = 1, - RetryDelay = TimeSpan.FromSeconds(3) + MaxRetriesPerIndexAction = 1, + ThrottlingDelay = TimeSpan.FromSeconds(3) }); // Keep 503ing to trigger delays @@ -1245,8 +1245,8 @@ public async Task Behavior_MaxRetryDelay() client.CreateIndexingBufferedSender( new SearchIndexingBufferedSenderOptions() { - MaxRetries = 10, - MaxRetryDelay = TimeSpan.FromSeconds(1) + MaxRetriesPerIndexAction = 10, + MaxThrottlingDelay = TimeSpan.FromSeconds(1) }); // Keep 503ing to trigger delays diff --git a/sdk/search/Azure.Search.Documents/tests/Samples/Sample05_IndexingDocuments.cs b/sdk/search/Azure.Search.Documents/tests/Samples/Sample05_IndexingDocuments.cs index f46aafae2b5a5..a621f8f798ebf 100644 --- a/sdk/search/Azure.Search.Documents/tests/Samples/Sample05_IndexingDocuments.cs +++ b/sdk/search/Azure.Search.Documents/tests/Samples/Sample05_IndexingDocuments.cs @@ -164,7 +164,7 @@ public async Task BufferedSender() { #region Snippet:Azure_Search_Documents_Tests_Samples_Sample05_IndexingDocuments_BufferedSender1 await using SearchIndexingBufferedSender indexer = - searchClient.CreateIndexingBufferedSender(); + new SearchIndexingBufferedSender(searchClient); await indexer.UploadDocumentsAsync(GenerateCatalog(count: 100000)); #endregion }