diff --git a/Cogworks.AzureSearch/Extensions/EnumerableExtensions.cs b/Cogworks.AzureSearch/Extensions/EnumerableExtensions.cs index 316484f..5a4f36c 100644 --- a/Cogworks.AzureSearch/Extensions/EnumerableExtensions.cs +++ b/Cogworks.AzureSearch/Extensions/EnumerableExtensions.cs @@ -7,5 +7,12 @@ internal static class EnumerableExtensions { public static bool HasAny(this IEnumerable items) => items != null && items.Any(); + + public static IEnumerable> ChunkBy(this IEnumerable items, int chunkSize) + => items + .Select((x, i) => new { Index = i, Value = x }) + .GroupBy(x => x.Index / chunkSize) + .Select(x => x.Select(v => v.Value).ToList()) + .ToList(); } } \ No newline at end of file diff --git a/Cogworks.AzureSearch/Repositories/AzureSearchRepository.cs b/Cogworks.AzureSearch/Repositories/AzureSearchRepository.cs index ac2d130..56ed0c6 100644 --- a/Cogworks.AzureSearch/Repositories/AzureSearchRepository.cs +++ b/Cogworks.AzureSearch/Repositories/AzureSearchRepository.cs @@ -44,6 +44,8 @@ public class AzureSearchRepository : IAzureSearchRepository azureIndexDefinition, AzureSearchClientOption azureSearchClientOption) { _azureIndexDefinition = azureIndexDefinition; @@ -128,26 +130,30 @@ public async Task AddOrUpdateDocumentsAsync( }; } - var batchActions = models + var chunkedBatchActions = models .Select(model => new IndexAction(model, IndexActionType.Upload)) + .ChunkBy(BatchOperationSize) .ToList(); - var batch = IndexBatch.New(batchActions); - - var indexResults = Enumerable.Empty(); + var indexResults = new List(); - try + foreach (var batchActions in chunkedBatchActions) { - var result = await _searchIndex.Documents.IndexAsync(batch); - indexResults = result.Results; - } - catch (IndexBatchException indexBatchException) - { - indexResults = indexBatchException.IndexingResults; - } - catch (Exception exception) - { - // todo: handle it proper + var batch = IndexBatch.New(batchActions); + + try + { + var result = await _searchIndex.Documents.IndexAsync(batch); + indexResults.AddRange(result.Results); + } + catch (IndexBatchException indexBatchException) + { + indexResults.AddRange(indexBatchException.IndexingResults); + } + catch (Exception exception) + { + // todo: handle it proper + } } return GetBatchOperationStatus(indexResults, "adding or updating"); @@ -173,25 +179,30 @@ public async Task TryRemoveDocumentsAsync(IE }; } - var batchActions = models + var chunkedBatchActions = models .Select(model => new IndexAction(model, IndexActionType.Delete)) + .ChunkBy(BatchOperationSize) .ToList(); - var batch = IndexBatch.New(batchActions); - var indexResults = Enumerable.Empty(); + var indexResults = new List(); - try - { - var result = await _searchIndex.Documents.IndexAsync(batch); - indexResults = result.Results; - } - catch (IndexBatchException indexBatchException) + foreach (var batchActions in chunkedBatchActions) { - indexResults = indexBatchException.IndexingResults; - } - catch (Exception exception) - { - // todo: handle it proper + var batch = IndexBatch.New(batchActions); + + try + { + var result = await _searchIndex.Documents.IndexAsync(batch); + indexResults.AddRange(result.Results); + } + catch (IndexBatchException indexBatchException) + { + indexResults.AddRange(indexBatchException.IndexingResults); + } + catch (Exception exception) + { + // todo: handle it proper + } } return GetBatchOperationStatus(indexResults, "removing");