diff --git a/src/Umbraco.Core/Constants-Sql.cs b/src/Umbraco.Core/Constants-Sql.cs new file mode 100644 index 000000000000..366dfbe44ab4 --- /dev/null +++ b/src/Umbraco.Core/Constants-Sql.cs @@ -0,0 +1,17 @@ +namespace Umbraco.Core +{ + public static partial class Constants + { + public static class Sql + { + /// + /// The maximum amount of parameters that can be used in a query. + /// + /// + /// The actual limit is 2100 (https://docs.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server), + /// but we want to ensure there's room for additional parameters if this value is used to create groups/batches. + /// + public const int MaxParameterCount = 2000; + } + } +} diff --git a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs index 77cc0d66019b..a50e3c2aaa9f 100644 --- a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs +++ b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs @@ -137,7 +137,7 @@ internal static IDbCommand[] GenerateBulkInsertCommands(this IUmbracoDatabase // Math.Floor(2100 / 8) = 262 record per command // 4168 / 262 = 15.908... = there will be 16 command in total // (if we have disabled db parameters, then all records will be included, in only one command) - var recordsPerCommand = paramsPerRecord == 0 ? int.MaxValue : Convert.ToInt32(Math.Floor(2000.00 / paramsPerRecord)); + var recordsPerCommand = paramsPerRecord == 0 ? int.MaxValue : Convert.ToInt32(Math.Floor((double)Constants.Sql.MaxParameterCount / paramsPerRecord)); var commandsCount = Convert.ToInt32(Math.Ceiling((double)records.Length / recordsPerCommand)); var commands = new IDbCommand[commandsCount]; diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs index c3d34cc3e9f8..d921bb6d5153 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs @@ -54,7 +54,7 @@ protected override IEnumerable PerformGetAll(params int[] ids) var entries = new List(); - foreach (var group in ids.InGroupsOf(2000)) + foreach (var group in ids.InGroupsOf(Constants.Sql.MaxParameterCount)) { var sql = Sql() .Select() diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs index 85814ef681cf..a6b08c2bfd2b 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs @@ -638,7 +638,7 @@ protected IDictionary GetPropertyCollections(List(versions, 2000, batch => + var allPropertyDataDtos = Database.FetchByGroups(versions, Constants.Sql.MaxParameterCount, batch => SqlContext.Sql() .Select() .From() @@ -647,7 +647,7 @@ protected IDictionary GetPropertyCollections(List x.PropertyTypeId).Distinct().ToList(); - var allPropertyTypeDtos = Database.FetchByGroups(allPropertyTypeIds, 2000, batch => + var allPropertyTypeDtos = Database.FetchByGroups(allPropertyTypeIds, Constants.Sql.MaxParameterCount, batch => SqlContext.Sql() .Select(r => r.Select(x => x.DataTypeDto)) .From() diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs index aff53e44b0af..3a74b9209b61 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs @@ -768,7 +768,7 @@ private void CopyTagData(int? sourceLanguageId, int? targetLanguageId, IReadOnly // note: important to use SqlNullableEquals for nullable types, cannot directly compare language identifiers var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0); - if (whereInArgsCount > 2000) + if (whereInArgsCount > Constants.Sql.MaxParameterCount) throw new NotSupportedException("Too many property/content types."); // delete existing relations (for target language) @@ -906,7 +906,7 @@ private void CopyPropertyData(int? sourceLanguageId, int? targetLanguageId, IRea // note: important to use SqlNullableEquals for nullable types, cannot directly compare language identifiers // var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0); - if (whereInArgsCount > 2000) + if (whereInArgsCount > Constants.Sql.MaxParameterCount) throw new NotSupportedException("Too many property/content types."); //first clear out any existing property data that might already exists under the target language @@ -1005,7 +1005,7 @@ private void RenormalizeDocumentEditedFlags(IReadOnlyCollection propertyTyp //based on the current variance of each item to see if it's 'edited' value should be true/false. var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0); - if (whereInArgsCount > 2000) + if (whereInArgsCount > Constants.Sql.MaxParameterCount) throw new NotSupportedException("Too many property/content types."); var propertySql = Sql() @@ -1094,14 +1094,20 @@ private void RenormalizeDocumentEditedFlags(IReadOnlyCollection propertyTyp } } - //lookup all matching rows in umbracoDocumentCultureVariation - var docCultureVariationsToUpdate = editedLanguageVersions.InGroupsOf(2000) - .SelectMany(_ => Database.Fetch( - Sql().Select().From() - .WhereIn(x => x.LanguageId, editedLanguageVersions.Keys.Select(x => x.langId).ToList()) - .WhereIn(x => x.NodeId, editedLanguageVersions.Keys.Select(x => x.nodeId)))) - //convert to dictionary with the same key type - .ToDictionary(x => (x.NodeId, (int?)x.LanguageId), x => x); + // lookup all matching rows in umbracoDocumentCultureVariation + // fetch in batches to account for maximum parameter count (distinct languages can't exceed 2000) + var languageIds = editedLanguageVersions.Keys.Select(x => x.langId).Distinct().ToArray(); + var nodeIds = editedLanguageVersions.Keys.Select(x => x.nodeId).Distinct(); + var docCultureVariationsToUpdate = nodeIds.InGroupsOf(Constants.Sql.MaxParameterCount - languageIds.Length) + .SelectMany(group => + { + var sql = Sql().Select().From() + .WhereIn(x => x.LanguageId, languageIds) + .WhereIn(x => x.NodeId, group); + + return Database.Fetch(sql); + }) + .ToDictionary(x => (x.NodeId, (int?)x.LanguageId), x => x); //convert to dictionary with the same key type var toUpdate = new List(); foreach (var ev in editedLanguageVersions) diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs index ac1f7c3f2a94..da957a728858 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs @@ -260,13 +260,12 @@ public IEnumerable GetDictionaryItemDescendants(Guid? parentId) Func>> getItemsFromParents = guids => { - //needs to be in groups of 2000 because we are doing an IN clause and there's a max parameter count that can be used. - return guids.InGroupsOf(2000) - .Select(@group => + return guids.InGroupsOf(Constants.Sql.MaxParameterCount) + .Select(group => { var sqlClause = GetBaseQuery(false) .Where(x => x.Parent != null) - .Where($"{SqlSyntax.GetQuotedColumnName("parent")} IN (@parentIds)", new { parentIds = @group }); + .WhereIn(x => x.Parent, group); var translator = new SqlTranslator(sqlClause, Query()); var sql = translator.Translate(); diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs index 5716fbe129c4..753630d186d0 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs @@ -1342,7 +1342,7 @@ private IDictionary GetContentSchedule(params in { var result = new Dictionary(); - var scheduleDtos = Database.FetchByGroups(contentIds, 2000, batch => Sql() + var scheduleDtos = Database.FetchByGroups(contentIds, Constants.Sql.MaxParameterCount, batch => Sql() .Select() .From() .WhereIn(x => x.NodeId, batch)); @@ -1391,7 +1391,7 @@ private IDictionary> GetContentVariations(List>(); - var dtos = Database.FetchByGroups(versions, 2000, batch + var dtos = Database.FetchByGroups(versions, Constants.Sql.MaxParameterCount, batch => Sql() .Select() .From() @@ -1420,7 +1420,7 @@ private IDictionary> GetDocumentVariations(List< { var ids = temps.Select(x => x.Id); - var dtos = Database.FetchByGroups(ids, 2000, batch => + var dtos = Database.FetchByGroups(ids, Constants.Sql.MaxParameterCount, batch => Sql() .Select() .From() diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs index 505cbfc816ea..07b82190f5d4 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs @@ -60,7 +60,7 @@ protected override IEnumerable PerformGetAll(params int[] ids) { if (ids.Any()) { - return Database.FetchByGroups(ids, 2000, batch => + return Database.FetchByGroups(ids, Constants.Sql.MaxParameterCount, batch => GetBaseQuery(false) .Where(x => x.NodeObjectType == NodeObjectTypeId) .WhereIn(x => x.NodeId, batch)) diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs index a7502a338af8..7bcc5f6b2db4 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs @@ -281,7 +281,7 @@ private IEnumerable BuildVariants(IEnumerable(v.Select(x => x.Id), 2000, GetVariantInfos); + var dtos = Database.FetchByGroups(v.Select(x => x.Id), Constants.Sql.MaxParameterCount, GetVariantInfos); // group by node id (each group contains all languages) var xdtos = dtos.GroupBy(x => x.NodeId).ToDictionary(x => x.Key, x => x); diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/MemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/MemberRepository.cs index 62f41aa72759..1982d6ebf921 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/MemberRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/MemberRepository.cs @@ -431,16 +431,13 @@ public IEnumerable FindMembersInRole(string roleName, string usernameTo var matchedMembers = Get(query).ToArray(); var membersInGroup = new List(); + //then we need to filter the matched members that are in the role - //since the max sql params are 2100 on sql server, we'll reduce that to be safe for potentially other servers and run the queries in batches - var inGroups = matchedMembers.InGroupsOf(1000); - foreach (var batch in inGroups) + foreach (var group in matchedMembers.Select(x => x.Id).InGroupsOf(Constants.Sql.MaxParameterCount)) { - var memberIdBatch = batch.Select(x => x.Id); - var sql = Sql().SelectAll().From() .Where(dto => dto.MemberGroup == memberGroup.Id) - .WhereIn(dto => dto.Member, memberIdBatch); + .WhereIn(dto => dto.Member, group); var memberIdsInGroup = Database.Fetch(sql) .Select(x => x.Member).ToArray(); diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs index 259f0b89c0ab..735496a0d187 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs @@ -38,44 +38,41 @@ public PermissionRepository(IScopeAccessor scopeAccessor, AppCaches cache, ILogg /// /// /// - /// This method will not support passing in more than 2000 group Ids + /// This method will not support passing in more than 2000 group IDs when also passing in entity IDs. /// public EntityPermissionCollection GetPermissionsForEntities(int[] groupIds, params int[] entityIds) { var result = new EntityPermissionCollection(); - foreach (var groupOfGroupIds in groupIds.InGroupsOf(2000)) + if (entityIds.Length == 0) { - //copy local - var localIds = groupOfGroupIds.ToArray(); - - if (entityIds.Length == 0) + foreach (var group in groupIds.InGroupsOf(Constants.Sql.MaxParameterCount)) { var sql = Sql() .SelectAll() .From() - .Where(dto => localIds.Contains(dto.UserGroupId)); + .Where(dto => group.Contains(dto.UserGroupId)); + var permissions = AmbientScope.Database.Fetch(sql); foreach (var permission in ConvertToPermissionList(permissions)) { result.Add(permission); } } - else + } + else + { + foreach (var group in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount - groupIds.Length)) { - //iterate in groups of 2000 since we don't want to exceed the max SQL param count - foreach (var groupOfEntityIds in entityIds.InGroupsOf(2000)) + var sql = Sql() + .SelectAll() + .From() + .Where(dto => groupIds.Contains(dto.UserGroupId) && group.Contains(dto.NodeId)); + + var permissions = AmbientScope.Database.Fetch(sql); + foreach (var permission in ConvertToPermissionList(permissions)) { - var ids = groupOfEntityIds; - var sql = Sql() - .SelectAll() - .From() - .Where(dto => localIds.Contains(dto.UserGroupId) && ids.Contains(dto.NodeId)); - var permissions = AmbientScope.Database.Fetch(sql); - foreach (var permission in ConvertToPermissionList(permissions)) - { - result.Add(permission); - } + result.Add(permission); } } } @@ -133,11 +130,10 @@ public void ReplacePermissions(int groupId, IEnumerable permissions, param var db = AmbientScope.Database; - //we need to batch these in groups of 2000 so we don't exceed the max 2100 limit var sql = "DELETE FROM umbracoUserGroup2NodePermission WHERE userGroupId = @groupId AND nodeId in (@nodeIds)"; - foreach (var idGroup in entityIds.InGroupsOf(2000)) + foreach (var group in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount)) { - db.Execute(sql, new { groupId, nodeIds = idGroup }); + db.Execute(sql, new { groupId, nodeIds = group }); } var toInsert = new List(); diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs index 24c1e31c2015..099d49fbf8ac 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs @@ -37,8 +37,9 @@ protected override IRedirectUrl PerformGet(Guid id) protected override IEnumerable PerformGetAll(params Guid[] ids) { - if (ids.Length > 2000) - throw new NotSupportedException("This repository does not support more than 2000 ids."); + if (ids.Length > Constants.Sql.MaxParameterCount) + throw new NotSupportedException($"This repository does not support more than {Constants.Sql.MaxParameterCount} ids."); + var sql = GetBaseQuery(false).WhereIn(x => x.Id, ids); var dtos = Database.Fetch(sql); return dtos.WhereNotNull().Select(Map); diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs index 69e4db5940a2..a7704272a85f 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs @@ -186,17 +186,17 @@ public IEnumerable GetMany(params TId[] ids) // can't query more than 2000 ids at a time... but if someone is really querying 2000+ entities, // the additional overhead of fetching them in groups is minimal compared to the lookup time of each group - const int maxParams = 2000; - if (ids.Length <= maxParams) + if (ids.Length <= Constants.Sql.MaxParameterCount) { return CachePolicy.GetAll(ids, PerformGetAll); } var entities = new List(); - foreach (var groupOfIds in ids.InGroupsOf(maxParams)) + foreach (var group in ids.InGroupsOf(Constants.Sql.MaxParameterCount)) { - entities.AddRange(CachePolicy.GetAll(groupOfIds.ToArray(), PerformGetAll)); + entities.AddRange(CachePolicy.GetAll(group.ToArray(), PerformGetAll)); } + return entities; } diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs index 279e03b19e37..83170d9dbcb4 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs @@ -39,7 +39,7 @@ protected override IEnumerable PerformGetAll(params int[] ids) { var dtos = ids.Length == 0 ? Database.Fetch(Sql().Select().From()) - : Database.FetchByGroups(ids, 2000, batch => Sql().Select().From().WhereIn(x => x.Id, batch)); + : Database.FetchByGroups(ids, Constants.Sql.MaxParameterCount, batch => Sql().Select().From().WhereIn(x => x.Id, batch)); return dtos.Select(TagFactory.BuildEntity).ToList(); } diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 77eeaaa85329..e2679c222351 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -132,6 +132,7 @@ + @@ -1665,4 +1666,4 @@ - + \ No newline at end of file diff --git a/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs b/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs index 619863089847..f66a1ee934d9 100644 --- a/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs +++ b/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs @@ -240,7 +240,7 @@ private void Map(IUserGroup source, UserGroupDisplay target, MapperContext conte // the entity service due to too many Sql parameters. var list = new List(); - foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(2000)) + foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(Constants.Sql.MaxParameterCount)) list.AddRange(_entityService.GetAll(UmbracoObjectTypes.Document, idGroup.ToArray())); contentEntities = list.ToArray(); }