diff --git a/src/Umbraco.Core/Constants-Sql.cs b/src/Umbraco.Core/Constants-Sql.cs
new file mode 100644
index 000000000000..366dfbe44ab4
--- /dev/null
+++ b/src/Umbraco.Core/Constants-Sql.cs
@@ -0,0 +1,17 @@
+namespace Umbraco.Core
+{
+ public static partial class Constants
+ {
+ public static class Sql
+ {
+ ///
+ /// The maximum amount of parameters that can be used in a query.
+ ///
+ ///
+ /// The actual limit is 2100 (https://docs.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server),
+ /// but we want to ensure there's room for additional parameters if this value is used to create groups/batches.
+ ///
+ public const int MaxParameterCount = 2000;
+ }
+ }
+}
diff --git a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs
index 77cc0d66019b..a50e3c2aaa9f 100644
--- a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs
+++ b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs
@@ -137,7 +137,7 @@ internal static IDbCommand[] GenerateBulkInsertCommands(this IUmbracoDatabase
// Math.Floor(2100 / 8) = 262 record per command
// 4168 / 262 = 15.908... = there will be 16 command in total
// (if we have disabled db parameters, then all records will be included, in only one command)
- var recordsPerCommand = paramsPerRecord == 0 ? int.MaxValue : Convert.ToInt32(Math.Floor(2000.00 / paramsPerRecord));
+ var recordsPerCommand = paramsPerRecord == 0 ? int.MaxValue : Convert.ToInt32(Math.Floor((double)Constants.Sql.MaxParameterCount / paramsPerRecord));
var commandsCount = Convert.ToInt32(Math.Ceiling((double)records.Length / recordsPerCommand));
var commands = new IDbCommand[commandsCount];
diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs
index c3d34cc3e9f8..d921bb6d5153 100644
--- a/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs
@@ -54,7 +54,7 @@ protected override IEnumerable PerformGetAll(params int[] ids)
var entries = new List();
- foreach (var group in ids.InGroupsOf(2000))
+ foreach (var group in ids.InGroupsOf(Constants.Sql.MaxParameterCount))
{
var sql = Sql()
.Select()
diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs
index 85814ef681cf..a6b08c2bfd2b 100644
--- a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs
@@ -638,7 +638,7 @@ protected IDictionary GetPropertyCollections(List(versions, 2000, batch =>
+ var allPropertyDataDtos = Database.FetchByGroups(versions, Constants.Sql.MaxParameterCount, batch =>
SqlContext.Sql()
.Select()
.From()
@@ -647,7 +647,7 @@ protected IDictionary GetPropertyCollections(List x.PropertyTypeId).Distinct().ToList();
- var allPropertyTypeDtos = Database.FetchByGroups(allPropertyTypeIds, 2000, batch =>
+ var allPropertyTypeDtos = Database.FetchByGroups(allPropertyTypeIds, Constants.Sql.MaxParameterCount, batch =>
SqlContext.Sql()
.Select(r => r.Select(x => x.DataTypeDto))
.From()
diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs
index aff53e44b0af..3a74b9209b61 100644
--- a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs
@@ -768,7 +768,7 @@ private void CopyTagData(int? sourceLanguageId, int? targetLanguageId, IReadOnly
// note: important to use SqlNullableEquals for nullable types, cannot directly compare language identifiers
var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0);
- if (whereInArgsCount > 2000)
+ if (whereInArgsCount > Constants.Sql.MaxParameterCount)
throw new NotSupportedException("Too many property/content types.");
// delete existing relations (for target language)
@@ -906,7 +906,7 @@ private void CopyPropertyData(int? sourceLanguageId, int? targetLanguageId, IRea
// note: important to use SqlNullableEquals for nullable types, cannot directly compare language identifiers
//
var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0);
- if (whereInArgsCount > 2000)
+ if (whereInArgsCount > Constants.Sql.MaxParameterCount)
throw new NotSupportedException("Too many property/content types.");
//first clear out any existing property data that might already exists under the target language
@@ -1005,7 +1005,7 @@ private void RenormalizeDocumentEditedFlags(IReadOnlyCollection propertyTyp
//based on the current variance of each item to see if it's 'edited' value should be true/false.
var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0);
- if (whereInArgsCount > 2000)
+ if (whereInArgsCount > Constants.Sql.MaxParameterCount)
throw new NotSupportedException("Too many property/content types.");
var propertySql = Sql()
@@ -1094,14 +1094,20 @@ private void RenormalizeDocumentEditedFlags(IReadOnlyCollection propertyTyp
}
}
- //lookup all matching rows in umbracoDocumentCultureVariation
- var docCultureVariationsToUpdate = editedLanguageVersions.InGroupsOf(2000)
- .SelectMany(_ => Database.Fetch(
- Sql().Select().From()
- .WhereIn(x => x.LanguageId, editedLanguageVersions.Keys.Select(x => x.langId).ToList())
- .WhereIn(x => x.NodeId, editedLanguageVersions.Keys.Select(x => x.nodeId))))
- //convert to dictionary with the same key type
- .ToDictionary(x => (x.NodeId, (int?)x.LanguageId), x => x);
+ // lookup all matching rows in umbracoDocumentCultureVariation
+ // fetch in batches to account for maximum parameter count (distinct languages can't exceed 2000)
+ var languageIds = editedLanguageVersions.Keys.Select(x => x.langId).Distinct().ToArray();
+ var nodeIds = editedLanguageVersions.Keys.Select(x => x.nodeId).Distinct();
+ var docCultureVariationsToUpdate = nodeIds.InGroupsOf(Constants.Sql.MaxParameterCount - languageIds.Length)
+ .SelectMany(group =>
+ {
+ var sql = Sql().Select().From()
+ .WhereIn(x => x.LanguageId, languageIds)
+ .WhereIn(x => x.NodeId, group);
+
+ return Database.Fetch(sql);
+ })
+ .ToDictionary(x => (x.NodeId, (int?)x.LanguageId), x => x); //convert to dictionary with the same key type
var toUpdate = new List();
foreach (var ev in editedLanguageVersions)
diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs
index ac1f7c3f2a94..da957a728858 100644
--- a/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs
@@ -260,13 +260,12 @@ public IEnumerable GetDictionaryItemDescendants(Guid? parentId)
Func>> getItemsFromParents = guids =>
{
- //needs to be in groups of 2000 because we are doing an IN clause and there's a max parameter count that can be used.
- return guids.InGroupsOf(2000)
- .Select(@group =>
+ return guids.InGroupsOf(Constants.Sql.MaxParameterCount)
+ .Select(group =>
{
var sqlClause = GetBaseQuery(false)
.Where(x => x.Parent != null)
- .Where($"{SqlSyntax.GetQuotedColumnName("parent")} IN (@parentIds)", new { parentIds = @group });
+ .WhereIn(x => x.Parent, group);
var translator = new SqlTranslator(sqlClause, Query());
var sql = translator.Translate();
diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs
index 5716fbe129c4..753630d186d0 100644
--- a/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs
@@ -1342,7 +1342,7 @@ private IDictionary GetContentSchedule(params in
{
var result = new Dictionary();
- var scheduleDtos = Database.FetchByGroups(contentIds, 2000, batch => Sql()
+ var scheduleDtos = Database.FetchByGroups(contentIds, Constants.Sql.MaxParameterCount, batch => Sql()
.Select()
.From()
.WhereIn(x => x.NodeId, batch));
@@ -1391,7 +1391,7 @@ private IDictionary> GetContentVariations(List>();
- var dtos = Database.FetchByGroups(versions, 2000, batch
+ var dtos = Database.FetchByGroups(versions, Constants.Sql.MaxParameterCount, batch
=> Sql()
.Select()
.From()
@@ -1420,7 +1420,7 @@ private IDictionary> GetDocumentVariations(List<
{
var ids = temps.Select(x => x.Id);
- var dtos = Database.FetchByGroups(ids, 2000, batch =>
+ var dtos = Database.FetchByGroups(ids, Constants.Sql.MaxParameterCount, batch =>
Sql()
.Select()
.From()
diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs
index 505cbfc816ea..07b82190f5d4 100644
--- a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs
@@ -60,7 +60,7 @@ protected override IEnumerable PerformGetAll(params int[] ids)
{
if (ids.Any())
{
- return Database.FetchByGroups(ids, 2000, batch =>
+ return Database.FetchByGroups(ids, Constants.Sql.MaxParameterCount, batch =>
GetBaseQuery(false)
.Where(x => x.NodeObjectType == NodeObjectTypeId)
.WhereIn(x => x.NodeId, batch))
diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs
index a7502a338af8..7bcc5f6b2db4 100644
--- a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs
@@ -281,7 +281,7 @@ private IEnumerable BuildVariants(IEnumerable(v.Select(x => x.Id), 2000, GetVariantInfos);
+ var dtos = Database.FetchByGroups(v.Select(x => x.Id), Constants.Sql.MaxParameterCount, GetVariantInfos);
// group by node id (each group contains all languages)
var xdtos = dtos.GroupBy(x => x.NodeId).ToDictionary(x => x.Key, x => x);
diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/MemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/MemberRepository.cs
index 62f41aa72759..1982d6ebf921 100644
--- a/src/Umbraco.Core/Persistence/Repositories/Implement/MemberRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/Implement/MemberRepository.cs
@@ -431,16 +431,13 @@ public IEnumerable FindMembersInRole(string roleName, string usernameTo
var matchedMembers = Get(query).ToArray();
var membersInGroup = new List();
+
//then we need to filter the matched members that are in the role
- //since the max sql params are 2100 on sql server, we'll reduce that to be safe for potentially other servers and run the queries in batches
- var inGroups = matchedMembers.InGroupsOf(1000);
- foreach (var batch in inGroups)
+ foreach (var group in matchedMembers.Select(x => x.Id).InGroupsOf(Constants.Sql.MaxParameterCount))
{
- var memberIdBatch = batch.Select(x => x.Id);
-
var sql = Sql().SelectAll().From()
.Where(dto => dto.MemberGroup == memberGroup.Id)
- .WhereIn(dto => dto.Member, memberIdBatch);
+ .WhereIn(dto => dto.Member, group);
var memberIdsInGroup = Database.Fetch(sql)
.Select(x => x.Member).ToArray();
diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs
index 259f0b89c0ab..735496a0d187 100644
--- a/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs
@@ -38,44 +38,41 @@ public PermissionRepository(IScopeAccessor scopeAccessor, AppCaches cache, ILogg
///
///
///
- /// This method will not support passing in more than 2000 group Ids
+ /// This method will not support passing in more than 2000 group IDs when also passing in entity IDs.
///
public EntityPermissionCollection GetPermissionsForEntities(int[] groupIds, params int[] entityIds)
{
var result = new EntityPermissionCollection();
- foreach (var groupOfGroupIds in groupIds.InGroupsOf(2000))
+ if (entityIds.Length == 0)
{
- //copy local
- var localIds = groupOfGroupIds.ToArray();
-
- if (entityIds.Length == 0)
+ foreach (var group in groupIds.InGroupsOf(Constants.Sql.MaxParameterCount))
{
var sql = Sql()
.SelectAll()
.From()
- .Where(dto => localIds.Contains(dto.UserGroupId));
+ .Where(dto => group.Contains(dto.UserGroupId));
+
var permissions = AmbientScope.Database.Fetch(sql);
foreach (var permission in ConvertToPermissionList(permissions))
{
result.Add(permission);
}
}
- else
+ }
+ else
+ {
+ foreach (var group in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount - groupIds.Length))
{
- //iterate in groups of 2000 since we don't want to exceed the max SQL param count
- foreach (var groupOfEntityIds in entityIds.InGroupsOf(2000))
+ var sql = Sql()
+ .SelectAll()
+ .From()
+ .Where(dto => groupIds.Contains(dto.UserGroupId) && group.Contains(dto.NodeId));
+
+ var permissions = AmbientScope.Database.Fetch(sql);
+ foreach (var permission in ConvertToPermissionList(permissions))
{
- var ids = groupOfEntityIds;
- var sql = Sql()
- .SelectAll()
- .From()
- .Where(dto => localIds.Contains(dto.UserGroupId) && ids.Contains(dto.NodeId));
- var permissions = AmbientScope.Database.Fetch(sql);
- foreach (var permission in ConvertToPermissionList(permissions))
- {
- result.Add(permission);
- }
+ result.Add(permission);
}
}
}
@@ -133,11 +130,10 @@ public void ReplacePermissions(int groupId, IEnumerable permissions, param
var db = AmbientScope.Database;
- //we need to batch these in groups of 2000 so we don't exceed the max 2100 limit
var sql = "DELETE FROM umbracoUserGroup2NodePermission WHERE userGroupId = @groupId AND nodeId in (@nodeIds)";
- foreach (var idGroup in entityIds.InGroupsOf(2000))
+ foreach (var group in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount))
{
- db.Execute(sql, new { groupId, nodeIds = idGroup });
+ db.Execute(sql, new { groupId, nodeIds = group });
}
var toInsert = new List();
diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs
index 24c1e31c2015..099d49fbf8ac 100644
--- a/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs
@@ -37,8 +37,9 @@ protected override IRedirectUrl PerformGet(Guid id)
protected override IEnumerable PerformGetAll(params Guid[] ids)
{
- if (ids.Length > 2000)
- throw new NotSupportedException("This repository does not support more than 2000 ids.");
+ if (ids.Length > Constants.Sql.MaxParameterCount)
+ throw new NotSupportedException($"This repository does not support more than {Constants.Sql.MaxParameterCount} ids.");
+
var sql = GetBaseQuery(false).WhereIn(x => x.Id, ids);
var dtos = Database.Fetch(sql);
return dtos.WhereNotNull().Select(Map);
diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs
index 69e4db5940a2..a7704272a85f 100644
--- a/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs
@@ -186,17 +186,17 @@ public IEnumerable GetMany(params TId[] ids)
// can't query more than 2000 ids at a time... but if someone is really querying 2000+ entities,
// the additional overhead of fetching them in groups is minimal compared to the lookup time of each group
- const int maxParams = 2000;
- if (ids.Length <= maxParams)
+ if (ids.Length <= Constants.Sql.MaxParameterCount)
{
return CachePolicy.GetAll(ids, PerformGetAll);
}
var entities = new List();
- foreach (var groupOfIds in ids.InGroupsOf(maxParams))
+ foreach (var group in ids.InGroupsOf(Constants.Sql.MaxParameterCount))
{
- entities.AddRange(CachePolicy.GetAll(groupOfIds.ToArray(), PerformGetAll));
+ entities.AddRange(CachePolicy.GetAll(group.ToArray(), PerformGetAll));
}
+
return entities;
}
diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs
index 279e03b19e37..83170d9dbcb4 100644
--- a/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs
@@ -39,7 +39,7 @@ protected override IEnumerable PerformGetAll(params int[] ids)
{
var dtos = ids.Length == 0
? Database.Fetch(Sql().Select().From())
- : Database.FetchByGroups(ids, 2000, batch => Sql().Select().From().WhereIn(x => x.Id, batch));
+ : Database.FetchByGroups(ids, Constants.Sql.MaxParameterCount, batch => Sql().Select().From().WhereIn(x => x.Id, batch));
return dtos.Select(TagFactory.BuildEntity).ToList();
}
diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj
index 77eeaaa85329..e2679c222351 100755
--- a/src/Umbraco.Core/Umbraco.Core.csproj
+++ b/src/Umbraco.Core/Umbraco.Core.csproj
@@ -132,6 +132,7 @@
+
@@ -1665,4 +1666,4 @@
-
+
\ No newline at end of file
diff --git a/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs b/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs
index 619863089847..f66a1ee934d9 100644
--- a/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs
+++ b/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs
@@ -240,7 +240,7 @@ private void Map(IUserGroup source, UserGroupDisplay target, MapperContext conte
// the entity service due to too many Sql parameters.
var list = new List();
- foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(2000))
+ foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(Constants.Sql.MaxParameterCount))
list.AddRange(_entityService.GetAll(UmbracoObjectTypes.Document, idGroup.ToArray()));
contentEntities = list.ToArray();
}