Merge remote-tracking branch 'origin/v8/dev' into v9/feature/merge-v8…

…_29-10-2021 # Conflicts: # src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs # src/Umbraco.Web.UI.Client/package.json # src/Umbraco.Web.UI.Client/src/views/common/overlays/user/user.html # src/Umbraco.Web.UI.Client/src/views/content/overlays/publish.html # src/Umbraco.Web.UI.Client/src/views/documenttypes/importdocumenttype.html # src/Umbraco.Web.UI/umbraco/config/lang/en.xml # src/Umbraco.Web.UI/umbraco/config/lang/en_us.xml # src/Umbraco.Web.UI/umbraco/config/lang/nl.xml
Nuklon · Oct 29, 2021 · 02bf616 · 02bf616
2 parents e9ae567 + b046098
commit 02bf616
Show file tree

Hide file tree

Showing 64 changed files with 678 additions and 505 deletions.
diff --git a/src/Umbraco.Core/Constants-Sql.cs b/src/Umbraco.Core/Constants-Sql.cs
@@ -0,0 +1,18 @@
+namespace Umbraco.Cms.Core
+{
+    public static partial class Constants
+    {
+        public static class Sql
+        {
+            /// <summary>
+            ///     The maximum amount of parameters that can be used in a query.
+            /// </summary>
+            /// <remarks>
+            ///     The actual limit is 2100
+            ///     (https://docs.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server),
+            ///     but we want to ensure there's room for additional parameters if this value is used to create groups/batches.
+            /// </remarks>
+            public const int MaxParameterCount = 2000;
+        }
+    }
+}
diff --git a/src/Umbraco.Core/Models/Mapping/UserMapDefinition.cs b/src/Umbraco.Core/Models/Mapping/UserMapDefinition.cs
@@ -251,7 +251,7 @@ private void Map(IUserGroup source, UserGroupDisplay target, MapperContext conte
                 // the entity service due to too many Sql parameters.
 
                 var list = new List<IEntitySlim>();
-                foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(2000))
+                foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(Constants.Sql.MaxParameterCount))
                     list.AddRange(_entityService.GetAll(UmbracoObjectTypes.Document, idGroup.ToArray()));
                 contentEntities = list.ToArray();
             }

diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj
@@ -1,61 +1,61 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
-    <PropertyGroup>
-        <TargetFramework>netstandard2.0</TargetFramework>
-        <RootNamespace>Umbraco.Cms.Core</RootNamespace>
-        <Product>Umbraco CMS</Product>
-        <PackageId>Umbraco.Cms.Core</PackageId>
-        <Title>Umbraco CMS Core</Title>
-        <Description>Contains the core assembly needed to run Umbraco Cms. This package only contains the assembly, and can be used for package development. Use the template in the Umbraco.Templates package to setup Umbraco</Description>
-        <Product>Umbraco CMS</Product>
-    </PropertyGroup>
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <RootNamespace>Umbraco.Cms.Core</RootNamespace>
+    <Product>Umbraco CMS</Product>
+    <PackageId>Umbraco.Cms.Core</PackageId>
+    <Title>Umbraco CMS Core</Title>
+    <Description>Contains the core assembly needed to run Umbraco Cms. This package only contains the assembly, and can be used for package development. Use the template in the Umbraco.Templates package to setup Umbraco</Description>
+    <Product>Umbraco CMS</Product>
+  </PropertyGroup>
 
-    <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
-        <DocumentationFile>bin\Release\Umbraco.Core.xml</DocumentationFile>
-    </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
+    <DocumentationFile>bin\Release\Umbraco.Core.xml</DocumentationFile>
+  </PropertyGroup>
 
-    <ItemGroup>
-        <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="5.0.0" />
-        <PackageReference Include="Microsoft.Extensions.FileProviders.Embedded" Version="5.0.10" />
-        <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="5.0.0" />
-        <PackageReference Include="Microsoft.Extensions.Logging" Version="5.0.0" />
-        <PackageReference Include="Microsoft.Extensions.Options" Version="5.0.0" />
-        <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="5.0.0" />
-        <PackageReference Include="Microsoft.Extensions.Options.DataAnnotations" Version="5.0.0" />
-        <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0">
-          <PrivateAssets>all</PrivateAssets>
-          <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-        </PackageReference>
-        <PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
-        <PackageReference Include="System.Reflection.Emit.Lightweight" Version="4.7.0" />
-        <PackageReference Include="System.Runtime.Caching" Version="5.0.0" />
-        <PackageReference Include="Umbraco.Code" Version="1.2.0">
-          <PrivateAssets>all</PrivateAssets>
-        </PackageReference>
-    </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="5.0.0"/>
+    <PackageReference Include="Microsoft.Extensions.FileProviders.Embedded" Version="5.0.10"/>
+    <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="5.0.0"/>
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="5.0.0"/>
+    <PackageReference Include="Microsoft.Extensions.Options" Version="5.0.0"/>
+    <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="5.0.0"/>
+    <PackageReference Include="Microsoft.Extensions.Options.DataAnnotations" Version="5.0.0"/>
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0"/>
+    <PackageReference Include="System.Reflection.Emit.Lightweight" Version="4.7.0"/>
+    <PackageReference Include="System.Runtime.Caching" Version="5.0.0"/>
+    <PackageReference Include="Umbraco.Code" Version="1.2.0">
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+  </ItemGroup>
 
-    <ItemGroup>
-        <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
-            <_Parameter1>Umbraco.Tests</_Parameter1>
-        </AssemblyAttribute>
-        <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
-            <_Parameter1>Umbraco.Tests.Common</_Parameter1>
-        </AssemblyAttribute>
-        <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
-            <_Parameter1>Umbraco.Tests.UnitTests</_Parameter1>
-        </AssemblyAttribute>
-        <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
-            <_Parameter1>Umbraco.Tests.Benchmarks</_Parameter1>
-        </AssemblyAttribute>
-        <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
-            <_Parameter1>Umbraco.Tests.Integration</_Parameter1>
-        </AssemblyAttribute>
-        <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
-            <_Parameter1>DynamicProxyGenAssembly2</_Parameter1>
-        </AssemblyAttribute>
-    </ItemGroup>
+  <ItemGroup>
+    <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
+      <_Parameter1>Umbraco.Tests</_Parameter1>
+    </AssemblyAttribute>
+    <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
+      <_Parameter1>Umbraco.Tests.Common</_Parameter1>
+    </AssemblyAttribute>
+    <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
+      <_Parameter1>Umbraco.Tests.UnitTests</_Parameter1>
+    </AssemblyAttribute>
+    <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
+      <_Parameter1>Umbraco.Tests.Benchmarks</_Parameter1>
+    </AssemblyAttribute>
+    <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
+      <_Parameter1>Umbraco.Tests.Integration</_Parameter1>
+    </AssemblyAttribute>
+    <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
+      <_Parameter1>DynamicProxyGenAssembly2</_Parameter1>
+    </AssemblyAttribute>
+  </ItemGroup>
 
-    <ItemGroup>
-      <EmbeddedResource Include="EmbeddedResources\**\*" />
-    </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="EmbeddedResources\**\*"/>
+  </ItemGroup>
 </Project>
diff --git a/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/AuditEntryRepository.cs b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/AuditEntryRepository.cs
@@ -53,7 +53,7 @@ protected override IEnumerable<IAuditEntry> PerformGetAll(params int[] ids)
 
             var entries = new List<IAuditEntry>();
 
-            foreach (var group in ids.InGroupsOf(2000))
+            foreach (var group in ids.InGroupsOf(Constants.Sql.MaxParameterCount))
             {
                 var sql = Sql()
                     .Select<AuditEntryDto>()

diff --git a/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/ContentRepositoryBase.cs b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/ContentRepositoryBase.cs
@@ -657,7 +657,7 @@ protected IDictionary<int, PropertyCollection> GetPropertyCollections<T>(List<Te
             // in the table?
 
             // get all PropertyDataDto for all definitions / versions
-            var allPropertyDataDtos = Database.FetchByGroups<PropertyDataDto, int>(versions, 2000, batch =>
+            var allPropertyDataDtos = Database.FetchByGroups<PropertyDataDto, int>(versions, Constants.Sql.MaxParameterCount, batch =>
                 SqlContext.Sql()
                     .Select<PropertyDataDto>()
                     .From<PropertyDataDto>()
@@ -666,7 +666,7 @@ protected IDictionary<int, PropertyCollection> GetPropertyCollections<T>(List<Te
 
             // get PropertyDataDto distinct PropertyTypeDto
             var allPropertyTypeIds = allPropertyDataDtos.Select(x => x.PropertyTypeId).Distinct().ToList();
-            var allPropertyTypeDtos = Database.FetchByGroups<PropertyTypeDto, int>(allPropertyTypeIds, 2000, batch =>
+            var allPropertyTypeDtos = Database.FetchByGroups<PropertyTypeDto, int>(allPropertyTypeIds, Constants.Sql.MaxParameterCount, batch =>
                 SqlContext.Sql()
                     .Select<PropertyTypeDto>(r => r.Select(x => x.DataTypeDto))
                     .From<PropertyTypeDto>()

diff --git a/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs
@@ -795,7 +795,7 @@ private void CopyTagData(int? sourceLanguageId, int? targetLanguageId, IReadOnly
             // note: important to use SqlNullableEquals for nullable types, cannot directly compare language identifiers
 
             var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0);
-            if (whereInArgsCount > 2000)
+            if (whereInArgsCount > Constants.Sql.MaxParameterCount)
                 throw new NotSupportedException("Too many property/content types.");
 
             // delete existing relations (for target language)
@@ -933,7 +933,7 @@ private void CopyPropertyData(int? sourceLanguageId, int? targetLanguageId, IRea
             // note: important to use SqlNullableEquals for nullable types, cannot directly compare language identifiers
             //
             var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0);
-            if (whereInArgsCount > 2000)
+            if (whereInArgsCount > Constants.Sql.MaxParameterCount)
                 throw new NotSupportedException("Too many property/content types.");
 
             //first clear out any existing property data that might already exists under the target language
@@ -1032,7 +1032,7 @@ private void RenormalizeDocumentEditedFlags(IReadOnlyCollection<int> propertyTyp
             //based on the current variance of each item to see if it's 'edited' value should be true/false.
 
             var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0);
-            if (whereInArgsCount > 2000)
+            if (whereInArgsCount > Constants.Sql.MaxParameterCount)
                 throw new NotSupportedException("Too many property/content types.");
 
             var propertySql = Sql()
@@ -1121,14 +1121,20 @@ private void RenormalizeDocumentEditedFlags(IReadOnlyCollection<int> propertyTyp
                 }
             }
 
-            //lookup all matching rows in umbracoDocumentCultureVariation
-            var docCultureVariationsToUpdate = editedLanguageVersions.InGroupsOf(2000)
-                .SelectMany(_ => Database.Fetch<DocumentCultureVariationDto>(
-                    Sql().Select<DocumentCultureVariationDto>().From<DocumentCultureVariationDto>()
-                            .WhereIn<DocumentCultureVariationDto>(x => x.LanguageId, editedLanguageVersions.Keys.Select(x => x.langId).ToList())
-                            .WhereIn<DocumentCultureVariationDto>(x => x.NodeId, editedLanguageVersions.Keys.Select(x => x.nodeId))))
-                //convert to dictionary with the same key type
-                .ToDictionary(x => (x.NodeId, (int?)x.LanguageId), x => x);
+            // lookup all matching rows in umbracoDocumentCultureVariation
+            // fetch in batches to account for maximum parameter count (distinct languages can't exceed 2000)
+            var languageIds = editedLanguageVersions.Keys.Select(x => x.langId).Distinct().ToArray();
+            var nodeIds = editedLanguageVersions.Keys.Select(x => x.nodeId).Distinct();
+            var docCultureVariationsToUpdate = nodeIds.InGroupsOf(Constants.Sql.MaxParameterCount - languageIds.Length)
+                .SelectMany(group =>
+                {
+                    var sql = Sql().Select<DocumentCultureVariationDto>().From<DocumentCultureVariationDto>()
+                        .WhereIn<DocumentCultureVariationDto>(x => x.LanguageId, languageIds)
+                        .WhereIn<DocumentCultureVariationDto>(x => x.NodeId, group);
+
+                    return Database.Fetch<DocumentCultureVariationDto>(sql);
+                })
+                .ToDictionary(x => (x.NodeId, (int?)x.LanguageId), x => x); //convert to dictionary with the same key type
 
             var toUpdate = new List<DocumentCultureVariationDto>();
             foreach (var ev in editedLanguageVersions)

diff --git a/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/DictionaryRepository.cs b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/DictionaryRepository.cs
@@ -263,13 +263,12 @@ public IEnumerable<IDictionaryItem> GetDictionaryItemDescendants(Guid? parentId)
 
             Func<Guid[], IEnumerable<IEnumerable<IDictionaryItem>>> getItemsFromParents = guids =>
             {
-                //needs to be in groups of 2000 because we are doing an IN clause and there's a max parameter count that can be used.
-                return guids.InGroupsOf(2000)
-                    .Select(@group =>
+                return guids.InGroupsOf(Constants.Sql.MaxParameterCount)
+                    .Select(group =>
                     {
                         var sqlClause = GetBaseQuery(false)
                             .Where<DictionaryDto>(x => x.Parent != null)
-                            .Where($"{SqlSyntax.GetQuotedColumnName("parent")} IN (@parentIds)", new { parentIds = @group });
+                            .WhereIn<DictionaryDto>(x => x.Parent, group);
 
                         var translator = new SqlTranslator<IDictionaryItem>(sqlClause, Query<IDictionaryItem>());
                         var sql = translator.Translate();

diff --git a/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/DocumentRepository.cs b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/DocumentRepository.cs
@@ -1390,7 +1390,7 @@ private IDictionary<int, ContentScheduleCollection> GetContentSchedule(params in
         {
             var result = new Dictionary<int, ContentScheduleCollection>();
 
-            var scheduleDtos = Database.FetchByGroups<ContentScheduleDto, int>(contentIds, 2000, batch => Sql()
+            var scheduleDtos = Database.FetchByGroups<ContentScheduleDto, int>(contentIds, Constants.Sql.MaxParameterCount, batch => Sql()
                 .Select<ContentScheduleDto>()
                 .From<ContentScheduleDto>()
                 .WhereIn<ContentScheduleDto>(x => x.NodeId, batch));
@@ -1440,7 +1440,7 @@ private IDictionary<int, List<ContentVariation>> GetContentVariations<T>(List<Te
             if (versions.Count == 0)
                 return new Dictionary<int, List<ContentVariation>>();
 
-            var dtos = Database.FetchByGroups<ContentVersionCultureVariationDto, int>(versions, 2000, batch
+            var dtos = Database.FetchByGroups<ContentVersionCultureVariationDto, int>(versions, Constants.Sql.MaxParameterCount, batch
                 => Sql()
                     .Select<ContentVersionCultureVariationDto>()
                     .From<ContentVersionCultureVariationDto>()
@@ -1469,7 +1469,7 @@ private IDictionary<int, List<DocumentVariation>> GetDocumentVariations<T>(List<
         {
             var ids = temps.Select(x => x.Id);
 
-            var dtos = Database.FetchByGroups<DocumentCultureVariationDto, int>(ids, 2000, batch =>
+            var dtos = Database.FetchByGroups<DocumentCultureVariationDto, int>(ids, Constants.Sql.MaxParameterCount, batch =>
                 Sql()
                     .Select<DocumentCultureVariationDto>()
                     .From<DocumentCultureVariationDto>()

diff --git a/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/EntityContainerRepository.cs b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/EntityContainerRepository.cs
@@ -62,7 +62,7 @@ protected override IEnumerable<EntityContainer> PerformGetAll(params int[] ids)
         {
             if (ids.Any())
             {
-                return Database.FetchByGroups<NodeDto, int>(ids, 2000, batch =>
+                return Database.FetchByGroups<NodeDto, int>(ids, Constants.Sql.MaxParameterCount, batch =>
                     GetBaseQuery(false)
                         .Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId)
                         .WhereIn<NodeDto>(x => x.NodeId, batch))

diff --git a/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/EntityRepository.cs b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/EntityRepository.cs
@@ -279,7 +279,7 @@ private IEnumerable<DocumentEntitySlim> BuildVariants(IEnumerable<DocumentEntity
             if (v == null) return entitiesList;
 
             // fetch all variant info dtos
-            var dtos = Database.FetchByGroups<VariantInfoDto, int>(v.Select(x => x.Id), 2000, GetVariantInfos);
+            var dtos = Database.FetchByGroups<VariantInfoDto, int>(v.Select(x => x.Id), Constants.Sql.MaxParameterCount, GetVariantInfos);
 
             // group by node id (each group contains all languages)
             var xdtos = dtos.GroupBy(x => x.NodeId).ToDictionary(x => x.Key, x => x);

diff --git a/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/EntityRepositoryBase.cs b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/EntityRepositoryBase.cs
@@ -209,16 +209,15 @@ public IEnumerable<TEntity> GetMany(params TId[] ids)
 
             // can't query more than 2000 ids at a time... but if someone is really querying 2000+ entities,
             // the additional overhead of fetching them in groups is minimal compared to the lookup time of each group
-            const int maxParams = 2000;
-            if (ids.Length <= maxParams)
+            if (ids.Length <= Constants.Sql.MaxParameterCount)
             {
                 return CachePolicy.GetAll(ids, PerformGetAll);
             }
 
             var entities = new List<TEntity>();
-            foreach (var groupOfIds in ids.InGroupsOf(maxParams))
+            foreach (var group in ids.InGroupsOf(Constants.Sql.MaxParameterCount))
             {
-                entities.AddRange(CachePolicy.GetAll(groupOfIds.ToArray(), PerformGetAll));
+                entities.AddRange(CachePolicy.GetAll(group.ToArray(), PerformGetAll));
             }
 
             return entities;

diff --git a/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/MemberRepository.cs b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/MemberRepository.cs
@@ -530,16 +530,13 @@ public IEnumerable<IMember> FindMembersInRole(string roleName, string usernameTo
             var matchedMembers = Get(query).ToArray();
 
             var membersInGroup = new List<IMember>();
+
             //then we need to filter the matched members that are in the role
-            //since the max sql params are 2100 on sql server, we'll reduce that to be safe for potentially other servers and run the queries in batches
-            var inGroups = matchedMembers.InGroupsOf(1000);
-            foreach (var batch in inGroups)
+            foreach (var group in matchedMembers.Select(x => x.Id).InGroupsOf(Constants.Sql.MaxParameterCount))
             {
-                var memberIdBatch = batch.Select(x => x.Id);
-
                 var sql = Sql().SelectAll().From<Member2MemberGroupDto>()
                     .Where<Member2MemberGroupDto>(dto => dto.MemberGroup == memberGroup.Id)
-                    .WhereIn<Member2MemberGroupDto>(dto => dto.Member, memberIdBatch);
+                    .WhereIn<Member2MemberGroupDto>(dto => dto.Member, group);
 
                 var memberIdsInGroup = Database.Fetch<Member2MemberGroupDto>(sql)
                     .Select(x => x.Member).ToArray();