Skip to content

Commit

Permalink
Merge pull request umbraco#11538 from umbraco/v9/feature/merge-v8_29-…
Browse files Browse the repository at this point in the history 
…10-2021

V9: Merge v8 29-10-2021
  • Loading branch information
@bergmania
bergmania authored Oct 29, 2021
2 parents e9ae567 + bf5f136 commit 7aea361
Show file tree
Hide file tree
Showing 69 changed files with 1,674 additions and 1,394 deletions.
45 changes: 12 additions & 33 deletions src/Umbraco.Core/Collections/StackQueue.cs
View file
Original file line number Diff line number Diff line change
@@ -1,60 +1,39 @@
using System.Collections.Generic;

namespace Umbraco.Core.Collections
namespace Umbraco.Cms.Core.Collections
{
/// <summary>
/// Collection that can be both a queue and a stack.
/// Collection that can be both a queue and a stack.
/// </summary>
/// <typeparam name="T"></typeparam>
public class StackQueue<T>
{
private readonly LinkedList<T> _linkedList = new LinkedList<T>();
private readonly LinkedList<T> _linkedList = new();

public void Clear()
{
_linkedList.Clear();
}
public int Count => _linkedList.Count;

public void Push(T obj)
{
_linkedList.AddFirst(obj);
}
public void Clear() => _linkedList.Clear();

public void Enqueue(T obj)
{
_linkedList.AddFirst(obj);
}
public void Push(T obj) => _linkedList.AddFirst(obj);

public void Enqueue(T obj) => _linkedList.AddFirst(obj);

public T Pop()
{
var obj = _linkedList.First.Value;
T obj = _linkedList.First.Value;
_linkedList.RemoveFirst();
return obj;
}

public T Dequeue()
{
var obj = _linkedList.Last.Value;
T obj = _linkedList.Last.Value;
_linkedList.RemoveLast();
return obj;
}

public T PeekStack()
{
return _linkedList.First.Value;
}
public T PeekStack() => _linkedList.First.Value;

public T PeekQueue()
{
return _linkedList.Last.Value;
}

public int Count
{
get
{
return _linkedList.Count;
}
}
public T PeekQueue() => _linkedList.Last.Value;
}
}
18 changes: 18 additions & 0 deletions src/Umbraco.Core/Constants-Sql.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
namespace Umbraco.Cms.Core
{
public static partial class Constants
{
public static class Sql
{
/// <summary>
/// The maximum amount of parameters that can be used in a query.
/// </summary>
/// <remarks>
/// The actual limit is 2100
/// (https://docs.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server),
/// but we want to ensure there's room for additional parameters if this value is used to create groups/batches.
/// </remarks>
public const int MaxParameterCount = 2000;
}
}
}
2 changes: 1 addition & 1 deletion src/Umbraco.Core/Models/Mapping/UserMapDefinition.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ private void Map(IUserGroup source, UserGroupDisplay target, MapperContext conte
// the entity service due to too many Sql parameters.

var list = new List<IEntitySlim>();
foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(2000))
foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(Constants.Sql.MaxParameterCount))
list.AddRange(_entityService.GetAll(UmbracoObjectTypes.Document, idGroup.ToArray()));
contentEntities = list.ToArray();
}
Expand Down
108 changes: 54 additions & 54 deletions src/Umbraco.Core/Umbraco.Core.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,61 +1,61 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<RootNamespace>Umbraco.Cms.Core</RootNamespace>
<Product>Umbraco CMS</Product>
<PackageId>Umbraco.Cms.Core</PackageId>
<Title>Umbraco CMS Core</Title>
<Description>Contains the core assembly needed to run Umbraco Cms. This package only contains the assembly, and can be used for package development. Use the template in the Umbraco.Templates package to setup Umbraco</Description>
<Product>Umbraco CMS</Product>
</PropertyGroup>
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<RootNamespace>Umbraco.Cms.Core</RootNamespace>
<Product>Umbraco CMS</Product>
<PackageId>Umbraco.Cms.Core</PackageId>
<Title>Umbraco CMS Core</Title>
<Description>Contains the core assembly needed to run Umbraco Cms. This package only contains the assembly, and can be used for package development. Use the template in the Umbraco.Templates package to setup Umbraco</Description>
<Product>Umbraco CMS</Product>
</PropertyGroup>

<PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
<DocumentationFile>bin\Release\Umbraco.Core.xml</DocumentationFile>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
<DocumentationFile>bin\Release\Umbraco.Core.xml</DocumentationFile>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.FileProviders.Embedded" Version="5.0.10" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Options.DataAnnotations" Version="5.0.0" />
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
<PackageReference Include="System.Reflection.Emit.Lightweight" Version="4.7.0" />
<PackageReference Include="System.Runtime.Caching" Version="5.0.0" />
<PackageReference Include="Umbraco.Code" Version="1.2.0">
<PrivateAssets>all</PrivateAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="5.0.0"/>
<PackageReference Include="Microsoft.Extensions.FileProviders.Embedded" Version="5.0.10"/>
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="5.0.0"/>
<PackageReference Include="Microsoft.Extensions.Logging" Version="5.0.0"/>
<PackageReference Include="Microsoft.Extensions.Options" Version="5.0.0"/>
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="5.0.0"/>
<PackageReference Include="Microsoft.Extensions.Options.DataAnnotations" Version="5.0.0"/>
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0"/>
<PackageReference Include="System.Reflection.Emit.Lightweight" Version="4.7.0"/>
<PackageReference Include="System.Runtime.Caching" Version="5.0.0"/>
<PackageReference Include="Umbraco.Code" Version="1.2.0">
<PrivateAssets>all</PrivateAssets>
</PackageReference>
</ItemGroup>

<ItemGroup>
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
<_Parameter1>Umbraco.Tests</_Parameter1>
</AssemblyAttribute>
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
<_Parameter1>Umbraco.Tests.Common</_Parameter1>
</AssemblyAttribute>
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
<_Parameter1>Umbraco.Tests.UnitTests</_Parameter1>
</AssemblyAttribute>
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
<_Parameter1>Umbraco.Tests.Benchmarks</_Parameter1>
</AssemblyAttribute>
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
<_Parameter1>Umbraco.Tests.Integration</_Parameter1>
</AssemblyAttribute>
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
<_Parameter1>DynamicProxyGenAssembly2</_Parameter1>
</AssemblyAttribute>
</ItemGroup>
<ItemGroup>
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
<_Parameter1>Umbraco.Tests</_Parameter1>
</AssemblyAttribute>
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
<_Parameter1>Umbraco.Tests.Common</_Parameter1>
</AssemblyAttribute>
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
<_Parameter1>Umbraco.Tests.UnitTests</_Parameter1>
</AssemblyAttribute>
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
<_Parameter1>Umbraco.Tests.Benchmarks</_Parameter1>
</AssemblyAttribute>
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
<_Parameter1>Umbraco.Tests.Integration</_Parameter1>
</AssemblyAttribute>
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
<_Parameter1>DynamicProxyGenAssembly2</_Parameter1>
</AssemblyAttribute>
</ItemGroup>

<ItemGroup>
<EmbeddedResource Include="EmbeddedResources\**\*" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="EmbeddedResources\**\*"/>
</ItemGroup>
</Project>
65 changes: 38 additions & 27 deletions src/Umbraco.Infrastructure/Persistence/NPocoDatabaseExtensions-Bulk.cs
View file
Original file line number Diff line number Diff line change
@@ -1,64 +1,76 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.Common;
using System.Data.SqlClient;
using System.Linq;
using NPoco;
using Umbraco.Cms.Core;
using Umbraco.Cms.Infrastructure.Persistence;

namespace Umbraco.Extensions
{
/// <summary>
/// Provides extension methods to NPoco Database class.
/// Provides extension methods to NPoco Database class.
/// </summary>
public static partial class NPocoDatabaseExtensions
{
/// <summary>
/// Configures NPoco's SqlBulkCopyHelper to use the correct SqlConnection and SqlTransaction instances from the underlying RetryDbConnection and ProfiledDbTransaction
/// Configures NPoco's SqlBulkCopyHelper to use the correct SqlConnection and SqlTransaction instances from the
/// underlying RetryDbConnection and ProfiledDbTransaction
/// </summary>
/// <remarks>
/// This is required to use NPoco's own <see cref="Database.InsertBulk{T}(IEnumerable{T})" /> method because we use wrapped DbConnection and DbTransaction instances.
/// NPoco's InsertBulk method only caters for efficient bulk inserting records for Sql Server, it does not cater for bulk inserting of records for
/// any other database type and in which case will just insert records one at a time.
/// NPoco's InsertBulk method also deals with updating the passed in entity's PK/ID once it's inserted whereas our own BulkInsertRecords methods
/// do not handle this scenario.
/// This is required to use NPoco's own <see cref="Database.InsertBulk{T}(IEnumerable{T})" /> method because we use
/// wrapped DbConnection and DbTransaction instances.
/// NPoco's InsertBulk method only caters for efficient bulk inserting records for Sql Server, it does not cater for
/// bulk inserting of records for
/// any other database type and in which case will just insert records one at a time.
/// NPoco's InsertBulk method also deals with updating the passed in entity's PK/ID once it's inserted whereas our own
/// BulkInsertRecords methods
/// do not handle this scenario.
/// </remarks>
public static void ConfigureNPocoBulkExtensions()
{

SqlBulkCopyHelper.SqlConnectionResolver = dbConn => GetTypedConnection<SqlConnection>(dbConn);
SqlBulkCopyHelper.SqlTransactionResolver = dbTran => GetTypedTransaction<SqlTransaction>(dbTran);
}


/// <summary>
/// Creates bulk-insert commands.
/// Creates bulk-insert commands.
/// </summary>
/// <typeparam name="T">The type of the records.</typeparam>
/// <param name="database">The database.</param>
/// <param name="records">The records.</param>
/// <returns>The sql commands to execute.</returns>
internal static IDbCommand[] GenerateBulkInsertCommands<T>(this IUmbracoDatabase database, T[] records)
{
if (database?.Connection == null) throw new ArgumentException("Null database?.connection.", nameof(database));
if (database?.Connection == null)
{
throw new ArgumentException("Null database?.connection.", nameof(database));
}

var pocoData = database.PocoDataFactory.ForType(typeof(T));
PocoData pocoData = database.PocoDataFactory.ForType(typeof(T));

// get columns to include, = number of parameters per row
var columns = pocoData.Columns.Where(c => IncludeColumn(pocoData, c)).ToArray();
KeyValuePair<string, PocoColumn>[] columns =
pocoData.Columns.Where(c => IncludeColumn(pocoData, c)).ToArray();
var paramsPerRecord = columns.Length;

// format columns to sql
var tableName = database.DatabaseType.EscapeTableName(pocoData.TableInfo.TableName);
var columnNames = string.Join(", ", columns.Select(c => tableName + "." + database.DatabaseType.EscapeSqlIdentifier(c.Key)));
var columnNames = string.Join(", ",
columns.Select(c => tableName + "." + database.DatabaseType.EscapeSqlIdentifier(c.Key)));

// example:
// assume 4168 records, each record containing 8 fields, ie 8 command parameters
// max 2100 parameter per command
// Math.Floor(2100 / 8) = 262 record per command
// 4168 / 262 = 15.908... = there will be 16 command in total
// (if we have disabled db parameters, then all records will be included, in only one command)
var recordsPerCommand = paramsPerRecord == 0 ? int.MaxValue : Convert.ToInt32(Math.Floor(2000.00 / paramsPerRecord));
var recordsPerCommand = paramsPerRecord == 0
? int.MaxValue
: Convert.ToInt32(Math.Floor((double)Constants.Sql.MaxParameterCount / paramsPerRecord));
var commandsCount = Convert.ToInt32(Math.Ceiling((double)records.Length / recordsPerCommand));

var commands = new IDbCommand[commandsCount];
Expand All @@ -67,43 +79,42 @@ internal static IDbCommand[] GenerateBulkInsertCommands<T>(this IUmbracoDatabase
var prefix = database.DatabaseType.GetParameterPrefix(database.ConnectionString);
for (var commandIndex = 0; commandIndex < commandsCount; commandIndex++)
{
var command = database.CreateCommand(database.Connection, CommandType.Text, string.Empty);
DbCommand command = database.CreateCommand(database.Connection, CommandType.Text, string.Empty);
var parameterIndex = 0;
var commandRecords = Math.Min(recordsPerCommand, recordsLeftToInsert);
var recordsValues = new string[commandRecords];
for (var commandRecordIndex = 0; commandRecordIndex < commandRecords; commandRecordIndex++, recordsIndex++, recordsLeftToInsert--)
for (var commandRecordIndex = 0;
commandRecordIndex < commandRecords;
commandRecordIndex++, recordsIndex++, recordsLeftToInsert--)
{
var record = records[recordsIndex];
T record = records[recordsIndex];
var recordValues = new string[columns.Length];
for (var columnIndex = 0; columnIndex < columns.Length; columnIndex++)
{
database.AddParameter(command, columns[columnIndex].Value.GetValue(record));
recordValues[columnIndex] = prefix + parameterIndex++;
}

recordsValues[commandRecordIndex] = "(" + string.Join(",", recordValues) + ")";
}

command.CommandText = $"INSERT INTO {tableName} ({columnNames}) VALUES {string.Join(", ", recordsValues)}";
command.CommandText =
$"INSERT INTO {tableName} ({columnNames}) VALUES {string.Join(", ", recordsValues)}";
commands[commandIndex] = command;
}

return commands;
}

/// <summary>
/// Determines whether a column should be part of a bulk-insert.
/// Determines whether a column should be part of a bulk-insert.
/// </summary>
/// <param name="pocoData">The PocoData object corresponding to the record's type.</param>
/// <param name="column">The column.</param>
/// <returns>A value indicating whether the column should be part of the bulk-insert.</returns>
/// <remarks>Columns that are primary keys and auto-incremental, or result columns, are excluded from bulk-inserts.</remarks>
public static bool IncludeColumn(PocoData pocoData, KeyValuePair<string, PocoColumn> column)
{
return column.Value.ResultColumn == false
&& (pocoData.TableInfo.AutoIncrement == false || column.Key != pocoData.TableInfo.PrimaryKey);
}



public static bool IncludeColumn(PocoData pocoData, KeyValuePair<string, PocoColumn> column) =>
column.Value.ResultColumn == false
&& (pocoData.TableInfo.AutoIncrement == false || column.Key != pocoData.TableInfo.PrimaryKey);
}
}
Loading

0 comments on commit 7aea361

Please sign in to comment.