From 276872bc627b633857e88d2e079f23f769c84a06 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 27 Jun 2016 08:30:18 +0100 Subject: [PATCH 01/23] Improve Threadpool throughput --- .../src/System/Threading/ExecutionContext.cs | 36 +- .../src/System/Threading/ThreadPool.cs | 560 ++++++++++-------- 2 files changed, 345 insertions(+), 251 deletions(-) diff --git a/src/mscorlib/src/System/Threading/ExecutionContext.cs b/src/mscorlib/src/System/Threading/ExecutionContext.cs index 0440368608c6..cbb4f6e0f3b4 100644 --- a/src/mscorlib/src/System/Threading/ExecutionContext.cs +++ b/src/mscorlib/src/System/Threading/ExecutionContext.cs @@ -38,6 +38,8 @@ namespace System.Threading [System.Runtime.InteropServices.ComVisible(true)] public delegate void ContextCallback(Object state); + internal delegate void ContextCallback(T state); + #if FEATURE_CORECLR [SecurityCritical] @@ -92,8 +94,33 @@ public static ExecutionContext Capture() [HandleProcessCorruptedStateExceptions] public static void Run(ExecutionContext executionContext, ContextCallback callback, Object state) { - if (executionContext == null) - throw new InvalidOperationException(Environment.GetResourceString("InvalidOperation_NullContext")); + if (executionContext == null) ThrowInvalidOperationNullContextException(); + + Thread currentThread = Thread.CurrentThread; + ExecutionContextSwitcher ecsw = default(ExecutionContextSwitcher); + try + { + EstablishCopyOnWriteScope(currentThread, ref ecsw); + ExecutionContext.Restore(currentThread, executionContext); + callback(state); + } + catch + { + // Note: we have a "catch" rather than a "finally" because we want + // to stop the first pass of EH here. That way we can restore the previous + // context before any of our callers' EH filters run. That means we need to + // end the scope separately in the non-exceptional case below. + ecsw.Undo(currentThread); + throw; + } + ecsw.Undo(currentThread); + } + + [SecurityCritical] + [HandleProcessCorruptedStateExceptions] + internal static void Run(ExecutionContext executionContext, ContextCallback callback, T state) + { + if (executionContext == null) ThrowInvalidOperationNullContextException(); Thread currentThread = Thread.CurrentThread; ExecutionContextSwitcher ecsw = default(ExecutionContextSwitcher); @@ -115,6 +142,11 @@ public static void Run(ExecutionContext executionContext, ContextCallback callba ecsw.Undo(currentThread); } + private static void ThrowInvalidOperationNullContextException() + { + throw new InvalidOperationException(Environment.GetResourceString("InvalidOperation_NullContext")); + } + [SecurityCritical] internal static void Restore(Thread currentThread, ExecutionContext executionContext) { diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index 2ee7b76f89d5..a77dc64c16ca 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -29,14 +29,12 @@ namespace System.Threading { using System.Security; - using System.Runtime.Remoting; using System.Security.Permissions; using System; using Microsoft.Win32; using System.Runtime.CompilerServices; using System.Runtime.ConstrainedExecution; using System.Runtime.InteropServices; - using System.Runtime.Versioning; using System.Collections.Generic; using System.Diagnostics.Contracts; using System.Diagnostics.CodeAnalysis; @@ -46,21 +44,25 @@ internal static class ThreadPoolGlobals { //Per-appDomain quantum (in ms) for which the thread keeps processing //requests in the current domain. - public static uint tpQuantum = 30U; + public static uint tpQuantum; - public static int processorCount = Environment.ProcessorCount; + public static int processorCount; - public static bool tpHosted = ThreadPool.IsThreadPoolHosted(); + public static bool tpHosted; public static volatile bool vmTpInitialized; public static bool enableWorkerTracking; [SecurityCritical] - public static ThreadPoolWorkQueue workQueue = new ThreadPoolWorkQueue(); + public static ThreadPoolWorkQueue workQueue; - [System.Security.SecuritySafeCritical] // static constructors should be safe to call - static ThreadPoolGlobals() + [SecurityCritical] + internal static void Initialize() { + tpQuantum = 30U; + processorCount = Environment.ProcessorCount; + tpHosted = ThreadPool.IsThreadPoolHosted(); + workQueue = new ThreadPoolWorkQueue(); } } @@ -154,33 +156,7 @@ public void LocalPush(IThreadPoolWorkItem obj) // We're going to increment the tail; if we'll overflow, then we need to reset our counts if (tail == int.MaxValue) { - bool lockTaken = false; - try - { - m_foreignLock.Enter(ref lockTaken); - - if (m_tailIndex == int.MaxValue) - { - // - // Rather than resetting to zero, we'll just mask off the bits we don't care about. - // This way we don't need to rearrange the items already in the queue; they'll be found - // correctly exactly where they are. One subtlety here is that we need to make sure that - // if head is currently < tail, it remains that way. This happens to just fall out from - // the bit-masking, because we only do this if tail == int.MaxValue, meaning that all - // bits are set, so all of the bits we're keeping will also be set. Thus it's impossible - // for the head to end up > than the tail, since you can't set any more bits than all of - // them. - // - m_headIndex = m_headIndex & m_mask; - m_tailIndex = tail = m_tailIndex & m_mask; - Contract.Assert(m_headIndex <= m_tailIndex); - } - } - finally - { - if (lockTaken) - m_foreignLock.Exit(true); - } + tail = LocalPushOverflow(tail); } // When there are at least 2 elements' worth of space, we can take the fast path. @@ -188,42 +164,79 @@ public void LocalPush(IThreadPoolWorkItem obj) { Volatile.Write(ref m_array[tail & m_mask], obj); m_tailIndex = tail + 1; + return; } - else + + LocalPushPreventSteal(obj, tail); + } + + private void LocalPushPreventSteal(IThreadPoolWorkItem obj, int tail) + { + // We need to contend with foreign pops, so we lock. + bool lockTaken = false; + try { - // We need to contend with foreign pops, so we lock. - bool lockTaken = false; - try + m_foreignLock.Enter(ref lockTaken); + + int head = m_headIndex; + int count = m_tailIndex - m_headIndex; + + // If there is still space (one left), just add the element. + if (count >= m_mask) { - m_foreignLock.Enter(ref lockTaken); + // We're full; expand the queue by doubling its size. + IThreadPoolWorkItem[] newArray = new IThreadPoolWorkItem[m_array.Length << 1]; + for (int i = 0; i < m_array.Length; i++) + newArray[i] = m_array[(i + head) & m_mask]; + + // Reset the field values, incl. the mask. + m_array = newArray; + m_headIndex = 0; + m_tailIndex = tail = count; + m_mask = (m_mask << 1) | 1; + } - int head = m_headIndex; - int count = m_tailIndex - m_headIndex; + Volatile.Write(ref m_array[tail & m_mask], obj); + m_tailIndex = tail + 1; + } + finally + { + if (lockTaken) + m_foreignLock.Exit(false); + } + } - // If there is still space (one left), just add the element. - if (count >= m_mask) - { - // We're full; expand the queue by doubling its size. - IThreadPoolWorkItem[] newArray = new IThreadPoolWorkItem[m_array.Length << 1]; - for (int i = 0; i < m_array.Length; i++) - newArray[i] = m_array[(i + head) & m_mask]; - - // Reset the field values, incl. the mask. - m_array = newArray; - m_headIndex = 0; - m_tailIndex = tail = count; - m_mask = (m_mask << 1) | 1; - } + private int LocalPushOverflow(int tail) + { + bool lockTaken = false; + try + { + m_foreignLock.Enter(ref lockTaken); - Volatile.Write(ref m_array[tail & m_mask], obj); - m_tailIndex = tail + 1; - } - finally + if (m_tailIndex == int.MaxValue) { - if (lockTaken) - m_foreignLock.Exit(false); + // + // Rather than resetting to zero, we'll just mask off the bits we don't care about. + // This way we don't need to rearrange the items already in the queue; they'll be found + // correctly exactly where they are. One subtlety here is that we need to make sure that + // if head is currently < tail, it remains that way. This happens to just fall out from + // the bit-masking, because we only do this if tail == int.MaxValue, meaning that all + // bits are set, so all of the bits we're keeping will also be set. Thus it's impossible + // for the head to end up > than the tail, since you can't set any more bits than all of + // them. + // + m_headIndex = m_headIndex & m_mask; + m_tailIndex = tail = m_tailIndex & m_mask; + Contract.Assert(m_headIndex <= m_tailIndex); } } + finally + { + if (lockTaken) + m_foreignLock.Exit(true); + } + + return tail; } [SuppressMessage("Microsoft.Concurrency", "CA8001", Justification = "Reviewed for thread safety")] @@ -241,6 +254,11 @@ public bool LocalFindAndPop(IThreadPoolWorkItem obj) return false; } + return LocalFindAndPopSteal(obj); + } + + private bool LocalFindAndPopSteal(IThreadPoolWorkItem obj) + { // Else, do an O(N) search for the work item. The theory of work stealing and our // inlining logic is that most waits will happen on recently queued work. And // since recently queued work will be close to the tail end (which is where we @@ -315,57 +333,76 @@ public bool LocalPop(out IThreadPoolWorkItem obj) m_array[idx] = null; return true; } - else - { - // Interaction with takes: 0 or 1 elements left. - bool lockTaken = false; - try - { - m_foreignLock.Enter(ref lockTaken); - if (m_headIndex <= tail) - { - // Element still available. Take it. - int idx = tail & m_mask; - obj = Volatile.Read(ref m_array[idx]); + bool skip; + bool result = LocalPopLocked(out obj, ref tail, out skip); + // continue if null in array + if (skip) continue; - // Check for nulls in the array. - if (obj == null) continue; + return result; + } + } - m_array[idx] = null; - return true; - } - else - { - // If we encountered a race condition and element was stolen, restore the tail. - m_tailIndex = tail + 1; - obj = null; - return false; - } - } - finally - { - if (lockTaken) - m_foreignLock.Exit(false); - } + private bool LocalPopLocked(out IThreadPoolWorkItem obj, ref int tail, out bool skip) + { + // Interaction with takes: 0 or 1 elements left. + bool lockTaken = false; + try + { + m_foreignLock.Enter(ref lockTaken); + + if (m_headIndex <= tail) + { + // Element still available. Take it. + int idx = tail & m_mask; + obj = Volatile.Read(ref m_array[idx]); + + // Check for nulls in the array. + if (obj == null) { + skip = true; + return false; + }; + + m_array[idx] = null; + skip = false; + return true; } + else + { + // If we encountered a race condition and element was stolen, restore the tail. + m_tailIndex = tail + 1; + skip = false; + obj = null; + return false; + } + } + finally + { + if (lockTaken) + m_foreignLock.Exit(false); } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool TrySteal(out IThreadPoolWorkItem obj, ref bool missedSteal) { return TrySteal(out obj, ref missedSteal, 0); // no blocking by default. } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool TrySteal(out IThreadPoolWorkItem obj, ref bool missedSteal, int millisecondsTimeout) { obj = null; + if (m_headIndex >= m_tailIndex) + return false; + return TryStealWithItems(ref obj, ref missedSteal, millisecondsTimeout); + } + + private bool TryStealWithItems(ref IThreadPoolWorkItem obj, ref bool missedSteal, int millisecondsTimeout) + { while (true) { - if (m_headIndex >= m_tailIndex) - return false; - bool taken = false; try { @@ -382,7 +419,12 @@ private bool TrySteal(out IThreadPoolWorkItem obj, ref bool missedSteal, int mil obj = Volatile.Read(ref m_array[idx]); // Check for nulls in the array. - if (obj == null) continue; + if (obj == null) + { + if (m_headIndex >= m_tailIndex) + return false; + continue; + }; m_array[idx] = null; return true; @@ -411,17 +453,22 @@ private bool TrySteal(out IThreadPoolWorkItem obj, ref bool missedSteal, int mil } } + // To seperate `indexes` and `Next` to reduce cache line false sharing between them + [StructLayout(LayoutKind.Explicit)] internal class QueueSegment { - // Holds a segment of the queue. Enqueues/Dequeues start at element 0, and work their way up. - internal readonly IThreadPoolWorkItem[] nodes; - private const int QueueSegmentLength = 256; - // Holds the indexes of the lowest and highest valid elements of the nodes array. // The low index is in the lower 16 bits, high index is in the upper 16 bits. // Use GetIndexes and CompareExchangeIndexes to manipulate this. + [FieldOffset(0)] private volatile int indexes; + // Holds a segment of the queue. Enqueues/Dequeues start at element 0, and work their way up. + [FieldOffset(8)] + internal readonly IThreadPoolWorkItem[] nodes; + private const int QueueSegmentLength = 256; + + [FieldOffset(64)] // The next segment in the queue. public volatile QueueSegment Next; @@ -557,11 +604,11 @@ public ThreadPoolWorkQueue() } [SecurityCritical] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public ThreadPoolWorkQueueThreadLocals EnsureCurrentThreadHasQueue() { - if (null == ThreadPoolWorkQueueThreadLocals.threadLocals) - ThreadPoolWorkQueueThreadLocals.threadLocals = new ThreadPoolWorkQueueThreadLocals(this); - return ThreadPoolWorkQueueThreadLocals.threadLocals; + var queue = ThreadPoolWorkQueueThreadLocals.threadLocals; + return null != queue ? queue : (ThreadPoolWorkQueueThreadLocals.threadLocals = new ThreadPoolWorkQueueThreadLocals(this)); } [SecurityCritical] @@ -573,7 +620,8 @@ internal void EnsureThreadRequested() // which is handled by RequestWorkerThread. // int count = numOutstandingThreadRequests; - while (count < ThreadPoolGlobals.processorCount) + var procCount = ThreadPoolGlobals.processorCount; + while (count < procCount) { int prev = Interlocked.CompareExchange(ref numOutstandingThreadRequests, count+1, count); if (prev == count) @@ -657,49 +705,57 @@ public void Dequeue(ThreadPoolWorkQueueThreadLocals tl, out IThreadPoolWorkItem WorkStealingQueue wsq = tl.workStealingQueue; if (wsq.LocalPop(out callback)) + { Contract.Assert(null != callback); + return; + } + + DequeueSeek(tl, ref callback, ref missedSteal); + } - if (null == callback) + private void DequeueSeek(ThreadPoolWorkQueueThreadLocals tl, ref IThreadPoolWorkItem callback, ref bool missedSteal) + { + QueueSegment tail = queueTail; + while (true) { - QueueSegment tail = queueTail; - while (true) + if (tail.TryDequeue(out callback)) { - if (tail.TryDequeue(out callback)) - { - Contract.Assert(null != callback); - break; - } + Contract.Assert(null != callback); + return; + } - if (null == tail.Next || !tail.IsUsedUp()) - { - break; - } - else - { - Interlocked.CompareExchange(ref queueTail, tail.Next, tail); - tail = queueTail; - } + if (null == tail.Next || !tail.IsUsedUp()) + { + break; + } + else + { + Interlocked.CompareExchange(ref queueTail, tail.Next, tail); + tail = queueTail; } } - if (null == callback) + DequeueSteal(tl, ref callback, ref missedSteal); + } + + private static void DequeueSteal(ThreadPoolWorkQueueThreadLocals tl, ref IThreadPoolWorkItem callback, ref bool missedSteal) + { + WorkStealingQueue wsq = tl.workStealingQueue; + WorkStealingQueue[] otherQueues = allThreadQueues.Current; + int i = tl.random.Next(otherQueues.Length); + int c = otherQueues.Length; + while (c > 0) { - WorkStealingQueue[] otherQueues = allThreadQueues.Current; - int i = tl.random.Next(otherQueues.Length); - int c = otherQueues.Length; - while (c > 0) + WorkStealingQueue otherQueue = Volatile.Read(ref otherQueues[i % otherQueues.Length]); + if (otherQueue != null && + otherQueue != wsq && + otherQueue.TrySteal(out callback, ref missedSteal)) { - WorkStealingQueue otherQueue = Volatile.Read(ref otherQueues[i % otherQueues.Length]); - if (otherQueue != null && - otherQueue != wsq && - otherQueue.TrySteal(out callback, ref missedSteal)) - { - Contract.Assert(null != callback); - break; - } - i++; - c--; + Contract.Assert(null != callback); + break; } + i++; + c--; } } @@ -1168,9 +1224,6 @@ internal interface IThreadPoolWorkItem internal sealed class QueueUserWorkItemCallback : IThreadPoolWorkItem { - [System.Security.SecuritySafeCritical] - static QueueUserWorkItemCallback() {} - private WaitCallback callback; private ExecutionContext context; private Object state; @@ -1197,6 +1250,7 @@ void MarkExecuted(bool aborted) [SecurityCritical] internal QueueUserWorkItemCallback(WaitCallback waitCallback, Object stateObj, ExecutionContext ec) { + Contract.Assert(waitCallback != null, "Null callback passed to QueueUserWorkItemCallback!"); callback = waitCallback; state = stateObj; context = ec; @@ -1217,7 +1271,7 @@ void IThreadPoolWorkItem.ExecuteWorkItem() } else { - ExecutionContext.Run(context, ccb, this, true); + ExecutionContext.Run(context, ccb, this); } } @@ -1232,23 +1286,17 @@ void IThreadPoolWorkItem.MarkAborted(ThreadAbortException tae) } [System.Security.SecurityCritical] - static internal ContextCallback ccb = new ContextCallback(WaitCallback_Context); + static internal ContextCallback ccb; [System.Security.SecurityCritical] - static private void WaitCallback_Context(Object state) + static internal void Initialize() { - QueueUserWorkItemCallback obj = (QueueUserWorkItemCallback)state; - WaitCallback wc = obj.callback as WaitCallback; - Contract.Assert(null != wc); - wc(obj.state); + ccb = (helper) => helper.callback(helper.state); } } internal sealed class QueueUserWorkItemCallbackDefaultContext : IThreadPoolWorkItem { - [System.Security.SecuritySafeCritical] - static QueueUserWorkItemCallbackDefaultContext() { } - private WaitCallback callback; private Object state; @@ -1274,6 +1322,7 @@ void MarkExecuted(bool aborted) [SecurityCritical] internal QueueUserWorkItemCallbackDefaultContext(WaitCallback waitCallback, Object stateObj) { + Contract.Assert(waitCallback != null, "Null callback passed to QueueUserWorkItemCallbackDefaultContext!"); callback = waitCallback; state = stateObj; } @@ -1284,7 +1333,7 @@ void IThreadPoolWorkItem.ExecuteWorkItem() #if DEBUG MarkExecuted(false); #endif - ExecutionContext.Run(ExecutionContext.PreAllocatedDefault, ccb, this, true); + ExecutionContext.Run(ExecutionContext.PreAllocatedDefault, ccb, this); } [SecurityCritical] @@ -1298,69 +1347,53 @@ void IThreadPoolWorkItem.MarkAborted(ThreadAbortException tae) } [System.Security.SecurityCritical] - static internal ContextCallback ccb = new ContextCallback(WaitCallback_Context); + static internal ContextCallback ccb; [System.Security.SecurityCritical] - static private void WaitCallback_Context(Object state) + static internal void Initialize() { - QueueUserWorkItemCallbackDefaultContext obj = (QueueUserWorkItemCallbackDefaultContext)state; - WaitCallback wc = obj.callback as WaitCallback; - Contract.Assert(null != wc); - obj.callback = null; - wc(obj.state); + ccb = (helper) => helper.callback(helper.state); } } internal class _ThreadPoolWaitOrTimerCallback { - [System.Security.SecuritySafeCritical] - static _ThreadPoolWaitOrTimerCallback() {} - WaitOrTimerCallback _waitOrTimerCallback; ExecutionContext _executionContext; Object _state; [System.Security.SecurityCritical] - static private ContextCallback _ccbt = new ContextCallback(WaitOrTimerCallback_Context_t); + static private ContextCallback<_ThreadPoolWaitOrTimerCallback> _ccbt; [System.Security.SecurityCritical] - static private ContextCallback _ccbf = new ContextCallback(WaitOrTimerCallback_Context_f); + static private ContextCallback<_ThreadPoolWaitOrTimerCallback> _ccbf; + + [System.Security.SecurityCritical] + static internal void Initialize() + { + _ccbt = (helper) => helper._waitOrTimerCallback(helper._state, true); + _ccbf = (helper) => helper._waitOrTimerCallback(helper._state, false); + } [System.Security.SecurityCritical] // auto-generated internal _ThreadPoolWaitOrTimerCallback(WaitOrTimerCallback waitOrTimerCallback, Object state, bool compressStack, ref StackCrawlMark stackMark) { + Contract.Assert(waitOrTimerCallback != null, "Null callback passed to _ThreadPoolWaitOrTimerCallback!"); + _waitOrTimerCallback = waitOrTimerCallback; _state = state; if (compressStack && !ExecutionContext.IsFlowSuppressed()) { // capture the exection context - _executionContext = ExecutionContext.Capture( - ref stackMark, - ExecutionContext.CaptureOptions.IgnoreSyncCtx | ExecutionContext.CaptureOptions.OptimizeDefaultCase); + _executionContext = ExecutionContext.FastCapture(); } } - - [System.Security.SecurityCritical] - static private void WaitOrTimerCallback_Context_t(Object state) - { - WaitOrTimerCallback_Context(state, true); - } - - [System.Security.SecurityCritical] - static private void WaitOrTimerCallback_Context_f(Object state) - { - WaitOrTimerCallback_Context(state, false); - } - - static private void WaitOrTimerCallback_Context(Object state, bool timedOut) - { - _ThreadPoolWaitOrTimerCallback helper = (_ThreadPoolWaitOrTimerCallback)state; - helper._waitOrTimerCallback(helper._state, timedOut); - } // call back helper [System.Security.SecurityCritical] // auto-generated static internal void PerformWaitOrTimerCallback(Object state, bool timedOut) { + ThreadPool.EnsureVMInitialized(); + _ThreadPoolWaitOrTimerCallback helper = (_ThreadPoolWaitOrTimerCallback)state; Contract.Assert(helper != null, "Null state passed to PerformWaitOrTimerCallback!"); // call directly if it is an unsafe call OR EC flow is suppressed @@ -1373,10 +1406,14 @@ static internal void PerformWaitOrTimerCallback(Object state, bool timedOut) { using (ExecutionContext executionContext = helper._executionContext.CreateCopy()) { - if (timedOut) - ExecutionContext.Run(executionContext, _ccbt, helper, true); - else - ExecutionContext.Run(executionContext, _ccbf, helper, true); + if (timedOut) + { + ExecutionContext.Run(executionContext, _ccbt, helper); + } + else + { + ExecutionContext.Run(executionContext, _ccbf, helper); + } } } } @@ -1619,79 +1656,86 @@ bool executeOnlyOnce return RegisterWaitForSingleObject(waitObject,callBack,state,(UInt32)tm,executeOnlyOnce,ref stackMark,false); } - [System.Security.SecuritySafeCritical] // auto-generated - [MethodImplAttribute(MethodImplOptions.NoInlining)] // Methods containing StackCrawlMark local var has to be marked non-inlineable + [System.Security.SecuritySafeCritical] // auto-generated public static bool QueueUserWorkItem( - WaitCallback callBack, // NOTE: we do not expose options that allow the callback to be queued as an APC - Object state - ) + WaitCallback callBack, // NOTE: we do not expose options that allow the callback to be queued as an APC + Object state) { - StackCrawlMark stackMark = StackCrawlMark.LookForMyCaller; - return QueueUserWorkItemHelper(callBack,state,ref stackMark,true); + if (callBack == null) return ThrowWaitCallbackNullException(); + + //The VM is responsible for the actual growing/shrinking of threads. + EnsureVMInitialized(); + + // If we are able to create the workitem, we need to get it in the queue without being interrupted by a ThreadAbortException. + try { } + finally + { + ExecutionContext context = !ExecutionContext.IsFlowSuppressed() ? ExecutionContext.FastCapture() : null; + + IThreadPoolWorkItem tpcallBack = context == ExecutionContext.PreAllocatedDefault ? + new QueueUserWorkItemCallbackDefaultContext(callBack, state) : + (IThreadPoolWorkItem)new QueueUserWorkItemCallback(callBack, state, context); + + //ThreadPool has per-appdomain managed queue of work-items. The VM is + //responsible for just scheduling threads into appdomains. After that + //work-items are dispatched from the managed queue. + ThreadPoolGlobals.workQueue.Enqueue(tpcallBack, true); + } + return true; } [System.Security.SecuritySafeCritical] // auto-generated - [MethodImplAttribute(MethodImplOptions.NoInlining)] // Methods containing StackCrawlMark local var has to be marked non-inlineable public static bool QueueUserWorkItem( - WaitCallback callBack // NOTE: we do not expose options that allow the callback to be queued as an APC - ) + WaitCallback callBack) // NOTE: we do not expose options that allow the callback to be queued as an APC { - StackCrawlMark stackMark = StackCrawlMark.LookForMyCaller; - return QueueUserWorkItemHelper(callBack,null,ref stackMark,true); - } - - [System.Security.SecurityCritical] // auto-generated_required - [MethodImplAttribute(MethodImplOptions.NoInlining)] // Methods containing StackCrawlMark local var has to be marked non-inlineable - public static bool UnsafeQueueUserWorkItem( - WaitCallback callBack, // NOTE: we do not expose options that allow the callback to be queued as an APC - Object state - ) - { - StackCrawlMark stackMark = StackCrawlMark.LookForMyCaller; - return QueueUserWorkItemHelper(callBack,state,ref stackMark,false); - } + if (callBack == null) return ThrowWaitCallbackNullException(); - //ThreadPool has per-appdomain managed queue of work-items. The VM is - //responsible for just scheduling threads into appdomains. After that - //work-items are dispatched from the managed queue. - [System.Security.SecurityCritical] // auto-generated - private static bool QueueUserWorkItemHelper(WaitCallback callBack, Object state, ref StackCrawlMark stackMark, bool compressStack ) - { - bool success = true; + //The VM is responsible for the actual growing/shrinking of threads. + EnsureVMInitialized(); - if (callBack != null) + // If we are able to create the workitem, we need to get it in the queue without being interrupted by a ThreadAbortException. + try { } + finally { - //The thread pool maintains a per-appdomain managed work queue. - //New thread pool entries are added in the managed queue. - //The VM is responsible for the actual growing/shrinking of - //threads. + ExecutionContext context = !ExecutionContext.IsFlowSuppressed() ? ExecutionContext.FastCapture() : null; - EnsureVMInitialized(); + IThreadPoolWorkItem tpcallBack = context == ExecutionContext.PreAllocatedDefault ? + new QueueUserWorkItemCallbackDefaultContext(callBack, null) : + (IThreadPoolWorkItem)new QueueUserWorkItemCallback(callBack, null, context); - // - // If we are able to create the workitem, we need to get it in the queue without being interrupted - // by a ThreadAbortException. - // - try { } - finally - { - ExecutionContext context = compressStack && !ExecutionContext.IsFlowSuppressed() ? - ExecutionContext.Capture(ref stackMark, ExecutionContext.CaptureOptions.IgnoreSyncCtx | ExecutionContext.CaptureOptions.OptimizeDefaultCase) : - null; + //ThreadPool has per-appdomain managed queue of work-items. The VM is + //responsible for just scheduling threads into appdomains. After that + //work-items are dispatched from the managed queue. + ThreadPoolGlobals.workQueue.Enqueue(tpcallBack, true); + } + return true; + } + + [System.Security.SecurityCritical] // auto-generated_required + public static bool UnsafeQueueUserWorkItem( + WaitCallback callBack, // NOTE: we do not expose options that allow the callback to be queued as an APC + Object state) + { + if (callBack == null) return ThrowWaitCallbackNullException(); - IThreadPoolWorkItem tpcallBack = context == ExecutionContext.PreAllocatedDefault ? - new QueueUserWorkItemCallbackDefaultContext(callBack, state) : - (IThreadPoolWorkItem)new QueueUserWorkItemCallback(callBack, state, context); + //The VM is responsible for the actual growing/shrinking of threads. + EnsureVMInitialized(); - ThreadPoolGlobals.workQueue.Enqueue(tpcallBack, true); - success = true; - } - } - else + // If we are able to create the workitem, we need to get it in the queue without being interrupted by a ThreadAbortException. + try { } + finally { - throw new ArgumentNullException("WaitCallback"); + //ThreadPool has per-appdomain managed queue of work-items. The VM is + //responsible for just scheduling threads into appdomains. After that + //work-items are dispatched from the managed queue. + ThreadPoolGlobals.workQueue.Enqueue(new QueueUserWorkItemCallback(callBack, state, null), true); } - return success; + return true; + } + + private static bool ThrowWaitCallbackNullException() + { + throw new ArgumentNullException("WaitCallback"); } [SecurityCritical] @@ -1834,17 +1878,33 @@ unsafe public static bool UnsafeQueueNativeOverlapped(NativeOverlapped* overlapp } [SecurityCritical] - private static void EnsureVMInitialized() + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void EnsureVMInitialized() { if (!ThreadPoolGlobals.vmTpInitialized) { - ThreadPool.InitializeVMTp(ref ThreadPoolGlobals.enableWorkerTracking); - ThreadPoolGlobals.vmTpInitialized = true; + //The thread pool maintains a per-appdomain managed work queue. + //New thread pool entries are added in the managed queue. + //The VM is responsible for the actual growing/shrinking of + //threads. + + InitalizeVM(); } } + [SecurityCritical] + private static void InitalizeVM() + { + ThreadPool.InitializeVMTp(ref ThreadPoolGlobals.enableWorkerTracking); + ThreadPoolGlobals.Initialize(); + QueueUserWorkItemCallback.Initialize(); + QueueUserWorkItemCallbackDefaultContext.Initialize(); + _ThreadPoolWaitOrTimerCallback.Initialize(); + ThreadPoolGlobals.vmTpInitialized = true; + } + // Native methods: - + [System.Security.SecurityCritical] // auto-generated [MethodImplAttribute(MethodImplOptions.InternalCall)] private static extern bool SetMinThreadsNative(int workerThreads, int completionPortThreads); @@ -1877,7 +1937,9 @@ private static void EnsureVMInitialized() internal static void NotifyWorkItemProgress() { if (!ThreadPoolGlobals.vmTpInitialized) - ThreadPool.InitializeVMTp(ref ThreadPoolGlobals.enableWorkerTracking); + { + EnsureVMInitialized(); + } NotifyWorkItemProgressNative(); } From 851ee72e006420cd4a281527fc8e67418c91fa95 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 27 Jun 2016 11:04:16 +0100 Subject: [PATCH 02/23] ThreadPool.SparseArray improvement for DequeueSteal --- .../src/System/Threading/ThreadPool.cs | 99 ++++++++++++++----- 1 file changed, 75 insertions(+), 24 deletions(-) diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index a77dc64c16ca..bd58e1af249d 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -71,42 +71,78 @@ internal sealed class ThreadPoolWorkQueue // Simple sparsely populated array to allow lock-free reading. internal class SparseArray where T : class { - private volatile T[] m_array; + private Snapshot m_current; + + internal sealed class Snapshot + { + public readonly T[] Data; + public readonly int Mask; + private int m_length; + + internal Snapshot(int initialSize, int initalLength = 0) + { + if ((initialSize >> 1) << 1 != initialSize) + { + throw new ArgumentOutOfRangeException(nameof(initialSize), "initialSize must be a power of 2"); + } + + Data = new T[initialSize]; + Mask = initialSize - 1; + m_length = initalLength; + } + + internal int ActiveLength => m_length; + + internal void IncrementLength() + { + m_length++; + } + } internal SparseArray(int initialSize) { - m_array = new T[initialSize]; + m_current = new Snapshot(initialSize); } - internal T[] Current + internal Snapshot Current { - get { return m_array; } + get { return m_current; } } internal int Add(T e) { while (true) { - T[] array = m_array; - lock (array) + var current = m_current; + lock (current) { + if (current != m_current) + { + // If there was a race condition, we start over again. + continue; + } + + var array = current.Data; + for (int i = 0; i < array.Length; i++) { if (array[i] == null) { + if (i + 1 > current.ActiveLength) + { + current.IncrementLength(); + } Volatile.Write(ref array[i], e); return i; } else if (i == array.Length - 1) { - // Must resize. If there was a race condition, we start over again. - if (array != m_array) - continue; + var newSnapshot = new Snapshot(array.Length * 2, array.Length + 1); + T[] newArray = newSnapshot.Data; - T[] newArray = new T[array.Length * 2]; Array.Copy(array, newArray, i + 1); newArray[i + 1] = e; - m_array = newArray; + m_current = newSnapshot; return i + 1; } } @@ -116,16 +152,29 @@ internal int Add(T e) internal void Remove(T e) { - T[] array = m_array; - lock (array) + while (true) { - for (int i = 0; i < m_array.Length; i++) + var current = m_current; + lock (current) { - if (m_array[i] == e) + if (current != m_current) { - Volatile.Write(ref m_array[i], null); - break; + // If there was a race condition, we start over again. + continue; + } + + var array = current.Data; + var length = current.ActiveLength; + + for (int i = 0; i < length; i++) + { + if (array[i] == e) + { + Volatile.Write(ref array[i], null); + break; + } } + break; } } } @@ -741,12 +790,14 @@ private void DequeueSeek(ThreadPoolWorkQueueThreadLocals tl, ref IThreadPoolWork private static void DequeueSteal(ThreadPoolWorkQueueThreadLocals tl, ref IThreadPoolWorkItem callback, ref bool missedSteal) { WorkStealingQueue wsq = tl.workStealingQueue; - WorkStealingQueue[] otherQueues = allThreadQueues.Current; - int i = tl.random.Next(otherQueues.Length); - int c = otherQueues.Length; - while (c > 0) + var otherQueues = allThreadQueues.Current; + var remaining = otherQueues.ActiveLength; + var i = tl.random.Next(remaining); + var data = otherQueues.Data; + var mask = otherQueues.Mask; + while (remaining > 0) { - WorkStealingQueue otherQueue = Volatile.Read(ref otherQueues[i % otherQueues.Length]); + WorkStealingQueue otherQueue = Volatile.Read(ref data[i & mask]); if (otherQueue != null && otherQueue != wsq && otherQueue.TrySteal(out callback, ref missedSteal)) @@ -755,7 +806,7 @@ private static void DequeueSteal(ThreadPoolWorkQueueThreadLocals tl, ref IThread break; } i++; - c--; + remaining--; } } @@ -1768,7 +1819,7 @@ internal static bool TryPopCustomWorkItem(IThreadPoolWorkItem workItem) [SecurityCritical] internal static IEnumerable GetQueuedWorkItems() { - return EnumerateQueuedWorkItems(ThreadPoolWorkQueue.allThreadQueues.Current, ThreadPoolGlobals.workQueue.queueTail); + return EnumerateQueuedWorkItems(ThreadPoolWorkQueue.allThreadQueues.Current.Data, ThreadPoolGlobals.workQueue.queueTail); } internal static IEnumerable EnumerateQueuedWorkItems(ThreadPoolWorkQueue.WorkStealingQueue[] wsQueues, ThreadPoolWorkQueue.QueueSegment globalQueueTail) From 50f8154ff6955c70884e124af58707482519434b Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 3 Jul 2016 16:36:00 +0100 Subject: [PATCH 03/23] Deterministic not random DequeueSteal search start --- src/mscorlib/src/System/Threading/ThreadPool.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index bd58e1af249d..3934caf09552 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -642,6 +642,8 @@ public bool TryDequeue(out IThreadPoolWorkItem node) internal volatile QueueSegment queueTail; internal bool loggingEnabled; + private static int NextSearchStart; + internal static SparseArray allThreadQueues = new SparseArray(16); private volatile int numOutstandingThreadRequests = 0; @@ -792,7 +794,11 @@ private static void DequeueSteal(ThreadPoolWorkQueueThreadLocals tl, ref IThread WorkStealingQueue wsq = tl.workStealingQueue; var otherQueues = allThreadQueues.Current; var remaining = otherQueues.ActiveLength; - var i = tl.random.Next(remaining); + // allThreadQueues.Data.Length is a power of 2, initally 16 + // Move next steal start on by 9 = (8 + 1) rather than 1 + // It means the search still progresses through all start points evenly in a deterministic manner + // However it also interleaves them to reduce collisions between threads + var i = Interlocked.Add(ref NextSearchStart, 9); var data = otherQueues.Data; var mask = otherQueues.Mask; while (remaining > 0) @@ -975,7 +981,6 @@ internal sealed class ThreadPoolWorkQueueThreadLocals public readonly ThreadPoolWorkQueue workQueue; public readonly ThreadPoolWorkQueue.WorkStealingQueue workStealingQueue; - public readonly Random random = new Random(Thread.CurrentThread.ManagedThreadId); public ThreadPoolWorkQueueThreadLocals(ThreadPoolWorkQueue tpq) { From 440e3030766308c5c4d0bdfa0326577ed33de8af Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Tue, 28 Jun 2016 04:22:51 +0100 Subject: [PATCH 04/23] Inline GetIndexes --- src/mscorlib/src/System/Threading/ThreadPool.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index 3934caf09552..1219403c011d 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -524,6 +524,7 @@ internal class QueueSegment const int SixteenBits = 0xffff; + [MethodImpl(MethodImplOptions.AggressiveInlining)] void GetIndexes(out int upper, out int lower) { int i = indexes; From 4bc20cb212659e42685719764403f968424ba5c4 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Tue, 28 Jun 2016 04:56:57 +0100 Subject: [PATCH 05/23] Inline index comparisions --- src/mscorlib/src/System/Threading/ThreadPool.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index 1219403c011d..1502a44dd279 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -538,6 +538,7 @@ void GetIndexes(out int upper, out int lower) Contract.Assert(lower >= 0); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] bool CompareExchangeIndexes(ref int prevUpper, int newUpper, ref int prevLower, int newLower) { Contract.Assert(newUpper >= newLower); @@ -564,7 +565,7 @@ public QueueSegment() nodes = new IThreadPoolWorkItem[QueueSegmentLength]; } - + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool IsUsedUp() { int upper, lower; From 3215ffd682661e3e2a250392dd8f857493037823 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 4 Jul 2016 04:50:21 +0100 Subject: [PATCH 06/23] Use ref rather than out for new call chain --- .../src/System/Threading/ThreadPool.cs | 46 ++++++++----------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index 1502a44dd279..e49ac95a57e0 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -294,8 +294,8 @@ public bool LocalFindAndPop(IThreadPoolWorkItem obj) // Fast path: check the tail. If equal, we can skip the lock. if (m_array[(m_tailIndex - 1) & m_mask] == obj) { - IThreadPoolWorkItem unused; - if (LocalPop(out unused)) + IThreadPoolWorkItem unused = null; + if (LocalPop(ref unused)) { Contract.Assert(unused == obj); return true; @@ -355,7 +355,7 @@ private bool LocalFindAndPopSteal(IThreadPoolWorkItem obj) } [SuppressMessage("Microsoft.Concurrency", "CA8001", Justification = "Reviewed for thread safety")] - public bool LocalPop(out IThreadPoolWorkItem obj) + public bool LocalPop(ref IThreadPoolWorkItem obj) { while (true) { @@ -363,7 +363,6 @@ public bool LocalPop(out IThreadPoolWorkItem obj) int tail = m_tailIndex; if (m_headIndex >= tail) { - obj = null; return false; } @@ -384,7 +383,7 @@ public bool LocalPop(out IThreadPoolWorkItem obj) } bool skip; - bool result = LocalPopLocked(out obj, ref tail, out skip); + bool result = LocalPopLocked(ref obj, ref tail, out skip); // continue if null in array if (skip) continue; @@ -392,7 +391,7 @@ public bool LocalPop(out IThreadPoolWorkItem obj) } } - private bool LocalPopLocked(out IThreadPoolWorkItem obj, ref int tail, out bool skip) + private bool LocalPopLocked(ref IThreadPoolWorkItem obj, ref int tail, out bool skip) { // Interaction with takes: 0 or 1 elements left. bool lockTaken = false; @@ -421,7 +420,6 @@ private bool LocalPopLocked(out IThreadPoolWorkItem obj, ref int tail, out bool // If we encountered a race condition and element was stolen, restore the tail. m_tailIndex = tail + 1; skip = false; - obj = null; return false; } } @@ -433,19 +431,15 @@ private bool LocalPopLocked(out IThreadPoolWorkItem obj, ref int tail, out bool } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TrySteal(out IThreadPoolWorkItem obj, ref bool missedSteal) + public bool TrySteal(ref IThreadPoolWorkItem obj, ref bool missedSteal) { - return TrySteal(out obj, ref missedSteal, 0); // no blocking by default. + return TrySteal(ref obj, ref missedSteal, 0); // no blocking by default. } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TrySteal(out IThreadPoolWorkItem obj, ref bool missedSteal, int millisecondsTimeout) + private bool TrySteal(ref IThreadPoolWorkItem obj, ref bool missedSteal, int millisecondsTimeout) { - obj = null; - if (m_headIndex >= m_tailIndex) - return false; - - return TryStealWithItems(ref obj, ref missedSteal, millisecondsTimeout); + return (m_headIndex >= m_tailIndex) ? false : TryStealWithItems(ref obj, ref missedSteal, millisecondsTimeout); } private bool TryStealWithItems(ref IThreadPoolWorkItem obj, ref bool missedSteal, int millisecondsTimeout) @@ -470,8 +464,7 @@ private bool TryStealWithItems(ref IThreadPoolWorkItem obj, ref bool missedSteal // Check for nulls in the array. if (obj == null) { - if (m_headIndex >= m_tailIndex) - return false; + if (m_headIndex >= m_tailIndex) return false; continue; }; @@ -482,7 +475,6 @@ private bool TryStealWithItems(ref IThreadPoolWorkItem obj, ref bool missedSteal { // Failed, restore head. m_headIndex = head; - obj = null; missedSteal = true; } } @@ -604,7 +596,7 @@ public bool TryEnqueue(IThreadPoolWorkItem node) } [SuppressMessage("Microsoft.Concurrency", "CA8001", Justification = "Reviewed for thread safety")] - public bool TryDequeue(out IThreadPoolWorkItem node) + public bool TryDequeue(ref IThreadPoolWorkItem node) { // // If there are nodes in this segment, increment the lower count, then take the @@ -617,7 +609,6 @@ public bool TryDequeue(out IThreadPoolWorkItem node) { if (lower == upper) { - node = null; return false; } @@ -751,13 +742,12 @@ internal bool LocalFindAndPop(IThreadPoolWorkItem callback) } [SecurityCritical] - public void Dequeue(ThreadPoolWorkQueueThreadLocals tl, out IThreadPoolWorkItem callback, out bool missedSteal) + public void Dequeue(ThreadPoolWorkQueueThreadLocals tl, ref IThreadPoolWorkItem callback, out bool missedSteal) { - callback = null; missedSteal = false; WorkStealingQueue wsq = tl.workStealingQueue; - if (wsq.LocalPop(out callback)) + if (wsq.LocalPop(ref callback)) { Contract.Assert(null != callback); return; @@ -771,7 +761,7 @@ private void DequeueSeek(ThreadPoolWorkQueueThreadLocals tl, ref IThreadPoolWork QueueSegment tail = queueTail; while (true) { - if (tail.TryDequeue(out callback)) + if (tail.TryDequeue(ref callback)) { Contract.Assert(null != callback); return; @@ -808,7 +798,7 @@ private static void DequeueSteal(ThreadPoolWorkQueueThreadLocals tl, ref IThread WorkStealingQueue otherQueue = Volatile.Read(ref data[i & mask]); if (otherQueue != null && otherQueue != wsq && - otherQueue.TrySteal(out callback, ref missedSteal)) + otherQueue.TrySteal(ref callback, ref missedSteal)) { Contract.Assert(null != callback); break; @@ -844,7 +834,7 @@ static internal bool Dispatch() // // Assume that we're going to need another thread if this one returns to the VM. We'll set this to // false later, but only if we're absolutely certain that the queue is empty. - // + bool needAnotherThread = true; IThreadPoolWorkItem workItem = null; try @@ -867,7 +857,7 @@ static internal bool Dispatch() finally { bool missedSteal = false; - workQueue.Dequeue(tl, out workItem, out missedSteal); + workQueue.Dequeue(tl, ref workItem, out missedSteal); if (workItem == null) { @@ -1006,7 +996,7 @@ private void CleanUp() finally { IThreadPoolWorkItem cb = null; - if (workStealingQueue.LocalPop(out cb)) + if (workStealingQueue.LocalPop(ref cb)) { Contract.Assert(null != cb); workQueue.Enqueue(cb, true); From 0956d7e22ef6ca9fb3619bb5249e580961d8a819 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 4 Jul 2016 06:18:49 +0100 Subject: [PATCH 07/23] Add queue on enqueue rather than dispatch --- .../src/System/Threading/ThreadPool.cs | 73 ++++++++++++++----- 1 file changed, 53 insertions(+), 20 deletions(-) diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index e49ac95a57e0..5a7d951ff824 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -442,6 +442,7 @@ private bool TrySteal(ref IThreadPoolWorkItem obj, ref bool missedSteal, int mil return (m_headIndex >= m_tailIndex) ? false : TryStealWithItems(ref obj, ref missedSteal, millisecondsTimeout); } + [MethodImpl(MethodImplOptions.NoInlining)] private bool TryStealWithItems(ref IThreadPoolWorkItem obj, ref bool missedSteal, int millisecondsTimeout) { while (true) @@ -651,6 +652,9 @@ public ThreadPoolWorkQueue() [MethodImpl(MethodImplOptions.AggressiveInlining)] public ThreadPoolWorkQueueThreadLocals EnsureCurrentThreadHasQueue() { + // Don't add thread work pool for non-threadpool threads + if (!Thread.CurrentThread.IsThreadPoolThread) return null; + var queue = ThreadPoolWorkQueueThreadLocals.threadLocals; return null != queue ? queue : (ThreadPoolWorkQueueThreadLocals.threadLocals = new ThreadPoolWorkQueueThreadLocals(this)); } @@ -703,7 +707,7 @@ public void Enqueue(IThreadPoolWorkItem callback, bool forceGlobal) { ThreadPoolWorkQueueThreadLocals tl = null; if (!forceGlobal) - tl = ThreadPoolWorkQueueThreadLocals.threadLocals; + tl = EnsureCurrentThreadHasQueue(); if (loggingEnabled) System.Diagnostics.Tracing.FrameworkEventSource.Log.ThreadPoolEnqueueWorkObject(callback); @@ -742,21 +746,19 @@ internal bool LocalFindAndPop(IThreadPoolWorkItem callback) } [SecurityCritical] - public void Dequeue(ThreadPoolWorkQueueThreadLocals tl, ref IThreadPoolWorkItem callback, out bool missedSteal) + public void Dequeue(WorkStealingQueue wsq, ref IThreadPoolWorkItem callback, out bool missedSteal) { missedSteal = false; - WorkStealingQueue wsq = tl.workStealingQueue; - - if (wsq.LocalPop(ref callback)) + if (wsq?.LocalPop(ref callback) ?? false) { Contract.Assert(null != callback); return; } - DequeueSeek(tl, ref callback, ref missedSteal); + DequeueSeek(wsq, ref callback, ref missedSteal); } - private void DequeueSeek(ThreadPoolWorkQueueThreadLocals tl, ref IThreadPoolWorkItem callback, ref bool missedSteal) + private void DequeueSeek(WorkStealingQueue wsq, ref IThreadPoolWorkItem callback, ref bool missedSteal) { QueueSegment tail = queueTail; while (true) @@ -778,24 +780,35 @@ private void DequeueSeek(ThreadPoolWorkQueueThreadLocals tl, ref IThreadPoolWork } } - DequeueSteal(tl, ref callback, ref missedSteal); + // allThreadQueues.Data.Length is a power of 2, initally 16 + // Move next steal start on by 9 = (8 + 1) rather than 1 + // It means the search still progresses through all start points evenly in a deterministic manner + // However it also interleaves them to reduce collisions between threads + var startIndex = Interlocked.Add(ref NextSearchStart, 9); + + if (wsq == null) + { + DequeueSteal(startIndex, ref callback, ref missedSteal); + } + else + { + DequeueStealWithQueue(wsq, startIndex, ref callback, ref missedSteal); + } } - private static void DequeueSteal(ThreadPoolWorkQueueThreadLocals tl, ref IThreadPoolWorkItem callback, ref bool missedSteal) + [MethodImpl(MethodImplOptions.NoInlining)] + private static void DequeueStealWithQueue(WorkStealingQueue wsq, int index, ref IThreadPoolWorkItem callback, ref bool missedSteal) { - WorkStealingQueue wsq = tl.workStealingQueue; var otherQueues = allThreadQueues.Current; var remaining = otherQueues.ActiveLength; - // allThreadQueues.Data.Length is a power of 2, initally 16 - // Move next steal start on by 9 = (8 + 1) rather than 1 - // It means the search still progresses through all start points evenly in a deterministic manner - // However it also interleaves them to reduce collisions between threads - var i = Interlocked.Add(ref NextSearchStart, 9); var data = otherQueues.Data; var mask = otherQueues.Mask; + while (remaining > 0) { - WorkStealingQueue otherQueue = Volatile.Read(ref data[i & mask]); + remaining--; + WorkStealingQueue otherQueue = Volatile.Read(ref data[index & mask]); + index++; if (otherQueue != null && otherQueue != wsq && otherQueue.TrySteal(ref callback, ref missedSteal)) @@ -803,8 +816,28 @@ private static void DequeueSteal(ThreadPoolWorkQueueThreadLocals tl, ref IThread Contract.Assert(null != callback); break; } - i++; + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void DequeueSteal(int index, ref IThreadPoolWorkItem callback, ref bool missedSteal) + { + var otherQueues = allThreadQueues.Current; + var remaining = otherQueues.ActiveLength; + var data = otherQueues.Data; + var mask = otherQueues.Mask; + + while (remaining > 0) + { remaining--; + WorkStealingQueue otherQueue = Volatile.Read(ref data[index & mask]); + index++; + if (otherQueue != null && + otherQueue.TrySteal(ref callback, ref missedSteal)) + { + Contract.Assert(null != callback); + break; + } } } @@ -840,9 +873,9 @@ static internal bool Dispatch() try { // - // Set up our thread-local data + // Get our thread-local queue // - ThreadPoolWorkQueueThreadLocals tl = workQueue.EnsureCurrentThreadHasQueue(); + WorkStealingQueue wsq = ThreadPoolWorkQueueThreadLocals.threadLocals?.workStealingQueue; // // Loop until our quantum expires. @@ -857,7 +890,7 @@ static internal bool Dispatch() finally { bool missedSteal = false; - workQueue.Dequeue(tl, ref workItem, out missedSteal); + workQueue.Dequeue(wsq, ref workItem, out missedSteal); if (workItem == null) { From 4e7eadbd47b0194e48c2951414c1eaf23ad1720f Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 23 Jul 2016 13:42:22 +0100 Subject: [PATCH 08/23] fix queues --- .../src/System/Threading/ThreadPool.cs | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index 5a7d951ff824..45ce71420337 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -76,18 +76,11 @@ internal class SparseArray where T : class internal sealed class Snapshot { public readonly T[] Data; - public readonly int Mask; private int m_length; internal Snapshot(int initialSize, int initalLength = 0) { - if ((initialSize >> 1) << 1 != initialSize) - { - throw new ArgumentOutOfRangeException(nameof(initialSize), "initialSize must be a power of 2"); - } - Data = new T[initialSize]; - Mask = initialSize - 1; m_length = initalLength; } @@ -800,45 +793,53 @@ private void DequeueSeek(WorkStealingQueue wsq, ref IThreadPoolWorkItem callback private static void DequeueStealWithQueue(WorkStealingQueue wsq, int index, ref IThreadPoolWorkItem callback, ref bool missedSteal) { var otherQueues = allThreadQueues.Current; - var remaining = otherQueues.ActiveLength; + var total = otherQueues.ActiveLength; var data = otherQueues.Data; - var mask = otherQueues.Mask; + var remaining = total; + index = index % total; while (remaining > 0) { remaining--; - WorkStealingQueue otherQueue = Volatile.Read(ref data[index & mask]); - index++; + WorkStealingQueue otherQueue = Volatile.Read(ref data[index]); + index = index + 1 == total ? 0 : index + 1; if (otherQueue != null && otherQueue != wsq && otherQueue.TrySteal(ref callback, ref missedSteal)) { Contract.Assert(null != callback); - break; + return; } } + + if (total != otherQueues.ActiveLength) missedSteal = true; } [MethodImpl(MethodImplOptions.NoInlining)] private static void DequeueSteal(int index, ref IThreadPoolWorkItem callback, ref bool missedSteal) { var otherQueues = allThreadQueues.Current; - var remaining = otherQueues.ActiveLength; + var total = otherQueues.ActiveLength; + // No local queue, may not be other queues + if (total == 0) return; var data = otherQueues.Data; - var mask = otherQueues.Mask; + var remaining = total; + index = index % total; while (remaining > 0) { remaining--; - WorkStealingQueue otherQueue = Volatile.Read(ref data[index & mask]); - index++; + WorkStealingQueue otherQueue = Volatile.Read(ref data[index]); + index = index + 1 == total ? 0 : index + 1; if (otherQueue != null && otherQueue.TrySteal(ref callback, ref missedSteal)) { Contract.Assert(null != callback); - break; + return; } } + + if (total != otherQueues.ActiveLength) missedSteal = true; } [SecurityCritical] From eddd2315d000b98d9020d67e6cc6e837bfb32eea Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 4 Jul 2016 08:33:55 +0100 Subject: [PATCH 09/23] Differentiate threadpool local and global enqueue --- .../src/System/Threading/ThreadPool.cs | 59 +++++++++++-------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index 45ce71420337..7ca856af7a9b 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -696,38 +696,51 @@ internal void MarkThreadRequestSatisfied() } [SecurityCritical] - public void Enqueue(IThreadPoolWorkItem callback, bool forceGlobal) + public void EnqueueGlobal(IThreadPoolWorkItem callback) { - ThreadPoolWorkQueueThreadLocals tl = null; - if (!forceGlobal) - tl = EnsureCurrentThreadHasQueue(); - if (loggingEnabled) System.Diagnostics.Tracing.FrameworkEventSource.Log.ThreadPoolEnqueueWorkObject(callback); - - if (null != tl) - { - tl.workStealingQueue.LocalPush(callback); - } - else + + QueueSegment head = queueHead; + QueueSegment newSegment = null; + + while (!head.TryEnqueue(callback)) { - QueueSegment head = queueHead; + if (newSegment == null) newSegment = new QueueSegment(); - while (!head.TryEnqueue(callback)) + if (Interlocked.CompareExchange(ref head.Next, newSegment, null) == null) { - Interlocked.CompareExchange(ref head.Next, new QueueSegment(), null); + newSegment = null; + } - while (head.Next != null) - { - Interlocked.CompareExchange(ref queueHead, head.Next, head); - head = queueHead; - } + while (head.Next != null) + { + Interlocked.CompareExchange(ref queueHead, head.Next, head); + head = queueHead; } } EnsureThreadRequested(); } + [SecurityCritical] + public void Enqueue(IThreadPoolWorkItem callback, bool forceGlobal) + { + ThreadPoolWorkQueueThreadLocals tl = null; + if (forceGlobal || (tl = EnsureCurrentThreadHasQueue()) == null) + { + EnqueueGlobal(callback); + return; + } + + if (loggingEnabled) + System.Diagnostics.Tracing.FrameworkEventSource.Log.ThreadPoolEnqueueWorkObject(callback); + + tl.workStealingQueue.LocalPush(callback); + + EnsureThreadRequested(); + } + [SecurityCritical] internal bool LocalFindAndPop(IThreadPoolWorkItem callback) { @@ -1033,7 +1046,7 @@ private void CleanUp() if (workStealingQueue.LocalPop(ref cb)) { Contract.Assert(null != cb); - workQueue.Enqueue(cb, true); + workQueue.EnqueueGlobal(cb); } else { @@ -1761,7 +1774,7 @@ public static bool QueueUserWorkItem( //ThreadPool has per-appdomain managed queue of work-items. The VM is //responsible for just scheduling threads into appdomains. After that //work-items are dispatched from the managed queue. - ThreadPoolGlobals.workQueue.Enqueue(tpcallBack, true); + ThreadPoolGlobals.workQueue.EnqueueGlobal(tpcallBack); } return true; } @@ -1788,7 +1801,7 @@ public static bool QueueUserWorkItem( //ThreadPool has per-appdomain managed queue of work-items. The VM is //responsible for just scheduling threads into appdomains. After that //work-items are dispatched from the managed queue. - ThreadPoolGlobals.workQueue.Enqueue(tpcallBack, true); + ThreadPoolGlobals.workQueue.EnqueueGlobal(tpcallBack); } return true; } @@ -1810,7 +1823,7 @@ public static bool UnsafeQueueUserWorkItem( //ThreadPool has per-appdomain managed queue of work-items. The VM is //responsible for just scheduling threads into appdomains. After that //work-items are dispatched from the managed queue. - ThreadPoolGlobals.workQueue.Enqueue(new QueueUserWorkItemCallback(callBack, state, null), true); + ThreadPoolGlobals.workQueue.EnqueueGlobal(new QueueUserWorkItemCallback(callBack, state, null)); } return true; } From 052e43d026c0f9c6fa8340fab8d5f4ebfb1564f4 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Tue, 19 Jul 2016 00:47:19 +0100 Subject: [PATCH 10/23] Base class for UserWorkItems --- .../src/System/Threading/Tasks/Task.cs | 10 +- .../src/System/Threading/ThreadPool.cs | 226 +++++++++++++----- 2 files changed, 172 insertions(+), 64 deletions(-) diff --git a/src/mscorlib/src/System/Threading/Tasks/Task.cs b/src/mscorlib/src/System/Threading/Tasks/Task.cs index 777c7b9c039e..34496c697bd0 100644 --- a/src/mscorlib/src/System/Threading/Tasks/Task.cs +++ b/src/mscorlib/src/System/Threading/Tasks/Task.cs @@ -6666,7 +6666,7 @@ private static Task[] GetActiveTasks() } - internal sealed class CompletionActionInvoker : IThreadPoolWorkItem + internal sealed class CompletionActionInvoker : DeferrableWorkItem { private readonly ITaskCompletionAction m_action; private readonly Task m_completingTask; @@ -6678,16 +6678,10 @@ internal CompletionActionInvoker(ITaskCompletionAction action, Task completingTa } [SecurityCritical] - void IThreadPoolWorkItem.ExecuteWorkItem() + public override void ExecuteWorkItem() { m_action.Invoke(m_completingTask); } - - [SecurityCritical] - void IThreadPoolWorkItem.MarkAborted(ThreadAbortException tae) - { - /* NOP */ - } } // Proxy class for better debugging experience diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index 7ca856af7a9b..aa6eef179086 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -1317,113 +1317,190 @@ internal interface IThreadPoolWorkItem void MarkAborted(ThreadAbortException tae); } - internal sealed class QueueUserWorkItemCallback : IThreadPoolWorkItem + internal abstract class DeferrableWorkItem : IThreadPoolWorkItem { - private WaitCallback callback; - private ExecutionContext context; - private Object state; + [SecurityCritical] + public virtual void ExecuteWorkItem() { /* noop */ } + + [SecurityCritical] + void IThreadPoolWorkItem.MarkAborted(ThreadAbortException tae) => MarkAborted(tae); + + [SecurityCritical] + internal virtual void MarkAborted(ThreadAbortException tae) { /* noop */ } + } + + internal abstract class UserWorkItem : DeferrableWorkItem + { + protected WaitCallback callback; + protected UserWorkItem(WaitCallback waitCallback) + { + Contract.Assert(waitCallback != null, "Null callback passed to UserWorkItem!"); + callback = waitCallback; + } + #if DEBUG - volatile int executed; + private volatile int executed; - ~QueueUserWorkItemCallback() + ~UserWorkItem() { Contract.Assert( - executed != 0 || Environment.HasShutdownStarted || AppDomain.CurrentDomain.IsFinalizingForUnload(), - "A QueueUserWorkItemCallback was never called!"); + executed != 0 || Environment.HasShutdownStarted || AppDomain.CurrentDomain.IsFinalizingForUnload(), + "A UserWorkItem was never called!"); } - void MarkExecuted(bool aborted) + protected void MarkExecuted(bool aborted) { GC.SuppressFinalize(this); Contract.Assert( 0 == Interlocked.Exchange(ref executed, 1) || aborted, - "A QueueUserWorkItemCallback was called twice!"); + "A UserWorkItem was called twice!"); + } + + [SecurityCritical] + internal override void MarkAborted(ThreadAbortException tae) + { + // this workitem didn't execute because we got a ThreadAbortException prior to the call to ExecuteWorkItem. + // This counts as being executed for our purposes. + MarkExecuted(true); } #endif + } + + internal sealed class QueueUserWorkItemCallback : UserWorkItem + { + private readonly ExecutionContext context; + private Object state; [SecurityCritical] internal QueueUserWorkItemCallback(WaitCallback waitCallback, Object stateObj, ExecutionContext ec) + : base(waitCallback) { - Contract.Assert(waitCallback != null, "Null callback passed to QueueUserWorkItemCallback!"); - callback = waitCallback; state = stateObj; context = ec; } [SecurityCritical] - void IThreadPoolWorkItem.ExecuteWorkItem() + public override void ExecuteWorkItem() { #if DEBUG MarkExecuted(false); #endif - // call directly if it is an unsafe call OR EC flow is suppressed - if (context == null) - { - WaitCallback cb = callback; - callback = null; - cb(state); - } - else + ExecutionContext.Run(context, ccb, this); + } + + [System.Security.SecurityCritical] + static internal ContextCallback ccb; + + [System.Security.SecurityCritical] + static internal void Initialize() + { + ccb = (helper) => { - ExecutionContext.Run(context, ccb, this); - } + var state = helper.state; + var callback = helper.callback; + // Detach state for early GC as it may be unreferenced early in the call chain + helper.state = null; + helper.callback = null; + callback(state); + }; } + } + + internal sealed class QueueUserWorkItemCallbackNoState : UserWorkItem + { + private ExecutionContext context; [SecurityCritical] - void IThreadPoolWorkItem.MarkAborted(ThreadAbortException tae) + internal QueueUserWorkItemCallbackNoState(WaitCallback waitCallback, ExecutionContext ec) + : base(waitCallback) + { + context = ec; + } + + [SecurityCritical] + public override void ExecuteWorkItem() { #if DEBUG - // this workitem didn't execute because we got a ThreadAbortException prior to the call to ExecuteWorkItem. - // This counts as being executed for our purposes. - MarkExecuted(true); + MarkExecuted(false); #endif + ExecutionContext.Run(context, ccb, this); } [System.Security.SecurityCritical] - static internal ContextCallback ccb; + static internal ContextCallback ccb; [System.Security.SecurityCritical] static internal void Initialize() { - ccb = (helper) => helper.callback(helper.state); + ccb = (helper) => + { + var callback = helper.callback; + helper.callback = null; + callback(null); + }; } } - internal sealed class QueueUserWorkItemCallbackDefaultContext : IThreadPoolWorkItem + internal sealed class QueueUserWorkItemCallbackNoContext : UserWorkItem { - private WaitCallback callback; private Object state; -#if DEBUG - private volatile int executed; + [SecurityCritical] + internal QueueUserWorkItemCallbackNoContext(WaitCallback waitCallback, Object stateObj) + : base(waitCallback) + { + state = stateObj; + } - ~QueueUserWorkItemCallbackDefaultContext() + [SecurityCritical] + public override void ExecuteWorkItem() { - Contract.Assert( - executed != 0 || Environment.HasShutdownStarted || AppDomain.CurrentDomain.IsFinalizingForUnload(), - "A QueueUserWorkItemCallbackDefaultContext was never called!"); +#if DEBUG + MarkExecuted(false); +#endif + var cb = callback; + var s = state; + // Detach state for early GC as it may be unreferenced early in the call chain + state = null; + callback = null; + cb(s); } + } - void MarkExecuted(bool aborted) + internal sealed class QueueUserWorkItemCallbackNoContextNoState : UserWorkItem + { + [SecurityCritical] + internal QueueUserWorkItemCallbackNoContextNoState(WaitCallback waitCallback) + : base(waitCallback) { - GC.SuppressFinalize(this); - Contract.Assert( - 0 == Interlocked.Exchange(ref executed, 1) || aborted, - "A QueueUserWorkItemCallbackDefaultContext was called twice!"); } + + [SecurityCritical] + public override void ExecuteWorkItem() + { +#if DEBUG + MarkExecuted(false); #endif + var cb = callback; + callback = null; + cb(null); + } + } + + internal sealed class QueueUserWorkItemCallbackDefaultContext : UserWorkItem + { + private Object state; [SecurityCritical] internal QueueUserWorkItemCallbackDefaultContext(WaitCallback waitCallback, Object stateObj) + : base(waitCallback) { - Contract.Assert(waitCallback != null, "Null callback passed to QueueUserWorkItemCallbackDefaultContext!"); - callback = waitCallback; state = stateObj; } [SecurityCritical] - void IThreadPoolWorkItem.ExecuteWorkItem() + public override void ExecuteWorkItem() { #if DEBUG MarkExecuted(false); @@ -1431,23 +1508,53 @@ void IThreadPoolWorkItem.ExecuteWorkItem() ExecutionContext.Run(ExecutionContext.PreAllocatedDefault, ccb, this); } + [System.Security.SecurityCritical] + static internal ContextCallback ccb; + + [System.Security.SecurityCritical] + static internal void Initialize() + { + ccb = (helper) => + { + var state = helper.state; + var callback = helper.callback; + // Detach state for early GC as it may be unreferenced early in the call chain + helper.state = null; + helper.callback = null; + callback(state); + }; + } + } + + internal sealed class QueueUserWorkItemCallbackDefaultContextNoState : UserWorkItem + { [SecurityCritical] - void IThreadPoolWorkItem.MarkAborted(ThreadAbortException tae) + internal QueueUserWorkItemCallbackDefaultContextNoState(WaitCallback waitCallback) + : base(waitCallback) + { + } + + [SecurityCritical] + public override void ExecuteWorkItem() { #if DEBUG - // this workitem didn't execute because we got a ThreadAbortException prior to the call to ExecuteWorkItem. - // This counts as being executed for our purposes. - MarkExecuted(true); + MarkExecuted(false); #endif + ExecutionContext.Run(ExecutionContext.PreAllocatedDefault, ccb, this); } [System.Security.SecurityCritical] - static internal ContextCallback ccb; + static internal ContextCallback ccb; [System.Security.SecurityCritical] static internal void Initialize() { - ccb = (helper) => helper.callback(helper.state); + ccb = (helper) => + { + var callback = helper.callback; + helper.callback = null; + callback(null); + }; } } @@ -1757,6 +1864,7 @@ public static bool QueueUserWorkItem( Object state) { if (callBack == null) return ThrowWaitCallbackNullException(); + if (state == null) return QueueUserWorkItem(callBack); //The VM is responsible for the actual growing/shrinking of threads. EnsureVMInitialized(); @@ -1768,8 +1876,10 @@ public static bool QueueUserWorkItem( ExecutionContext context = !ExecutionContext.IsFlowSuppressed() ? ExecutionContext.FastCapture() : null; IThreadPoolWorkItem tpcallBack = context == ExecutionContext.PreAllocatedDefault ? - new QueueUserWorkItemCallbackDefaultContext(callBack, state) : - (IThreadPoolWorkItem)new QueueUserWorkItemCallback(callBack, state, context); + new QueueUserWorkItemCallbackDefaultContext(callBack, state) : + (context == null ? (IThreadPoolWorkItem) + new QueueUserWorkItemCallbackNoContext(callBack, state) : + new QueueUserWorkItemCallback(callBack, state, context)); //ThreadPool has per-appdomain managed queue of work-items. The VM is //responsible for just scheduling threads into appdomains. After that @@ -1795,8 +1905,10 @@ public static bool QueueUserWorkItem( ExecutionContext context = !ExecutionContext.IsFlowSuppressed() ? ExecutionContext.FastCapture() : null; IThreadPoolWorkItem tpcallBack = context == ExecutionContext.PreAllocatedDefault ? - new QueueUserWorkItemCallbackDefaultContext(callBack, null) : - (IThreadPoolWorkItem)new QueueUserWorkItemCallback(callBack, null, context); + new QueueUserWorkItemCallbackDefaultContextNoState(callBack) : + (context == null ? (IThreadPoolWorkItem) + new QueueUserWorkItemCallbackNoContextNoState(callBack) : + new QueueUserWorkItemCallbackNoState(callBack, context)); //ThreadPool has per-appdomain managed queue of work-items. The VM is //responsible for just scheduling threads into appdomains. After that @@ -1823,7 +1935,7 @@ public static bool UnsafeQueueUserWorkItem( //ThreadPool has per-appdomain managed queue of work-items. The VM is //responsible for just scheduling threads into appdomains. After that //work-items are dispatched from the managed queue. - ThreadPoolGlobals.workQueue.EnqueueGlobal(new QueueUserWorkItemCallback(callBack, state, null)); + ThreadPoolGlobals.workQueue.EnqueueGlobal(new QueueUserWorkItemCallbackNoContext(callBack, state)); } return true; } @@ -1993,7 +2105,9 @@ private static void InitalizeVM() ThreadPool.InitializeVMTp(ref ThreadPoolGlobals.enableWorkerTracking); ThreadPoolGlobals.Initialize(); QueueUserWorkItemCallback.Initialize(); + QueueUserWorkItemCallbackNoState.Initialize(); QueueUserWorkItemCallbackDefaultContext.Initialize(); + QueueUserWorkItemCallbackDefaultContextNoState.Initialize(); _ThreadPoolWaitOrTimerCallback.Initialize(); ThreadPoolGlobals.vmTpInitialized = true; } From eb2b1cfb85013445e05f0a5c873c00778c555a8e Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 23 Jul 2016 01:51:36 +0100 Subject: [PATCH 11/23] Use Generic ExecutionContext.Run --- .../Runtime/CompilerServices/AsyncMethodBuilder.cs | 8 ++++---- src/mscorlib/src/System/Threading/Tasks/Task.cs | 10 ++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/mscorlib/src/System/Runtime/CompilerServices/AsyncMethodBuilder.cs b/src/mscorlib/src/System/Runtime/CompilerServices/AsyncMethodBuilder.cs index 05850605b841..0955625fdd08 100644 --- a/src/mscorlib/src/System/Runtime/CompilerServices/AsyncMethodBuilder.cs +++ b/src/mscorlib/src/System/Runtime/CompilerServices/AsyncMethodBuilder.cs @@ -1062,7 +1062,7 @@ internal void RunWithCapturedContext() try { // Use the context and callback to invoke m_stateMachine.MoveNext. - ExecutionContext.Run(m_context, InvokeMoveNextCallback, m_stateMachine, preserveSyncCtx: true); + ExecutionContext.Run(m_context, InvokeMoveNextCallback, m_stateMachine); } finally { m_context.Dispose(); } } @@ -1091,11 +1091,11 @@ internal MoveNextRunner(IAsyncStateMachine stateMachine) internal void RunWithDefaultContext() { Contract.Assert(m_stateMachine != null, "The state machine must have been set before calling Run."); - ExecutionContext.Run(ExecutionContext.PreAllocatedDefault, InvokeMoveNextCallback, m_stateMachine, preserveSyncCtx: true); + ExecutionContext.Run(ExecutionContext.PreAllocatedDefault, InvokeMoveNextCallback, m_stateMachine); } /// Gets a delegate to the InvokeMoveNext method. - protected static ContextCallback InvokeMoveNextCallback + protected static ContextCallback InvokeMoveNextCallback { [SecuritySafeCritical] get { return s_invokeMoveNext ?? (s_invokeMoveNext = InvokeMoveNext); } @@ -1103,7 +1103,7 @@ protected static ContextCallback InvokeMoveNextCallback /// Cached delegate used with ExecutionContext.Run. [SecurityCritical] - private static ContextCallback s_invokeMoveNext; // lazily-initialized due to SecurityCritical attribution + private static ContextCallback s_invokeMoveNext; // lazily-initialized due to SecurityCritical attribution /// Invokes the MoveNext method on the supplied IAsyncStateMachine. /// The IAsyncStateMachine machine instance. diff --git a/src/mscorlib/src/System/Threading/Tasks/Task.cs b/src/mscorlib/src/System/Threading/Tasks/Task.cs index 34496c697bd0..bf7467da3271 100644 --- a/src/mscorlib/src/System/Threading/Tasks/Task.cs +++ b/src/mscorlib/src/System/Threading/Tasks/Task.cs @@ -2790,8 +2790,8 @@ private void ExecuteWithThreadLocal(ref Task currentTaskSlot) // Lazily initialize the callback delegate; benign race condition var callback = s_ecCallback; - if (callback == null) s_ecCallback = callback = new ContextCallback(ExecutionContextCallback); - ExecutionContext.Run(ec, callback, this, true); + if (callback == null) s_ecCallback = callback = new ContextCallback(ExecutionContextCallback); + ExecutionContext.Run(ec, callback, this); } if (AsyncCausalityTracer.LoggingOn) @@ -2820,13 +2820,11 @@ private void ExecuteWithThreadLocal(ref Task currentTaskSlot) // Cached callback delegate that's lazily initialized due to ContextCallback being SecurityCritical [SecurityCritical] - private static ContextCallback s_ecCallback; + private static ContextCallback s_ecCallback; [SecurityCritical] - private static void ExecutionContextCallback(object obj) + private static void ExecutionContextCallback(Task task) { - Task task = obj as Task; - Contract.Assert(task != null, "expected a task object"); task.Execute(); } From 2ec54c8af91f82befa551305c58c64adedeeac75 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 23 Jul 2016 15:23:35 +0100 Subject: [PATCH 12/23] Inline simple props --- src/mscorlib/src/System/Threading/Tasks/Task.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mscorlib/src/System/Threading/Tasks/Task.cs b/src/mscorlib/src/System/Threading/Tasks/Task.cs index bf7467da3271..1e2079b122d2 100644 --- a/src/mscorlib/src/System/Threading/Tasks/Task.cs +++ b/src/mscorlib/src/System/Threading/Tasks/Task.cs @@ -1671,6 +1671,7 @@ internal ManualResetEventSlim CompletedEvent /// internal bool IsSelfReplicatingRoot { + [MethodImpl(MethodImplOptions.AggressiveInlining)] get { // Return true if self-replicating bit is set and child replica bit is not set @@ -1732,6 +1733,7 @@ public bool IsFaulted /// internal ExecutionContext CapturedContext { + [MethodImpl(MethodImplOptions.AggressiveInlining)] get { if ((m_stateFlags & TASK_STATE_EXECUTIONCONTEXT_IS_NULL) == TASK_STATE_EXECUTIONCONTEXT_IS_NULL) @@ -1743,6 +1745,7 @@ internal ExecutionContext CapturedContext return m_contingentProperties?.m_capturedContext ?? ExecutionContext.PreAllocatedDefault; } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] set { // There is no need to atomically set this bit because this set() method is only called during construction, and therefore there should be no contending accesses to m_stateFlags From 9ab01dc3918cf8865ea9a78c716d34d20d691073 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 23 Jul 2016 19:34:15 +0100 Subject: [PATCH 13/23] Inline AsyncCausalityTracer logging to remove it --- .../src/System/Threading/Tasks/Task.cs | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/mscorlib/src/System/Threading/Tasks/Task.cs b/src/mscorlib/src/System/Threading/Tasks/Task.cs index 1e2079b122d2..b4a342513aaa 100644 --- a/src/mscorlib/src/System/Threading/Tasks/Task.cs +++ b/src/mscorlib/src/System/Threading/Tasks/Task.cs @@ -3589,7 +3589,9 @@ internal void FinishContinuations() if (singleAction != null) { AwaitTaskContinuation.RunOrScheduleAction(singleAction, bCanInlineContinuations, ref t_currentTask); - LogFinishCompletionNotification(); + + if (AsyncCausalityTracer.LoggingOn) + AsyncCausalityTracer.TraceSynchronousWorkCompletion(CausalityTraceLevel.Required, CausalitySynchronousWork.CompletionNotification); return; } @@ -3605,7 +3607,9 @@ internal void FinishContinuations() { ThreadPool.UnsafeQueueCustomWorkItem(new CompletionActionInvoker(singleTaskCompletionAction, this), forceGlobal: false); } - LogFinishCompletionNotification(); + + if (AsyncCausalityTracer.LoggingOn) + AsyncCausalityTracer.TraceSynchronousWorkCompletion(CausalityTraceLevel.Required, CausalitySynchronousWork.CompletionNotification); return; } @@ -3614,7 +3618,9 @@ internal void FinishContinuations() if (singleTaskContinuation != null) { singleTaskContinuation.Run(this, bCanInlineContinuations); - LogFinishCompletionNotification(); + + if (AsyncCausalityTracer.LoggingOn) + AsyncCausalityTracer.TraceSynchronousWorkCompletion(CausalityTraceLevel.Required, CausalitySynchronousWork.CompletionNotification); return; } @@ -3623,7 +3629,8 @@ internal void FinishContinuations() if (continuations == null) { - LogFinishCompletionNotification(); + if (AsyncCausalityTracer.LoggingOn) + AsyncCausalityTracer.TraceSynchronousWorkCompletion(CausalityTraceLevel.Required, CausalitySynchronousWork.CompletionNotification); return; // Not a single or a list; just return } @@ -3700,16 +3707,11 @@ internal void FinishContinuations() } } - LogFinishCompletionNotification(); + if (AsyncCausalityTracer.LoggingOn) + AsyncCausalityTracer.TraceSynchronousWorkCompletion(CausalityTraceLevel.Required, CausalitySynchronousWork.CompletionNotification); } } - private void LogFinishCompletionNotification() - { - if (AsyncCausalityTracer.LoggingOn) - AsyncCausalityTracer.TraceSynchronousWorkCompletion(CausalityTraceLevel.Required, CausalitySynchronousWork.CompletionNotification); - } - #region Continuation methods #region Action continuation From 2c4faff48c742c3ecde9eb5fc1337ff3f1da4a1c Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 23 Jul 2016 23:18:14 +0100 Subject: [PATCH 14/23] Inline + simplify stack guard --- src/mscorlib/src/System/Threading/Tasks/Task.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/mscorlib/src/System/Threading/Tasks/Task.cs b/src/mscorlib/src/System/Threading/Tasks/Task.cs index b4a342513aaa..715edb18bcf4 100644 --- a/src/mscorlib/src/System/Threading/Tasks/Task.cs +++ b/src/mscorlib/src/System/Threading/Tasks/Task.cs @@ -1367,14 +1367,10 @@ internal static Task InternalCurrentIfAttached(TaskCreationOptions creationOptio /// internal static StackGuard CurrentStackGuard { + [MethodImpl(MethodImplOptions.AggressiveInlining)] get { - StackGuard sg = t_stackGuard; - if (sg == null) - { - t_stackGuard = sg = new StackGuard(); - } - return sg; + return t_stackGuard ?? (t_stackGuard = new StackGuard()); } } From e6987a37aa620a4d0c5142a504bce7b730c5344a Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 24 Jul 2016 15:05:05 +0100 Subject: [PATCH 15/23] ExecuteWorkItem -> abstract --- src/mscorlib/src/System/Threading/ThreadPool.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index aa6eef179086..33f8e8d5c18f 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -1320,7 +1320,7 @@ internal interface IThreadPoolWorkItem internal abstract class DeferrableWorkItem : IThreadPoolWorkItem { [SecurityCritical] - public virtual void ExecuteWorkItem() { /* noop */ } + public abstract void ExecuteWorkItem(); [SecurityCritical] void IThreadPoolWorkItem.MarkAborted(ThreadAbortException tae) => MarkAborted(tae); From d1e12114a4a652e6a9c3501f2177ca9c1a40d2b1 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 24 Jul 2016 16:20:23 +0100 Subject: [PATCH 16/23] Specialize TaskContinuations --- .../Runtime/CompilerServices/TaskAwaiter.cs | 3 +- .../src/System/Threading/Tasks/Task.cs | 54 +- .../Threading/Tasks/TaskContinuation.cs | 962 ++++++++++++++---- 3 files changed, 786 insertions(+), 233 deletions(-) diff --git a/src/mscorlib/src/System/Runtime/CompilerServices/TaskAwaiter.cs b/src/mscorlib/src/System/Runtime/CompilerServices/TaskAwaiter.cs index ea6bb96e16e8..64a21727d248 100644 --- a/src/mscorlib/src/System/Runtime/CompilerServices/TaskAwaiter.cs +++ b/src/mscorlib/src/System/Runtime/CompilerServices/TaskAwaiter.cs @@ -206,7 +206,6 @@ private static void ThrowForNonSuccess(Task task) internal static void OnCompletedInternal(Task task, Action continuation, bool continueOnCapturedContext, bool flowExecutionContext) { if (continuation == null) throw new ArgumentNullException("continuation"); - StackCrawlMark stackMark = StackCrawlMark.LookForMyCaller; // If TaskWait* ETW events are enabled, trace a beginning event for this await // and set up an ending event to be traced when the asynchronous await completes. @@ -216,7 +215,7 @@ internal static void OnCompletedInternal(Task task, Action continuation, bool co } // Set the continuation onto the awaited task. - task.SetContinuationForAwait(continuation, continueOnCapturedContext, flowExecutionContext, ref stackMark); + task.SetContinuationForAwait(continuation, continueOnCapturedContext, flowExecutionContext); } /// diff --git a/src/mscorlib/src/System/Threading/Tasks/Task.cs b/src/mscorlib/src/System/Threading/Tasks/Task.cs index 715edb18bcf4..bd28f19b33f5 100644 --- a/src/mscorlib/src/System/Threading/Tasks/Task.cs +++ b/src/mscorlib/src/System/Threading/Tasks/Task.cs @@ -2924,11 +2924,10 @@ public ConfiguredTaskAwaitable ConfigureAwait(bool continueOnCapturedContext) /// true to attempt to marshal the continuation back to the original context captured; otherwise, false. /// /// Whether to flow ExecutionContext across the await. - /// A stack crawl mark tied to execution context. /// The awaiter was not properly initialized. [SecurityCritical] internal void SetContinuationForAwait( - Action continuationAction, bool continueOnCapturedContext, bool flowExecutionContext, ref StackCrawlMark stackMark) + Action continuationAction, bool continueOnCapturedContext, bool flowExecutionContext) { Contract.Requires(continuationAction != null); @@ -2937,6 +2936,9 @@ internal void SetContinuationForAwait( // continuationAction directly without wrapping it. TaskContinuation tc = null; + // Capture Context + ExecutionContext context = flowExecutionContext ? ExecutionContext.FastCapture() : null; + // If the user wants the continuation to run on the current "context" if there is one... if (continueOnCapturedContext) { @@ -2949,7 +2951,7 @@ internal void SetContinuationForAwait( var syncCtx = SynchronizationContext.CurrentNoFlow; if (syncCtx != null && syncCtx.GetType() != typeof(SynchronizationContext)) { - tc = new SynchronizationContextAwaitTaskContinuation(syncCtx, continuationAction, flowExecutionContext, ref stackMark); + tc = GetSynchronizationContextAwaitTaskContinuation(syncCtx, continuationAction, context); } else { @@ -2958,7 +2960,7 @@ internal void SetContinuationForAwait( var scheduler = TaskScheduler.InternalCurrent; if (scheduler != null && scheduler != TaskScheduler.Default) { - tc = new TaskSchedulerAwaitTaskContinuation(scheduler, continuationAction, flowExecutionContext, ref stackMark); + tc = GetTaskSchedulerAwaitTaskContinuation(scheduler, continuationAction, context); } } } @@ -2970,7 +2972,7 @@ internal void SetContinuationForAwait( // ExecutionContext, we need to capture it and wrap it in an AwaitTaskContinuation. // Otherwise, we're targeting the default scheduler and we don't need to flow ExecutionContext, so // we don't actually need a continuation object. We can just store/queue the action itself. - tc = new AwaitTaskContinuation(continuationAction, flowExecutionContext: true, stackMark: ref stackMark); + tc = GetAwaitTaskContinuation(continuationAction, context); } // Now register the continuation, and if we couldn't register it because the task is already completing, @@ -2989,6 +2991,48 @@ internal void SetContinuationForAwait( } } + private TaskContinuation GetSynchronizationContextAwaitTaskContinuation(SynchronizationContext syncCtx, Action continuationAction, ExecutionContext context) + { + if (context == ExecutionContext.PreAllocatedDefault) + { + return new SynchronizationContextAwaitTaskContinuation(syncCtx, continuationAction); + } + else if (context == null) + { + return new SynchronizationContextAwaitTaskContinuationNoContext(syncCtx, continuationAction); + } + + return new SynchronizationContextAwaitTaskContinuationWithContext(syncCtx, continuationAction, context); + } + + private TaskContinuation GetTaskSchedulerAwaitTaskContinuation(TaskScheduler scheduler, Action continuationAction, ExecutionContext context) + { + if (context == ExecutionContext.PreAllocatedDefault) + { + return new TaskSchedulerAwaitTaskContinuation(scheduler, continuationAction); + } + else if (context == null) + { + return new TaskSchedulerAwaitTaskContinuationNoContext(scheduler, continuationAction); + } + + return new TaskSchedulerAwaitTaskContinuationWithContext(scheduler, continuationAction, context); + } + + private TaskContinuation GetAwaitTaskContinuation(Action continuationAction, ExecutionContext context) + { + if (context == ExecutionContext.PreAllocatedDefault) + { + return new AwaitTaskContinuation(continuationAction); + } + else if (context == null) + { + return new AwaitTaskContinuationNoContext(continuationAction); + } + + return new AwaitTaskContinuationWithContext(continuationAction, context); + } + /// Creates an awaitable that asynchronously yields back to the current context when awaited. /// /// A context that, when awaited, will asynchronously transition back into the current context at the diff --git a/src/mscorlib/src/System/Threading/Tasks/TaskContinuation.cs b/src/mscorlib/src/System/Threading/Tasks/TaskContinuation.cs index 4c035dfddb45..7e4784c0fc7d 100644 --- a/src/mscorlib/src/System/Threading/Tasks/TaskContinuation.cs +++ b/src/mscorlib/src/System/Threading/Tasks/TaskContinuation.cs @@ -214,13 +214,21 @@ internal override void InnerInvoke() } } - // For performance reasons, we don't just have a single way of representing - // a continuation object. Rather, we have a hierarchy of types: + // For performance reasons, we don't just have a single way of representing a continuation object. + // Rather, we have a hierarchy of types: + // // - TaskContinuation: abstract base that provides a virtual Run method - // - StandardTaskContinuation: wraps a task,options,and scheduler, and overrides Run to process the task with that configuration - // - AwaitTaskContinuation: base for continuations created through TaskAwaiter; targets default scheduler by default - // - TaskSchedulerAwaitTaskContinuation: awaiting with a non-default TaskScheduler - // - SynchronizationContextAwaitTaskContinuation: awaiting with a "current" sync ctx + // - StandardTaskContinuation: wraps a task, options, and scheduler, and overrides Run to process the task with that configuration + // TaskAwaiter await continuations: Execution Context | TaskScheduler | Sync Context + // - AwaitTaskContinuation: Default | Default | None + // - TaskSchedulerAwaitTaskContinuation: Default | Non-Default | None + // - SynchronizationContextAwaitTaskContinuation: Default | Sync-ctx | "Current" + // - AwaitTaskContinuationNoContext: Suppressed | Default | None + // - TaskSchedulerAwaitTaskContinuationNoContext: Suppressed | Non-Default | None + // - SynchronizationContextAwaitTaskContinuationNoContext: Suppressed | Sync-ctx | "Current" + // - AwaitTaskContinuationWithContext: Non-Default | Default | None + // - TaskSchedulerAwaitTaskContinuationWithContext: Non-Default | Non-Default | None + // - SynchronizationContextAwaitTaskContinuationWithContext: Non-Default | Sync-ctx | "Current" /// Represents a continuation. internal abstract class TaskContinuation @@ -280,7 +288,6 @@ protected static void InlineIfPossibleOrElseQueue(Task task, bool needsProtectio } internal abstract Delegate[] GetDelegateContinuationsForDebugger(); - } /// Provides the standard implementation of a task continuation. @@ -378,169 +385,9 @@ internal override Delegate[] GetDelegateContinuationsForDebugger() } } - /// Task continuation for awaiting with a current synchronization context. - internal sealed class SynchronizationContextAwaitTaskContinuation : AwaitTaskContinuation - { - /// SendOrPostCallback delegate to invoke the action. - private readonly static SendOrPostCallback s_postCallback = state => ((Action)state)(); // can't use InvokeAction as it's SecurityCritical - /// Cached delegate for PostAction - [SecurityCritical] - private static ContextCallback s_postActionCallback; - /// The context with which to run the action. - private readonly SynchronizationContext m_syncContext; - - /// Initializes the SynchronizationContextAwaitTaskContinuation. - /// The synchronization context with which to invoke the action. Must not be null. - /// The action to invoke. Must not be null. - /// Whether to capture and restore ExecutionContext. - /// The captured stack mark. - [SecurityCritical] - internal SynchronizationContextAwaitTaskContinuation( - SynchronizationContext context, Action action, bool flowExecutionContext, ref StackCrawlMark stackMark) : - base(action, flowExecutionContext, ref stackMark) - { - Contract.Assert(context != null); - m_syncContext = context; - } - - /// Inlines or schedules the continuation. - /// The antecedent task, which is ignored. - /// true if inlining is permitted; otherwise, false. - [SecuritySafeCritical] - internal sealed override void Run(Task task, bool canInlineContinuationTask) - { - // If we're allowed to inline, run the action on this thread. - if (canInlineContinuationTask && - m_syncContext == SynchronizationContext.CurrentNoFlow) - { - RunCallback(GetInvokeActionCallback(), m_action, ref Task.t_currentTask); - } - // Otherwise, Post the action back to the SynchronizationContext. - else - { - TplEtwProvider etwLog = TplEtwProvider.Log; - if (etwLog.IsEnabled()) - { - m_continuationId = Task.NewId(); - etwLog.AwaitTaskContinuationScheduled((task.ExecutingTaskScheduler ?? TaskScheduler.Default).Id, task.Id, m_continuationId); - } - RunCallback(GetPostActionCallback(), this, ref Task.t_currentTask); - } - // Any exceptions will be handled by RunCallback. - } - - /// Calls InvokeOrPostAction(false) on the supplied SynchronizationContextAwaitTaskContinuation. - /// The SynchronizationContextAwaitTaskContinuation. - [SecurityCritical] - private static void PostAction(object state) - { - var c = (SynchronizationContextAwaitTaskContinuation)state; - - TplEtwProvider etwLog = TplEtwProvider.Log; - if (etwLog.TasksSetActivityIds && c.m_continuationId != 0) - { - c.m_syncContext.Post(s_postCallback, GetActionLogDelegate(c.m_continuationId, c.m_action)); - } - else - { - c.m_syncContext.Post(s_postCallback, c.m_action); // s_postCallback is manually cached, as the compiler won't in a SecurityCritical method - } - } - - private static Action GetActionLogDelegate(int continuationId, Action action) - { - return () => - { - Guid savedActivityId; - Guid activityId = TplEtwProvider.CreateGuidForTaskID(continuationId); - System.Diagnostics.Tracing.EventSource.SetCurrentThreadActivityId(activityId, out savedActivityId); - try { action(); } - finally { System.Diagnostics.Tracing.EventSource.SetCurrentThreadActivityId(savedActivityId); } - }; - } - - /// Gets a cached delegate for the PostAction method. - /// - /// A delegate for PostAction, which expects a SynchronizationContextAwaitTaskContinuation - /// to be passed as state. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [SecurityCritical] - private static ContextCallback GetPostActionCallback() - { - ContextCallback callback = s_postActionCallback; - if (callback == null) { s_postActionCallback = callback = PostAction; } // lazily initialize SecurityCritical delegate - return callback; - } - } - - /// Task continuation for awaiting with a task scheduler. - internal sealed class TaskSchedulerAwaitTaskContinuation : AwaitTaskContinuation - { - /// The scheduler on which to run the action. - private readonly TaskScheduler m_scheduler; - - /// Initializes the TaskSchedulerAwaitTaskContinuation. - /// The task scheduler with which to invoke the action. Must not be null. - /// The action to invoke. Must not be null. - /// Whether to capture and restore ExecutionContext. - /// The captured stack mark. - [SecurityCritical] - internal TaskSchedulerAwaitTaskContinuation( - TaskScheduler scheduler, Action action, bool flowExecutionContext, ref StackCrawlMark stackMark) : - base(action, flowExecutionContext, ref stackMark) - { - Contract.Assert(scheduler != null); - m_scheduler = scheduler; - } - - /// Inlines or schedules the continuation. - /// The antecedent task, which is ignored. - /// true if inlining is permitted; otherwise, false. - internal sealed override void Run(Task ignored, bool canInlineContinuationTask) - { - // If we're targeting the default scheduler, we can use the faster path provided by the base class. - if (m_scheduler == TaskScheduler.Default) - { - base.Run(ignored, canInlineContinuationTask); - } - else - { - // We permit inlining if the caller allows us to, and - // either we're on a thread pool thread (in which case we're fine running arbitrary code) - // or we're already on the target scheduler (in which case we'll just ask the scheduler - // whether it's ok to run here). We include the IsThreadPoolThread check here, whereas - // we don't in AwaitTaskContinuation.Run, since here it expands what's allowed as opposed - // to in AwaitTaskContinuation.Run where it restricts what's allowed. - bool inlineIfPossible = canInlineContinuationTask && - (TaskScheduler.InternalCurrent == m_scheduler || Thread.CurrentThread.IsThreadPoolThread); - - // Create the continuation task task. If we're allowed to inline, try to do so. - // The target scheduler may still deny us from executing on this thread, in which case this'll be queued. - var task = CreateTask(state => { - try { ((Action)state)(); } - catch (Exception exc) { ThrowAsyncIfNecessary(exc); } - }, m_action, m_scheduler); - - if (inlineIfPossible) - { - InlineIfPossibleOrElseQueue(task, needsProtection: false); - } - else - { - // We need to run asynchronously, so just schedule the task. - try { task.ScheduleAndStart(needsProtection: false); } - catch (TaskSchedulerException) { } // No further action is necessary, as ScheduleAndStart already transitioned task to faulted - } - } - } - } - - /// Base task continuation class used for await continuations. + /// Base task continuation class used for await continuations. Uses the default ExecutionContext and default TaskScheduler. internal class AwaitTaskContinuation : TaskContinuation, IThreadPoolWorkItem { - /// The ExecutionContext with which to run the continuation. - private readonly ExecutionContext m_capturedContext; /// The action to invoke. protected readonly Action m_action; @@ -551,30 +398,10 @@ internal class AwaitTaskContinuation : TaskContinuation, IThreadPoolWorkItem /// Whether to capture and restore ExecutionContext. /// The captured stack mark with which to construct an ExecutionContext. [SecurityCritical] - internal AwaitTaskContinuation(Action action, bool flowExecutionContext, ref StackCrawlMark stackMark) - { - Contract.Requires(action != null); - m_action = action; - if (flowExecutionContext) - { - m_capturedContext = ExecutionContext.Capture( - ref stackMark, - ExecutionContext.CaptureOptions.IgnoreSyncCtx | ExecutionContext.CaptureOptions.OptimizeDefaultCase); - } - } - - /// Initializes the continuation. - /// The action to invoke. Must not be null. - /// Whether to capture and restore ExecutionContext. - [SecurityCritical] - internal AwaitTaskContinuation(Action action, bool flowExecutionContext) + internal AwaitTaskContinuation(Action action) { Contract.Requires(action != null); m_action = action; - if (flowExecutionContext) - { - m_capturedContext = ExecutionContext.FastCapture(); - } } /// Creates a task to run the action with the specified state on the specified scheduler. @@ -582,16 +409,16 @@ internal AwaitTaskContinuation(Action action, bool flowExecutionContext) /// The state to pass to the action. Must not be null. /// The scheduler to target. /// The created task. - protected Task CreateTask(Action action, object state, TaskScheduler scheduler) + protected virtual Task CreateTask(Action action, object state, TaskScheduler scheduler) { Contract.Requires(action != null); Contract.Requires(scheduler != null); return new Task( - action, state, null, default(CancellationToken), - TaskCreationOptions.None, InternalTaskOptions.QueuedByRuntime, scheduler) - { - CapturedContext = m_capturedContext + action, state, null, default(CancellationToken), + TaskCreationOptions.None, InternalTaskOptions.QueuedByRuntime, scheduler) + { + CapturedContext = ExecutionContext.PreAllocatedDefault }; } @@ -621,7 +448,7 @@ internal override void Run(Task task, bool canInlineContinuationTask) // We couldn't inline, so now we need to schedule it ThreadPool.UnsafeQueueCustomWorkItem(this, forceGlobal: false); - } + } } /// @@ -658,7 +485,7 @@ internal static bool IsValidLocationForInlining /// IThreadPoolWorkItem override, which is the entry function for this when the ThreadPool scheduler decides to run it. [SecurityCritical] - void ExecuteWorkItemHelper() + private void ExecuteWorkItemHelper() { var etwLog = TplEtwProvider.Log; Guid savedActivityId = Guid.Empty; @@ -672,20 +499,8 @@ void ExecuteWorkItemHelper() // We're not inside of a task, so t_currentTask doesn't need to be specially maintained. // We're on a thread pool thread with no higher-level callers, so exceptions can just propagate. - // If there's no execution context, just invoke the delegate. - if (m_capturedContext == null) - { - m_action(); - } - // If there is an execution context, get the cached delegate and run the action under the context. - else - { - try - { - ExecutionContext.Run(m_capturedContext, GetInvokeActionCallback(), m_action, true); - } - finally { m_capturedContext.Dispose(); } - } + ExecutionContext.Run(ExecutionContext.PreAllocatedDefault, GetInvokeActionCallback(), m_action); + } finally { @@ -697,13 +512,12 @@ void ExecuteWorkItemHelper() } [SecurityCritical] - void IThreadPoolWorkItem.ExecuteWorkItem() + public virtual void ExecuteWorkItem() { // inline the fast path - if (m_capturedContext == null && !TplEtwProvider.Log.IsEnabled() - ) + if (!TplEtwProvider.Log.IsEnabled()) { - m_action(); + ExecutionContext.Run(ExecutionContext.PreAllocatedDefault, GetInvokeActionCallback(), m_action); } else { @@ -719,7 +533,7 @@ void IThreadPoolWorkItem.MarkAborted(ThreadAbortException tae) { /* nop */ } /// Cached delegate that invokes an Action passed as an object parameter. [SecurityCritical] - private static ContextCallback s_invokeActionCallback; + private static ContextCallback s_invokeActionCallback; /// Runs an action provided as an object parameter. /// The Action to invoke. @@ -728,9 +542,9 @@ void IThreadPoolWorkItem.MarkAborted(ThreadAbortException tae) { /* nop */ } [MethodImpl(MethodImplOptions.AggressiveInlining)] [SecurityCritical] - protected static ContextCallback GetInvokeActionCallback() + protected static ContextCallback GetInvokeActionCallback() { - ContextCallback callback = s_invokeActionCallback; + ContextCallback callback = s_invokeActionCallback; if (callback == null) { s_invokeActionCallback = callback = InvokeAction; } // lazily initialize SecurityCritical delegate return callback; } @@ -740,7 +554,7 @@ protected static ContextCallback GetInvokeActionCallback() /// The state to pass to the callback. /// A reference to Task.t_currentTask. [SecurityCritical] - protected void RunCallback(ContextCallback callback, object state, ref Task currentTask) + protected virtual void RunCallback(ContextCallback callback, Action state, ref Task currentTask) { Contract.Requires(callback != null); Contract.Assert(currentTask == Task.t_currentTask); @@ -752,10 +566,7 @@ protected void RunCallback(ContextCallback callback, object state, ref Task curr { if (prevCurrentTask != null) currentTask = null; - // If there's no captured context, just run the callback directly. - if (m_capturedContext == null) callback(state); - // Otherwise, use the captured context to do so. - else ExecutionContext.Run(m_capturedContext, callback, state, true); + ExecutionContext.Run(ExecutionContext.PreAllocatedDefault, callback, state); } catch (Exception exc) // we explicitly do not request handling of dangerous exceptions like AVs { @@ -765,9 +576,6 @@ protected void RunCallback(ContextCallback callback, object state, ref Task curr { // Restore the current task information if (prevCurrentTask != null) currentTask = prevCurrentTask; - - // Clean up after the execution context, which is only usable once. - if (m_capturedContext != null) m_capturedContext.Dispose(); } } @@ -821,7 +629,7 @@ internal static void RunOrScheduleAction(Action action, bool allowInlining, ref [SecurityCritical] internal static void UnsafeScheduleAction(Action action, Task task) { - AwaitTaskContinuation atc = new AwaitTaskContinuation(action, flowExecutionContext: false); + AwaitTaskContinuationNoContext atc = new AwaitTaskContinuationNoContext(action); var etwLog = TplEtwProvider.Log; if (etwLog.IsEnabled() && task != null) @@ -864,4 +672,706 @@ internal override Delegate[] GetDelegateContinuationsForDebugger() } } + /// Task continuation for awaiting with no ExecutionContext and default TaskScheduler. + internal class AwaitTaskContinuationNoContext : AwaitTaskContinuation + { + /// Initializes the continuation. + /// The action to invoke. Must not be null. + /// Whether to capture and restore ExecutionContext. + /// The captured stack mark with which to construct an ExecutionContext. + [SecurityCritical] + internal AwaitTaskContinuationNoContext(Action action) + : base(action) + { + } + + /// Creates a task to run the action with the specified state on the specified scheduler. + /// The action to run. Must not be null. + /// The state to pass to the action. Must not be null. + /// The scheduler to target. + /// The created task. + protected override Task CreateTask(Action action, object state, TaskScheduler scheduler) + { + Contract.Requires(action != null); + Contract.Requires(scheduler != null); + + return new Task( + action, state, null, default(CancellationToken), + TaskCreationOptions.None, InternalTaskOptions.QueuedByRuntime, scheduler) + { + CapturedContext = null + }; + } + + /// IThreadPoolWorkItem override, which is the entry function for this when the ThreadPool scheduler decides to run it. + [SecurityCritical] + void ExecuteWorkItemHelper() + { + var etwLog = TplEtwProvider.Log; + Guid savedActivityId = Guid.Empty; + if (etwLog.TasksSetActivityIds && m_continuationId != 0) + { + Guid activityId = TplEtwProvider.CreateGuidForTaskID(m_continuationId); + System.Diagnostics.Tracing.EventSource.SetCurrentThreadActivityId(activityId, out savedActivityId); + } + try + { + // We're not inside of a task, so t_currentTask doesn't need to be specially maintained. + // We're on a thread pool thread with no higher-level callers, so exceptions can just propagate. + + // If there's no execution context, just invoke the delegate. + m_action(); + } + finally + { + if (etwLog.TasksSetActivityIds && m_continuationId != 0) + { + System.Diagnostics.Tracing.EventSource.SetCurrentThreadActivityId(savedActivityId); + } + } + } + + [SecurityCritical] + public override void ExecuteWorkItem() + { + // inline the fast path + if (!TplEtwProvider.Log.IsEnabled()) + { + m_action(); + } + else + { + ExecuteWorkItemHelper(); + } + } + + /// Runs the callback synchronously with the provided state. + /// The callback to run. + /// The state to pass to the callback. + /// A reference to Task.t_currentTask. + [SecurityCritical] + protected override void RunCallback(ContextCallback callback, Action state, ref Task currentTask) + { + Contract.Requires(callback != null); + Contract.Assert(currentTask == Task.t_currentTask); + + // Pretend there's no current task, so that no task is seen as a parent + // and TaskScheduler.Current does not reflect false information + var prevCurrentTask = currentTask; + try + { + if (prevCurrentTask != null) currentTask = null; + + // There's no captured context, just run the callback directly. + callback(state); + } + catch (Exception exc) // we explicitly do not request handling of dangerous exceptions like AVs + { + ThrowAsyncIfNecessary(exc); + } + finally + { + // Restore the current task information + if (prevCurrentTask != null) currentTask = prevCurrentTask; + } + } + } + + /// Task continuation for awaiting with a custom ExecutionContext and default TaskScheduler. + internal class AwaitTaskContinuationWithContext : AwaitTaskContinuation + { + /// The ExecutionContext with which to run the continuation. + protected readonly ExecutionContext m_capturedContext; + + /// Initializes the continuation. + /// The action to invoke. Must not be null. + [SecurityCritical] + internal AwaitTaskContinuationWithContext(Action action, ExecutionContext capturedContext) + : base(action) + { + Contract.Requires(capturedContext != null); + m_capturedContext = capturedContext; + } + + /// Creates a task to run the action with the specified state on the specified scheduler. + /// The action to run. Must not be null. + /// The state to pass to the action. Must not be null. + /// The scheduler to target. + /// The created task. + protected override Task CreateTask(Action action, object state, TaskScheduler scheduler) + { + Contract.Requires(action != null); + Contract.Requires(scheduler != null); + + return new Task( + action, state, null, default(CancellationToken), + TaskCreationOptions.None, InternalTaskOptions.QueuedByRuntime, scheduler) + { + CapturedContext = m_capturedContext + }; + } + + /// IThreadPoolWorkItem override, which is the entry function for this when the ThreadPool scheduler decides to run it. + [SecurityCritical] + void ExecuteWorkItemHelper() + { + var etwLog = TplEtwProvider.Log; + Guid savedActivityId = Guid.Empty; + if (etwLog.TasksSetActivityIds && m_continuationId != 0) + { + Guid activityId = TplEtwProvider.CreateGuidForTaskID(m_continuationId); + System.Diagnostics.Tracing.EventSource.SetCurrentThreadActivityId(activityId, out savedActivityId); + } + try + { + // We're not inside of a task, so t_currentTask doesn't need to be specially maintained. + // We're on a thread pool thread with no higher-level callers, so exceptions can just propagate. + try + { + ExecutionContext.Run(m_capturedContext, GetInvokeActionCallback(), m_action); + } + finally + { + m_capturedContext.Dispose(); + } + } + finally + { + if (etwLog.TasksSetActivityIds && m_continuationId != 0) + { + System.Diagnostics.Tracing.EventSource.SetCurrentThreadActivityId(savedActivityId); + } + } + } + + [SecurityCritical] + public override void ExecuteWorkItem() + { + // inline the fast path + if (!TplEtwProvider.Log.IsEnabled()) + { + try + { + ExecutionContext.Run(m_capturedContext, GetInvokeActionCallback(), m_action); + } + finally + { + m_capturedContext.Dispose(); + } + } + else + { + ExecuteWorkItemHelper(); + } + } + + /// Runs the callback synchronously with the provided state. + /// The callback to run. + /// The state to pass to the callback. + /// A reference to Task.t_currentTask. + [SecurityCritical] + protected override void RunCallback(ContextCallback callback, Action state, ref Task currentTask) + { + Contract.Requires(callback != null); + Contract.Assert(currentTask == Task.t_currentTask); + + // Pretend there's no current task, so that no task is seen as a parent + // and TaskScheduler.Current does not reflect false information + var prevCurrentTask = currentTask; + try + { + if (prevCurrentTask != null) currentTask = null; + + ExecutionContext.Run(m_capturedContext, callback, state); + } + catch (Exception exc) // we explicitly do not request handling of dangerous exceptions like AVs + { + ThrowAsyncIfNecessary(exc); + } + finally + { + // Restore the current task information + if (prevCurrentTask != null) currentTask = prevCurrentTask; + + // Clean up after the execution context, which is only usable once. + m_capturedContext.Dispose(); + } + } + } + + /// Task continuation for awaiting with a current synchronization context and default ExecutionContext. + internal sealed class SynchronizationContextAwaitTaskContinuation : AwaitTaskContinuation + { + /// SendOrPostCallback delegate to invoke the action. + private readonly static SendOrPostCallback s_postCallback = state => ((Action)state)(); // can't use InvokeAction as it's SecurityCritical + /// Cached delegate for PostAction + [SecurityCritical] + private static ContextCallback s_postActionCallback; + + /// The context with which to run the action. + private readonly SynchronizationContext m_syncContext; + + [SecurityCritical] + internal SynchronizationContextAwaitTaskContinuation(SynchronizationContext syncContext, Action action) + : base(action) + { + Contract.Assert(syncContext != null); + m_syncContext = syncContext; + } + + /// Inlines or schedules the continuation. + /// The antecedent task, which is ignored. + /// true if inlining is permitted; otherwise, false. + [SecuritySafeCritical] + internal sealed override void Run(Task task, bool canInlineContinuationTask) + { + // If we're allowed to inline, run the action on this thread. + if (canInlineContinuationTask && + m_syncContext == SynchronizationContext.CurrentNoFlow) + { + RunCallback(GetInvokeActionCallback(), m_action, ref Task.t_currentTask); + } + // Post the action back to the SynchronizationContext. + else + { + // Otherwise, Post the action back to the SynchronizationContext. + TplEtwProvider etwLog = TplEtwProvider.Log; + if (etwLog.IsEnabled()) + { + m_continuationId = Task.NewId(); + etwLog.AwaitTaskContinuationScheduled((task.ExecutingTaskScheduler ?? TaskScheduler.Default).Id, task.Id, m_continuationId); + } + RunCallback(GetPostActionCallback(), this, ref Task.t_currentTask); + } + // Any exceptions will be handled by RunCallback. + } + + /// Runs the callback synchronously with the provided state. + /// The callback to run. + /// The state to pass to the callback. + /// A reference to Task.t_currentTask. + [SecurityCritical] + private void RunCallback(ContextCallback callback, SynchronizationContextAwaitTaskContinuation state, ref Task currentTask) + { + Contract.Requires(callback != null); + Contract.Assert(currentTask == Task.t_currentTask); + + // Pretend there's no current task, so that no task is seen as a parent + // and TaskScheduler.Current does not reflect false information + var prevCurrentTask = currentTask; + try + { + if (prevCurrentTask != null) currentTask = null; + + ExecutionContext.Run(ExecutionContext.PreAllocatedDefault, callback, state); + } + catch (Exception exc) // we explicitly do not request handling of dangerous exceptions like AVs + { + ThrowAsyncIfNecessary(exc); + } + finally + { + // Restore the current task information + if (prevCurrentTask != null) currentTask = prevCurrentTask; + } + } + + /// Calls InvokeOrPostAction(false) on the supplied SynchronizationContextAwaitTaskContinuation. + /// The SynchronizationContextAwaitTaskContinuation. + [SecurityCritical] + private static void PostAction(SynchronizationContextAwaitTaskContinuation state) + { + TplEtwProvider etwLog = TplEtwProvider.Log; + if (etwLog.TasksSetActivityIds && state.m_continuationId != 0) + { + state.m_syncContext.Post(s_postCallback, GetActionLogDelegate(state.m_continuationId, state.m_action)); + } + else + { + state.m_syncContext.Post(s_postCallback, state.m_action); // s_postCallback is manually cached, as the compiler won't in a SecurityCritical method + } + } + + internal static Action GetActionLogDelegate(int continuationId, Action action) + { + return () => + { + Guid savedActivityId; + Guid activityId = TplEtwProvider.CreateGuidForTaskID(continuationId); + System.Diagnostics.Tracing.EventSource.SetCurrentThreadActivityId(activityId, out savedActivityId); + try { action(); } + finally { System.Diagnostics.Tracing.EventSource.SetCurrentThreadActivityId(savedActivityId); } + }; + } + + /// Gets a cached delegate for the PostAction method. + /// + /// A delegate for PostAction, which expects a SynchronizationContextAwaitTaskContinuation + /// to be passed as state. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [SecurityCritical] + private static ContextCallback GetPostActionCallback() + { + // lazily initialize SecurityCritical delegate + return s_postActionCallback ?? (s_postActionCallback = PostAction); + } + } + + /// Task continuation for awaiting with a current synchronization context and no ExecutionContext. + internal sealed class SynchronizationContextAwaitTaskContinuationNoContext : AwaitTaskContinuationNoContext + { + /// SendOrPostCallback delegate to invoke the action. + private readonly static SendOrPostCallback s_postCallback = state => ((Action)state)(); // can't use InvokeAction as it's SecurityCritical + /// Cached delegate for PostAction + [SecurityCritical] + private static ContextCallback s_postActionCallback; + + /// The context with which to run the action. + private readonly SynchronizationContext m_syncContext; + + [SecurityCritical] + internal SynchronizationContextAwaitTaskContinuationNoContext(SynchronizationContext syncContext, Action action) + : base(action) + { + Contract.Assert(syncContext != null); + m_syncContext = syncContext; + } + + /// Inlines or schedules the continuation. + /// The antecedent task, which is ignored. + /// true if inlining is permitted; otherwise, false. + [SecuritySafeCritical] + internal sealed override void Run(Task task, bool canInlineContinuationTask) + { + // If we're allowed to inline, run the action on this thread. + if (canInlineContinuationTask && + m_syncContext == SynchronizationContext.CurrentNoFlow) + { + RunCallback(GetInvokeActionCallback(), m_action, ref Task.t_currentTask); + } + // Post the action back to the SynchronizationContext. + else + { + TplEtwProvider etwLog = TplEtwProvider.Log; + if (etwLog.IsEnabled()) + { + m_continuationId = Task.NewId(); + etwLog.AwaitTaskContinuationScheduled((task.ExecutingTaskScheduler ?? TaskScheduler.Default).Id, task.Id, m_continuationId); + } + RunCallback(GetPostActionCallback(), this, ref Task.t_currentTask); + } + // Any exceptions will be handled by RunCallback. + } + + /// Runs the callback synchronously with the provided state. + /// The callback to run. + /// The state to pass to the callback. + /// A reference to Task.t_currentTask. + [SecurityCritical] + private void RunCallback(ContextCallback callback, SynchronizationContextAwaitTaskContinuationNoContext state, ref Task currentTask) + { + Contract.Requires(callback != null); + Contract.Assert(currentTask == Task.t_currentTask); + + // Pretend there's no current task, so that no task is seen as a parent + // and TaskScheduler.Current does not reflect false information + var prevCurrentTask = currentTask; + try + { + if (prevCurrentTask != null) currentTask = null; + + // There's no captured context, just run the callback directly. + callback(state); + } + catch (Exception exc) // we explicitly do not request handling of dangerous exceptions like AVs + { + ThrowAsyncIfNecessary(exc); + } + finally + { + // Restore the current task information + if (prevCurrentTask != null) currentTask = prevCurrentTask; + } + } + + /// Calls InvokeOrPostAction(false) on the supplied SynchronizationContextAwaitTaskContinuation. + /// The SynchronizationContextAwaitTaskContinuation. + [SecurityCritical] + private static void PostAction(SynchronizationContextAwaitTaskContinuationNoContext state) + { + TplEtwProvider etwLog = TplEtwProvider.Log; + if (etwLog.TasksSetActivityIds && state.m_continuationId != 0) + { + state.m_syncContext.Post(s_postCallback, SynchronizationContextAwaitTaskContinuation.GetActionLogDelegate(state.m_continuationId, state.m_action)); + } + else + { + state.m_syncContext.Post(s_postCallback, state.m_action); // s_postCallback is manually cached, as the compiler won't in a SecurityCritical method + } + } + + /// Gets a cached delegate for the PostAction method. + /// + /// A delegate for PostAction, which expects a SynchronizationContextAwaitTaskContinuation + /// to be passed as state. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [SecurityCritical] + private static ContextCallback GetPostActionCallback() + { + // lazily initialize SecurityCritical delegate + return s_postActionCallback ?? (s_postActionCallback = PostAction); + } + } + + /// Task continuation for awaiting with a current synchronization context and a custom ExecutionContext. + internal sealed class SynchronizationContextAwaitTaskContinuationWithContext : AwaitTaskContinuationWithContext + { + /// SendOrPostCallback delegate to invoke the action. + private readonly static SendOrPostCallback s_postCallback = state => ((Action)state)(); // can't use InvokeAction as it's SecurityCritical + /// Cached delegate for PostAction + [SecurityCritical] + private static ContextCallback s_postActionCallback; + + /// The context with which to run the action. + private readonly SynchronizationContext m_syncContext; + + [SecurityCritical] + internal SynchronizationContextAwaitTaskContinuationWithContext(SynchronizationContext syncContext, Action action, ExecutionContext executionContext) + : base(action, executionContext) + { + Contract.Assert(syncContext != null); + m_syncContext = syncContext; + } + + /// Inlines or schedules the continuation. + /// The antecedent task, which is ignored. + /// true if inlining is permitted; otherwise, false. + [SecuritySafeCritical] + internal sealed override void Run(Task task, bool canInlineContinuationTask) + { + // If we're allowed to inline, run the action on this thread. + if (canInlineContinuationTask && + m_syncContext == SynchronizationContext.CurrentNoFlow) + { + RunCallback(GetInvokeActionCallback(), m_action, ref Task.t_currentTask); + } + // Otherwise, Post the action back to the SynchronizationContext. + else + { + TplEtwProvider etwLog = TplEtwProvider.Log; + if (etwLog.IsEnabled()) + { + m_continuationId = Task.NewId(); + etwLog.AwaitTaskContinuationScheduled((task.ExecutingTaskScheduler ?? TaskScheduler.Default).Id, task.Id, m_continuationId); + } + RunCallback(GetPostActionCallback(), this, ref Task.t_currentTask); + } + // Any exceptions will be handled by RunCallback. + } + + /// Calls InvokeOrPostAction(false) on the supplied SynchronizationContextAwaitTaskContinuation. + /// The SynchronizationContextAwaitTaskContinuation. + [SecurityCritical] + private static void PostAction(SynchronizationContextAwaitTaskContinuationWithContext state) + { + TplEtwProvider etwLog = TplEtwProvider.Log; + if (etwLog.TasksSetActivityIds && state.m_continuationId != 0) + { + state.m_syncContext.Post(s_postCallback, SynchronizationContextAwaitTaskContinuation.GetActionLogDelegate(state.m_continuationId, state.m_action)); + } + else + { + state.m_syncContext.Post(s_postCallback, state.m_action); // s_postCallback is manually cached, as the compiler won't in a SecurityCritical method + } + } + + /// Runs the callback synchronously with the provided state. + /// The callback to run. + /// The state to pass to the callback. + /// A reference to Task.t_currentTask. + [SecurityCritical] + private void RunCallback(ContextCallback callback, SynchronizationContextAwaitTaskContinuationWithContext state, ref Task currentTask) + { + Contract.Requires(callback != null); + Contract.Assert(currentTask == Task.t_currentTask); + + // Pretend there's no current task, so that no task is seen as a parent + // and TaskScheduler.Current does not reflect false information + var prevCurrentTask = currentTask; + try + { + if (prevCurrentTask != null) currentTask = null; + + ExecutionContext.Run(m_capturedContext, callback, state); + } + catch (Exception exc) // we explicitly do not request handling of dangerous exceptions like AVs + { + ThrowAsyncIfNecessary(exc); + } + finally + { + // Restore the current task information + if (prevCurrentTask != null) currentTask = prevCurrentTask; + + // Clean up after the execution context, which is only usable once. + m_capturedContext.Dispose(); + } + } + + /// Gets a cached delegate for the PostAction method. + /// + /// A delegate for PostAction, which expects a SynchronizationContextAwaitTaskContinuation + /// to be passed as state. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [SecurityCritical] + private static ContextCallback GetPostActionCallback() + { + // lazily initialize SecurityCritical delegate + return s_postActionCallback ?? (s_postActionCallback = PostAction); + } + } + + /// Task continuation for awaiting with a task scheduler and default ExecutionContext. + internal sealed class TaskSchedulerAwaitTaskContinuation : AwaitTaskContinuation + { + /// The scheduler on which to run the action. + private readonly TaskScheduler m_scheduler; + + [SecurityCritical] + internal TaskSchedulerAwaitTaskContinuation(TaskScheduler scheduler, Action action) + : base(action) + { + m_scheduler = scheduler; + } + + /// Inlines or schedules the continuation. + /// The antecedent task, which is ignored. + /// true if inlining is permitted; otherwise, false. + internal sealed override void Run(Task ignored, bool canInlineContinuationTask) + { + // We permit inlining if the caller allows us to, and + // either we're on a thread pool thread (in which case we're fine running arbitrary code) + // or we're already on the target scheduler (in which case we'll just ask the scheduler + // whether it's ok to run here). We include the IsThreadPoolThread check here, whereas + // we don't in AwaitTaskContinuation.Run, since here it expands what's allowed as opposed + // to in AwaitTaskContinuation.Run where it restricts what's allowed. + bool inlineIfPossible = canInlineContinuationTask && + (TaskScheduler.InternalCurrent == m_scheduler || Thread.CurrentThread.IsThreadPoolThread); + + // Create the continuation task task. If we're allowed to inline, try to do so. + // The target scheduler may still deny us from executing on this thread, in which case this'll be queued. + var task = CreateTask(state => { + try { ((Action)state)(); } + catch (Exception exc) { ThrowAsyncIfNecessary(exc); } + }, m_action, m_scheduler); + + if (inlineIfPossible) + { + InlineIfPossibleOrElseQueue(task, needsProtection: false); + } + else + { + // We need to run asynchronously, so just schedule the task. + try { task.ScheduleAndStart(needsProtection: false); } + catch (TaskSchedulerException) { } // No further action is necessary, as ScheduleAndStart already transitioned task to faulted + } + } + } + + /// Task continuation for awaiting with a task scheduler and no ExecutionContext. + internal sealed class TaskSchedulerAwaitTaskContinuationNoContext : AwaitTaskContinuationNoContext + { + /// The scheduler on which to run the action. + private readonly TaskScheduler m_scheduler; + + [SecurityCritical] + internal TaskSchedulerAwaitTaskContinuationNoContext(TaskScheduler scheduler, Action action) + : base(action) + { + m_scheduler = scheduler; + } + + /// Inlines or schedules the continuation. + /// The antecedent task, which is ignored. + /// true if inlining is permitted; otherwise, false. + internal sealed override void Run(Task ignored, bool canInlineContinuationTask) + { + // We permit inlining if the caller allows us to, and + // either we're on a thread pool thread (in which case we're fine running arbitrary code) + // or we're already on the target scheduler (in which case we'll just ask the scheduler + // whether it's ok to run here). We include the IsThreadPoolThread check here, whereas + // we don't in AwaitTaskContinuation.Run, since here it expands what's allowed as opposed + // to in AwaitTaskContinuation.Run where it restricts what's allowed. + bool inlineIfPossible = canInlineContinuationTask && + (TaskScheduler.InternalCurrent == m_scheduler || Thread.CurrentThread.IsThreadPoolThread); + + // Create the continuation task task. If we're allowed to inline, try to do so. + // The target scheduler may still deny us from executing on this thread, in which case this'll be queued. + var task = CreateTask(state => { + try { ((Action)state)(); } + catch (Exception exc) { ThrowAsyncIfNecessary(exc); } + }, m_action, m_scheduler); + + if (inlineIfPossible) + { + InlineIfPossibleOrElseQueue(task, needsProtection: false); + } + else + { + // We need to run asynchronously, so just schedule the task. + try { task.ScheduleAndStart(needsProtection: false); } + catch (TaskSchedulerException) { } // No further action is necessary, as ScheduleAndStart already transitioned task to faulted + } + } + } + + /// Task continuation for awaiting with a task scheduler and a custom ExecutionContext. + internal sealed class TaskSchedulerAwaitTaskContinuationWithContext : AwaitTaskContinuationWithContext + { + /// The scheduler on which to run the action. + private readonly TaskScheduler m_scheduler; + + [SecurityCritical] + internal TaskSchedulerAwaitTaskContinuationWithContext(TaskScheduler scheduler, Action action, ExecutionContext executionContext) + : base(action, executionContext) + { + m_scheduler = scheduler; + } + + /// Inlines or schedules the continuation. + /// The antecedent task, which is ignored. + /// true if inlining is permitted; otherwise, false. + internal sealed override void Run(Task ignored, bool canInlineContinuationTask) + { + // We permit inlining if the caller allows us to, and + // either we're on a thread pool thread (in which case we're fine running arbitrary code) + // or we're already on the target scheduler (in which case we'll just ask the scheduler + // whether it's ok to run here). We include the IsThreadPoolThread check here, whereas + // we don't in AwaitTaskContinuation.Run, since here it expands what's allowed as opposed + // to in AwaitTaskContinuation.Run where it restricts what's allowed. + bool inlineIfPossible = canInlineContinuationTask && + (TaskScheduler.InternalCurrent == m_scheduler || Thread.CurrentThread.IsThreadPoolThread); + + // Create the continuation task task. If we're allowed to inline, try to do so. + // The target scheduler may still deny us from executing on this thread, in which case this'll be queued. + var task = CreateTask(state => { + try { ((Action)state)(); } + catch (Exception exc) { ThrowAsyncIfNecessary(exc); } + }, m_action, m_scheduler); + + if (inlineIfPossible) + { + InlineIfPossibleOrElseQueue(task, needsProtection: false); + } + else + { + // We need to run asynchronously, so just schedule the task. + try { task.ScheduleAndStart(needsProtection: false); } + catch (TaskSchedulerException) { } // No further action is necessary, as ScheduleAndStart already transitioned task to faulted + } + } + } } From ccf0c8d58acb20ed6ae0e121f53b1a3ee48873dc Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 24 Jul 2016 17:00:35 +0100 Subject: [PATCH 17/23] Strong type ExecutionConext.Runs --- src/mscorlib/src/System/IO/Stream.cs | 16 +++++++--------- .../CompilerServices/AsyncMethodBuilder.cs | 4 ++-- .../System/Threading/CancellationTokenSource.cs | 10 ++++------ .../src/System/Threading/ExecutionContext.cs | 14 +++++++------- src/mscorlib/src/System/Threading/Overlapped.cs | 7 +++---- src/mscorlib/src/System/Threading/Thread.cs | 15 +++++++-------- src/mscorlib/src/System/Threading/Timer.cs | 13 +++++-------- 7 files changed, 35 insertions(+), 44 deletions(-) diff --git a/src/mscorlib/src/System/IO/Stream.cs b/src/mscorlib/src/System/IO/Stream.cs index 3de9492310b3..432568b54c2d 100644 --- a/src/mscorlib/src/System/IO/Stream.cs +++ b/src/mscorlib/src/System/IO/Stream.cs @@ -652,16 +652,15 @@ public ReadWriteTask( } [SecurityCritical] // necessary for CoreCLR - private static void InvokeAsyncCallback(object completedTask) + private static void InvokeAsyncCallback(ReadWriteTask completedTask) { - var rwc = (ReadWriteTask)completedTask; - var callback = rwc._callback; - rwc._callback = null; - callback(rwc); + var callback = completedTask._callback; + completedTask._callback = null; + callback(completedTask); } [SecurityCritical] // necessary for CoreCLR - private static ContextCallback s_invokeAsyncCallback; + private static ContextCallback s_invokeAsyncCallback; [SecuritySafeCritical] // necessary for ExecutionContext.Run void ITaskCompletionAction.Invoke(Task completingTask) @@ -680,10 +679,9 @@ void ITaskCompletionAction.Invoke(Task completingTask) { _context = null; - var invokeAsyncCallback = s_invokeAsyncCallback; - if (invokeAsyncCallback == null) s_invokeAsyncCallback = invokeAsyncCallback = InvokeAsyncCallback; // benign race condition + var invokeAsyncCallback = s_invokeAsyncCallback ?? (s_invokeAsyncCallback = InvokeAsyncCallback); // benign race condition - using(context) ExecutionContext.Run(context, invokeAsyncCallback, this, true); + using(context) ExecutionContext.Run(context, invokeAsyncCallback, this); } } diff --git a/src/mscorlib/src/System/Runtime/CompilerServices/AsyncMethodBuilder.cs b/src/mscorlib/src/System/Runtime/CompilerServices/AsyncMethodBuilder.cs index 0955625fdd08..a16f40ecdcca 100644 --- a/src/mscorlib/src/System/Runtime/CompilerServices/AsyncMethodBuilder.cs +++ b/src/mscorlib/src/System/Runtime/CompilerServices/AsyncMethodBuilder.cs @@ -1108,9 +1108,9 @@ protected static ContextCallback InvokeMoveNextCallback /// Invokes the MoveNext method on the supplied IAsyncStateMachine. /// The IAsyncStateMachine machine instance. [SecurityCritical] // necessary for ContextCallback in CoreCLR - private static void InvokeMoveNext(object stateMachine) + private static void InvokeMoveNext(IAsyncStateMachine stateMachine) { - ((IAsyncStateMachine)stateMachine).MoveNext(); + stateMachine.MoveNext(); } } diff --git a/src/mscorlib/src/System/Threading/CancellationTokenSource.cs b/src/mscorlib/src/System/Threading/CancellationTokenSource.cs index 954cd3834469..43e49f35385e 100644 --- a/src/mscorlib/src/System/Threading/CancellationTokenSource.cs +++ b/src/mscorlib/src/System/Threading/CancellationTokenSource.cs @@ -1030,7 +1030,7 @@ internal CancellationCallbackInfo( // Cached callback delegate that's lazily initialized due to ContextCallback being SecurityCritical [SecurityCritical] - private static ContextCallback s_executionContextCallback; + private static ContextCallback s_executionContextCallback; /// /// InternalExecuteCallbackSynchronously_GeneralPath @@ -1042,8 +1042,8 @@ internal void ExecuteCallback() if (TargetExecutionContext != null) { // Lazily initialize the callback delegate; benign race condition - var callback = s_executionContextCallback; - if (callback == null) s_executionContextCallback = callback = new ContextCallback(ExecutionContextCallback); + var callback = s_executionContextCallback ?? + (s_executionContextCallback = new ContextCallback(ExecutionContextCallback)); ExecutionContext.Run( TargetExecutionContext, @@ -1060,10 +1060,8 @@ internal void ExecuteCallback() // the worker method to actually run the callback // The signature is such that it can be used as a 'ContextCallback' [SecurityCritical] - private static void ExecutionContextCallback(object obj) + private static void ExecutionContextCallback(CancellationCallbackInfo callbackInfo) { - CancellationCallbackInfo callbackInfo = obj as CancellationCallbackInfo; - Contract.Assert(callbackInfo != null); callbackInfo.Callback(callbackInfo.StateForCallback); } } diff --git a/src/mscorlib/src/System/Threading/ExecutionContext.cs b/src/mscorlib/src/System/Threading/ExecutionContext.cs index cbb4f6e0f3b4..6595d6f7ebbf 100644 --- a/src/mscorlib/src/System/Threading/ExecutionContext.cs +++ b/src/mscorlib/src/System/Threading/ExecutionContext.cs @@ -90,6 +90,13 @@ public static ExecutionContext Capture() return Thread.CurrentThread.ExecutionContext ?? ExecutionContext.Default; } + [SecurityCritical] + [FriendAccessAllowed] + internal static void Run(ExecutionContext executionContext, ContextCallback callback, Object state, bool preserveSyncCtx) + { + Run(executionContext, callback, state); + } + [SecurityCritical] [HandleProcessCorruptedStateExceptions] public static void Run(ExecutionContext executionContext, ContextCallback callback, Object state) @@ -304,13 +311,6 @@ internal static ExecutionContext FastCapture() return Capture(); } - [SecurityCritical] - [FriendAccessAllowed] - internal static void Run(ExecutionContext executionContext, ContextCallback callback, Object state, bool preserveSyncCtx) - { - Run(executionContext, callback, state); - } - [SecurityCritical] internal bool IsDefaultFTContext(bool ignoreSyncCtx) { diff --git a/src/mscorlib/src/System/Threading/Overlapped.cs b/src/mscorlib/src/System/Threading/Overlapped.cs index 3f9fbc4989db..7cb5ce55c19a 100644 --- a/src/mscorlib/src/System/Threading/Overlapped.cs +++ b/src/mscorlib/src/System/Threading/Overlapped.cs @@ -86,11 +86,10 @@ internal _IOCompletionCallback(IOCompletionCallback ioCompletionCallback, ref St #if FEATURE_CORECLR [System.Security.SecurityCritical] // auto-generated #endif - static internal ContextCallback _ccb = new ContextCallback(IOCompletionCallback_Context); + static internal ContextCallback<_IOCompletionCallback> _ccb = new ContextCallback<_IOCompletionCallback>(IOCompletionCallback_Context); [System.Security.SecurityCritical] - static internal void IOCompletionCallback_Context(Object state) + static internal void IOCompletionCallback_Context(_IOCompletionCallback helper) { - _IOCompletionCallback helper = (_IOCompletionCallback)state; Contract.Assert(helper != null,"_IOCompletionCallback cannot be null"); helper._ioCompletionCallback(helper._errorCode, helper._numBytes, helper._pOVERLAP); } @@ -124,7 +123,7 @@ static unsafe internal void PerformIOCompletionCallback(uint errorCode, // Error helper._numBytes = numBytes; helper._pOVERLAP = pOVERLAP; using (ExecutionContext executionContext = helper._executionContext.CreateCopy()) - ExecutionContext.Run(executionContext, _ccb, helper, true); + ExecutionContext.Run(executionContext, _ccb, helper); } //Quickly check the VM again, to see if a packet has arrived. diff --git a/src/mscorlib/src/System/Threading/Thread.cs b/src/mscorlib/src/System/Threading/Thread.cs index e62cfae9febb..8008f760da39 100644 --- a/src/mscorlib/src/System/Threading/Thread.cs +++ b/src/mscorlib/src/System/Threading/Thread.cs @@ -54,19 +54,18 @@ internal void SetExecutionContextHelper(ExecutionContext ec) } [System.Security.SecurityCritical] - static internal ContextCallback _ccb = new ContextCallback(ThreadStart_Context); + static internal ContextCallback _ccb = new ContextCallback(ThreadStart_Context); [System.Security.SecurityCritical] - static private void ThreadStart_Context(Object state) + static private void ThreadStart_Context(ThreadHelper threadHelper) { - ThreadHelper t = (ThreadHelper)state; - if (t._start is ThreadStart) + if (threadHelper._start is ThreadStart) { - ((ThreadStart)t._start)(); + ((ThreadStart)threadHelper._start)(); } else { - ((ParameterizedThreadStart)t._start)(t._startArg); + ((ParameterizedThreadStart)threadHelper._start)(threadHelper._startArg); } } @@ -81,7 +80,7 @@ internal void ThreadStart(object obj) _startArg = obj; if (_executionContext != null) { - ExecutionContext.Run(_executionContext, _ccb, (Object)this); + ExecutionContext.Run(_executionContext, _ccb, this); } else { @@ -99,7 +98,7 @@ internal void ThreadStart() { if (_executionContext != null) { - ExecutionContext.Run(_executionContext, _ccb, (Object)this); + ExecutionContext.Run(_executionContext, _ccb, this); } else { diff --git a/src/mscorlib/src/System/Threading/Timer.cs b/src/mscorlib/src/System/Threading/Timer.cs index cb08c6e033bb..b87d77c40fcc 100644 --- a/src/mscorlib/src/System/Threading/Timer.cs +++ b/src/mscorlib/src/System/Threading/Timer.cs @@ -695,26 +695,23 @@ internal void CallCallback() using (ExecutionContext executionContext = m_executionContext.IsPreAllocatedDefault ? m_executionContext : m_executionContext.CreateCopy()) { - ContextCallback callback = s_callCallbackInContext; - if (callback == null) - s_callCallbackInContext = callback = new ContextCallback(CallCallbackInContext); + var callback = s_callCallbackInContext ?? + (s_callCallbackInContext = new ContextCallback(CallCallbackInContext)); ExecutionContext.Run( executionContext, callback, - this, // state - true); // ignoreSyncCtx + this); } } } [SecurityCritical] - private static ContextCallback s_callCallbackInContext; + private static ContextCallback s_callCallbackInContext; [SecurityCritical] - private static void CallCallbackInContext(object state) + private static void CallCallbackInContext(TimerQueueTimer t) { - TimerQueueTimer t = (TimerQueueTimer)state; t.m_timerCallback(t.m_state); } } From fdd9fbadf4cba2d3428795ab96839c2fb43d175f Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 24 Jul 2016 20:36:11 +0100 Subject: [PATCH 18/23] Fast-path EC.Restore & AsyncLocal Skip EC.Restore when not changing from defaults Early bail from GetLocalValue when EC Default Fast-path SetLocalValue adding first value --- .../CompilerServices/AsyncMethodBuilder.cs | 15 ++-- .../src/System/Threading/ExecutionContext.cs | 79 ++++++++++++------- src/mscorlib/src/System/Threading/Thread.cs | 2 +- 3 files changed, 59 insertions(+), 37 deletions(-) diff --git a/src/mscorlib/src/System/Runtime/CompilerServices/AsyncMethodBuilder.cs b/src/mscorlib/src/System/Runtime/CompilerServices/AsyncMethodBuilder.cs index a16f40ecdcca..796464e1d6ad 100644 --- a/src/mscorlib/src/System/Runtime/CompilerServices/AsyncMethodBuilder.cs +++ b/src/mscorlib/src/System/Runtime/CompilerServices/AsyncMethodBuilder.cs @@ -73,12 +73,11 @@ public void Start(ref TStateMachine stateMachine) where TStateMac // This allows us to undo any ExecutionContext changes made in MoveNext, // so that they won't "leak" out of the first await. - Thread currentThread = Thread.CurrentThread; - ExecutionContextSwitcher ecs = default(ExecutionContextSwitcher); RuntimeHelpers.PrepareConstrainedRegions(); + Thread currentThread = Thread.CurrentThread; + ExecutionContextSwitcher ecs = new ExecutionContextSwitcher(currentThread); try { - ExecutionContext.EstablishCopyOnWriteScope(currentThread, ref ecs); stateMachine.MoveNext(); } finally @@ -308,12 +307,11 @@ public void Start(ref TStateMachine stateMachine) where TStateMac // This allows us to undo any ExecutionContext changes made in MoveNext, // so that they won't "leak" out of the first await. - Thread currentThread = Thread.CurrentThread; - ExecutionContextSwitcher ecs = default(ExecutionContextSwitcher); RuntimeHelpers.PrepareConstrainedRegions(); + Thread currentThread = Thread.CurrentThread; + ExecutionContextSwitcher ecs = new ExecutionContextSwitcher(currentThread); try { - ExecutionContext.EstablishCopyOnWriteScope(currentThread, ref ecs); stateMachine.MoveNext(); } finally @@ -464,12 +462,11 @@ public void Start(ref TStateMachine stateMachine) where TStateMac // This allows us to undo any ExecutionContext changes made in MoveNext, // so that they won't "leak" out of the first await. - Thread currentThread = Thread.CurrentThread; - ExecutionContextSwitcher ecs = default(ExecutionContextSwitcher); RuntimeHelpers.PrepareConstrainedRegions(); + Thread currentThread = Thread.CurrentThread; + ExecutionContextSwitcher ecs = new ExecutionContextSwitcher(currentThread); try { - ExecutionContext.EstablishCopyOnWriteScope(currentThread, ref ecs); stateMachine.MoveNext(); } finally diff --git a/src/mscorlib/src/System/Threading/ExecutionContext.cs b/src/mscorlib/src/System/Threading/ExecutionContext.cs index 6595d6f7ebbf..c7517cae8d42 100644 --- a/src/mscorlib/src/System/Threading/ExecutionContext.cs +++ b/src/mscorlib/src/System/Threading/ExecutionContext.cs @@ -45,8 +45,16 @@ namespace System.Threading [SecurityCritical] internal struct ExecutionContextSwitcher { - internal ExecutionContext m_ec; - internal SynchronizationContext m_sc; + private ExecutionContext m_ec; + private SynchronizationContext m_sc; + + public ExecutionContextSwitcher(Thread currentThread) + { + Contract.Assert(currentThread == Thread.CurrentThread); + + m_sc = currentThread.SynchronizationContext; + m_ec = currentThread.ExecutionContext; + } internal void Undo(Thread currentThread) { @@ -57,11 +65,8 @@ internal void Undo(Thread currentThread) { currentThread.SynchronizationContext = m_sc; } - - if (currentThread.ExecutionContext != m_ec) - { - ExecutionContext.Restore(currentThread, m_ec); - } + + ExecutionContext.Restore(currentThread, m_ec); } } @@ -87,7 +92,7 @@ private ExecutionContext(Dictionary localValues, IAsyncLoca [SecuritySafeCritical] public static ExecutionContext Capture() { - return Thread.CurrentThread.ExecutionContext ?? ExecutionContext.Default; + return Thread.CurrentThread.ExecutionContext; } [SecurityCritical] @@ -104,10 +109,9 @@ public static void Run(ExecutionContext executionContext, ContextCallback callba if (executionContext == null) ThrowInvalidOperationNullContextException(); Thread currentThread = Thread.CurrentThread; - ExecutionContextSwitcher ecsw = default(ExecutionContextSwitcher); + ExecutionContextSwitcher ecsw = new ExecutionContextSwitcher(currentThread); try { - EstablishCopyOnWriteScope(currentThread, ref ecsw); ExecutionContext.Restore(currentThread, executionContext); callback(state); } @@ -130,10 +134,9 @@ internal static void Run(ExecutionContext executionContext, ContextCallback(1); + values[local] = newValue; + + if (!needChangeNotifications) + { + Thread.CurrentThread.ExecutionContext = new ExecutionContext(values, Array.Empty()); + } + else + { + Thread.CurrentThread.ExecutionContext = new ExecutionContext(values, new IAsyncLocal[] { local }); + + local.OnValueChanged(null, newValue, false); + } + + return; + } object previousValue; bool hadPreviousValue = current.m_localValues.TryGetValue(local, out previousValue); @@ -258,7 +281,9 @@ internal static void SetLocalValue(IAsyncLocal local, object newValue, bool need Dictionary newValues = new Dictionary(current.m_localValues.Count + (hadPreviousValue ? 0 : 1)); foreach (KeyValuePair pair in current.m_localValues) + { newValues.Add(pair.Key, pair.Value); + } newValues[local] = newValue; diff --git a/src/mscorlib/src/System/Threading/Thread.cs b/src/mscorlib/src/System/Threading/Thread.cs index 8008f760da39..71671b1608b8 100644 --- a/src/mscorlib/src/System/Threading/Thread.cs +++ b/src/mscorlib/src/System/Threading/Thread.cs @@ -357,7 +357,7 @@ private void Start(ref StackCrawlMark stackMark) #if FEATURE_CORECLR internal ExecutionContext ExecutionContext { - get { return m_ExecutionContext; } + get { return m_ExecutionContext ?? (m_ExecutionContext = ExecutionContext.PreAllocatedDefault); } set { m_ExecutionContext = value; } } From cd60ec72e29f9b16410ea7898da113bcc099d046 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 25 Jul 2016 04:17:11 +0100 Subject: [PATCH 19/23] Thread LocalQueues --- src/mscorlib/src/System/Threading/Thread.cs | 6 +++++ .../src/System/Threading/ThreadPool.cs | 24 +++++++++---------- src/vm/object.h | 1 + 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/mscorlib/src/System/Threading/Thread.cs b/src/mscorlib/src/System/Threading/Thread.cs index 71671b1608b8..8aa5a17914d0 100644 --- a/src/mscorlib/src/System/Threading/Thread.cs +++ b/src/mscorlib/src/System/Threading/Thread.cs @@ -134,6 +134,7 @@ public sealed class Thread : CriticalFinalizerObject, _Thread private ExecutionContext m_ExecutionContext; // this call context follows the logical thread #if FEATURE_CORECLR private SynchronizationContext m_SynchronizationContext; // On CoreCLR, this is maintained separately from ExecutionContext + private ThreadPoolWorkQueueThreadLocals m_LocalQueues; #endif private String m_Name; @@ -366,6 +367,11 @@ internal SynchronizationContext SynchronizationContext get { return m_SynchronizationContext; } set { m_SynchronizationContext = value; } } + internal ThreadPoolWorkQueueThreadLocals LocalQueues + { + get { return m_LocalQueues; } + set { m_LocalQueues = value; } + } #else // !FEATURE_CORECLR [ReliabilityContract(Consistency.WillNotCorruptState, Cer.Success)] internal ExecutionContext.Reader GetExecutionContextReader() diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index 33f8e8d5c18f..11b80944cc6b 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -646,10 +646,11 @@ public ThreadPoolWorkQueue() public ThreadPoolWorkQueueThreadLocals EnsureCurrentThreadHasQueue() { // Don't add thread work pool for non-threadpool threads - if (!Thread.CurrentThread.IsThreadPoolThread) return null; + var currentThread = Thread.CurrentThread; + if (!currentThread.IsThreadPoolThread) return null; - var queue = ThreadPoolWorkQueueThreadLocals.threadLocals; - return null != queue ? queue : (ThreadPoolWorkQueueThreadLocals.threadLocals = new ThreadPoolWorkQueueThreadLocals(this)); + var queue = currentThread.LocalQueues; + return null != queue ? queue : (currentThread.LocalQueues = new ThreadPoolWorkQueueThreadLocals(this)); } [SecurityCritical] @@ -744,7 +745,7 @@ public void Enqueue(IThreadPoolWorkItem callback, bool forceGlobal) [SecurityCritical] internal bool LocalFindAndPop(IThreadPoolWorkItem callback) { - ThreadPoolWorkQueueThreadLocals tl = ThreadPoolWorkQueueThreadLocals.threadLocals; + ThreadPoolWorkQueueThreadLocals tl = Thread.CurrentThread.LocalQueues; if (null == tl) return false; @@ -858,6 +859,7 @@ private static void DequeueSteal(int index, ref IThreadPoolWorkItem callback, re [SecurityCritical] static internal bool Dispatch() { + var currentThread = Thread.CurrentThread; var workQueue = ThreadPoolGlobals.workQueue; // // The clock is ticking! We have ThreadPoolGlobals.tpQuantum milliseconds to get some work done, and then @@ -886,16 +888,17 @@ static internal bool Dispatch() IThreadPoolWorkItem workItem = null; try { - // - // Get our thread-local queue - // - WorkStealingQueue wsq = ThreadPoolWorkQueueThreadLocals.threadLocals?.workStealingQueue; // // Loop until our quantum expires. // while ((Environment.TickCount - quantumStartTime) < ThreadPoolGlobals.tpQuantum) { + // + // Get our thread-local queue, may have been created by work item + // + var wsq = currentThread.LocalQueues?.workStealingQueue; + // // Dequeue and EnsureThreadRequested must be protected from ThreadAbortException. // These are fast, so this will not delay aborts/AD-unloads for very long. @@ -1014,9 +1017,6 @@ static internal bool Dispatch() // Holds a WorkStealingQueue, and remmoves it from the list when this object is no longer referened. internal sealed class ThreadPoolWorkQueueThreadLocals { - [ThreadStatic] - [SecurityCritical] - public static ThreadPoolWorkQueueThreadLocals threadLocals; public readonly ThreadPoolWorkQueue workQueue; public readonly ThreadPoolWorkQueue.WorkStealingQueue workStealingQueue; @@ -2019,7 +2019,7 @@ internal static IEnumerable EnumerateQueuedWorkItems(Thread [SecurityCritical] internal static IEnumerable GetLocallyQueuedWorkItems() { - return EnumerateQueuedWorkItems(new ThreadPoolWorkQueue.WorkStealingQueue[] { ThreadPoolWorkQueueThreadLocals.threadLocals.workStealingQueue }, null); + return EnumerateQueuedWorkItems(new ThreadPoolWorkQueue.WorkStealingQueue[] { Thread.CurrentThread.LocalQueues.workStealingQueue }, null); } [SecurityCritical] diff --git a/src/vm/object.h b/src/vm/object.h index 1722c1654122..8bda037c1332 100644 --- a/src/vm/object.h +++ b/src/vm/object.h @@ -2074,6 +2074,7 @@ class ThreadBaseObject : public Object #ifdef FEATURE_CORECLR OBJECTREF m_ExecutionContext; OBJECTREF m_SynchronizationContext; + OBJECTREF m_LocalQueues; #else EXECUTIONCONTEXTREF m_ExecutionContext; #endif From a66a59d5cf30a8cfcac42bb6e2428f4bced14fe5 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Mon, 25 Jul 2016 13:50:24 +0100 Subject: [PATCH 20/23] Task local --- .../src/System/Threading/Tasks/Task.cs | 91 +++++++++++++------ .../Threading/Tasks/TaskContinuation.cs | 45 +++++++-- src/mscorlib/src/System/Threading/Thread.cs | 7 +- .../src/System/Threading/ThreadPool.cs | 32 +++++-- src/vm/object.h | 2 +- 5 files changed, 130 insertions(+), 47 deletions(-) diff --git a/src/mscorlib/src/System/Threading/Tasks/Task.cs b/src/mscorlib/src/System/Threading/Tasks/Task.cs index bd28f19b33f5..c277cc4e4039 100644 --- a/src/mscorlib/src/System/Threading/Tasks/Task.cs +++ b/src/mscorlib/src/System/Threading/Tasks/Task.cs @@ -142,11 +142,13 @@ public enum TaskStatus [DebuggerDisplay("Id = {Id}, Status = {Status}, Method = {DebuggerDisplayMethodDescription}")] public class Task : IThreadPoolWorkItem, IAsyncResult, IDisposable { +#if !FEATURE_CORECLR [ThreadStatic] internal static Task t_currentTask; // The currently executing task. [ThreadStatic] private static StackGuard t_stackGuard; // The stack guard object for this thread +#endif internal static int s_taskIdCounter; //static counter used to generate unique task IDs private readonly static TaskFactory s_factory = new TaskFactory(); @@ -1348,7 +1350,11 @@ public static int? CurrentId /// internal static Task InternalCurrent { +#if FEATURE_CORECLR + get { return Thread.CurrentThread.ThreadTaskLocals.CurrentTask; } +#else get { return t_currentTask; } +#endif } /// @@ -1370,7 +1376,11 @@ internal static StackGuard CurrentStackGuard [MethodImpl(MethodImplOptions.AggressiveInlining)] get { +#if FEATURE_CORECLR + return Thread.CurrentThread.ThreadTaskLocals.StackGuard; +#else return t_stackGuard ?? (t_stackGuard = new StackGuard()); +#endif } } @@ -2726,7 +2736,18 @@ internal bool ExecuteEntry(bool bPreventDoubleExecution) if (!IsCancellationRequested && !IsCanceled) { - ExecuteWithThreadLocal(ref t_currentTask); + var etwLog = TplEtwProvider.Log; +#if FEATURE_CORECLR + if (!etwLog.IsEnabled()) + ExecuteWithThreadLocal(ref Thread.CurrentThread.ThreadTaskLocals.CurrentTask); + else + ExecuteWithThreadLocalTraced(ref Thread.CurrentThread.ThreadTaskLocals.CurrentTask, etwLog); +#else + if (!etwLog.IsEnabled()) + ExecuteWithThreadLocal(ref t_currentTask); + else + ExecuteWithThreadLocalTraced(ref t_currentTask, etwLog); +#endif } else if (!IsCanceled) { @@ -2747,25 +2768,9 @@ private void ExecuteWithThreadLocal(ref Task currentTaskSlot) // Remember the current task so we can restore it after running, and then Task previousTask = currentTaskSlot; - // ETW event for Task Started - var etwLog = TplEtwProvider.Log; - Guid savedActivityID = new Guid(); - bool etwIsEnabled = etwLog.IsEnabled(); - if (etwIsEnabled) - { - if (etwLog.TasksSetActivityIds) - EventSource.SetCurrentThreadActivityId(TplEtwProvider.CreateGuidForTaskID(this.Id), out savedActivityID); - // previousTask holds the actual "current task" we want to report in the event - if (previousTask != null) - etwLog.TaskStarted(previousTask.m_taskScheduler.Id, previousTask.Id, this.Id); - else - etwLog.TaskStarted(TaskScheduler.Current.Id, 0, this.Id); - } - if (AsyncCausalityTracer.LoggingOn) AsyncCausalityTracer.TraceSynchronousWorkStart(CausalityTraceLevel.Required, this.Id, CausalitySynchronousWork.Execution); - try { // place the current task into TLS. @@ -2801,19 +2806,41 @@ private void ExecuteWithThreadLocal(ref Task currentTaskSlot) finally { currentTaskSlot = previousTask; - + } + } + + // A trick so we can refer to the TLS slot with a byref. + [SecurityCritical] + private void ExecuteWithThreadLocalTraced(ref Task currentTaskSlot, TplEtwProvider etwLog) + { + // ETW event for Task Started + Guid savedActivityID = new Guid(); + + if (etwLog.TasksSetActivityIds) + EventSource.SetCurrentThreadActivityId(TplEtwProvider.CreateGuidForTaskID(this.Id), out savedActivityID); + // previousTask holds the actual "current task" we want to report in the event + if (currentTaskSlot != null) + etwLog.TaskStarted(currentTaskSlot.m_taskScheduler.Id, currentTaskSlot.Id, this.Id); + else + etwLog.TaskStarted(TaskScheduler.Current.Id, 0, this.Id); + + + try + { + ExecuteWithThreadLocal(ref currentTaskSlot); + } + finally + { // ETW event for Task Completed - if (etwIsEnabled) - { - // previousTask holds the actual "current task" we want to report in the event - if (previousTask != null) - etwLog.TaskCompleted(previousTask.m_taskScheduler.Id, previousTask.Id, this.Id, IsFaulted); - else - etwLog.TaskCompleted(TaskScheduler.Current.Id, 0, this.Id, IsFaulted); - if (etwLog.TasksSetActivityIds) - EventSource.SetCurrentThreadActivityId(savedActivityID); - } + // previousTask holds the actual "current task" we want to report in the event + if (currentTaskSlot != null) + etwLog.TaskCompleted(currentTaskSlot.m_taskScheduler.Id, currentTaskSlot.Id, this.Id, IsFaulted); + else + etwLog.TaskCompleted(TaskScheduler.Current.Id, 0, this.Id, IsFaulted); + + if (etwLog.TasksSetActivityIds) + EventSource.SetCurrentThreadActivityId(savedActivityID); } } @@ -3628,7 +3655,11 @@ internal void FinishContinuations() Action singleAction = continuationObject as Action; if (singleAction != null) { +#if FEATURE_CORECLR + AwaitTaskContinuation.RunOrScheduleAction(singleAction, bCanInlineContinuations, ref Thread.CurrentThread.ThreadTaskLocals.CurrentTask); +#else AwaitTaskContinuation.RunOrScheduleAction(singleAction, bCanInlineContinuations, ref t_currentTask); +#endif if (AsyncCausalityTracer.LoggingOn) AsyncCausalityTracer.TraceSynchronousWorkCompletion(CausalityTraceLevel.Required, CausalitySynchronousWork.CompletionNotification); @@ -3717,7 +3748,11 @@ internal void FinishContinuations() Action ad = currentContinuation as Action; if (ad != null) { +#if FEATURE_CORECLR + AwaitTaskContinuation.RunOrScheduleAction(ad, bCanInlineContinuations, ref Thread.CurrentThread.ThreadTaskLocals.CurrentTask); +#else AwaitTaskContinuation.RunOrScheduleAction(ad, bCanInlineContinuations, ref t_currentTask); +#endif } else { diff --git a/src/mscorlib/src/System/Threading/Tasks/TaskContinuation.cs b/src/mscorlib/src/System/Threading/Tasks/TaskContinuation.cs index 7e4784c0fc7d..95b0e54aa0a0 100644 --- a/src/mscorlib/src/System/Threading/Tasks/TaskContinuation.cs +++ b/src/mscorlib/src/System/Threading/Tasks/TaskContinuation.cs @@ -435,7 +435,12 @@ internal override void Run(Task task, bool canInlineContinuationTask) // running arbitrary amounts of work in suspected "bad locations", like UI threads. if (canInlineContinuationTask && IsValidLocationForInlining) { - RunCallback(GetInvokeActionCallback(), m_action, ref Task.t_currentTask); // any exceptions from m_action will be handled by s_callbackRunAction + // any exceptions from m_action will be handled by s_callbackRunAction +#if FEATURE_CORECLR + RunCallback(GetInvokeActionCallback(), m_action, ref Thread.CurrentThread.ThreadTaskLocals.CurrentTask); +#else + RunCallback(GetInvokeActionCallback(), m_action, ref Task.t_currentTask); +#endif } else { @@ -557,7 +562,7 @@ protected static ContextCallback GetInvokeActionCallback() protected virtual void RunCallback(ContextCallback callback, Action state, ref Task currentTask) { Contract.Requires(callback != null); - Contract.Assert(currentTask == Task.t_currentTask); + Contract.Assert(currentTask == Task.InternalCurrent); // Pretend there's no current task, so that no task is seen as a parent // and TaskScheduler.Current does not reflect false information @@ -598,7 +603,7 @@ protected virtual void RunCallback(ContextCallback callback, Action stat [SecurityCritical] internal static void RunOrScheduleAction(Action action, bool allowInlining, ref Task currentTask) { - Contract.Assert(currentTask == Task.t_currentTask); + Contract.Assert(currentTask == Task.InternalCurrent); // If we're not allowed to run here, schedule the action if (!allowInlining || !IsValidLocationForInlining) @@ -753,7 +758,7 @@ public override void ExecuteWorkItem() protected override void RunCallback(ContextCallback callback, Action state, ref Task currentTask) { Contract.Requires(callback != null); - Contract.Assert(currentTask == Task.t_currentTask); + Contract.Assert(currentTask == Task.InternalCurrent); // Pretend there's no current task, so that no task is seen as a parent // and TaskScheduler.Current does not reflect false information @@ -873,7 +878,7 @@ public override void ExecuteWorkItem() protected override void RunCallback(ContextCallback callback, Action state, ref Task currentTask) { Contract.Requires(callback != null); - Contract.Assert(currentTask == Task.t_currentTask); + Contract.Assert(currentTask == Task.InternalCurrent); // Pretend there's no current task, so that no task is seen as a parent // and TaskScheduler.Current does not reflect false information @@ -929,7 +934,11 @@ internal sealed override void Run(Task task, bool canInlineContinuationTask) if (canInlineContinuationTask && m_syncContext == SynchronizationContext.CurrentNoFlow) { +#if FEATURE_CORECLR + RunCallback(GetInvokeActionCallback(), m_action, ref Thread.CurrentThread.ThreadTaskLocals.CurrentTask); +#else RunCallback(GetInvokeActionCallback(), m_action, ref Task.t_currentTask); +#endif } // Post the action back to the SynchronizationContext. else @@ -941,7 +950,11 @@ internal sealed override void Run(Task task, bool canInlineContinuationTask) m_continuationId = Task.NewId(); etwLog.AwaitTaskContinuationScheduled((task.ExecutingTaskScheduler ?? TaskScheduler.Default).Id, task.Id, m_continuationId); } +#if FEATURE_CORECLR + RunCallback(GetPostActionCallback(), this, ref Thread.CurrentThread.ThreadTaskLocals.CurrentTask); +#else RunCallback(GetPostActionCallback(), this, ref Task.t_currentTask); +#endif } // Any exceptions will be handled by RunCallback. } @@ -954,7 +967,7 @@ internal sealed override void Run(Task task, bool canInlineContinuationTask) private void RunCallback(ContextCallback callback, SynchronizationContextAwaitTaskContinuation state, ref Task currentTask) { Contract.Requires(callback != null); - Contract.Assert(currentTask == Task.t_currentTask); + Contract.Assert(currentTask == Task.InternalCurrent); // Pretend there's no current task, so that no task is seen as a parent // and TaskScheduler.Current does not reflect false information @@ -1048,7 +1061,11 @@ internal sealed override void Run(Task task, bool canInlineContinuationTask) if (canInlineContinuationTask && m_syncContext == SynchronizationContext.CurrentNoFlow) { +#if FEATURE_CORECLR + RunCallback(GetInvokeActionCallback(), m_action, ref Thread.CurrentThread.ThreadTaskLocals.CurrentTask); +#else RunCallback(GetInvokeActionCallback(), m_action, ref Task.t_currentTask); +#endif } // Post the action back to the SynchronizationContext. else @@ -1059,7 +1076,11 @@ internal sealed override void Run(Task task, bool canInlineContinuationTask) m_continuationId = Task.NewId(); etwLog.AwaitTaskContinuationScheduled((task.ExecutingTaskScheduler ?? TaskScheduler.Default).Id, task.Id, m_continuationId); } +#if FEATURE_CORECLR + RunCallback(GetPostActionCallback(), this, ref Thread.CurrentThread.ThreadTaskLocals.CurrentTask); +#else RunCallback(GetPostActionCallback(), this, ref Task.t_currentTask); +#endif } // Any exceptions will be handled by RunCallback. } @@ -1072,7 +1093,7 @@ internal sealed override void Run(Task task, bool canInlineContinuationTask) private void RunCallback(ContextCallback callback, SynchronizationContextAwaitTaskContinuationNoContext state, ref Task currentTask) { Contract.Requires(callback != null); - Contract.Assert(currentTask == Task.t_currentTask); + Contract.Assert(currentTask == Task.InternalCurrent); // Pretend there's no current task, so that no task is seen as a parent // and TaskScheduler.Current does not reflect false information @@ -1155,7 +1176,11 @@ internal sealed override void Run(Task task, bool canInlineContinuationTask) if (canInlineContinuationTask && m_syncContext == SynchronizationContext.CurrentNoFlow) { +#if FEATURE_CORECLR + RunCallback(GetInvokeActionCallback(), m_action, ref Thread.CurrentThread.ThreadTaskLocals.CurrentTask); +#else RunCallback(GetInvokeActionCallback(), m_action, ref Task.t_currentTask); +#endif } // Otherwise, Post the action back to the SynchronizationContext. else @@ -1166,7 +1191,11 @@ internal sealed override void Run(Task task, bool canInlineContinuationTask) m_continuationId = Task.NewId(); etwLog.AwaitTaskContinuationScheduled((task.ExecutingTaskScheduler ?? TaskScheduler.Default).Id, task.Id, m_continuationId); } +#if FEATURE_CORECLR + RunCallback(GetPostActionCallback(), this, ref Thread.CurrentThread.ThreadTaskLocals.CurrentTask); +#else RunCallback(GetPostActionCallback(), this, ref Task.t_currentTask); +#endif } // Any exceptions will be handled by RunCallback. } @@ -1195,7 +1224,7 @@ private static void PostAction(SynchronizationContextAwaitTaskContinuationWithCo private void RunCallback(ContextCallback callback, SynchronizationContextAwaitTaskContinuationWithContext state, ref Task currentTask) { Contract.Requires(callback != null); - Contract.Assert(currentTask == Task.t_currentTask); + Contract.Assert(currentTask == Task.InternalCurrent); // Pretend there's no current task, so that no task is seen as a parent // and TaskScheduler.Current does not reflect false information diff --git a/src/mscorlib/src/System/Threading/Thread.cs b/src/mscorlib/src/System/Threading/Thread.cs index 8aa5a17914d0..05a0acf8f14a 100644 --- a/src/mscorlib/src/System/Threading/Thread.cs +++ b/src/mscorlib/src/System/Threading/Thread.cs @@ -134,7 +134,7 @@ public sealed class Thread : CriticalFinalizerObject, _Thread private ExecutionContext m_ExecutionContext; // this call context follows the logical thread #if FEATURE_CORECLR private SynchronizationContext m_SynchronizationContext; // On CoreCLR, this is maintained separately from ExecutionContext - private ThreadPoolWorkQueueThreadLocals m_LocalQueues; + private ThreadTaskLocals m_ThreadTaskLocals; #endif private String m_Name; @@ -367,10 +367,9 @@ internal SynchronizationContext SynchronizationContext get { return m_SynchronizationContext; } set { m_SynchronizationContext = value; } } - internal ThreadPoolWorkQueueThreadLocals LocalQueues + internal ThreadTaskLocals ThreadTaskLocals { - get { return m_LocalQueues; } - set { m_LocalQueues = value; } + get { return m_ThreadTaskLocals ?? (m_ThreadTaskLocals = new ThreadTaskLocals()); } } #else // !FEATURE_CORECLR [ReliabilityContract(Consistency.WillNotCorruptState, Cer.Success)] diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index 11b80944cc6b..9b4cddf2a8f9 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -39,6 +39,7 @@ namespace System.Threading using System.Diagnostics.Contracts; using System.Diagnostics.CodeAnalysis; using System.Diagnostics.Tracing; + using Tasks; internal static class ThreadPoolGlobals { @@ -66,6 +67,25 @@ internal static void Initialize() } } + internal sealed class ThreadTaskLocals + { + private readonly StackGuard m_stackGuard = new StackGuard(); + private ThreadPoolWorkQueueThreadLocals m_localQueues; + + internal Task CurrentTask; + + public StackGuard StackGuard + { + get { return m_stackGuard; } + } + + public ThreadPoolWorkQueueThreadLocals LocalQueues + { + get { return m_localQueues; } + set { m_localQueues = value; } + } + } + internal sealed class ThreadPoolWorkQueue { // Simple sparsely populated array to allow lock-free reading. @@ -649,8 +669,8 @@ public ThreadPoolWorkQueueThreadLocals EnsureCurrentThreadHasQueue() var currentThread = Thread.CurrentThread; if (!currentThread.IsThreadPoolThread) return null; - var queue = currentThread.LocalQueues; - return null != queue ? queue : (currentThread.LocalQueues = new ThreadPoolWorkQueueThreadLocals(this)); + var queue = currentThread.ThreadTaskLocals.LocalQueues; + return null != queue ? queue : (currentThread.ThreadTaskLocals.LocalQueues = new ThreadPoolWorkQueueThreadLocals(this)); } [SecurityCritical] @@ -745,7 +765,7 @@ public void Enqueue(IThreadPoolWorkItem callback, bool forceGlobal) [SecurityCritical] internal bool LocalFindAndPop(IThreadPoolWorkItem callback) { - ThreadPoolWorkQueueThreadLocals tl = Thread.CurrentThread.LocalQueues; + ThreadPoolWorkQueueThreadLocals tl = Thread.CurrentThread.ThreadTaskLocals.LocalQueues; if (null == tl) return false; @@ -859,7 +879,7 @@ private static void DequeueSteal(int index, ref IThreadPoolWorkItem callback, re [SecurityCritical] static internal bool Dispatch() { - var currentThread = Thread.CurrentThread; + var threadLocals = Thread.CurrentThread.ThreadTaskLocals; var workQueue = ThreadPoolGlobals.workQueue; // // The clock is ticking! We have ThreadPoolGlobals.tpQuantum milliseconds to get some work done, and then @@ -897,7 +917,7 @@ static internal bool Dispatch() // // Get our thread-local queue, may have been created by work item // - var wsq = currentThread.LocalQueues?.workStealingQueue; + var wsq = threadLocals.LocalQueues?.workStealingQueue; // // Dequeue and EnsureThreadRequested must be protected from ThreadAbortException. @@ -2019,7 +2039,7 @@ internal static IEnumerable EnumerateQueuedWorkItems(Thread [SecurityCritical] internal static IEnumerable GetLocallyQueuedWorkItems() { - return EnumerateQueuedWorkItems(new ThreadPoolWorkQueue.WorkStealingQueue[] { Thread.CurrentThread.LocalQueues.workStealingQueue }, null); + return EnumerateQueuedWorkItems(new ThreadPoolWorkQueue.WorkStealingQueue[] { Thread.CurrentThread.ThreadTaskLocals.LocalQueues?.workStealingQueue }, null); } [SecurityCritical] diff --git a/src/vm/object.h b/src/vm/object.h index 8bda037c1332..6d086c1f07b5 100644 --- a/src/vm/object.h +++ b/src/vm/object.h @@ -2074,7 +2074,7 @@ class ThreadBaseObject : public Object #ifdef FEATURE_CORECLR OBJECTREF m_ExecutionContext; OBJECTREF m_SynchronizationContext; - OBJECTREF m_LocalQueues; + OBJECTREF m_ThreadTaskLocals; #else EXECUTIONCONTEXTREF m_ExecutionContext; #endif From d4016724d4c0e2f53154b6b9d9b2489c283a61f0 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Thu, 28 Jul 2016 00:10:13 +0100 Subject: [PATCH 21/23] fix race condition --- src/mscorlib/src/System/Threading/ThreadPool.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index 9b4cddf2a8f9..0a9892a8be63 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -141,11 +141,11 @@ internal int Add(T e) { if (array[i] == null) { + Volatile.Write(ref array[i], e); if (i + 1 > current.ActiveLength) { current.IncrementLength(); } - Volatile.Write(ref array[i], e); return i; } else if (i == array.Length - 1) From c6d7a2026368a1f2728bd94917553616e983d521 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Thu, 28 Jul 2016 02:30:11 +0100 Subject: [PATCH 22/23] Deal with integer overflows --- .../src/System/Threading/ThreadPool.cs | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index 0a9892a8be63..d1e9a4580033 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -148,17 +148,16 @@ internal int Add(T e) } return i; } - else if (i == array.Length - 1) - { - var newSnapshot = new Snapshot(array.Length * 2, array.Length + 1); - T[] newArray = newSnapshot.Data; - - Array.Copy(array, newArray, i + 1); - newArray[i + 1] = e; - m_current = newSnapshot; - return i + 1; - } } + + var oldLength = array.Length; + var newSnapshot = new Snapshot(oldLength * 2, oldLength + 1); + T[] newArray = newSnapshot.Data; + + Array.Copy(array, newArray, oldLength); + newArray[oldLength + 1] = e; + m_current = newSnapshot; + return oldLength + 1; } } } @@ -829,14 +828,17 @@ private static void DequeueStealWithQueue(WorkStealingQueue wsq, int index, ref var otherQueues = allThreadQueues.Current; var total = otherQueues.ActiveLength; var data = otherQueues.Data; + Contract.Assert(data.Length >= total); + var remaining = total; - index = index % total; + // Only positive indices + index = (index & 0x7fff) % total; while (remaining > 0) { remaining--; WorkStealingQueue otherQueue = Volatile.Read(ref data[index]); - index = index + 1 == total ? 0 : index + 1; + index = (index + 1 >= total) ? 0 : index + 1; if (otherQueue != null && otherQueue != wsq && otherQueue.TrySteal(ref callback, ref missedSteal)) @@ -857,14 +859,17 @@ private static void DequeueSteal(int index, ref IThreadPoolWorkItem callback, re // No local queue, may not be other queues if (total == 0) return; var data = otherQueues.Data; + Contract.Assert(data.Length >= total); + var remaining = total; - index = index % total; + // Only positive indices + index = (index & 0x7fff) % total; while (remaining > 0) { remaining--; WorkStealingQueue otherQueue = Volatile.Read(ref data[index]); - index = index + 1 == total ? 0 : index + 1; + index = (index + 1 == total) ? 0 : index + 1; if (otherQueue != null && otherQueue.TrySteal(ref callback, ref missedSteal)) { From 87d2fae57a11a1b4d8bb1b5315e7beb778a5a010 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 13 Aug 2016 00:00:38 +0100 Subject: [PATCH 23/23] Enqueue falsesharing --- src/mscorlib/src/System/Threading/ThreadPool.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mscorlib/src/System/Threading/ThreadPool.cs b/src/mscorlib/src/System/Threading/ThreadPool.cs index d1e9a4580033..c8f953155fe3 100644 --- a/src/mscorlib/src/System/Threading/ThreadPool.cs +++ b/src/mscorlib/src/System/Threading/ThreadPool.cs @@ -518,11 +518,11 @@ internal class QueueSegment private volatile int indexes; // Holds a segment of the queue. Enqueues/Dequeues start at element 0, and work their way up. - [FieldOffset(8)] + [FieldOffset(64)] internal readonly IThreadPoolWorkItem[] nodes; private const int QueueSegmentLength = 256; - [FieldOffset(64)] + [FieldOffset(128)] // The next segment in the queue. public volatile QueueSegment Next;