diff --git a/Documentation/design-docs/jump-stubs.md b/Documentation/design-docs/jump-stubs.md index 86bf0ac13480..54f5ecb60228 100644 --- a/Documentation/design-docs/jump-stubs.md +++ b/Documentation/design-docs/jump-stubs.md @@ -188,8 +188,9 @@ still reach their intended target with a rel32 offset, so jump stubs are not expected to be required in most cases. If this attempt to create a jump stub fails, then the generated code -cannot be used, and we hit a fatal error; we have no mechanism currently -to recover from this failure, or to prevent it. +cannot be used, and the VM restarts the compilation with reserving +extra space in the code heap for jump stubs. The reserved extra space +ensures that the retry succeeds with high probability. There are several problems with this system: 1. Because the VM doesn't know whether a `IMAGE_REL_BASED_REL32` @@ -205,8 +206,6 @@ code because the JIT generates `IMAGE_REL_BASED_REL32` relocations for intra-function jumps and calls that it expects and, in fact, requires, not be replaced with jump stubs, because it doesn't expect the register used by jump stubs (RAX) to be trashed. -3. We don't have any mechanism to recover if a jump stub can't be -allocated. In the NGEN case, rel32 calls are guaranteed to always reach, as PE image files are limited to 2GB in size, meaning a rel32 offset is @@ -217,8 +216,8 @@ jump stubs, as described later. ### Failure mitigation -There are several possible mitigations for JIT failure to allocate jump -stubs. +There are several possible alternative mitigations for JIT failure to +allocate jump stubs. 1. When we get into "rel32 overflow" mode, the JIT could always generate large calls, and never generate rel32 offsets. This is obviously somewhat expensive, as every external call, such as every call to a JIT @@ -469,19 +468,9 @@ bytes allocated, to reserve space for one jump stub per FixupPrecode in the chunk. When the FixupPrecode is patched, for LCG methods it will use the pre-allocated space if a jump stub is required. -For the non-LCG, non-FixupPrecode cases, we need a different solution. -It would be easy to similarly allocate additional space for each type of -precode with the precode itself. This might prove expensive. An -alternative would be to ensure, by design, that somehow shared jump stub -space is available, perhaps by reserving it in a shared area when the -precode is allocated, and falling back to a mechanism where the precode -reserves its own jump stub space if shared jump stub space cannot be -allocated. - -A possibly better implementation would be to reserve, but not allocate, -jump stub space at the end of the code heap, similar to how -CodeHeapReserveForJumpStubs works, but instead the reserve amount should -be computed precisely. +For non-LCG, we are reserving, but not allocating, a space at the end +of the code heap. This is similar and in addition to the reservation done by +COMPlus_CodeHeapReserveForJumpStubs. (See https://github.com/dotnet/coreclr/pull/15296). ## Ready2Run diff --git a/src/debug/daccess/fntableaccess.h b/src/debug/daccess/fntableaccess.h index b5ea5452f65e..36e5d8f67324 100644 --- a/src/debug/daccess/fntableaccess.h +++ b/src/debug/daccess/fntableaccess.h @@ -41,9 +41,7 @@ struct FakeHeapList DWORD_PTR mapBase; // changed from PBYTE DWORD_PTR pHdrMap; // changed from DWORD* size_t maxCodeHeapSize; - DWORD cBlocks; - bool bFull; // Heap is considered full do not use for new allocations - bool bFullForJumpStubs; // Heap is considered full do not use for new allocations of jump stubs + size_t reserveForJumpStubs; }; typedef struct _FakeHpRealCodeHdr diff --git a/src/inc/clrconfigvalues.h b/src/inc/clrconfigvalues.h index 8c79b93010f5..24a18f1b3625 100644 --- a/src/inc/clrconfigvalues.h +++ b/src/inc/clrconfigvalues.h @@ -599,7 +599,7 @@ RETAIL_CONFIG_STRING_INFO(INTERNAL_WinMDPath, W("WinMDPath"), "Path for Windows // Loader heap // CONFIG_DWORD_INFO_EX(INTERNAL_LoaderHeapCallTracing, W("LoaderHeapCallTracing"), 0, "Loader heap troubleshooting", CLRConfig::REGUTIL_default) -RETAIL_CONFIG_DWORD_INFO(INTERNAL_CodeHeapReserveForJumpStubs, W("CodeHeapReserveForJumpStubs"), 2, "Percentage of code heap to reserve for jump stubs") +RETAIL_CONFIG_DWORD_INFO(INTERNAL_CodeHeapReserveForJumpStubs, W("CodeHeapReserveForJumpStubs"), 1, "Percentage of code heap to reserve for jump stubs") RETAIL_CONFIG_DWORD_INFO(INTERNAL_NGenReserveForJumpStubs, W("NGenReserveForJumpStubs"), 0, "Percentage of ngen image size to reserve for jump stubs") RETAIL_CONFIG_DWORD_INFO(INTERNAL_BreakOnOutOfMemoryWithinRange, W("BreakOnOutOfMemoryWithinRange"), 0, "Break before out of memory within range exception is thrown") diff --git a/src/inc/loaderheap.h b/src/inc/loaderheap.h index 4333505e8398..86a0e900d5a6 100644 --- a/src/inc/loaderheap.h +++ b/src/inc/loaderheap.h @@ -417,7 +417,7 @@ class UnlockedLoaderHeap #endif protected: - void *UnlockedAllocMemForCode_NoThrow(size_t dwHeaderSize, size_t dwCodeSize, DWORD dwCodeAlignment); + void *UnlockedAllocMemForCode_NoThrow(size_t dwHeaderSize, size_t dwCodeSize, DWORD dwCodeAlignment, size_t dwReserveForJumpStubs); void UnlockedSetReservedRegion(BYTE* dwReservedRegionAddress, SIZE_T dwReservedRegionSize, BOOL fReleaseMemory); }; @@ -838,10 +838,10 @@ class ExplicitControlLoaderHeap : public UnlockedLoaderHeap public: - void *AllocMemForCode_NoThrow(size_t dwHeaderSize, size_t dwCodeSize, DWORD dwCodeAlignment) + void *AllocMemForCode_NoThrow(size_t dwHeaderSize, size_t dwCodeSize, DWORD dwCodeAlignment, size_t dwReserveForJumpStubs) { WRAPPER_NO_CONTRACT; - return UnlockedAllocMemForCode_NoThrow(dwHeaderSize, dwCodeSize, dwCodeAlignment); + return UnlockedAllocMemForCode_NoThrow(dwHeaderSize, dwCodeSize, dwCodeAlignment, dwReserveForJumpStubs); } void SetReservedRegion(BYTE* dwReservedRegionAddress, SIZE_T dwReservedRegionSize, BOOL fReleaseMemory) diff --git a/src/utilcode/loaderheap.cpp b/src/utilcode/loaderheap.cpp index ca1ed666f421..90b23c3411d3 100644 --- a/src/utilcode/loaderheap.cpp +++ b/src/utilcode/loaderheap.cpp @@ -1731,7 +1731,7 @@ void *UnlockedLoaderHeap::UnlockedAllocAlignedMem(size_t dwRequestedSize, -void *UnlockedLoaderHeap::UnlockedAllocMemForCode_NoThrow(size_t dwHeaderSize, size_t dwCodeSize, DWORD dwCodeAlignment) +void *UnlockedLoaderHeap::UnlockedAllocMemForCode_NoThrow(size_t dwHeaderSize, size_t dwCodeSize, DWORD dwCodeAlignment, size_t dwReserveForJumpStubs) { CONTRACT(void*) { @@ -1753,7 +1753,7 @@ void *UnlockedLoaderHeap::UnlockedAllocMemForCode_NoThrow(size_t dwHeaderSize, s // // Thus, we'll request as much heap growth as is needed for the worst case (we request an extra dwCodeAlignment - 1 bytes) - S_SIZE_T cbAllocSize = S_SIZE_T(dwHeaderSize) + S_SIZE_T(dwCodeSize) + S_SIZE_T(dwCodeAlignment - 1); + S_SIZE_T cbAllocSize = S_SIZE_T(dwHeaderSize) + S_SIZE_T(dwCodeSize) + S_SIZE_T(dwCodeAlignment - 1) + S_SIZE_T(dwReserveForJumpStubs); if( cbAllocSize.IsOverflow() ) { RETURN NULL; diff --git a/src/vm/amd64/cgenamd64.cpp b/src/vm/amd64/cgenamd64.cpp index 20dca22e36bc..4e1f38ad128e 100644 --- a/src/vm/amd64/cgenamd64.cpp +++ b/src/vm/amd64/cgenamd64.cpp @@ -692,7 +692,8 @@ UMEntryThunk* UMEntryThunk::Decode(LPVOID pCallback) return (UMEntryThunk*)pThunkCode->m_uet; } -INT32 rel32UsingJumpStub(INT32 UNALIGNED * pRel32, PCODE target, MethodDesc *pMethod, LoaderAllocator *pLoaderAllocator /* = NULL */) +INT32 rel32UsingJumpStub(INT32 UNALIGNED * pRel32, PCODE target, MethodDesc *pMethod, + LoaderAllocator *pLoaderAllocator /* = NULL */, bool throwOnOutOfMemoryWithinRange /*= true*/) { CONTRACTL { @@ -721,11 +722,31 @@ INT32 rel32UsingJumpStub(INT32 UNALIGNED * pRel32, PCODE target, MethodDesc *pMe TADDR hiAddr = baseAddr + INT32_MAX; if (hiAddr < baseAddr) hiAddr = UINT64_MAX; // overflow + // Always try to allocate with throwOnOutOfMemoryWithinRange:false first to conserve reserveForJumpStubs until when + // it is really needed. LoaderCodeHeap::CreateCodeHeap and EEJitManager::CanUseCodeHeap won't use the reserved + // space when throwOnOutOfMemoryWithinRange is false. + // + // The reserved space should be only used by jump stubs for precodes and other similar code fragments. It should + // not be used by JITed code. And since the accounting of the reserved space is not precise, we are conservative + // and try to save the reserved space until it is really needed to avoid throwing out of memory within range exception. PCODE jumpStubAddr = ExecutionManager::jumpStub(pMethod, target, (BYTE *)loAddr, (BYTE *)hiAddr, - pLoaderAllocator); + pLoaderAllocator, + /* throwOnOutOfMemoryWithinRange */ false); + if (jumpStubAddr == NULL) + { + if (!throwOnOutOfMemoryWithinRange) + return 0; + + jumpStubAddr = ExecutionManager::jumpStub(pMethod, + target, + (BYTE *)loAddr, + (BYTE *)hiAddr, + pLoaderAllocator, + /* throwOnOutOfMemoryWithinRange */ true); + } offset = jumpStubAddr - baseAddr; diff --git a/src/vm/amd64/cgencpu.h b/src/vm/amd64/cgencpu.h index b83437039bac..9136b168aa4f 100644 --- a/src/vm/amd64/cgencpu.h +++ b/src/vm/amd64/cgencpu.h @@ -379,7 +379,8 @@ void EncodeLoadAndJumpThunk (LPBYTE pBuffer, LPVOID pv, LPVOID pTarget); // Get Rel32 destination, emit jumpStub if necessary -INT32 rel32UsingJumpStub(INT32 UNALIGNED * pRel32, PCODE target, MethodDesc *pMethod, LoaderAllocator *pLoaderAllocator = NULL); +INT32 rel32UsingJumpStub(INT32 UNALIGNED * pRel32, PCODE target, MethodDesc *pMethod, + LoaderAllocator *pLoaderAllocator = NULL, bool throwOnOutOfMemoryWithinRange = true); // Get Rel32 destination, emit jumpStub if necessary into a preallocated location INT32 rel32UsingPreallocatedJumpStub(INT32 UNALIGNED * pRel32, PCODE target, PCODE jumpStubAddr); diff --git a/src/vm/codeman.cpp b/src/vm/codeman.cpp index a19ab8e0ae21..de6ee7994f50 100644 --- a/src/vm/codeman.cpp +++ b/src/vm/codeman.cpp @@ -42,8 +42,6 @@ #include "perfmap.h" #endif -#define MAX_M_ALLOCATED (16 * 1024) - // Default number of jump stubs in a jump stub block #define DEFAULT_JUMPSTUBS_PER_BLOCK 32 @@ -564,7 +562,6 @@ Need EEJitManager::m_CodeHeapCritSec to be set NewCodeHeap allocCodeRaw GetCodeHeapList -GetCodeHeap RemoveCodeHeapFromDomainList DeleteCodeHeap AddRangeToJitHeapCache @@ -2056,12 +2053,36 @@ VOID EEJitManager::EnsureJumpStubReserve(BYTE * pImageBase, SIZE_T imageSize, SI } #endif // _TARGET_AMD64_ +static size_t GetDefaultReserveForJumpStubs(size_t codeHeapSize) +{ + LIMITED_METHOD_CONTRACT; + +#ifdef _TARGET_AMD64_ + // + // Keep a small default reserve at the end of the codeheap for jump stubs. It should reduce + // chance that we won't be able allocate jump stub because of lack of suitable address space. + // + static ConfigDWORD configCodeHeapReserveForJumpStubs; + int percentReserveForJumpStubs = configCodeHeapReserveForJumpStubs.val(CLRConfig::INTERNAL_CodeHeapReserveForJumpStubs); + + size_t reserveForJumpStubs = percentReserveForJumpStubs * (codeHeapSize / 100); + + size_t minReserveForJumpStubs = sizeof(CodeHeader) + + sizeof(JumpStubBlockHeader) + (size_t) DEFAULT_JUMPSTUBS_PER_BLOCK * BACK_TO_BACK_JUMP_ALLOCATE_SIZE + + CODE_SIZE_ALIGN + BYTES_PER_BUCKET; + + return max(reserveForJumpStubs, minReserveForJumpStubs); +#else + return 0; +#endif +} + HeapList* LoaderCodeHeap::CreateCodeHeap(CodeHeapRequestInfo *pInfo, LoaderHeap *pJitMetaHeap) { CONTRACT(HeapList *) { THROWS; GC_NOTRIGGER; - POSTCONDITION(CheckPointer(RETVAL)); + POSTCONDITION(CheckPointer(RETVAL) || !pInfo->getThrowOnOutOfMemoryWithinRange()); } CONTRACT_END; size_t * pPrivatePCLBytes = NULL; @@ -2101,11 +2122,19 @@ HeapList* LoaderCodeHeap::CreateCodeHeap(CodeHeapRequestInfo *pInfo, LoaderHeap { if (loAddr != NULL || hiAddr != NULL) { +#ifdef _DEBUG + // Always exercise the fallback path in the caller when forced relocs are turned on + if (!pInfo->getThrowOnOutOfMemoryWithinRange() && PEDecoder::GetForceRelocs()) + RETURN NULL; +#endif pBaseAddr = ClrVirtualAllocWithinRange(loAddr, hiAddr, reserveSize, MEM_RESERVE, PAGE_NOACCESS); if (!pBaseAddr) { + // Conserve emergency jump stub reserve until when it is really needed + if (!pInfo->getThrowOnOutOfMemoryWithinRange()) + RETURN NULL; #ifdef _TARGET_AMD64_ pBaseAddr = ExecutionManager::GetEEJitManager()->AllocateFromEmergencyJumpStubReserve(loAddr, hiAddr, &reserveSize); if (!pBaseAddr) @@ -2138,17 +2167,13 @@ HeapList* LoaderCodeHeap::CreateCodeHeap(CodeHeapRequestInfo *pInfo, LoaderHeap pHp->endAddress = pHp->startAddress; pHp->maxCodeHeapSize = heapSize; + pHp->reserveForJumpStubs = fAllocatedFromEmergencyJumpStubReserve ? pHp->maxCodeHeapSize : GetDefaultReserveForJumpStubs(pHp->maxCodeHeapSize); _ASSERTE(heapSize >= initialRequestSize); // We do not need to memset this memory, since ClrVirtualAlloc() guarantees that the memory is zero. // Furthermore, if we avoid writing to it, these pages don't come into our working set - pHp->bFull = fAllocatedFromEmergencyJumpStubReserve; - pHp->bFullForJumpStubs = false; - - pHp->cBlocks = 0; - pHp->mapBase = ROUND_DOWN_TO_PAGE(pHp->startAddress); // round down to next lower page align pHp->pHdrMap = (DWORD*)(void*)pJitMetaHeap->AllocMem(S_SIZE_T(nibbleMapSize)); @@ -2165,7 +2190,7 @@ HeapList* LoaderCodeHeap::CreateCodeHeap(CodeHeapRequestInfo *pInfo, LoaderHeap RETURN pHp; } -void * LoaderCodeHeap::AllocMemForCode_NoThrow(size_t header, size_t size, DWORD alignment) +void * LoaderCodeHeap::AllocMemForCode_NoThrow(size_t header, size_t size, DWORD alignment, size_t reserveForJumpStubs) { CONTRACTL { NOTHROW; @@ -2174,7 +2199,7 @@ void * LoaderCodeHeap::AllocMemForCode_NoThrow(size_t header, size_t size, DWORD if (m_cbMinNextPad > (SSIZE_T)header) header = m_cbMinNextPad; - void * p = m_LoaderHeap.AllocMemForCode_NoThrow(header, size, alignment); + void * p = m_LoaderHeap.AllocMemForCode_NoThrow(header, size, alignment, reserveForJumpStubs); if (p == NULL) return NULL; @@ -2199,6 +2224,7 @@ void CodeHeapRequestInfo::Init() m_pAllocator = m_pMD->GetLoaderAllocatorForCode(); m_isDynamicDomain = (m_pMD != NULL) ? m_pMD->IsLCGMethod() : false; m_isCollectible = m_pAllocator->IsCollectible() ? true : false; + m_throwOnOutOfMemoryWithinRange = true; } #ifdef WIN64EXCEPTIONS @@ -2260,7 +2286,7 @@ HeapList* EEJitManager::NewCodeHeap(CodeHeapRequestInfo *pInfo, DomainCodeHeapLi THROWS; GC_NOTRIGGER; PRECONDITION(m_CodeHeapCritSec.OwnedByCurrentThread()); - POSTCONDITION(CheckPointer(RETVAL)); + POSTCONDITION(CheckPointer(RETVAL) || !pInfo->getThrowOnOutOfMemoryWithinRange()); } CONTRACT_END; size_t initialRequestSize = pInfo->getRequestSize(); @@ -2278,7 +2304,7 @@ HeapList* EEJitManager::NewCodeHeap(CodeHeapRequestInfo *pInfo, DomainCodeHeapLi // we bump up the reserve size for the 64-bit platforms if (!pInfo->IsDynamicDomain()) { - minReserveSize *= 4; // CodeHeaps are larger on AMD64 (256 KB to 1024 KB) + minReserveSize *= 8; // CodeHeaps are larger on AMD64 (256 KB to 2048 KB) } } #endif @@ -2314,6 +2340,11 @@ HeapList* EEJitManager::NewCodeHeap(CodeHeapRequestInfo *pInfo, DomainCodeHeapLi pHp = LoaderCodeHeap::CreateCodeHeap(pInfo, pJitMetaHeap); } + if (pHp == NULL) + { + _ASSERTE(!pInfo->getThrowOnOutOfMemoryWithinRange()); + RETURN(NULL); + } _ASSERTE (pHp != NULL); _ASSERTE (pHp->maxCodeHeapSize >= initialRequestSize); @@ -2368,125 +2399,84 @@ HeapList* EEJitManager::NewCodeHeap(CodeHeapRequestInfo *pInfo, DomainCodeHeapLi void* EEJitManager::allocCodeRaw(CodeHeapRequestInfo *pInfo, size_t header, size_t blockSize, unsigned align, - HeapList ** ppCodeHeap /* Writeback, Can be null */ ) + HeapList ** ppCodeHeap) { CONTRACT(void *) { THROWS; GC_NOTRIGGER; PRECONDITION(m_CodeHeapCritSec.OwnedByCurrentThread()); - POSTCONDITION(CheckPointer(RETVAL)); + POSTCONDITION(CheckPointer(RETVAL) || !pInfo->getThrowOnOutOfMemoryWithinRange()); } CONTRACT_END; - pInfo->setRequestSize(header+blockSize+(align-1)); - - // Initialize the writeback value to NULL if a non-NULL pointer was provided - if (ppCodeHeap) - *ppCodeHeap = NULL; - - void * mem = NULL; + pInfo->setRequestSize(header+blockSize+(align-1)+pInfo->getReserveForJumpStubs()); - bool bForJumpStubs = (pInfo->m_loAddr != 0) || (pInfo->m_hiAddr != 0); - bool bUseCachedDynamicCodeHeap = pInfo->IsDynamicDomain(); - - HeapList * pCodeHeap; + void * mem = NULL; + HeapList * pCodeHeap = NULL; + DomainCodeHeapList *pList = NULL; - for (;;) + // Avoid going through the full list in the common case - try to use the most recently used codeheap + if (pInfo->IsDynamicDomain()) { - // Avoid going through the full list in the common case - try to use the most recently used codeheap - if (bUseCachedDynamicCodeHeap) - { - pCodeHeap = (HeapList *)pInfo->m_pAllocator->m_pLastUsedDynamicCodeHeap; - pInfo->m_pAllocator->m_pLastUsedDynamicCodeHeap = NULL; - } - else - { - pCodeHeap = (HeapList *)pInfo->m_pAllocator->m_pLastUsedCodeHeap; - pInfo->m_pAllocator->m_pLastUsedCodeHeap = NULL; - } - - - // If we will use a cached code heap for jump stubs, ensure that the code heap meets the loAddr and highAddr constraint - if (bForJumpStubs && pCodeHeap && !CanUseCodeHeap(pInfo, pCodeHeap)) - { - pCodeHeap = NULL; - } + pCodeHeap = (HeapList *)pInfo->m_pAllocator->m_pLastUsedDynamicCodeHeap; + pInfo->m_pAllocator->m_pLastUsedDynamicCodeHeap = NULL; + } + else + { + pCodeHeap = (HeapList *)pInfo->m_pAllocator->m_pLastUsedCodeHeap; + pInfo->m_pAllocator->m_pLastUsedCodeHeap = NULL; + } - // If we don't have a cached code heap or can't use it, get a code heap - if (pCodeHeap == NULL) - { - pCodeHeap = GetCodeHeap(pInfo); - if (pCodeHeap == NULL) - break; - } + // If we will use a cached code heap, ensure that the code heap meets the constraints + if (pCodeHeap && CanUseCodeHeap(pInfo, pCodeHeap)) + { + mem = (pCodeHeap->pHeap)->AllocMemForCode_NoThrow(header, blockSize, align, pInfo->getReserveForJumpStubs()); + } -#ifdef _WIN64 - if (!bForJumpStubs) + if (mem == NULL) + { + pList = GetCodeHeapList(pInfo, pInfo->m_pAllocator); + if (pList != NULL) { - // - // Keep a small reserve at the end of the codeheap for jump stubs. It should reduce - // chance that we won't be able allocate jump stub because of lack of suitable address space. - // - // It is not a perfect solution. Ideally, we would be able to either ensure that jump stub - // allocation won't fail or handle jump stub allocation gracefully (see DevDiv #381823 and - // related bugs for details). - // - static ConfigDWORD configCodeHeapReserveForJumpStubs; - int percentReserveForJumpStubs = configCodeHeapReserveForJumpStubs.val(CLRConfig::INTERNAL_CodeHeapReserveForJumpStubs); - - size_t reserveForJumpStubs = percentReserveForJumpStubs * (pCodeHeap->maxCodeHeapSize / 100); - - size_t minReserveForJumpStubs = sizeof(CodeHeader) + - sizeof(JumpStubBlockHeader) + (size_t) DEFAULT_JUMPSTUBS_PER_BLOCK * BACK_TO_BACK_JUMP_ALLOCATE_SIZE + - CODE_SIZE_ALIGN + BYTES_PER_BUCKET; - - // Reserve only if the size can fit a cluster of jump stubs - if (reserveForJumpStubs > minReserveForJumpStubs) + for (int i = 0; i < pList->m_CodeHeapList.Count(); i++) { - size_t occupiedSize = pCodeHeap->endAddress - pCodeHeap->startAddress; + pCodeHeap = pList->m_CodeHeapList[i]; - if (occupiedSize + pInfo->getRequestSize() + reserveForJumpStubs > pCodeHeap->maxCodeHeapSize) + // Validate that the code heap can be used for the current request + if (CanUseCodeHeap(pInfo, pCodeHeap)) { - pCodeHeap->SetHeapFull(); - continue; + mem = (pCodeHeap->pHeap)->AllocMemForCode_NoThrow(header, blockSize, align, pInfo->getReserveForJumpStubs()); + if (mem != NULL) + break; } } } -#endif - - mem = (pCodeHeap->pHeap)->AllocMemForCode_NoThrow(header, blockSize, align); - if (mem != NULL) - break; - - // The current heap couldn't handle our request. Mark it as full. - if (bForJumpStubs) - pCodeHeap->SetHeapFullForJumpStubs(); - else - pCodeHeap->SetHeapFull(); - } - if (mem == NULL) - { - // Let us create a new heap. - - DomainCodeHeapList *pList = GetCodeHeapList(pInfo, pInfo->m_pAllocator); - if (pList == NULL) + if (mem == NULL) { - // not found so need to create the first one - pList = CreateCodeHeapList(pInfo); - _ASSERTE(pList == GetCodeHeapList(pInfo, pInfo->m_pAllocator)); - } - _ASSERTE(pList); + // Let us create a new heap. + if (pList == NULL) + { + // not found so need to create the first one + pList = CreateCodeHeapList(pInfo); + _ASSERTE(pList == GetCodeHeapList(pInfo, pInfo->m_pAllocator)); + } + _ASSERTE(pList); - pCodeHeap = NewCodeHeap(pInfo, pList); - _ASSERTE(pCodeHeap); + pCodeHeap = NewCodeHeap(pInfo, pList); + if (pCodeHeap == NULL) + { + _ASSERTE(!pInfo->getThrowOnOutOfMemoryWithinRange()); + RETURN(NULL); + } - mem = (pCodeHeap->pHeap)->AllocMemForCode_NoThrow(header, blockSize, align); - if (mem == NULL) - ThrowOutOfMemory(); - _ASSERTE(mem); + mem = (pCodeHeap->pHeap)->AllocMemForCode_NoThrow(header, blockSize, align, pInfo->getReserveForJumpStubs()); + if (mem == NULL) + ThrowOutOfMemory(); + _ASSERTE(mem); + } } - if (bUseCachedDynamicCodeHeap) + if (pInfo->IsDynamicDomain()) { pInfo->m_pAllocator->m_pLastUsedDynamicCodeHeap = pCodeHeap; } @@ -2495,10 +2485,8 @@ void* EEJitManager::allocCodeRaw(CodeHeapRequestInfo *pInfo, pInfo->m_pAllocator->m_pLastUsedCodeHeap = pCodeHeap; } - - // Record the pCodeHeap value into ppCodeHeap, if a non-NULL pointer was provided - if (ppCodeHeap) - *ppCodeHeap = pCodeHeap; + // Record the pCodeHeap value into ppCodeHeap + *ppCodeHeap = pCodeHeap; _ASSERTE((TADDR)mem >= pCodeHeap->startAddress); @@ -2511,7 +2499,7 @@ void* EEJitManager::allocCodeRaw(CodeHeapRequestInfo *pInfo, RETURN(mem); } -CodeHeader* EEJitManager::allocCode(MethodDesc* pMD, size_t blockSize, CorJitAllocMemFlag flag +CodeHeader* EEJitManager::allocCode(MethodDesc* pMD, size_t blockSize, size_t reserveForJumpStubs, CorJitAllocMemFlag flag #ifdef WIN64EXCEPTIONS , UINT nUnwindInfos , TADDR * pModuleBase @@ -2565,6 +2553,7 @@ CodeHeader* EEJitManager::allocCode(MethodDesc* pMD, size_t blockSize, CorJitAll requestInfo.SetDynamicDomain(); } #endif + requestInfo.setReserveForJumpStubs(reserveForJumpStubs); #if defined(USE_INDIRECT_CODEHEADER) SIZE_T realHeaderSize = offsetof(RealCodeHeader, unwindInfos[0]) + (sizeof(T_RUNTIME_FUNCTION) * nUnwindInfos); @@ -2675,49 +2664,6 @@ EEJitManager::DomainCodeHeapList *EEJitManager::GetCodeHeapList(CodeHeapRequestI return pList; } -HeapList* EEJitManager::GetCodeHeap(CodeHeapRequestInfo *pInfo) -{ - CONTRACT(HeapList *) { - NOTHROW; - GC_NOTRIGGER; - PRECONDITION(m_CodeHeapCritSec.OwnedByCurrentThread()); - } CONTRACT_END; - - HeapList *pResult = NULL; - - _ASSERTE(pInfo->m_pAllocator != NULL); - - // loop through the m_DomainCodeHeaps to find the AppDomain - // if not found, then create it - DomainCodeHeapList *pList = GetCodeHeapList(pInfo, pInfo->m_pAllocator); - if (pList) - { - // Set pResult to the largest non-full HeapList - // that also satisfies the [loAddr..hiAddr] constraint - for (int i=0; i < pList->m_CodeHeapList.Count(); i++) - { - HeapList *pCurrent = pList->m_CodeHeapList[i]; - - // Validate that the code heap can be used for the current request - if(CanUseCodeHeap(pInfo, pCurrent)) - { - if (pResult == NULL) - { - // pCurrent is the first (and possibly only) heap that would satistfy - pResult = pCurrent; - } - // We use the initial creation size as a discriminator (i.e largest heap) - else if (pResult->maxCodeHeapSize < pCurrent->maxCodeHeapSize) - { - pResult = pCurrent; - } - } - } - } - - RETURN (pResult); -} - bool EEJitManager::CanUseCodeHeap(CodeHeapRequestInfo *pInfo, HeapList *pCodeHeap) { CONTRACTL { @@ -2730,81 +2676,80 @@ bool EEJitManager::CanUseCodeHeap(CodeHeapRequestInfo *pInfo, HeapList *pCodeHea if ((pInfo->m_loAddr == 0) && (pInfo->m_hiAddr == 0)) { - if (!pCodeHeap->IsHeapFull()) + // We have no constraint so this non empty heap will be able to satisfy our request + if (pInfo->IsDynamicDomain()) { - // We have no constraint so this non empty heap will be able to satistfy our request - if (pInfo->IsDynamicDomain()) + _ASSERTE(pCodeHeap->reserveForJumpStubs == 0); + retVal = true; + } + else + { + BYTE * lastAddr = (BYTE *) pCodeHeap->startAddress + pCodeHeap->maxCodeHeapSize; + + BYTE * loRequestAddr = (BYTE *) pCodeHeap->endAddress; + BYTE * hiRequestAddr = loRequestAddr + pInfo->getRequestSize() + BYTES_PER_BUCKET; + if (hiRequestAddr <= lastAddr - pCodeHeap->reserveForJumpStubs) { retVal = true; } - else - { - BYTE * lastAddr = (BYTE *) pCodeHeap->startAddress + pCodeHeap->maxCodeHeapSize; - - BYTE * loRequestAddr = (BYTE *) pCodeHeap->endAddress; - BYTE * hiRequestAddr = loRequestAddr + pInfo->getRequestSize() + BYTES_PER_BUCKET; - if (hiRequestAddr <= lastAddr) - { - retVal = true; - } - } } } else { - if (!pCodeHeap->IsHeapFullForJumpStubs()) - { - // We also check to see if an allocation in this heap would satistfy - // the [loAddr..hiAddr] requirement + // We also check to see if an allocation in this heap would satisfy + // the [loAddr..hiAddr] requirement - // Calculate the byte range that can ever be returned by - // an allocation in this HeapList element - // - BYTE * firstAddr = (BYTE *) pCodeHeap->startAddress; - BYTE * lastAddr = (BYTE *) pCodeHeap->startAddress + pCodeHeap->maxCodeHeapSize; + // Calculate the byte range that can ever be returned by + // an allocation in this HeapList element + // + BYTE * firstAddr = (BYTE *) pCodeHeap->startAddress; + BYTE * lastAddr = (BYTE *) pCodeHeap->startAddress + pCodeHeap->maxCodeHeapSize; - _ASSERTE(pCodeHeap->startAddress <= pCodeHeap->endAddress); - _ASSERTE(firstAddr <= lastAddr); + _ASSERTE(pCodeHeap->startAddress <= pCodeHeap->endAddress); + _ASSERTE(firstAddr <= lastAddr); - if (pInfo->IsDynamicDomain()) - { - // We check to see if every allocation in this heap - // will satistfy the [loAddr..hiAddr] requirement. - // - // Dynamic domains use a free list allocator, - // thus we can receive any address in the range - // when calling AllocMemory with a DynamicDomain + if (pInfo->IsDynamicDomain()) + { + _ASSERTE(pCodeHeap->reserveForJumpStubs == 0); + + // We check to see if every allocation in this heap + // will satisfy the [loAddr..hiAddr] requirement. + // + // Dynamic domains use a free list allocator, + // thus we can receive any address in the range + // when calling AllocMemory with a DynamicDomain - // [firstaddr .. lastAddr] must be entirely within - // [pInfo->m_loAddr .. pInfo->m_hiAddr] - // - if ((pInfo->m_loAddr <= firstAddr) && - (lastAddr <= pInfo->m_hiAddr)) - { - // This heap will always satisfy our constraint - retVal = true; - } + // [firstaddr .. lastAddr] must be entirely within + // [pInfo->m_loAddr .. pInfo->m_hiAddr] + // + if ((pInfo->m_loAddr <= firstAddr) && + (lastAddr <= pInfo->m_hiAddr)) + { + // This heap will always satisfy our constraint + retVal = true; } - else // non-DynamicDomain + } + else // non-DynamicDomain + { + // Calculate the byte range that would be allocated for the + // next allocation request into [loRequestAddr..hiRequestAddr] + // + BYTE * loRequestAddr = (BYTE *) pCodeHeap->endAddress; + BYTE * hiRequestAddr = loRequestAddr + pInfo->getRequestSize() + BYTES_PER_BUCKET; + _ASSERTE(loRequestAddr <= hiRequestAddr); + + // loRequestAddr and hiRequestAddr must be entirely within + // [pInfo->m_loAddr .. pInfo->m_hiAddr] + // + if ((pInfo->m_loAddr <= loRequestAddr) && + (hiRequestAddr <= pInfo->m_hiAddr)) { - // Calculate the byte range that would be allocated for the - // next allocation request into [loRequestAddr..hiRequestAddr] - // - BYTE * loRequestAddr = (BYTE *) pCodeHeap->endAddress; - BYTE * hiRequestAddr = loRequestAddr + pInfo->getRequestSize() + BYTES_PER_BUCKET; - _ASSERTE(loRequestAddr <= hiRequestAddr); - - // loRequestAddr and hiRequestAddr must be entirely within - // [pInfo->m_loAddr .. pInfo->m_hiAddr] - // additionally hiRequestAddr must also be less than - // or equal to lastAddr - // - if ((pInfo->m_loAddr <= loRequestAddr) && - (hiRequestAddr <= pInfo->m_hiAddr) && - (hiRequestAddr <= lastAddr)) + // Additionally hiRequestAddr must also be less than or equal to lastAddr. + // If throwOnOutOfMemoryWithinRange is not set, conserve reserveForJumpStubs until when it is really needed. + if (hiRequestAddr <= lastAddr - (pInfo->getThrowOnOutOfMemoryWithinRange() ? 0 : pCodeHeap->reserveForJumpStubs)) { - // This heap will be able to satistfy our constraint - retVal = true; + // This heap will be able to satisfy our constraint + retVal = true; } } } @@ -2927,23 +2872,24 @@ EE_ILEXCEPTION* EEJitManager::allocEHInfo(CodeHeader* pCodeHeader, unsigned numC JumpStubBlockHeader * EEJitManager::allocJumpStubBlock(MethodDesc* pMD, DWORD numJumps, BYTE * loAddr, BYTE * hiAddr, - LoaderAllocator *pLoaderAllocator) + LoaderAllocator *pLoaderAllocator, + bool throwOnOutOfMemoryWithinRange) { CONTRACT(JumpStubBlockHeader *) { THROWS; GC_NOTRIGGER; PRECONDITION(loAddr < hiAddr); PRECONDITION(pLoaderAllocator != NULL); - POSTCONDITION(CheckPointer(RETVAL)); + POSTCONDITION(CheckPointer(RETVAL) || !throwOnOutOfMemoryWithinRange); } CONTRACT_END; _ASSERTE((sizeof(JumpStubBlockHeader) % CODE_SIZE_ALIGN) == 0); - _ASSERTE(numJumps < MAX_M_ALLOCATED); size_t blockSize = sizeof(JumpStubBlockHeader) + (size_t) numJumps * BACK_TO_BACK_JUMP_ALLOCATE_SIZE; HeapList *pCodeHeap = NULL; CodeHeapRequestInfo requestInfo(pMD, pLoaderAllocator, loAddr, hiAddr); + requestInfo.setThrowOnOutOfMemoryWithinRange(throwOnOutOfMemoryWithinRange); TADDR mem; JumpStubBlockHeader * pBlock; @@ -2953,6 +2899,11 @@ JumpStubBlockHeader * EEJitManager::allocJumpStubBlock(MethodDesc* pMD, DWORD n CrstHolder ch(&m_CodeHeapCritSec); mem = (TADDR) allocCodeRaw(&requestInfo, sizeof(TADDR), blockSize, CODE_SIZE_ALIGN, &pCodeHeap); + if (mem == NULL) + { + _ASSERTE(!throwOnOutOfMemoryWithinRange); + RETURN(NULL); + } // CodeHeader comes immediately before the block CodeHeader * pCodeHdr = (CodeHeader *) (mem - sizeof(CodeHeader)); @@ -2993,6 +2944,12 @@ void * EEJitManager::allocCodeFragmentBlock(size_t blockSize, unsigned alignment HeapList *pCodeHeap = NULL; CodeHeapRequestInfo requestInfo(NULL, pLoaderAllocator, NULL, NULL); +#ifdef _TARGET_AMD64_ + // CodeFragments are pretty much always Precodes that may need to be patched with jump stubs at some point in future + // We will assume the worst case that every FixupPrecode will need to be patched and reserve the jump stubs accordingly + requestInfo.setReserveForJumpStubs((blockSize / 8) * JUMP_ALLOCATE_SIZE); +#endif + TADDR mem; // Scope the lock @@ -3006,6 +2963,9 @@ void * EEJitManager::allocCodeFragmentBlock(size_t blockSize, unsigned alignment pCodeHdr->SetStubCodeBlockKind(kind); NibbleMapSet(pCodeHeap, (TADDR)mem, TRUE); + + // Record the jump stub reservation + pCodeHeap->reserveForJumpStubs += requestInfo.getReserveForJumpStubs(); } RETURN((void *)mem); @@ -3217,7 +3177,6 @@ void EEJitManager::RemoveJitData (CodeHeader * pCHdr, size_t GCinfo_len, size_t } _ASSERTE(pHp && pHp->pHdrMap); - _ASSERTE(pHp && pHp->cBlocks); // Better to just return than AV? if (pHp == NULL) @@ -3864,8 +3823,6 @@ void EEJitManager::NibbleMapSet(HeapList * pHp, TADDR pCode, BOOL bSet) // It is important for this update to be atomic. Synchronization would be required with FindMethodCode otherwise. *(pMap+index) = ((*(pMap+index))&mask)|value; - - pHp->cBlocks += (bSet ? 1 : -1); } #endif // !DACCESS_COMPILE @@ -4949,7 +4906,8 @@ void ExecutionManager::Unload(LoaderAllocator *pLoaderAllocator) PCODE ExecutionManager::jumpStub(MethodDesc* pMD, PCODE target, BYTE * loAddr, BYTE * hiAddr, - LoaderAllocator *pLoaderAllocator) + LoaderAllocator *pLoaderAllocator, + bool throwOnOutOfMemoryWithinRange) { CONTRACT(PCODE) { THROWS; @@ -4957,7 +4915,7 @@ PCODE ExecutionManager::jumpStub(MethodDesc* pMD, PCODE target, MODE_ANY; PRECONDITION(pLoaderAllocator != NULL || pMD != NULL); PRECONDITION(loAddr < hiAddr); - POSTCONDITION(RETVAL != NULL); + POSTCONDITION((RETVAL != NULL) || !throwOnOutOfMemoryWithinRange); } CONTRACT_END; PCODE jumpStub = NULL; @@ -5021,7 +4979,12 @@ PCODE ExecutionManager::jumpStub(MethodDesc* pMD, PCODE target, // If we get here we need to create a new jump stub // add or change the jump stub table to point at the new one - jumpStub = getNextJumpStub(pMD, target, loAddr, hiAddr, pLoaderAllocator); // this statement can throw + jumpStub = getNextJumpStub(pMD, target, loAddr, hiAddr, pLoaderAllocator, throwOnOutOfMemoryWithinRange); // this statement can throw + if (jumpStub == NULL) + { + _ASSERTE(!throwOnOutOfMemoryWithinRange); + RETURN(NULL); + } _ASSERTE(((TADDR)loAddr <= jumpStub) && (jumpStub <= (TADDR)hiAddr)); @@ -5032,14 +4995,16 @@ PCODE ExecutionManager::jumpStub(MethodDesc* pMD, PCODE target, } PCODE ExecutionManager::getNextJumpStub(MethodDesc* pMD, PCODE target, - BYTE * loAddr, BYTE * hiAddr, LoaderAllocator *pLoaderAllocator) + BYTE * loAddr, BYTE * hiAddr, + LoaderAllocator *pLoaderAllocator, + bool throwOnOutOfMemoryWithinRange) { CONTRACT(PCODE) { THROWS; GC_NOTRIGGER; PRECONDITION(pLoaderAllocator != NULL); PRECONDITION(m_JumpStubCrst.OwnedByCurrentThread()); - POSTCONDITION(RETVAL != NULL); + POSTCONDITION((RETVAL != NULL) || !throwOnOutOfMemoryWithinRange); } CONTRACT_END; DWORD numJumpStubs = DEFAULT_JUMPSTUBS_PER_BLOCK; // a block of 32 JumpStubs @@ -5112,7 +5077,12 @@ PCODE ExecutionManager::getNextJumpStub(MethodDesc* pMD, PCODE target, // // note that this can throw an OOM exception - curBlock = ExecutionManager::GetEEJitManager()->allocJumpStubBlock(pMD, numJumpStubs, loAddr, hiAddr, pLoaderAllocator); + curBlock = ExecutionManager::GetEEJitManager()->allocJumpStubBlock(pMD, numJumpStubs, loAddr, hiAddr, pLoaderAllocator, throwOnOutOfMemoryWithinRange); + if (curBlock == NULL) + { + _ASSERTE(!throwOnOutOfMemoryWithinRange); + RETURN(NULL); + } jumpStub = (BYTE *) curBlock + sizeof(JumpStubBlockHeader) + ((size_t) curBlock->m_used * BACK_TO_BACK_JUMP_ALLOCATE_SIZE); diff --git a/src/vm/codeman.h b/src/vm/codeman.h index afef682e2a18..983e2ca555c9 100644 --- a/src/vm/codeman.h +++ b/src/vm/codeman.h @@ -365,9 +365,11 @@ struct CodeHeapRequestInfo const BYTE * m_hiAddr; // hihest address to use to satisfy our request (0 -- don't care) size_t m_requestSize; // minimum size that must be made available size_t m_reserveSize; // Amount that VirtualAlloc will reserved + size_t m_reserveForJumpStubs; // Amount to reserve for jump stubs (won't be allocated) bool m_isDynamicDomain; bool m_isCollectible; - + bool m_throwOnOutOfMemoryWithinRange; + bool IsDynamicDomain() { return m_isDynamicDomain; } void SetDynamicDomain() { m_isDynamicDomain = true; } @@ -378,20 +380,26 @@ struct CodeHeapRequestInfo size_t getReserveSize() { return m_reserveSize; } void setReserveSize(size_t reserveSize) { m_reserveSize = reserveSize; } + + size_t getReserveForJumpStubs() { return m_reserveForJumpStubs; } + void setReserveForJumpStubs(size_t size) { m_reserveForJumpStubs = size; } + + bool getThrowOnOutOfMemoryWithinRange() { return m_throwOnOutOfMemoryWithinRange; } + void setThrowOnOutOfMemoryWithinRange(bool value) { m_throwOnOutOfMemoryWithinRange = value; } void Init(); CodeHeapRequestInfo(MethodDesc *pMD) : m_pMD(pMD), m_pAllocator(0), m_loAddr(0), m_hiAddr(0), - m_requestSize(0), m_reserveSize(0) + m_requestSize(0), m_reserveSize(0), m_reserveForJumpStubs(0) { WRAPPER_NO_CONTRACT; Init(); } CodeHeapRequestInfo(MethodDesc *pMD, LoaderAllocator* pAllocator, BYTE * loAddr, BYTE * hiAddr) : m_pMD(pMD), m_pAllocator(pAllocator), m_loAddr(loAddr), m_hiAddr(hiAddr), - m_requestSize(0), m_reserveSize(0) + m_requestSize(0), m_reserveSize(0), m_reserveForJumpStubs(0) { WRAPPER_NO_CONTRACT; Init(); } }; @@ -433,7 +441,7 @@ class CodeHeap // Alloc the specified numbers of bytes for code. Returns NULL if the request does not fit // Space for header is reserved immediately before. It is not included in size. - virtual void* AllocMemForCode_NoThrow(size_t header, size_t size, DWORD alignment) = 0; + virtual void* AllocMemForCode_NoThrow(size_t header, size_t size, DWORD alignment, size_t reserveForJumpStubs) = 0; #ifdef DACCESS_COMPILE virtual void EnumMemoryRegions(CLRDataEnumMemoryFlags flags) = 0; @@ -467,9 +475,7 @@ typedef struct _HeapList PTR_DWORD pHdrMap; // bit array used to find the start of methods size_t maxCodeHeapSize;// Size of the entire contiguous block of memory - DWORD cBlocks; // Number of allocations - bool bFull; // Heap is considered full do not use for new allocations - bool bFullForJumpStubs; // Heap is considered full do not use for new allocations of jump stubs + size_t reserveForJumpStubs; // Amount of memory reserved for jump stubs in this block #if defined(_TARGET_AMD64_) BYTE CLRPersonalityRoutine[JUMP_ALLOCATE_SIZE]; // jump thunk to personality routine @@ -483,18 +489,6 @@ typedef struct _HeapList void SetNext(PTR_HeapList next) { hpNext = next; } - void SetHeapFull() - { VolatileStore(&bFull, true); } - - bool IsHeapFull() - { return VolatileLoad(&bFull); } - - void SetHeapFullForJumpStubs() - { VolatileStore(&bFullForJumpStubs, true); } - - bool IsHeapFullForJumpStubs() - { return VolatileLoad(&bFullForJumpStubs); } - } HeapList; //----------------------------------------------------------------------------- @@ -527,7 +521,7 @@ class LoaderCodeHeap : CodeHeap WRAPPER_NO_CONTRACT; } - virtual void* AllocMemForCode_NoThrow(size_t header, size_t size, DWORD alignment) DAC_EMPTY_RET(NULL); + virtual void* AllocMemForCode_NoThrow(size_t header, size_t size, DWORD alignment, size_t reserveForJumpStubs) DAC_EMPTY_RET(NULL); #ifdef DACCESS_COMPILE virtual void EnumMemoryRegions(CLRDataEnumMemoryFlags flags) @@ -1015,7 +1009,7 @@ class EEJitManager : public IJitManager BOOL LoadJIT(); - CodeHeader* allocCode(MethodDesc* pFD, size_t blockSize, CorJitAllocMemFlag flag + CodeHeader* allocCode(MethodDesc* pFD, size_t blockSize, size_t reserveForJumpStubs, CorJitAllocMemFlag flag #ifdef WIN64EXCEPTIONS , UINT nUnwindInfos , TADDR * pModuleBase @@ -1025,7 +1019,8 @@ class EEJitManager : public IJitManager EE_ILEXCEPTION* allocEHInfo(CodeHeader* pCodeHeader, unsigned numClauses, size_t * pAllocationSize); JumpStubBlockHeader* allocJumpStubBlock(MethodDesc* pMD, DWORD numJumps, BYTE * loAddr, BYTE * hiAddr, - LoaderAllocator *pLoaderAllocator); + LoaderAllocator *pLoaderAllocator, + bool throwOnOutOfMemoryWithinRange); void * allocCodeFragmentBlock(size_t blockSize, unsigned alignment, LoaderAllocator *pLoaderAllocator, StubCodeBlockKind kind); #endif // !DACCESS_COMPILE && !CROSSGEN_COMPILE @@ -1091,11 +1086,10 @@ private : #ifndef DACCESS_COMPILE #ifndef CROSSGEN_COMPILE HeapList* NewCodeHeap(CodeHeapRequestInfo *pInfo, DomainCodeHeapList *pADHeapList); - HeapList* GetCodeHeap(CodeHeapRequestInfo *pInfo); bool CanUseCodeHeap(CodeHeapRequestInfo *pInfo, HeapList *pCodeHeap); void* allocCodeRaw(CodeHeapRequestInfo *pInfo, size_t header, size_t blockSize, unsigned align, - HeapList ** ppCodeHeap /* Writeback, Can be null */ ); + HeapList ** ppCodeHeap); DomainCodeHeapList *GetCodeHeapList(CodeHeapRequestInfo *pInfo, LoaderAllocator *pAllocator, BOOL fDynamicOnly = FALSE); DomainCodeHeapList *CreateCodeHeapList(CodeHeapRequestInfo *pInfo); @@ -1357,7 +1351,8 @@ class ExecutionManager PCODE target, BYTE * loAddr, BYTE * hiAddr, - LoaderAllocator *pLoaderAllocator = NULL); + LoaderAllocator *pLoaderAllocator = NULL, + bool throwOnOutOfMemoryWithinRange = true); #endif private: @@ -1430,7 +1425,8 @@ class ExecutionManager static PCODE getNextJumpStub(MethodDesc* pMD, PCODE target, BYTE * loAddr, BYTE * hiAddr, - LoaderAllocator *pLoaderAllocator); + LoaderAllocator *pLoaderAllocator, + bool throwOnOutOfMemoryWithinRange); #endif private: diff --git a/src/vm/crossgencompile.cpp b/src/vm/crossgencompile.cpp index 411029becda8..4cb2b5a06de7 100644 --- a/src/vm/crossgencompile.cpp +++ b/src/vm/crossgencompile.cpp @@ -263,7 +263,8 @@ BOOL Runtime_Test_For_SSE2() #endif #ifdef _TARGET_AMD64_ -INT32 rel32UsingJumpStub(INT32 UNALIGNED * pRel32, PCODE target, MethodDesc *pMethod, LoaderAllocator *pLoaderAllocator /* = NULL */) +INT32 rel32UsingJumpStub(INT32 UNALIGNED * pRel32, PCODE target, MethodDesc *pMethod, + LoaderAllocator *pLoaderAllocator /* = NULL */, bool throwOnOutOfMemoryWithinRange /*= true*/) { // crossgen does not have jump stubs return 0; diff --git a/src/vm/dynamicmethod.cpp b/src/vm/dynamicmethod.cpp index 2d0fa9ce566d..129ae1998cfb 100644 --- a/src/vm/dynamicmethod.cpp +++ b/src/vm/dynamicmethod.cpp @@ -322,64 +322,21 @@ HeapList* HostCodeHeap::CreateCodeHeap(CodeHeapRequestInfo *pInfo, EEJitManager GC_NOTRIGGER; MODE_ANY; INJECT_FAULT(COMPlusThrowOM()); - POSTCONDITION(CheckPointer(RETVAL)); + POSTCONDITION(CheckPointer(RETVAL) || !pInfo->getThrowOnOutOfMemoryWithinRange()); } CONTRACT_END; - size_t MaxCodeHeapSize = pInfo->getRequestSize(); - size_t ReserveBlockSize = MaxCodeHeapSize + sizeof(HeapList); + NewHolder pCodeHeap(new HostCodeHeap(pJitManager)); - ReserveBlockSize += sizeof(TrackAllocation) + GetOsPageSize(); // make sure we have enough for the allocation - // take a conservative size for the nibble map, we may change that later if appropriate - size_t nibbleMapSize = ROUND_UP_TO_PAGE(HEAP2MAPSIZE(ROUND_UP_TO_PAGE(ALIGN_UP(ReserveBlockSize, VIRTUAL_ALLOC_RESERVE_GRANULARITY)))); - size_t heapListSize = (sizeof(HeapList) + CODE_SIZE_ALIGN - 1) & (~(CODE_SIZE_ALIGN - 1)); - size_t otherData = heapListSize; - // make conservative estimate of the memory needed for otherData - size_t reservedData = (otherData + HOST_CODEHEAP_SIZE_ALIGN - 1) & (~(HOST_CODEHEAP_SIZE_ALIGN - 1)); - - NewHolder pCodeHeap(new HostCodeHeap(ReserveBlockSize + nibbleMapSize + reservedData, pJitManager, pInfo)); - LOG((LF_BCL, LL_INFO10, "Level2 - CodeHeap creation {0x%p} - requested 0x%p, size available 0x%p, private data 0x%p, nibble map 0x%p\n", - (HostCodeHeap*)pCodeHeap, ReserveBlockSize, pCodeHeap->m_TotalBytesAvailable, reservedData, nibbleMapSize)); + HeapList *pHp = pCodeHeap->InitializeHeapList(pInfo); + if (pHp == NULL) + { + _ASSERTE(!pInfo->getThrowOnOutOfMemoryWithinRange()); + RETURN NULL; + } - BYTE *pBuffer = pCodeHeap->InitCodeHeapPrivateData(ReserveBlockSize, reservedData, nibbleMapSize); - _ASSERTE(IS_ALIGNED(pBuffer, GetOsPageSize())); LOG((LF_BCL, LL_INFO100, "Level2 - CodeHeap creation {0x%p} - base addr 0x%p, size available 0x%p, nibble map ptr 0x%p\n", - (HostCodeHeap*)pCodeHeap, pCodeHeap->m_pBaseAddr, pCodeHeap->m_TotalBytesAvailable, pBuffer)); - - void* pHdrMap = pBuffer; - - HeapList *pHp = (HeapList*)pCodeHeap->AllocMemory(otherData, 0); - pHp->pHeap = (PTR_CodeHeap)pCodeHeap; - // wire it back - pCodeHeap->m_pHeapList = (PTR_HeapList)pHp; - // assign beginning of nibble map - pHp->pHdrMap = (PTR_DWORD)(DWORD*)pHdrMap; - - TrackAllocation *pTracker = *((TrackAllocation**)pHp - 1); - LOG((LF_BCL, LL_INFO100, "Level2 - CodeHeap creation {0x%p} - size available 0x%p, private data ptr [0x%p, 0x%p]\n", - (HostCodeHeap*)pCodeHeap, pCodeHeap->m_TotalBytesAvailable, pTracker, pTracker->size)); - - // need to update the reserved data - pCodeHeap->m_ReservedData += pTracker->size; - - pHp->startAddress = dac_cast(pCodeHeap->m_pBaseAddr) + pTracker->size; - pHp->mapBase = ROUND_DOWN_TO_PAGE(pHp->startAddress); // round down to next lower page align - pHp->endAddress = pHp->startAddress; - - pHp->maxCodeHeapSize = pCodeHeap->m_TotalBytesAvailable - pTracker->size; - _ASSERTE(pHp->maxCodeHeapSize >= MaxCodeHeapSize); - - // We do not need to memset this memory, since ClrVirtualAlloc() guarantees that the memory is zero. - // Furthermore, if we avoid writing to it, these pages don't come into our working set - - pHp->bFull = FALSE; - pHp->cBlocks = 0; -#ifdef _WIN64 - emitJump((LPBYTE)pHp->CLRPersonalityRoutine, (void *)ProcessCLRException); -#endif - - // zero the ref count as now starts the real counter - pCodeHeap->m_AllocationCount = 0; + (HostCodeHeap*)pCodeHeap, pCodeHeap->m_pBaseAddr, pCodeHeap->m_TotalBytesAvailable, pCodeHeap->m_pHeapList->pHdrMap)); pCodeHeap.SuppressRelease(); @@ -387,7 +344,7 @@ HeapList* HostCodeHeap::CreateCodeHeap(CodeHeapRequestInfo *pInfo, EEJitManager RETURN pHp; } -HostCodeHeap::HostCodeHeap(size_t ReserveBlockSize, EEJitManager *pJitManager, CodeHeapRequestInfo *pInfo) +HostCodeHeap::HostCodeHeap(EEJitManager *pJitManager) { CONTRACTL { @@ -398,45 +355,31 @@ HostCodeHeap::HostCodeHeap(size_t ReserveBlockSize, EEJitManager *pJitManager, C } CONTRACTL_END; - // reserve ReserveBlockSize rounded-up to VIRTUAL_ALLOC_RESERVE_GRANULARITY of memory - ReserveBlockSize = ALIGN_UP(ReserveBlockSize, VIRTUAL_ALLOC_RESERVE_GRANULARITY); - - if (pInfo->m_loAddr != NULL || pInfo->m_hiAddr != NULL) - { - m_pBaseAddr = ClrVirtualAllocWithinRange(pInfo->m_loAddr, pInfo->m_hiAddr, - ReserveBlockSize, MEM_RESERVE, PAGE_NOACCESS); - if (!m_pBaseAddr) - ThrowOutOfMemoryWithinRange(); - } - else - { - m_pBaseAddr = ClrVirtualAllocExecutable(ReserveBlockSize, MEM_RESERVE, PAGE_NOACCESS); - if (!m_pBaseAddr) - ThrowOutOfMemory(); - } - - m_pLastAvailableCommittedAddr = m_pBaseAddr; - m_TotalBytesAvailable = ReserveBlockSize; + m_pBaseAddr = NULL; + m_pLastAvailableCommittedAddr = NULL; + m_TotalBytesAvailable = 0; + m_ApproximateLargestBlock = 0; m_AllocationCount = 0; - m_ReservedData = 0; + m_pHeapList = NULL; m_pJitManager = (PTR_EEJitManager)pJitManager; m_pFreeList = NULL; - m_pAllocator = pInfo->m_pAllocator; + m_pAllocator = NULL; m_pNextHeapToRelease = NULL; - LOG((LF_BCL, LL_INFO100, "Level2 - CodeHeap creation {0x%p, vt(0x%x)} - base addr 0x%p, total size 0x%p\n", - this, *(size_t*)this, m_pBaseAddr, m_TotalBytesAvailable)); } HostCodeHeap::~HostCodeHeap() { LIMITED_METHOD_CONTRACT; + if (m_pHeapList != NULL && m_pHeapList->pHdrMap != NULL) + delete[] m_pHeapList->pHdrMap; + if (m_pBaseAddr) ClrVirtualFree(m_pBaseAddr, 0, MEM_RELEASE); LOG((LF_BCL, LL_INFO10, "Level1 - CodeHeap destroyed {0x%p}\n", this)); } -BYTE* HostCodeHeap::InitCodeHeapPrivateData(size_t ReserveBlockSize, size_t otherData, size_t nibbleMapSize) +HeapList* HostCodeHeap::InitializeHeapList(CodeHeapRequestInfo *pInfo) { CONTRACTL { @@ -446,43 +389,79 @@ BYTE* HostCodeHeap::InitCodeHeapPrivateData(size_t ReserveBlockSize, size_t othe } CONTRACTL_END; - size_t nibbleNewSize = ROUND_UP_TO_PAGE(HEAP2MAPSIZE(ROUND_UP_TO_PAGE(m_TotalBytesAvailable))); - if (m_TotalBytesAvailable - nibbleNewSize < ReserveBlockSize + otherData) + size_t ReserveBlockSize = pInfo->getRequestSize(); + + // Add TrackAllocation, HeapList and very conservative padding to make sure we have enough for the allocation + ReserveBlockSize += sizeof(TrackAllocation) + sizeof(HeapList) + HOST_CODEHEAP_SIZE_ALIGN + 0x100; + + // reserve ReserveBlockSize rounded-up to VIRTUAL_ALLOC_RESERVE_GRANULARITY of memory + ReserveBlockSize = ALIGN_UP(ReserveBlockSize, VIRTUAL_ALLOC_RESERVE_GRANULARITY); + + if (pInfo->m_loAddr != NULL || pInfo->m_hiAddr != NULL) { - // the new allocation for the nibble map would notleave enough room for the requested memory, bail out - nibbleNewSize = nibbleMapSize; + m_pBaseAddr = ClrVirtualAllocWithinRange(pInfo->m_loAddr, pInfo->m_hiAddr, + ReserveBlockSize, MEM_RESERVE, PAGE_NOACCESS); + if (!m_pBaseAddr) + { + if (pInfo->getThrowOnOutOfMemoryWithinRange()) + ThrowOutOfMemoryWithinRange(); + return NULL; + } } + else + { + // top up the ReserveBlockSize to suggested minimum + ReserveBlockSize = max(ReserveBlockSize, pInfo->getReserveSize()); - BYTE *pAddress = (BYTE*)ROUND_DOWN_TO_PAGE(dac_cast(m_pLastAvailableCommittedAddr) + - m_TotalBytesAvailable - nibbleNewSize); - _ASSERTE(m_pLastAvailableCommittedAddr + m_TotalBytesAvailable >= pAddress + nibbleNewSize); - if (NULL == ClrVirtualAlloc(pAddress, nibbleNewSize, MEM_COMMIT, PAGE_EXECUTE_READWRITE)) - ThrowOutOfMemory(); - m_TotalBytesAvailable = pAddress - m_pLastAvailableCommittedAddr; - _ASSERTE(m_TotalBytesAvailable >= ReserveBlockSize + otherData); - return pAddress; -} + m_pBaseAddr = ClrVirtualAllocExecutable(ReserveBlockSize, MEM_RESERVE, PAGE_NOACCESS); + if (!m_pBaseAddr) + ThrowOutOfMemory(); + } - // used to flag a block that is too small -#define UNUSABLE_BLOCK ((size_t)-1) - -size_t HostCodeHeap::GetPadding(TrackAllocation *pCurrent, size_t size, DWORD alignment) -{ - LIMITED_METHOD_CONTRACT; - if (pCurrent->size < size) - return UNUSABLE_BLOCK; - size_t padding = 0; - if (alignment) + m_pLastAvailableCommittedAddr = m_pBaseAddr; + m_TotalBytesAvailable = ReserveBlockSize; + m_ApproximateLargestBlock = ReserveBlockSize; + m_pAllocator = pInfo->m_pAllocator; + + TrackAllocation *pTracker = AllocMemory_NoThrow(0, sizeof(HeapList), sizeof(void*), 0); + if (pTracker == NULL) { - size_t pointer = (size_t)((BYTE*)pCurrent + sizeof(TrackAllocation)); - padding = ((pointer + (size_t)alignment - 1) & (~((size_t)alignment - 1))) - pointer; + // This should only ever happen with fault injection + _ASSERTE(g_pConfig->ShouldInjectFault(INJECTFAULT_DYNAMICCODEHEAP)); + ThrowOutOfMemory(); } - if (pCurrent->size < size + padding) - return UNUSABLE_BLOCK; - return padding; + + HeapList* pHp = (HeapList *)(pTracker + 1); + + pHp->hpNext = NULL; + pHp->pHeap = (PTR_CodeHeap)this; + // wire it back + m_pHeapList = (PTR_HeapList)pHp; + + LOG((LF_BCL, LL_INFO100, "Level2 - CodeHeap creation {0x%p} - size available 0x%p, private data ptr [0x%p, 0x%p]\n", + (HostCodeHeap*)this, m_TotalBytesAvailable, pTracker, pTracker->size)); + + // It is imporant to exclude the CLRPersonalityRoutine from the tracked range + pHp->startAddress = dac_cast(m_pBaseAddr) + pTracker->size; + pHp->mapBase = ROUND_DOWN_TO_PAGE(pHp->startAddress); // round down to next lower page align + pHp->pHdrMap = NULL; + pHp->endAddress = pHp->startAddress; + + pHp->maxCodeHeapSize = m_TotalBytesAvailable - pTracker->size; + pHp->reserveForJumpStubs = 0; + +#ifdef _WIN64 + emitJump((LPBYTE)pHp->CLRPersonalityRoutine, (void *)ProcessCLRException); +#endif + + size_t nibbleMapSize = HEAP2MAPSIZE(ROUND_UP_TO_PAGE(pHp->maxCodeHeapSize)); + pHp->pHdrMap = new DWORD[nibbleMapSize / sizeof(DWORD)]; + ZeroMemory(pHp->pHdrMap, nibbleMapSize); + + return pHp; } -void* HostCodeHeap::AllocFromFreeList(size_t size, DWORD alignment) +HostCodeHeap::TrackAllocation* HostCodeHeap::AllocFromFreeList(size_t header, size_t size, DWORD alignment, size_t reserveForJumpStubs) { CONTRACTL { @@ -500,19 +479,16 @@ void* HostCodeHeap::AllocFromFreeList(size_t size, DWORD alignment) TrackAllocation *pPrevious = NULL; while (pCurrent) { - // GetPadding will return UNUSABLE_BLOCK if the current block is not big enough - size_t padding = GetPadding(pCurrent, size, alignment); - if (UNUSABLE_BLOCK != padding) + BYTE* pPointer = ALIGN_UP((BYTE*)(pCurrent + 1) + header, alignment); + size_t realSize = ALIGN_UP(pPointer + size, sizeof(void*)) - (BYTE*)pCurrent; + if (pCurrent->size >= realSize + reserveForJumpStubs) { // found a block LOG((LF_BCL, LL_INFO100, "Level2 - CodeHeap [0x%p] - Block found, size 0x%X\n", this, pCurrent->size)); - size_t realSize = size + padding; - BYTE *pPointer = (BYTE*)pCurrent + sizeof(TrackAllocation) + padding; - _ASSERTE((size_t)(pPointer - (BYTE*)pCurrent) >= sizeof(TrackAllocation)); // The space left is not big enough for a new block, let's just // update the TrackAllocation record for the current block - if (pCurrent->size - realSize <= sizeof(TrackAllocation)) + if (pCurrent->size - realSize < max(HOST_CODEHEAP_SIZE_ALIGN, sizeof(TrackAllocation))) { LOG((LF_BCL, LL_INFO100, "Level2 - CodeHeap [0x%p] - Item removed %p, size 0x%X\n", this, pCurrent, pCurrent->size)); // remove current @@ -545,13 +521,10 @@ void* HostCodeHeap::AllocFromFreeList(size_t size, DWORD alignment) pCurrent->size = realSize; } - // now fill all the padding data correctly pCurrent->pHeap = this; - // store the location of the TrackAllocation record right before pPointer - *((void**)pPointer - 1) = pCurrent; LOG((LF_BCL, LL_INFO100, "Level2 - CodeHeap [0x%p] - Allocation returned %p, size 0x%X - data -> %p\n", this, pCurrent, pCurrent->size, pPointer)); - return pPointer; + return pCurrent; } pPrevious = pCurrent; pCurrent = pCurrent->pNext; @@ -656,7 +629,7 @@ void HostCodeHeap::AddToFreeList(TrackAllocation *pBlockToInsert) m_pFreeList, m_pFreeList->size)); } -void* HostCodeHeap::AllocMemForCode_NoThrow(size_t header, size_t size, DWORD alignment) +void* HostCodeHeap::AllocMemForCode_NoThrow(size_t header, size_t size, DWORD alignment, size_t reserveForJumpStubs) { CONTRACTL { @@ -667,45 +640,36 @@ void* HostCodeHeap::AllocMemForCode_NoThrow(size_t header, size_t size, DWORD al CONTRACTL_END; _ASSERTE(header == sizeof(CodeHeader)); + _ASSERTE(alignment <= HOST_CODEHEAP_SIZE_ALIGN); - // The code allocator has to guarantee that there is only one entrypoint per nibble map entry. + // The code allocator has to guarantee that there is only one entrypoint per nibble map entry. // It is guaranteed because of HostCodeHeap allocator always aligns the size up to HOST_CODEHEAP_SIZE_ALIGN, // and because the size of nibble map entries (BYTES_PER_BUCKET) is smaller than HOST_CODEHEAP_SIZE_ALIGN. // Assert the later fact here. _ASSERTE(HOST_CODEHEAP_SIZE_ALIGN >= BYTES_PER_BUCKET); - BYTE * pMem = (BYTE *)AllocMemory_NoThrow(size + sizeof(CodeHeader) + (alignment - 1), sizeof(void *)); - if (pMem == NULL) - return NULL; + header += sizeof(TrackAllocation*); - BYTE * pCode = (BYTE *)ALIGN_UP(pMem + sizeof(CodeHeader), alignment); + TrackAllocation* pTracker = AllocMemory_NoThrow(header, size, alignment, reserveForJumpStubs); + if (pTracker == NULL) + return NULL; - // Update tracker to account for the alignment we have just added - TrackAllocation *pTracker = *((TrackAllocation **)pMem - 1); + BYTE * pCode = ALIGN_UP((BYTE*)(pTracker + 1) + header, alignment); - CodeHeader * pHdr = dac_cast(pCode) - 1; + // Pointer to the TrackAllocation record is stored just before the code header + CodeHeader * pHdr = (CodeHeader *)pCode - 1; *((TrackAllocation **)(pHdr) - 1) = pTracker; - return pCode; -} + _ASSERTE(pCode + size <= (BYTE*)pTracker + pTracker->size); -void* HostCodeHeap::AllocMemory(size_t size, DWORD alignment) -{ - CONTRACTL - { - THROWS; - GC_NOTRIGGER; - MODE_ANY; - } - CONTRACTL_END; + // ref count the whole heap + m_AllocationCount++; + LOG((LF_BCL, LL_INFO100, "Level2 - CodeHeap [0x%p] - ref count %d\n", this, m_AllocationCount)); - void *pAllocation = AllocMemory_NoThrow(size, alignment); - if (!pAllocation) - ThrowOutOfMemory(); - return pAllocation; + return pCode; } -void* HostCodeHeap::AllocMemory_NoThrow(size_t size, DWORD alignment) +HostCodeHeap::TrackAllocation* HostCodeHeap::AllocMemory_NoThrow(size_t header, size_t size, DWORD alignment, size_t reserveForJumpStubs) { CONTRACTL { @@ -725,18 +689,15 @@ void* HostCodeHeap::AllocMemory_NoThrow(size_t size, DWORD alignment) } #endif // _DEBUG - // honor alignment (should assert the value is proper) - if (alignment) - size = (size + (size_t)alignment - 1) & (~((size_t)alignment - 1)); - // align size to HOST_CODEHEAP_SIZE_ALIGN always - size = (size + HOST_CODEHEAP_SIZE_ALIGN - 1) & (~(HOST_CODEHEAP_SIZE_ALIGN - 1)); - - size += sizeof(TrackAllocation); + // Skip walking the free list if the cached size of the largest block is not enough + size_t totalRequiredSize = ALIGN_UP(sizeof(TrackAllocation) + header + size + (alignment - 1) + reserveForJumpStubs, sizeof(void*)); + if (totalRequiredSize > m_ApproximateLargestBlock) + return NULL; LOG((LF_BCL, LL_INFO100, "Level2 - CodeHeap [0x%p] - Allocation requested 0x%X\n", this, size)); - void *pAddr = AllocFromFreeList(size, alignment); - if (!pAddr) + TrackAllocation* pTracker = AllocFromFreeList(header, size, alignment, reserveForJumpStubs); + if (!pTracker) { // walk free list to end to find available space size_t availableInFreeList = 0; @@ -751,8 +712,9 @@ void* HostCodeHeap::AllocMemory_NoThrow(size_t size, DWORD alignment) { availableInFreeList = pLastBlock->size; } - _ASSERTE(size > availableInFreeList); - size_t sizeToCommit = size - availableInFreeList; + + _ASSERTE(totalRequiredSize > availableInFreeList); + size_t sizeToCommit = totalRequiredSize - availableInFreeList; sizeToCommit = ROUND_UP_TO_PAGE(size); // round up to page if (m_pLastAvailableCommittedAddr + sizeToCommit <= m_pBaseAddr + m_TotalBytesAvailable) @@ -768,20 +730,18 @@ void* HostCodeHeap::AllocMemory_NoThrow(size_t size, DWORD alignment) pBlockToInsert->size = sizeToCommit; m_pLastAvailableCommittedAddr += sizeToCommit; AddToFreeList(pBlockToInsert); - pAddr = AllocFromFreeList(size, alignment); + pTracker = AllocFromFreeList(header, size, alignment, reserveForJumpStubs); + _ASSERTE(pTracker != NULL); } else { LOG((LF_BCL, LL_INFO100, "Level2 - CodeHeap [0x%p] - allocation failed:\n\tm_pLastAvailableCommittedAddr: 0x%X\n\tsizeToCommit: 0x%X\n\tm_pBaseAddr: 0x%X\n\tm_TotalBytesAvailable: 0x%X\n", this, m_pLastAvailableCommittedAddr, sizeToCommit, m_pBaseAddr, m_TotalBytesAvailable)); - return NULL; + // Update largest available block size + m_ApproximateLargestBlock = totalRequiredSize - 1; } } - _ASSERTE(pAddr); - // ref count the whole heap - m_AllocationCount++; - LOG((LF_BCL, LL_INFO100, "Level2 - CodeHeap [0x%p] - ref count %d\n", this, m_AllocationCount)); - return pAddr; + return pTracker; } #endif //!DACCESS_COMPILE @@ -857,6 +817,8 @@ void HostCodeHeap::FreeMemForCode(void * codeStart) TrackAllocation *pTracker = HostCodeHeap::GetTrackAllocation((TADDR)codeStart); AddToFreeList(pTracker); + m_ApproximateLargestBlock += pTracker->size; + m_AllocationCount--; LOG((LF_BCL, LL_INFO100, "Level2 - CodeHeap released [0x%p, vt(0x%x)] - ref count %d\n", this, *(size_t*)this, m_AllocationCount)); diff --git a/src/vm/dynamicmethod.h b/src/vm/dynamicmethod.h index 7fd63e59b90e..69cd9a9f9668 100644 --- a/src/vm/dynamicmethod.h +++ b/src/vm/dynamicmethod.h @@ -247,7 +247,7 @@ class HostCodeHeap : CodeHeap PTR_BYTE m_pBaseAddr; PTR_BYTE m_pLastAvailableCommittedAddr; size_t m_TotalBytesAvailable; - size_t m_ReservedData; + size_t m_ApproximateLargestBlock; // Heap ref count DWORD m_AllocationCount; @@ -260,10 +260,6 @@ class HostCodeHeap : CodeHeap TrackAllocation *pNext; }; size_t size; - - // the location of this TrackAllocation record will be stored right before the start of the allocated memory - // if there is padding between them it will be stored in that padding, otherwise it will be stored in this pad field - void *pad; }; TrackAllocation *m_pFreeList; @@ -275,18 +271,16 @@ class HostCodeHeap : CodeHeap static HeapList* CreateCodeHeap(CodeHeapRequestInfo *pInfo, EEJitManager *pJitManager); private: - HostCodeHeap(size_t ReserveBlockSize, EEJitManager *pJitManager, CodeHeapRequestInfo *pInfo); - BYTE* InitCodeHeapPrivateData(size_t ReserveBlockSize, size_t otherData, size_t nibbleMapSize); - void* AllocFromFreeList(size_t size, DWORD alignment); + HostCodeHeap(EEJitManager *pJitManager); + HeapList* InitializeHeapList(CodeHeapRequestInfo *pInfo); + TrackAllocation* AllocFromFreeList(size_t header, size_t size, DWORD alignment, size_t reserveForJumpStubs); void AddToFreeList(TrackAllocation *pBlockToInsert); - static size_t GetPadding(TrackAllocation *pCurrent, size_t size, DWORD alignement); - void* AllocMemory(size_t size, DWORD alignment); - void* AllocMemory_NoThrow(size_t size, DWORD alignment); + TrackAllocation* AllocMemory_NoThrow(size_t header, size_t size, DWORD alignment, size_t reserveForJumpStubs); public: // Space for header is reserved immediately before. It is not included in size. - virtual void* AllocMemForCode_NoThrow(size_t header, size_t size, DWORD alignment) DAC_EMPTY_RET(NULL); + virtual void* AllocMemForCode_NoThrow(size_t header, size_t size, DWORD alignment, size_t reserveForJumpStubs) DAC_EMPTY_RET(NULL); virtual ~HostCodeHeap() DAC_EMPTY(); diff --git a/src/vm/jitinterface.cpp b/src/vm/jitinterface.cpp index 51ddadad3ce8..94abe3aaa84a 100644 --- a/src/vm/jitinterface.cpp +++ b/src/vm/jitinterface.cpp @@ -11262,7 +11262,17 @@ void CEEJitInfo::recordRelocation(void * location, // When m_fAllowRel32 == FALSE, the JIT will use a REL32s for direct code targets only. // Use jump stub. // - delta = rel32UsingJumpStub(fixupLocation, (PCODE)target, m_pMethodBeingCompiled); + delta = rel32UsingJumpStub(fixupLocation, (PCODE)target, m_pMethodBeingCompiled, NULL, false /* throwOnOutOfMemoryWithinRange */); + if (delta == 0) + { + // This forces the JIT to retry the method, which allows us to reserve more space for jump stubs and have a higher chance that + // we will find space for them. + m_fRel32Overflow = TRUE; + } + + // Keep track of conservative estimate of how much memory may be needed by jump stubs. We will use it to reserve extra memory + // on retry to increase chances that the retry succeeds. + m_reserveForJumpStubs = max(0x400, m_reserveForJumpStubs + 0x10); } } @@ -11727,7 +11737,7 @@ void CEEJitInfo::allocMem ( COMPlusThrowHR(CORJIT_OUTOFMEM); } - m_CodeHeader = m_jitManager->allocCode(m_pMethodBeingCompiled, totalSize.Value(), flag + m_CodeHeader = m_jitManager->allocCode(m_pMethodBeingCompiled, totalSize.Value(), GetReserveForJumpStubs(), flag #ifdef WIN64EXCEPTIONS , m_totalUnwindInfos , &m_moduleBase @@ -12563,6 +12573,7 @@ PCODE UnsafeJitFunction(MethodDesc* ftn, COR_ILMETHOD_DECODER* ILHeader, CORJIT_ #endif BOOL fAllowRel32 = g_fAllowRel32 | fForceRel32Overflow; + size_t reserveForJumpStubs = 0; // For determinism, never try to use the REL32 in compilation process if (IsCompilationProcess()) @@ -12588,6 +12599,7 @@ PCODE UnsafeJitFunction(MethodDesc* ftn, COR_ILMETHOD_DECODER* ILHeader, CORJIT_ if (fForceRel32Overflow) jitInfo.SetRel32Overflow(fAllowRel32); jitInfo.SetAllowRel32(fAllowRel32); + jitInfo.SetReserveForJumpStubs(reserveForJumpStubs); #endif MethodDesc * pMethodForSecurity = jitInfo.GetMethodForSecurity(ftnHnd); @@ -12744,8 +12756,9 @@ PCODE UnsafeJitFunction(MethodDesc* ftn, COR_ILMETHOD_DECODER* ILHeader, CORJIT_ // Disallow rel32 relocs in future. g_fAllowRel32 = FALSE; - _ASSERTE(fAllowRel32 != FALSE); fAllowRel32 = FALSE; + + reserveForJumpStubs = jitInfo.GetReserveForJumpStubs(); continue; } #endif // _TARGET_AMD64_ && !CROSSGEN_COMPILE diff --git a/src/vm/jitinterface.h b/src/vm/jitinterface.h index deeb814f8275..4b52d9851aae 100644 --- a/src/vm/jitinterface.h +++ b/src/vm/jitinterface.h @@ -1360,12 +1360,30 @@ class CEEJitInfo : public CEEInfo LIMITED_METHOD_CONTRACT; return m_fRel32Overflow; } + + size_t GetReserveForJumpStubs() + { + LIMITED_METHOD_CONTRACT; + return m_reserveForJumpStubs; + } + + void SetReserveForJumpStubs(size_t value) + { + LIMITED_METHOD_CONTRACT; + m_reserveForJumpStubs = value; + } #else BOOL JitAgain() { LIMITED_METHOD_CONTRACT; return FALSE; } + + size_t GetReserveForJumpStubs() + { + LIMITED_METHOD_CONTRACT; + return 0; + } #endif CEEJitInfo(MethodDesc* fd, COR_ILMETHOD_DECODER* header, @@ -1385,6 +1403,7 @@ class CEEJitInfo : public CEEInfo #ifdef _TARGET_AMD64_ m_fAllowRel32(FALSE), m_fRel32Overflow(FALSE), + m_reserveForJumpStubs(0), #endif m_GCinfo_len(0), m_EHinfo_len(0), @@ -1469,6 +1488,7 @@ protected : BOOL m_fAllowRel32; // Use 32-bit PC relative address modes BOOL m_fRel32Overflow; // Overflow while trying to use encode 32-bit PC relative address. // The code will need to be regenerated with m_fRel32Allowed == FALSE. + size_t m_reserveForJumpStubs; // Space to reserve for jump stubs when allocating code #endif #if defined(_DEBUG)