Skip to content

Commit

Permalink
i#4418 vmheap: Make vmm bitmap dynamic and up vmheap to 8G (#4420)
Browse files Browse the repository at this point in the history 
Moves the vmm bitmap from a static array to being placed at the top of
the vmm region itself.  This allows for separate sizes for vmcode vs
vmheap, and makes it easier to increase vmheap sizes.  This eliminates
the previous maximum of 2G for -vmheap_size.

Changes the default -vmheap_size from 2G to 8G for 64-bit.

Fixes #4418
  • Loading branch information
@derekbruening
derekbruening authored Aug 27, 2020
1 parent fba43e5 commit 70be2df
Show file tree
Hide file tree
Showing 4 changed files with 62 additions and 40 deletions.
84 changes: 53 additions & 31 deletions core/heap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -628,12 +628,9 @@ static void
report_low_on_memory(which_vmm_t which, oom_source_t source,
heap_error_code_t os_error_code);

/* Maximum reservation is 512MB for 32-bit or 2GB for 64-bit. */
#ifdef X64
# define MAX_VMM_HEAP_UNIT_SIZE (2U * 1024 * 1024 * 1024)
#else
# define MAX_VMM_HEAP_UNIT_SIZE (512U * 1024 * 1024)
#endif
#define MAX_VMCODE_SIZE (2ULL * 1024 * 1024 * 1024)
#define MAX_VMHEAP_SIZE (IF_X64_ELSE(128ULL, (4ULL - 1)) * 1024 * 1024 * 1024)

/* We should normally have only one large unit, so this is in fact
* the maximum we should count on in one process
*/
Expand All @@ -657,18 +654,11 @@ typedef struct {
currently the bitmap_t is used with no write intent only for ASSERTs. */
uint num_free_blocks; /* currently free blocks */
const char *name;
/* Bitmap uses 4KB static data for granularity 64KB and static maximum 2GB on Windows,
* and 64KB on Linux where granularity is 4KB. These amounts are halved for
* 32-bit, so 1KB Windows and 16KB Linux.
* We could make this dynamic to save some of the 64K for 64-bit: the default
* is 512M so we waste 48K.
/* We dynamically allocate the bitmap to allow for different sizes for
* vmcode and vmheap and to allow for large vmheap sizes.
* We place it at start_addr, or the writable equivalent for vmcode.
*/
/* Since we expect only two of these, for now it is ok for users
to have static max rather than dynamically allocating with
exact size - however this field is left last in the structure
in case we do want to save some memory
*/
bitmap_element_t blocks[BITMAP_INDEX(MAX_VMM_HEAP_UNIT_SIZE / MIN_VMM_BLOCK_SIZE)];
bitmap_element_t *blocks;
} vm_heap_t;

/* We keep our heap management structs on the heap for selfprot (case 8074).
Expand Down Expand Up @@ -718,6 +708,13 @@ static bool vmm_heap_exited = false; /* FIXME: used only to thwart stack_free fr

#define MEMORY_FILE_NAME "dynamorio_dual_map"

static vm_addr_t
vmm_heap_reserve_blocks(vm_heap_t *vmh, size_t size_in, byte *base, which_vmm_t which);

static bool
vmm_heap_commit(vm_addr_t p, size_t size, uint prot, heap_error_code_t *error_code,
which_vmm_t which);

static inline uint
vmm_addr_to_block(vm_heap_t *vmh, vm_addr_t p)
{
Expand Down Expand Up @@ -963,6 +960,17 @@ vmm_place_vmcode(vm_heap_t *vmh, size_t size, heap_error_code_t *error_code)
ASSERT(ALIGNED(vmh->start_addr, DYNAMO_OPTION(vmm_block_size)));
}

/* Does not return. */
static void
vmm_heap_unit_init_failed(vm_heap_t *vmh, heap_error_code_t error_code, const char *name)
{
LOG(GLOBAL, LOG_HEAP, 1, "vmm_heap_unit_init %s: failed to allocate memory!\n", name);
vmm_heap_initialize_unusable(vmh);
/* We couldn't even reserve initial virtual memory - we're out of luck. */
report_low_on_memory(VMM_HEAP, OOM_INIT, error_code);
ASSERT_NOT_REACHED();
}

static void
vmm_heap_unit_init(vm_heap_t *vmh, size_t size, bool is_vmcode, const char *name)
{
Expand All @@ -975,7 +983,6 @@ vmm_heap_unit_init(vm_heap_t *vmh, size_t size, bool is_vmcode, const char *name
d_r_mutex_lock(&vmh->lock);
d_r_mutex_unlock(&vmh->lock);
size = ALIGN_FORWARD(size, DYNAMO_OPTION(vmm_block_size));
ASSERT(size <= MAX_VMM_HEAP_UNIT_SIZE);
vmh->alloc_size = size;
vmh->start_addr = NULL;
vmh->name = name;
Expand Down Expand Up @@ -1027,28 +1034,38 @@ vmm_heap_unit_init(vm_heap_t *vmh, size_t size, bool is_vmcode, const char *name
}

if (vmh->start_addr == 0) {
LOG(GLOBAL, LOG_HEAP, 1, "vmm_heap_unit_init %s: failed to allocate memory!\n",
name);
vmm_heap_initialize_unusable(vmh);
/* we couldn't even reserve initial virtual memory - we're out of luck */
/* XXX case 7373: make sure we tag as a potential
* interoperability issue, in staging mode we should probably
* get out from the process since we haven't really started yet
*/
report_low_on_memory(VMM_HEAP, OOM_INIT, error_code);
vmm_heap_unit_init_failed(vmh, error_code, name);
ASSERT_NOT_REACHED();
}
vmh->end_addr = vmh->start_addr + size;
ASSERT_TRUNCATE(vmh->num_blocks, uint, size / DYNAMO_OPTION(vmm_block_size));
vmh->num_blocks = (uint)(size / DYNAMO_OPTION(vmm_block_size));
size_t blocks_sz_bytes = BITMAP_INDEX(vmh->num_blocks) * sizeof(bitmap_element_t);
blocks_sz_bytes = ALIGN_FORWARD(blocks_sz_bytes, DYNAMO_OPTION(vmm_block_size));
/* We place the bitmap at the start of the (writable) vmm region. */
vmh->blocks = (bitmap_element_t *)vmh->start_addr;
if (is_vmcode)
vmh->blocks = (bitmap_element_t *)vmcode_get_writable_addr((byte *)vmh->blocks);
vmh->num_free_blocks = vmh->num_blocks;
LOG(GLOBAL, LOG_HEAP, 1,
"vmm_heap_unit_init %s reservation: [" PFX "," PFX ") total=%d free=%d\n", name,
vmh->start_addr, vmh->end_addr, vmh->num_blocks, vmh->num_free_blocks);

/* make sure static bitmap_t size is properly aligned on block boundaries */
ASSERT(ALIGNED(MAX_VMM_HEAP_UNIT_SIZE, DYNAMO_OPTION(vmm_block_size)));
/* Make sure the vmm area is properly aligned on block boundaries.
* The size was aligned above.
*/
ASSERT(ALIGNED(vmh->blocks, DYNAMO_OPTION(vmm_block_size)));

which_vmm_t which = VMM_HEAP | (is_vmcode ? VMM_REACHABLE : 0);
/* We have to commit first which our code does support. */
vmm_heap_commit((vm_addr_t)vmh->blocks, blocks_sz_bytes, MEMPROT_READ | MEMPROT_WRITE,
&error_code, which);
if (error_code != 0) {
vmm_heap_unit_init_failed(vmh, error_code, name);
ASSERT_NOT_REACHED();
}
bitmap_initialize_free(vmh->blocks, vmh->num_blocks);
vmm_heap_reserve_blocks(vmh, blocks_sz_bytes, vmh->start_addr, which);
DOLOG(1, LOG_HEAP, { vmm_dump_map(vmh); });
ASSERT(bitmap_check_consistency(vmh->blocks, vmh->num_blocks, vmh->num_free_blocks));
}
Expand Down Expand Up @@ -1908,6 +1925,11 @@ vmh_exit(vm_heap_t *vmh, bool contains_stacks)
/* current stack */
perstack * ((doing_detach IF_APP_EXPORTS(|| dr_api_exit)) ? 0 : 1);
}
/* Our bitmap does not get freed. */
size_t blocks_sz_bytes =
ALIGN_FORWARD_UINT(BITMAP_INDEX(vmh->num_blocks) * sizeof(bitmap_element_t),
DYNAMO_OPTION(vmm_block_size));
unfreed_blocks += (uint)(blocks_sz_bytes / DYNAMO_OPTION(vmm_block_size));
/* XXX: On detach, arch_thread_exit should explicitly mark as
* left behind all TPCs needed so then we can assert even for
* detach.
Expand Down Expand Up @@ -2142,10 +2164,10 @@ heap_check_option_compatibility()
bool ret = false;

ret = check_param_bounds(&dynamo_options.vm_size, MIN_VMM_HEAP_UNIT_SIZE,
MAX_VMM_HEAP_UNIT_SIZE, "vm_size") ||
MAX_VMCODE_SIZE, "vm_size") ||
ret;
ret = check_param_bounds(&dynamo_options.vmheap_size, MIN_VMM_HEAP_UNIT_SIZE,
MAX_VMM_HEAP_UNIT_SIZE, "vmheap_size") ||
MAX_VMHEAP_SIZE, "vmheap_size") ||
ret;
#ifdef INTERNAL
/* if max_heap_unit_size is too small you may get a funny message
Expand Down
6 changes: 3 additions & 3 deletions core/options.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2725,14 +2725,14 @@ unit_test_options(void)

# ifdef X64
/* Sanity-check pointer-sized integer values handling >int sizes. */
set_dynamo_options(&dynamo_options, "-vmheap_size 16384M -stack_size 64K");
set_dynamo_options(&dynamo_options, "-vmheap_size 16384M -persist_short_digest 8K");
EXPECT_EQ(dynamo_options.vmheap_size, 16 * 1024 * 1024 * 1024ULL);
char opstring[MAX_OPTIONS_STRING];
/* Ensure we print it back out with the shortest value+suffix.
* We include -stack_size to avoid printing out "0G" or sthg.
* We include a smaller option to ensure we avoid printing out "0G".
*/
get_dynamo_options_string(&dynamo_options, opstring, sizeof(opstring), true);
EXPECT_EQ(0, strcmp(opstring, "-stack_size 64K -vmheap_size 16G "));
EXPECT_EQ(0, strcmp(opstring, "-vmheap_size 16G -persist_short_digest 8K "));
# endif

SELF_PROTECT_OPTIONS();
Expand Down
7 changes: 3 additions & 4 deletions core/optionsx.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1568,12 +1568,11 @@ OPTION_DEFAULT(uint_size, vm_size,
IF_X64_ELSE(IF_WINDOWS_ELSE(512, 1024UL), 128) * 1024 * 1024,
"capacity of virtual memory region reserved (maximum supported is "
"512MB for 32-bit and 2GB for 64-bit) for code and reachable heap")
OPTION_DEFAULT(uint_size, vmheap_size, IF_X64_ELSE(2048UL, 128) * 1024 * 1024,
/* XXX: default value is currently not good enough for sqlserver,
OPTION_DEFAULT(uint_size, vmheap_size, IF_X64_ELSE(8192ULL, 128) * 1024 * 1024,
/* XXX: default value is currently not good enough for 32-bit sqlserver,
* for which we need more than 256MB.
*/
"capacity of virtual memory region reserved (maximum supported is "
"512MB for 32-bit and 2GB for 64-bit) for unreachable heap")
"capacity of virtual memory region reserved for unreachable heap")

/* We hardcode an address in the mmap_text region here, but verify via
* in vmk_init().
Expand Down
5 changes: 3 additions & 2 deletions suite/tests/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2150,10 +2150,11 @@ if (CLIENT_INTERFACE)
tobuild_ci(client.alloc client-interface/alloc.c ""
# For the subtest "targeting upper 2GB of low 4GB" we move -vm_base out of the
# way. We also raise the checklevel so we get memory filling for client allocs.
"-no_vm_base_near_app -vm_base 0x120000000 -checklevel 3" "")
# We keep vmheap_size at 2G to more easily trigger OOM.
"-no_vm_base_near_app -vm_base 0x120000000 -vmheap_size 2G -checklevel 3" "")
# A second test for i#4335's OOM despite a reset. We leave checklevel for CI speed.
tobuild_ci(client.alloc-noreset client-interface/alloc.c ""
"-no_enable_reset -no_vm_base_near_app -vm_base 0x120000000" "")
"-no_enable_reset -no_vm_base_near_app -vm_base 0x120000000 -vmheap_size 2G" "")
# XXX i#1312, i#3504: The test doesn't currently compile with CFLAGS_AVX512 but
# attempts to test features based on it. The compile flag needs to be added.
tobuild_ci(client.cleancall client-interface/cleancall.c "" "" "")
Expand Down

0 comments on commit 70be2df

Please sign in to comment.