Skip to content

Commit

Permalink
Separate marking and scanning of the object
Browse files Browse the repository at this point in the history 
Make sure in `gc_mark_obj` that the object will only be scanned once.
  • Loading branch information
@yuyichao
yuyichao committed Jan 9, 2017
1 parent 762a90c commit a4b8ce4
Showing 1 changed file with 101 additions and 81 deletions.
182 changes: 101 additions & 81 deletions src/gc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -447,12 +447,11 @@ STATIC_INLINE void gc_update_heap_size(int64_t sz_ub, int64_t sz_est)
static inline uint16_t gc_setmark_big(jl_ptls_t ptls, jl_taggedvalue_t *o,
int8_t mark_mode, uintptr_t tag)
{
assert(!gc_marked(tag));
if (gc_verifying) {
o->bits.gc = mark_mode;
return mark_mode;
}
if (gc_marked(tag))
return (uint16_t)tag & 3;
assert(find_region(o) == NULL);
bigval_t *hdr = bigval_header(o);
if (mark_reset_age) {
Expand Down Expand Up @@ -490,6 +489,7 @@ static inline uint16_t gc_setmark_pool_(jl_ptls_t ptls, jl_taggedvalue_t *o,
int8_t mark_mode, region_t *r,
uintptr_t tag)
{
assert(!gc_marked(tag));
#ifdef MEMDEBUG
return gc_setmark_big(ptls, o, mark_mode, tag);
#endif
Expand All @@ -498,8 +498,6 @@ static inline uint16_t gc_setmark_pool_(jl_ptls_t ptls, jl_taggedvalue_t *o,
o->bits.gc = mark_mode;
return mark_mode;
}
if (gc_marked(tag))
return (uint16_t)tag & 3;
jl_gc_pagemeta_t *page = page_metadata_(o, r);
if (mark_reset_age) {
// Reset the object as if it was just allocated
Expand Down Expand Up @@ -536,12 +534,14 @@ static inline uint16_t gc_setmark_pool_(jl_ptls_t ptls, jl_taggedvalue_t *o,
static inline uint16_t gc_setmark_pool(jl_ptls_t ptls, jl_taggedvalue_t *o,
int8_t mark_mode, uintptr_t tag)
{
assert(!gc_marked(tag));
return gc_setmark_pool_(ptls, o, mark_mode, find_region(o), tag);
}

static inline uint16_t gc_setmark(jl_ptls_t ptls, jl_value_t *v,
int sz, uintptr_t tag)
{
assert(!gc_marked(tag));
jl_taggedvalue_t *o = jl_astaggedvalue(v);
if (sz <= GC_MAX_SZCLASS)
return gc_setmark_pool(ptls, o, GC_MARKED, tag);
Expand All @@ -554,6 +554,8 @@ inline void gc_setmark_buf(jl_ptls_t ptls, void *o,
{
jl_taggedvalue_t *buf = jl_astaggedvalue(o);
uintptr_t tag = buf->header;
if (gc_marked(tag))
return;
// If the object is larger than the max pool size it can't be a pool object.
// This should be accurate most of the time but there might be corner cases
// where the size estimate is a little off so we do a pool lookup to make
Expand Down Expand Up @@ -1190,7 +1192,8 @@ void gc_queue_binding(jl_binding_t *bnd)
arraylist_push(&ptls->heap.rem_bindings, bnd);
}

static int gc_scan_obj(jl_ptls_t ptls, jl_value_t *v, int d, uintptr_t tag);
static void gc_scan_obj(jl_ptls_t ptls, jl_value_t *v, int d, uintptr_t tag);
static uint16_t gc_mark_obj(jl_ptls_t ptls, jl_value_t *v, uintptr_t tag);
#ifdef JL_DEBUG_BUILD
static void *volatile gc_findval; // for usage from gdb, for finding the gc-root for a value
#endif
Expand All @@ -1204,9 +1207,15 @@ static inline int gc_push_root(jl_ptls_t ptls, void *v, int d) // v isa jl_value
assert(v != NULL);
jl_taggedvalue_t *o = jl_astaggedvalue(v);
verify_val(v);
uintptr_t tag = o->header;
if (!gc_marked(tag))
return !gc_old(gc_scan_obj(ptls, (jl_value_t*)v, d, tag));
const uintptr_t tag = o->header;
if (!gc_marked(tag)) {
uint16_t mark_res = gc_mark_obj(ptls, (jl_value_t*)v, tag);
assert(gc_marked(o->header));
if (mark_res >> 8)
gc_scan_obj(ptls, (jl_value_t*)v, d,
gc_set_bits(tag, mark_res & 0xff));
return !gc_old(mark_res);
}
return !gc_old(tag);
}

Expand Down Expand Up @@ -1387,43 +1396,37 @@ JL_DLLEXPORT void jl_gc_lookfor(jl_value_t *v) { lookforme = v; }
*/

#define MAX_MARK_DEPTH 400
// mark v and recurse on its children (or store them on the mark stack when recursion depth becomes too high)
// it does so assuming the gc bits of v are "bits" and returns the new bits of v
// if v becomes GC_OLD_MARKED and some of its children are GC_MARKED (young),
// v is added to the remset
static int gc_scan_obj(jl_ptls_t ptls, jl_value_t *v, int d, uintptr_t tag)
// Scan an marked object `v` and recursively mark its children.
// The object will be queued on the mark stack when recursion depth
// becomes too high.
// It does so assuming that the tag of the (marked) object is `tag`.
// If `v` is `GC_OLD_MARKED` and some of its children are `GC_MARKED` (young),
// `v` is added to the remset
static void gc_scan_obj(jl_ptls_t ptls, jl_value_t *v, int d, uintptr_t tag)
{
assert(v != NULL);
assert(gc_marked(tag));
jl_datatype_t *vt = (jl_datatype_t*)(tag & ~(uintptr_t)15);
gc_assert_datatype(vt);
#ifdef JL_DEBUG_BUILD
gc_assert_datatype(vt); // should have checked in `gc_mark_obj`
#endif
int refyoung = 0, nptr = 0;
// Do not initialize `bits` to catch branches forgetting to set `bits`
// using compiler warnings.
int bits;
const int8_t bits = tag & 0xf;

if (vt == jl_weakref_type) {
bits = gc_setmark(ptls, v, sizeof(jl_weakref_t), tag) & 0xff;
goto ret;
}
else if (vt == jl_string_type) {
bits = gc_setmark(ptls, v, jl_string_len(v) + sizeof(size_t) + 1,
tag) & 0xff;
goto ret;
}
if (vt->layout->pointerfree) {
int sz = jl_datatype_size(vt);
bits = gc_setmark(ptls, v, sz, tag) & 0xff;
goto ret;
}
assert(vt != jl_symbol_type);
// weakref should not be marked
if (vt == jl_weakref_type)
return;
// fast path
if (vt->layout->pointerfree)
return;
d++;
if (d >= MAX_MARK_DEPTH)
goto queue_the_root;

// some values have special representations
if (vt == jl_simplevector_type) {
size_t l = jl_svec_len(v);
bits = gc_setmark(ptls, v, l * sizeof(void*) + sizeof(jl_svec_t),
tag) & 0xff;
if (d >= MAX_MARK_DEPTH)
goto queue_the_root;
jl_value_t **data = jl_svec_data(v);
nptr += l;
for(size_t i=0; i < l; i++) {
Expand All @@ -1436,25 +1439,7 @@ static int gc_scan_obj(jl_ptls_t ptls, jl_value_t *v, int d, uintptr_t tag)
}
else if (vt->name == jl_array_typename) {
jl_array_t *a = (jl_array_t*)v;
jl_taggedvalue_t *o = jl_astaggedvalue(v);
jl_array_flags_t flags = a->flags;
uint16_t mark_res = (flags.pooled ?
gc_setmark_pool(ptls, o, GC_MARKED, tag) :
gc_setmark_big(ptls, o, GC_MARKED, tag));
bits = mark_res & 0xff;
uint8_t tag_changed = mark_res >> 8;
if (flags.how == 2 && tag_changed) {
objprofile_count(jl_malloc_tag, bits == GC_OLD_MARKED,
array_nbytes(a));
if (bits == GC_OLD_MARKED) {
perm_scanned_bytes += array_nbytes(a);
}
else {
scanned_bytes += array_nbytes(a);
}
}
if (d >= MAX_MARK_DEPTH)
goto queue_the_root;
if (flags.how == 3) {
jl_value_t *owner = jl_array_data_owner(a);
refyoung |= gc_push_root(ptls, owner, d);
Expand All @@ -1467,7 +1452,7 @@ static int gc_scan_obj(jl_ptls_t ptls, jl_value_t *v, int d, uintptr_t tag)
gc_setmark_buf(ptls, (char*)a->data - a->offset*a->elsize,
bits, array_nbytes(a));
}
if (flags.ptrarray && a->data!=NULL) {
if (flags.ptrarray && a->data != NULL) {
size_t l = jl_array_len(a);
if (l > 100000 && d > MAX_MARK_DEPTH-10) {
// don't mark long arrays at high depth, to try to avoid
Expand All @@ -1491,36 +1476,17 @@ static int gc_scan_obj(jl_ptls_t ptls, jl_value_t *v, int d, uintptr_t tag)
}
else if (vt == jl_module_type) {
// should increase nptr here
bits = gc_setmark(ptls, v, sizeof(jl_module_t), tag) & 0xff;
if (d >= MAX_MARK_DEPTH)
goto queue_the_root;
refyoung |= gc_mark_module(ptls, (jl_module_t*)v, d, bits);
}
else if (vt == jl_task_type) {
// ditto nptr
bits = gc_setmark(ptls, v, sizeof(jl_task_t), tag) & 0xff;
if (d >= MAX_MARK_DEPTH)
goto queue_the_root;
gc_mark_task(ptls, (jl_task_t*)v, d, bits);
// tasks should always be remarked since we do not trigger the write barrier
// for stores to stack slots
refyoung = 1;
}
else if (vt == jl_symbol_type) {
// symbols have their own allocator and are never freed
bits = GC_OLD_MARKED;
}
else {
size_t dtsz = jl_datatype_size(vt);
bits = gc_setmark(ptls, v, dtsz, tag) & 0xff;
if (d >= MAX_MARK_DEPTH)
goto queue_the_root;

int nf = (int)jl_datatype_nfields(vt);
// TODO check if there is a perf improvement for objects with a lot of fields
// int fdsz = sizeof(void*)*nf;
// void** children = alloca(fdsz);
// int ci = 0;
for(int i=0; i < nf; i++) {
if (jl_field_isptr(vt, i)) {
nptr++;
Expand All @@ -1529,30 +1495,84 @@ static int gc_scan_obj(jl_ptls_t ptls, jl_value_t *v, int d, uintptr_t tag)
jl_value_t *fld = *slot;
if (fld) {
verify_parent2("object", v, slot, "field(%d)", i);
//children[ci++] = fld;
refyoung |= gc_push_root(ptls, fld, d);
}
}
}
//while(ci)
// refyoung |= gc_push_root(ptls, children[--ci], d);
}

ret:
if (gc_verifying)
return bits;
if ((bits == GC_OLD_MARKED) && refyoung) {
if ((bits == GC_OLD_MARKED) && refyoung && !gc_verifying) {
ptls->heap.remset_nptr += nptr;
// v is an old object referencing young objects
arraylist_push(ptls->heap.remset, v);
}
return bits;
return;

queue_the_root:
if (mark_sp >= mark_stack_size)
grow_mark_stack();
mark_stack[mark_sp++] = (jl_value_t*)v;
return bits;
}

// Mark an object (without scanning it)
// The top `int8_t` of the return value is set to `1` if the object was not
// marked before (another thread might have marked it). The bottom `int8_t`
// of the return value is the new GC bits.
static uint16_t gc_mark_obj(jl_ptls_t ptls, jl_value_t *v, uintptr_t tag)
{
assert(v != NULL);
assert(!gc_marked(tag));
jl_datatype_t *vt = (jl_datatype_t*)(tag & ~(uintptr_t)15);
gc_assert_datatype(vt);
// Do not initialize `mark_res` to catch branches forgetting to set `mark_res`
// using compiler warnings.
uint16_t mark_res;

// some values have special representations
if (vt == jl_simplevector_type) {
size_t l = jl_svec_len(v);
mark_res = gc_setmark(ptls, v, l * sizeof(void*) + sizeof(jl_svec_t),
tag);
}
else if (vt->name == jl_array_typename) {
jl_array_t *a = (jl_array_t*)v;
jl_taggedvalue_t *o = jl_astaggedvalue(v);
jl_array_flags_t flags = a->flags;
mark_res = (flags.pooled ? gc_setmark_pool(ptls, o, GC_MARKED, tag) :
gc_setmark_big(ptls, o, GC_MARKED, tag));
if (flags.how == 2 && (mark_res >> 8)) {
uint8_t bits = mark_res & 0xff;
objprofile_count(jl_malloc_tag, bits == GC_OLD_MARKED,
array_nbytes(a));
if (bits == GC_OLD_MARKED) {
perm_scanned_bytes += array_nbytes(a);
}
else {
scanned_bytes += array_nbytes(a);
}
}
}
else if (vt == jl_module_type) {
mark_res = gc_setmark(ptls, v, sizeof(jl_module_t), tag);
}
else if (vt == jl_task_type) {
mark_res = gc_setmark(ptls, v, sizeof(jl_task_t), tag);
}
else if (vt == jl_symbol_type) {
// symbols have their own allocator and are never freed
mark_res = GC_OLD_MARKED;
}
else if (vt == jl_string_type) {
mark_res = gc_setmark(ptls, v, jl_string_len(v) + sizeof(size_t) + 1,
tag);
}
else {
mark_res = gc_setmark(ptls, v, jl_datatype_size(vt), tag);
}
if (gc_verifying)
return mark_res | (1 << 8);
return mark_res;
}

void visit_mark_stack(jl_ptls_t ptls)
Expand Down

0 comments on commit a4b8ce4

Please sign in to comment.