Skip to content

Commit

Permalink
Rewrite GC page metadata for sweeping fast path
Browse files Browse the repository at this point in the history 
* Delete `gc_bits` and `allocd` which are not enough to accurately
  record the necessary information.
* Add `has_marked` and `has_young` to identify free pages and untouched pages.

Fixes #15543
  • Loading branch information
@yuyichao
yuyichao committed Jun 2, 2016
1 parent cc31133 commit 28c9abd
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 31 deletions.
5 changes: 3 additions & 2 deletions src/gc-debug.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,8 +348,8 @@ static void gc_scrub_range(char *stack_lo, char *stack_hi)
continue;
jl_gc_pagemeta_t *pg = page_metadata(tag);
// Make sure the sweep rebuild the freelist
pg->allocd = 1;
pg->gc_bits = 0x3;
pg->has_marked = 1;
pg->has_young = 1;
// Find the age bit
char *page_begin = gc_page_data(tag) + GC_PAGE_OFFSET;
int obj_id = (((char*)tag) - page_begin) / osize;
Expand All @@ -358,6 +358,7 @@ static void gc_scrub_range(char *stack_lo, char *stack_hi)
// (especially on 32bit where it's more likely to have pointer-like
// bit patterns)
*ages &= ~(1 << (obj_id % 8));
// set mark to GC_MARKED_NOESC (young and marked)
memset(tag, 0xff, osize);
}
}
Expand Down
57 changes: 31 additions & 26 deletions src/gc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,6 @@ static int64_t scanned_bytes; // young bytes scanned while marking
static int64_t perm_scanned_bytes; // old bytes scanned while marking
static int prev_sweep_mask = GC_MARKED;

static size_t array_nbytes(jl_array_t*);
#define inc_sat(v,s) v = (v) >= s ? s : (v)+1

// Full collection heuristics
Expand Down Expand Up @@ -415,8 +414,9 @@ static inline int gc_setmark_pool(void *o, int mark_mode)
objprofile_count(jl_typeof(jl_valueof(o)),
mark_mode == GC_MARKED, page->osize);
}
assert(mark_mode & GC_MARKED);
page->has_marked = 1;
_gc_setmark(o, mark_mode);
page->gc_bits |= mark_mode;
verify_val(jl_valueof(o));
return mark_mode;
}
Expand Down Expand Up @@ -667,15 +667,15 @@ static void sweep_malloced_arrays(void)
// pool allocation
static inline gcval_t *reset_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, gcval_t *fl)
{
pg->gc_bits = 0;
pg->nfree = (GC_PAGE_SZ - GC_PAGE_OFFSET) / p->osize;
jl_tls_states_t *ptls = jl_all_tls_states[pg->thread_n];
pg->pool_n = p - ptls->heap.norm_pools;
memset(pg->ages, 0, GC_PAGE_SZ / 8 / p->osize + 1);
gcval_t *beg = (gcval_t*)(pg->data + GC_PAGE_OFFSET);
gcval_t *end = (gcval_t*)((char*)beg + (pg->nfree - 1)*p->osize);
end->next = fl;
pg->allocd = 0;
pg->has_young = 0;
pg->has_marked = 0;
pg->fl_begin_offset = GC_PAGE_OFFSET;
pg->fl_end_offset = (char*)end - (char*)beg + GC_PAGE_OFFSET;
return beg;
Expand Down Expand Up @@ -724,7 +724,7 @@ static inline void *__pool_alloc(jl_gc_pool_t *p, int osize, int end_offset)
jl_gc_pagemeta_t *pg = page_metadata(v);
assert(pg->osize == p->osize);
pg->nfree = 0;
pg->allocd = 1;
pg->has_young = 1;
if (next)
p->nfree = page_metadata(next)->nfree;
}
Expand All @@ -745,7 +745,7 @@ static inline void *__pool_alloc(jl_gc_pool_t *p, int osize, int end_offset)
jl_gc_pagemeta_t *pg = page_metadata(v);
assert(pg->osize == p->osize);
pg->nfree = 0;
pg->allocd = 1;
pg->has_young = 1;
p->newpages = v->next;
}
v->flags = 0;
Expand Down Expand Up @@ -873,12 +873,15 @@ static gcval_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, gcval_t **pfl
freedall = 1;
old_nfree += pg->nfree;

if (pg->gc_bits == GC_MARKED) {
// this page only contains GC_MARKED and free cells
// if we are doing a quick sweep and nothing has been allocated inside since last sweep
// we can skip it
if (sweep_mask == GC_MARKED_NOESC && !pg->allocd) {
// the position of the freelist begin/end in this page is stored in its metadata
if (!pg->has_marked)
goto free_page;
// For quick sweep, we might be able to skip the page if the page doesn't
// have any young live cell before marking.
if (sweep_mask == GC_MARKED_NOESC && !pg->has_young) {
// TODO handle `prev_sweep_mask == GC_MARKED` with additional counters
if (prev_sweep_mask == GC_MARKED_NOESC) {
// the position of the freelist begin/end in this page
// is stored in its metadata
if (pg->fl_begin_offset != (uint16_t)-1) {
*pfl = page_pfl_beg(pg);
pfl = prev_pfl = (gcval_t**)page_pfl_end(pg);
Expand All @@ -888,11 +891,10 @@ static gcval_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, gcval_t **pfl
goto free_page;
}
}
else if (pg->gc_bits == GC_CLEAN) {
goto free_page;
}

{ // scope to avoid clang goto errors
int has_marked = 0;
int has_young = 0;
int pg_nfree = 0;
gcval_t **pfl_begin = NULL;
uint8_t msk = 1; // mask for the age bit in the current age byte
Expand All @@ -906,14 +908,19 @@ static gcval_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, gcval_t **pfl
*ages &= ~msk;
}
else { // marked young or old
if (*ages & msk) { // old enough
if (*ages & msk || bits == GC_MARKED) { // old enough
// `!age && bits == GC_MARKED` is possible for
// non-first-class objects like `jl_binding_t`
if (sweep_mask == GC_MARKED || bits == GC_MARKED_NOESC) {
gc_bits(v) = GC_QUEUED; // promote
bits = gc_bits(v) = GC_QUEUED; // promote
}
}
else if ((sweep_mask & bits) == sweep_mask) {
gc_bits(v) = GC_CLEAN; // unmark
else {
assert(bits == GC_MARKED_NOESC);
bits = gc_bits(v) = GC_CLEAN; // unmark
has_young = 1;
}
has_marked |= (bits & GC_MARKED) != 0;
*ages |= msk;
freedall = 0;
}
Expand All @@ -925,12 +932,14 @@ static gcval_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, gcval_t **pfl
}
}

assert(!freedall);
pg->has_marked = has_marked;
pg->has_young = has_young;
pg->fl_begin_offset = pfl_begin ? (char*)pfl_begin - data : (uint16_t)-1;
pg->fl_end_offset = pfl_begin ? (char*)pfl - data : (uint16_t)-1;

pg->nfree = pg_nfree;
page_done++;
pg->allocd = 0;
}
free_page:
pg_freedall += freedall;
Expand Down Expand Up @@ -966,10 +975,6 @@ static gcval_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, gcval_t **pfl
nfree += obj_per_page;
}
else {
if (sweep_mask == GC_MARKED)
pg->gc_bits = GC_CLEAN;
if (sweep_mask == GC_MARKED_NOESC)
pg->gc_bits = GC_MARKED;
nfree += pg->nfree;
}

Expand Down Expand Up @@ -1024,7 +1029,7 @@ static int gc_sweep_inc(int sweep_mask)
gcval_t *last = p->freelist;
if (last) {
jl_gc_pagemeta_t *pg = page_metadata(last);
pg->allocd = 1;
pg->has_young = 1;
pg->nfree = p->nfree;
}
p->freelist = NULL;
Expand All @@ -1034,7 +1039,7 @@ static int gc_sweep_inc(int sweep_mask)
if (last) {
jl_gc_pagemeta_t *pg = page_metadata(last);
pg->nfree = (GC_PAGE_SZ - ((char*)last - gc_page_data(last))) / p->osize;
pg->allocd = 1;
pg->has_young = 1;
}
p->newpages = NULL;
}
Expand Down
21 changes: 18 additions & 3 deletions src/gc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,9 +138,24 @@ typedef struct _mallocarray_t {
// pool page metadata
typedef struct {
struct {
uint16_t pool_n : 8; // index (into norm_pool) of pool that owns this page
uint16_t allocd : 1; // true if an allocation happened in this page since last sweep
uint16_t gc_bits : 2; // this is a bitwise | of all gc_bits in this page
// index of pool that owns this page
uint16_t pool_n : 8;
// Whether any cell in the page is marked
// This bit is set before sweeping iff there's live cells in the page.
// Note that before marking or after sweeping there can be live
// (and young) cells in the page for `!has_marked`.
uint16_t has_marked: 1;
// Whether any cell was live and young **before sweeping**.
// For a normal sweep (quick sweep that is NOT preceded by a
// full sweep) this bit is set iff there are young or newly dead
// objects in the page and the page needs to be swept.
//
// For a full sweep, this bit should be ignored.
//
// For a quick sweep preceded by a full sweep. If this bit is set,
// the page needs to be swept. If this bit is not set, there could
// still be old dead objects in the page.
uint16_t has_young: 1;
};
uint16_t nfree; // number of free objects in this page.
// invalid if pool that owns this page is allocating objects from this page.
Expand Down

0 comments on commit 28c9abd

Please sign in to comment.