diff --git a/src/gc.c b/src/gc.c index 99606b32e83d6..4a4b34d36f39a 100644 --- a/src/gc.c +++ b/src/gc.c @@ -409,10 +409,13 @@ static inline int gc_setmark_pool(void *o, int mark_mode) mark_mode = GC_MARKED; } if (!(bits & GC_MARKED)) { - if (mark_mode == GC_MARKED) + if (mark_mode == GC_MARKED) { perm_scanned_bytes += page->osize; - else + page->nold++; + } + else { scanned_bytes += page->osize; + } objprofile_count(jl_typeof(jl_valueof(o)), mark_mode == GC_MARKED, page->osize); } @@ -880,8 +883,8 @@ static gcval_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, gcval_t **pfl // For quick sweep, we might be able to skip the page if the page doesn't // have any young live cell before marking. if (sweep_mask == GC_MARKED_NOESC && !pg->has_young) { - // TODO handle `prev_sweep_mask == GC_MARKED` with additional counters - if (prev_sweep_mask == GC_MARKED_NOESC) { + assert(prev_sweep_mask == GC_MARKED_NOESC || pg->prev_nold >= pg->nold); + if (prev_sweep_mask == GC_MARKED_NOESC || pg->prev_nold == pg->nold) { // the position of the freelist begin/end in this page // is stored in its metadata if (pg->fl_begin_offset != (uint16_t)-1) { @@ -897,6 +900,7 @@ static gcval_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, gcval_t **pfl { // scope to avoid clang goto errors int has_marked = 0; int has_young = 0; + int16_t prev_nold = 0; int pg_nfree = 0; gcval_t **pfl_begin = NULL; uint8_t msk = 1; // mask for the age bit in the current age byte @@ -916,6 +920,7 @@ static gcval_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, gcval_t **pfl if (sweep_mask == GC_MARKED || bits == GC_MARKED_NOESC) { bits = gc_bits(v) = GC_QUEUED; // promote } + prev_nold++; } else { assert(bits == GC_MARKED_NOESC); @@ -941,6 +946,10 @@ static gcval_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, gcval_t **pfl pg->fl_end_offset = pfl_begin ? (char*)pfl - data : (uint16_t)-1; pg->nfree = pg_nfree; + if (sweep_mask == GC_MARKED) { + pg->nold = 0; + pg->prev_nold = prev_nold; + } page_done++; } free_page: diff --git a/src/gc.h b/src/gc.h index 971d4dcd03324..993b2f5063f3a 100644 --- a/src/gc.h +++ b/src/gc.h @@ -154,11 +154,17 @@ typedef struct { // // For a quick sweep preceded by a full sweep. If this bit is set, // the page needs to be swept. If this bit is not set, there could - // still be old dead objects in the page. + // still be old dead objects in the page and `nold` and `prev_nold` + // should be used to determine if the page needs to be swept. uint16_t has_young: 1; }; - uint16_t nfree; // number of free objects in this page. - // invalid if pool that owns this page is allocating objects from this page. + // number of old objects in this page + uint16_t nold; + // number of old objects in this page during the previous full sweep + uint16_t prev_nold; + // number of free objects in this page. + // invalid if pool that owns this page is allocating objects from this page. + uint16_t nfree; uint16_t osize; // size of each object in this page uint16_t fl_begin_offset; // offset of first free object in this page uint16_t fl_end_offset; // offset of last free object in this page