diff --git a/src/builtins.c b/src/builtins.c index cfe2b74ed6bd8..225707fb7b46c 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -432,15 +432,14 @@ JL_CALLABLE(jl_f_apply) } } jl_value_t **newargs; - if (n > jl_page_size/sizeof(jl_value_t*)) { + int onstack = (n < jl_page_size/sizeof(jl_value_t*)); + JL_GC_PUSHARGS(newargs, onstack ? n : 1); + if (!onstack) { // put arguments on the heap if there are too many jl_value_t *argarr = (jl_value_t*)jl_alloc_cell_1d(n); - JL_GC_PUSH1(&argarr); + newargs[0] = argarr; newargs = jl_cell_data(argarr); } - else { - JL_GC_PUSHARGS(newargs, n); - } n = 0; for(i=1; i < nargs; i++) { if (jl_is_tuple(args[i])) { diff --git a/src/gc.c b/src/gc.c index 6b267f20125c7..df01e26eafa03 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1379,24 +1379,36 @@ void jl_gc_setmark(jl_value_t *v) // TODO rename this as it is misleading now // perm_scanned_bytes = s; } -static void gc_mark_stack(jl_value_t* ta, jl_gcframe_t *s, ptrint_t offset, int d) +static void gc_mark_stack(jl_value_t* ta, jl_gcframe_t *s, char *bottom, char* top, int d) { + // this function assumes !_stack_grows_up + int verify = (ta != (jl_value_t*)jl_root_task); + ptrint_t offset = (ptrint_t)top - (ptrint_t)jl_stackbase; while (s != NULL) { - s = (jl_gcframe_t*)((char*)s + offset); jl_value_t ***rts = (jl_value_t***)(((void**)s)+2); + assert(!verify || ((char*)s + offset < top && (char*)s + offset >= bottom)); + s = (jl_gcframe_t*)((char*)s + offset); // this part of the stack has been relocated size_t nr = s->nroots>>1; if (s->nroots & 1) { for(size_t i=0; i < nr; i++) { - jl_value_t **ptr = (jl_value_t**)((char*)rts[i] + offset); + jl_value_t ***pptr = &rts[i]; + assert(!verify || ((char*)pptr + offset < top && (char*)pptr + offset >= bottom)); + pptr = (jl_value_t***)((char*)pptr + offset); // this part of the stack has been relocated + jl_value_t **ptr = *pptr; + assert(!verify || ((char*)ptr + offset < top && (char*)ptr + offset >= bottom)); + ptr = (jl_value_t**)((char*)ptr + offset); // this part of the stack has been relocated if (*ptr != NULL) gc_push_root(*ptr, d); } } else { for(size_t i=0; i < nr; i++) { - if (rts[i] != NULL) { + jl_value_t **ptr = (jl_value_t**)&rts[i]; + assert(!verify || ((char*)ptr + offset < top && (char*)ptr + offset >= bottom)); + ptr = (jl_value_t**)((char*)ptr + offset); // this part of the stack has been relocated + if (*ptr != NULL) { verify_parent("task", ta, &rts[i], "stack(%d)", i); - gc_push_root(rts[i], d); + gc_push_root(*ptr, d); } } } @@ -1404,6 +1416,50 @@ static void gc_mark_stack(jl_value_t* ta, jl_gcframe_t *s, ptrint_t offset, int } } +static void gc_mark_offset_root_stack(jl_value_t* ta, jl_gcframe_t *s, char *bottom, char* top, int d) +{ + // this function assumes !_stack_grows_up + ptrint_t offset = (ptrint_t)top - (ptrint_t)jl_stackbase; + while (s != NULL) { + jl_value_t ***rts = (jl_value_t***)(((void**)s)+2); + size_t *pnr = &s->nroots; assert((char*)pnr == (char*)s); + if ((char*)pnr + offset < top && (char*)pnr + offset >= bottom) + pnr = (size_t*)((char*)pnr + offset); // this part of the stack has been relocated + //else + // return gc_mark_stack(ta, s, (char*)s, jl_stackbase, d); // the rest of the stack hasn't been relocated + size_t nr = *pnr; + if (nr & 1) { + nr >>= 1; + for(size_t i=0; i < nr; i++) { + jl_value_t ***pptr = &rts[i]; + if ((char*)pptr + offset < top && (char*)pptr + offset >= bottom) + pptr = (jl_value_t***)((char*)pptr + offset); // this part of the stack has been relocated + jl_value_t **ptr = *pptr; + if ((char*)ptr + offset < top && (char*)ptr + offset >= bottom) + ptr = (jl_value_t**)((char*)ptr + offset); // this part of the stack has been relocated + if (*ptr != NULL) + gc_push_root(*ptr, d); + } + } + else { + nr >>= 1; + for(size_t i=0; i < nr; i++) { + jl_value_t **ptr = (jl_value_t**)&rts[i]; + if ((char*)ptr + offset < top && (char*)ptr + offset >= bottom) + ptr = (jl_value_t**)((char*)ptr + offset); // this part of the stack has been relocated + if (*ptr != NULL) { + verify_parent("task", ta, &rts[i], "stack(%d)", i); + gc_push_root(*ptr, d); + } + } + } + jl_gcframe_t **ps = &s->prev; + if ((char*)ps + offset < top && (char*)ps + offset >= bottom) + ps = (jl_gcframe_t**)((char*)ps + offset); // this part of the stack has been relocated + s = *ps; + } +} + __attribute__((noinline)) static int gc_mark_module(jl_module_t *m, int d) { size_t i; @@ -1442,24 +1498,29 @@ __attribute__((noinline)) static int gc_mark_module(jl_module_t *m, int d) static void gc_mark_task_stack(jl_task_t *ta, int d) { - if (ta->stkbuf != NULL || ta == jl_current_task) { - if (ta->stkbuf != NULL) { - gc_setmark_buf(ta->stkbuf, gc_bits(ta)); - } + if (ta->stkbuf != NULL) + gc_setmark_buf(ta->stkbuf, gc_bits(ta)); + + if (ta == jl_current_task) { + gc_mark_stack((jl_value_t*)ta, jl_pgcstack, NULL, jl_stackbase, d); + } #ifdef COPY_STACKS - ptrint_t offset; - if (ta == jl_current_task) { - offset = 0; - gc_mark_stack((jl_value_t*)ta, jl_pgcstack, offset, d); + else if (ta->stkbuf) { + if (ta == jl_root_task) { + if (ta->ssize > 0) + gc_mark_offset_root_stack((jl_value_t*)ta, ta->gcstack, ta->stkbuf, ta->stkbuf + ta->ssize, d); + else + gc_mark_stack((jl_value_t*)ta, ta->gcstack, NULL, jl_stackbase, d); // no part of this stack has been relocated } else { - offset = (char *)ta->stkbuf - ((char *)jl_stackbase - ta->ssize); - gc_mark_stack((jl_value_t*)ta, ta->gcstack, offset, d); + gc_mark_stack((jl_value_t*)ta, ta->gcstack, ta->stkbuf, ta->stkbuf + ta->ssize, d); } + } #else - gc_mark_stack((jl_value_t*)ta, ta->gcstack, 0, d); -#endif + else if (ta->gcstack) { + gc_mark_stack((jl_value_t*)ta, ta->gcstack, ta->stkbuf, ta->stkbuf + ta->ssize, d); } +#endif } #if 0 diff --git a/src/task.c b/src/task.c index 962d8a14fae52..53859dd147f80 100644 --- a/src/task.c +++ b/src/task.c @@ -166,7 +166,7 @@ static void NOINLINE save_stack(jl_task_t *t) if (t->state == done_sym || t->state == failed_sym) return; volatile char *_x; - size_t nb = (char*)jl_stackbase - (char*)&_x; + size_t nb = (char*)jl_stackbase > (char*)&_x ? (char*)jl_stackbase - (char*)&_x : 0; char *buf; if (t->stkbuf == NULL || t->bufsz < nb) { buf = (char*)allocb(nb); @@ -252,7 +252,7 @@ DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel) _julia_init(rel); #ifdef COPY_STACKS char __stk; - jl_stackbase = (char*)(((uptrint_t)&__stk + sizeof(__stk))&-16); // also ensures stackbase is 16-byte aligned + jl_stackbase = (char*)(((uptrint_t)&__stk - 4096 + sizeof(__stk))&-16); // also ensures stackbase is 16-byte aligned set_base_ctx(&__stk); // separate function, to record the size of a stack frame #endif }