diff --git a/base/boot.jl b/base/boot.jl index de72d9f7e8277b..b5facac1b183c8 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -298,7 +298,7 @@ eval(:(Core.call(::Type{GlobalRef}, m::Module, s::Symbol) = $(Expr(:new, :Global Module(name::Symbol=:anonymous, std_imports::Bool=true) = ccall(:jl_f_new_module, Any, (Any, Bool), name, std_imports)::Module -Task(f::ANY) = ccall(:jl_new_task, Any, (Any, Int), f::Function, 0)::Task +Task(f::ANY, reserved_stack::Int=0) = ccall(:jl_new_task, Any, (Any, Int), f::Function, reserved_stack)::Task # simple convert for use by constructors of types in Core # note that there is no actual conversion defined here, diff --git a/src/gc.c b/src/gc.c index 23d7d98e80ac07..67a663953b0b2c 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1562,17 +1562,15 @@ static void gc_mark_task_stack(jl_task_t *ta, int d) if (ta == jl_current_task) { gc_mark_stack((jl_value_t*)ta, jl_pgcstack, 0, d); } - else if (ta == jl_root_task) { + else if (!ta->copy_stack) { gc_mark_stack((jl_value_t*)ta, ta->gcstack, 0, d); } - else if (ta->stkbuf != NULL && ta->stkbuf != (void*)(intptr_t)-1) { #ifdef COPY_STACKS + else if (ta->stkbuf != NULL && ta->stkbuf != (void*)(intptr_t)-1) { ptrint_t offset = (char *)ta->stkbuf + ta->ssize - (char *)jl_stackbase; -#else - ptrint_t offset = 0; -#endif gc_mark_stack((jl_value_t*)ta, ta->gcstack, offset, d); } +#endif } NOINLINE static void gc_mark_task(jl_task_t *ta, int d) @@ -1808,7 +1806,7 @@ double clock_now(void); extern jl_module_t *jl_old_base_module; extern jl_array_t *jl_module_init_order; -extern jl_value_t *jl_unprotect_stack_func; +extern jl_value_t *jl_task_cleanup_func; static int inc_count = 0; static int quick_count = 0; @@ -1846,7 +1844,7 @@ static void pre_mark(void) } jl_mark_box_caches(); - gc_push_root(jl_unprotect_stack_func, 0); + gc_push_root(jl_task_cleanup_func, 0); gc_push_root(jl_bottom_func, 0); gc_push_root(jl_typetype_type, 0); diff --git a/src/julia.h b/src/julia.h index 5b63ae89cdfa99..1ab3a34d1a1ac6 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1389,7 +1389,8 @@ typedef struct _jl_task_t { #endif jl_jmp_buf ctx; // saved thread state void *stkbuf; // malloc'd memory - int ssize; // sizeof the portion of stack used in stkbuf + int ssize:31; // sizeof stkbuf + int copy_stack:1; // current exception handler jl_handler_t *eh; diff --git a/src/signals-unix.c b/src/signals-unix.c index 814a87fc9ff535..b0fdde0c2b68f1 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -14,17 +14,12 @@ void fpe_handler(int arg) static int is_addr_on_stack(void *addr) { -#ifdef COPY_STACKS - if (jl_current_task == jl_root_task) - return ((char*)addr > (char*)jl_stack_lo-3000000 && - (char*)addr < (char*)jl_stack_hi); - else + if (jl_current_task->copy_stack) return ((char*)addr > (char*)jl_stackbase - JL_STACK_SIZE && (char*)addr < (char*)jl_stackbase); -#else - return ((char*)addr > (char*)jl_current_task->stkbuf && - (char*)addr < (char*)jl_current_task->stkbuf + jl_current_task->ssize); -#endif + else + return ((char*)addr > (char*)jl_current_task->stkbuf && + (char*)addr < (char*)jl_current_task->stkbuf + jl_current_task->ssize); } #ifndef SIGINFO diff --git a/src/task.c b/src/task.c index 0519088575d01d..d73afe886a8ce4 100644 --- a/src/task.c +++ b/src/task.c @@ -118,12 +118,14 @@ static void *malloc_stack(size_t bufsz) return stk; } -#ifndef COPY_STACKS static void free_stack(void *stkbuf, size_t bufsz) { munmap(stkbuf, bufsz); } #endif + +#ifndef MINSIGSTKSZ +#define MINSIGSTKSZ 131072 // 128k #endif static jl_sym_t *done_sym; @@ -136,7 +138,7 @@ DLLEXPORT JL_THREAD jl_task_t * volatile jl_current_task; JL_THREAD jl_task_t *jl_root_task; DLLEXPORT JL_THREAD jl_value_t *jl_exception_in_transit; DLLEXPORT JL_THREAD jl_gcframe_t *jl_pgcstack = NULL; -static void jl_new_fiber(jl_task_t *t); +static void jl_new_fiber(jl_task_t *t, size_t ssize); #ifdef HAVE_UNW_CONTEXT static unw_context_t jl_base_uctx; @@ -156,7 +158,7 @@ static JL_THREAD LPVOID jl_basefiber; static void NOINLINE save_stack(jl_task_t *t) { - if (t->state == done_sym || t->state == failed_sym || t == jl_root_task) + if (t->state == done_sym || t->state == failed_sym) return; volatile char *_x; size_t nb = (char*)jl_stackbase - (char*)&_x; @@ -185,7 +187,7 @@ void NOINLINE NORETURN restore_stack(jl_task_t *t, char *p) if ((char*)&_x > _x) { p = (char*)alloca((char*)&_x - _x); } - restore_stack(t, p); + restore_stack(t, p); // pass p as a parameter so that the compiler can't avoid the alloca } assert(t->stkbuf != NULL); memcpy(_x, t->stkbuf, t->ssize); @@ -205,9 +207,9 @@ static void NORETURN finish_task(jl_task_t *t, jl_value_t *resultval) t->result = resultval; jl_gc_wb(t, t->result); #ifdef COPY_STACKS - // early free of stkbuf + // early free of stkbuf, if we aren't running on it right now void *stkbuf = t->stkbuf; - if (stkbuf != NULL) { + if (t->copy_stack && stkbuf != NULL) { t->stkbuf = (void*)(intptr_t)-1; free(stkbuf - sizeof(size_t)); } @@ -282,44 +284,31 @@ static void ctx_switch(jl_task_t *t) jl_current_task = t; -#if defined(_OS_WINDOWS_) && !defined(COPY_STACKS) - (void)lastt; - if (!t->fiber) { - LPVOID jl_fiber = CreateFiberEx(t->ssize, t->ssize, FIBER_FLAG_FLOAT_SWITCH, start_fiber, NULL); - if (jl_fiber == NULL) - jl_error("CreateFiberEx failed"); - t->fiber = jl_fiber; - } - SwitchToFiber(t->fiber); - -#else if (jl_setjmp(lastt->ctx, 0)) return; // store the old context, return when execution resumes here #ifdef _OS_WINDOWS_ if (t->fiber != lastt->fiber) { SwitchToFiber(t->fiber); - if (jl_current_task == jl_root_task || lastt == jl_current_task) +#ifdef COPY_STACKS + if (!jl_current_task->copy_stack || lastt == jl_current_task) return; +#endif t = jl_current_task; } #endif -#ifdef COPY_STACKS - // swap stacks - save_stack(lastt); - if (t != jl_root_task && t->stkbuf) { - // task already exists - restore_stack(t, NULL); // resume at jl_setjmp of the other task after restoring the stack (doesn't return) - } +#ifdef COPY_STACKS // swap stacks + if (lastt->copy_stack) // may need to save the old copy-stack + save_stack(lastt); + if (t->copy_stack && t->stkbuf) // task already exists, resume from the copy-stack + restore_stack(t, lastt->copy_stack ? NULL : (void*)(intptr_t)1); // resume at jl_setjmp of the other task after restoring the stack (doesn't return) +#endif -#else - if (!t->stkbuf) { // task not started yet, jump to start_task - jl_new_fiber(t); // (doesn't return) + if (!t->copy_stack && !t->stkbuf) { // task not started yet, jump to start_task + jl_new_fiber(t, t->ssize); // (doesn't return) } -#endif jl_longjmp(t->ctx, 1); // resume at jl_setjmp of the other task (doesn't return) -#endif //JL_SIGATOMIC_END(); } @@ -714,7 +703,7 @@ DLLEXPORT void jl_throw_with_superfluous_argument(jl_value_t *e, int line) jl_throw(e); } -jl_value_t *jl_unprotect_stack_func; +jl_value_t *jl_task_cleanup_func; DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) { @@ -725,7 +714,11 @@ DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) if (ssize == 0) // unspecified -- pick some default size ssize = JL_STACK_SIZE; #endif - ssize = LLT_ALIGN(ssize, pagesz); + if (ssize != 0) { + if (ssize < MINSIGSTKSZ) + ssize = MINSIGSTKSZ; + ssize = LLT_ALIGN(ssize, pagesz); + } t->ssize = ssize; t->current_module = NULL; t->parent = jl_current_task; @@ -741,36 +734,40 @@ DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) t->eh = NULL; t->gcstack = NULL; t->stkbuf = NULL; + t->copy_stack = (ssize == 0); +#if defined(JL_DEBUG_BUILD) + if (!t->copy_stack) + memset(&t->ctx, 0, sizeof(t->ctx)); +#endif #ifdef COPY_STACKS - memcpy(&t->ctx, &jl_basectx, sizeof(t->ctx)); -#elif JL_DEBUG - memset(&t->ctx, 0, sizeof(t->ctx)); + if (t->copy_stack) + memcpy(&t->ctx, &jl_basectx, sizeof(t->ctx)); #endif + #ifdef _OS_WINDOWS_ -#ifdef COPY_STACKS - t->fiber = jl_basefiber; -#else t->fiber = NULL; +#ifdef COPY_STACKS + if (t->copy_stack) + t->fiber = jl_basefiber; #endif #endif - jl_gc_add_finalizer((jl_value_t*)t, (jl_function_t*)jl_unprotect_stack_func); + jl_gc_add_finalizer((jl_value_t*)t, (jl_function_t*)jl_task_cleanup_func); return t; } -static void jl_unprotect_stack(jl_task_t *t) +static void jl_task_cleanup(jl_task_t *t) { void *stk = t->stkbuf; if (stk && stk != (void*)(intptr_t)-1) { t->stkbuf = (void*)(intptr_t)-1; -#ifdef COPY_STACKS - free(stk - sizeof(size_t)); -#else + if (t->copy_stack) + free(stk - sizeof(size_t)); + else #ifdef _OS_WINDOWS_ - DeleteFiber(t->fiber); + DeleteFiber(t->fiber); #else - free_stack(stk, t->ssize); -#endif + free_stack(stk, t->ssize); #endif } } @@ -810,23 +807,32 @@ void jl_init_tasks(void) failed_sym = jl_symbol("failed"); runnable_sym = jl_symbol("runnable"); - jl_unprotect_stack_func = jl_box_voidpointer(&jl_unprotect_stack); + jl_task_cleanup_func = jl_box_voidpointer(&jl_task_cleanup); } #ifdef _OS_WINDOWS_ -static VOID NOINLINE NORETURN CALLBACK start_fiber(PVOID lpParameter) -{ #ifdef COPY_STACKS +static VOID NOINLINE NORETURN CALLBACK start_basefiber(PVOID lpParameter) +{ jl_stackbase = &lpParameter; if (jl_setjmp(jl_basectx, 0)) start_task(); SwitchToFiber(jl_current_task->fiber); start_task(); -#else +} +#endif +static VOID NOINLINE NORETURN CALLBACK start_fiber(PVOID lpParameter) +{ jl_current_task->stkbuf = &lpParameter; start_task(); -#endif +} +static void jl_new_fiber(jl_task_t *t, size_t ssize) +{ + LPVOID jl_fiber = CreateFiberEx(ssize, ssize, FIBER_FLAG_FLOAT_SWITCH, start_fiber, NULL); + if (jl_fiber == NULL) + jl_error("CreateFiberEx failed"); + t->fiber = jl_fiber; } static void jl_init_basefiber(size_t ssize) { @@ -834,7 +840,7 @@ static void jl_init_basefiber(size_t ssize) if (jl_current_task->fiber == NULL) jl_error("ConvertThreadToFiberEx failed"); #ifdef COPY_STACKS - jl_basefiber = CreateFiberEx(ssize, ssize, FIBER_FLAG_FLOAT_SWITCH, start_fiber, NULL); + jl_basefiber = CreateFiberEx(ssize, ssize, FIBER_FLAG_FLOAT_SWITCH, start_basefiber, NULL); if (jl_basefiber == NULL) jl_error("CreateFiberEx failed"); SwitchToFiber(jl_basefiber); /* initializes jl_stackbase and jl_basectx */ @@ -852,9 +858,8 @@ static void start_basefiber() abort(); // function not used #endif } -static void jl_new_fiber(jl_task_t *t) +static void jl_new_fiber(jl_task_t *t, size_t ssize) { - size_t ssize = t->ssize; char *stk = malloc_stack(ssize); jl_base_uctx.uc_stack.ss_sp = stk; jl_base_uctx.uc_stack.ss_size = ssize; @@ -879,10 +884,7 @@ static void jl_init_basefiber(size_t ssize) if (r != 0) jl_error("getcontext failed"); #ifdef COPY_STACKS - size_t oldssize = jl_root_task->ssize; - jl_root_task->ssize = ssize; - jl_new_fiber(jl_root_task); - jl_root_task->ssize = oldssize; + jl_new_fiber(jl_root_task, ssize); #endif } #endif @@ -910,9 +912,8 @@ static void start_basefiber() #else #error please define how to simulate a CALL on this platform #endif -static void jl_new_fiber(jl_task_t *t) +static void jl_new_fiber(jl_task_t *t, size_t ssize) { - size_t ssize = t->ssize; char *stkbuf = malloc_stack(ssize); char *stk = stkbuf; stk += ssize; @@ -942,13 +943,8 @@ static void jl_init_basefiber(size_t ssize) if (r != 0) jl_error("unw_init_local failed"); #ifdef COPY_STACKS - size_t oldssize = jl_root_task->ssize; - if (jl_setjmp(jl_root_task->ctx, 0)) { - jl_root_task->ssize = oldssize; - return; - } - jl_root_task->ssize = ssize; - jl_new_fiber(jl_root_task); // (doesn't return) + if (jl_setjmp(jl_root_task->ctx, 0)) return; + jl_new_fiber(jl_root_task, ssize); // (doesn't return) #endif } #endif @@ -964,9 +960,8 @@ static void start_basefiber() abort(); // function not used #endif } -static void jl_new_fiber(jl_task_t *t) +static void jl_new_fiber(jl_task_t *t, size_t ssize) { - size_t ssize = t->ssize; char *stkbuf = malloc_stack(ssize); char *stk = stkbuf; stk += ssize; @@ -1008,13 +1003,8 @@ static void jl_new_fiber(jl_task_t *t) static void jl_init_basefiber(size_t ssize) { #ifdef COPY_STACKS - size_t oldssize = jl_root_task->ssize; - if (jl_setjmp(jl_root_task->ctx, 0)) { - jl_root_task->ssize = oldssize; - return; - } - jl_root_task->ssize = ssize; - jl_new_fiber(jl_root_task); // (doesn't return) + if (jl_setjmp(jl_root_task->ctx, 0)) return; + jl_new_fiber(jl_root_task, ssize); // (doesn't return) #endif } #endif @@ -1025,12 +1015,11 @@ static void start_fiber() if (jl_setjmp(jl_current_task->ctx, 0)) start_task(); } -static void jl_new_fiber(jl_task_t *t) +static void jl_new_fiber(jl_task_t *t, size_t ssize) { stack_t uc_stack, osigstk; struct sigaction sa, osa; sigset_t set, oset; - size_t ssize = t->ssize; char *stk = malloc_stack(ssize); // setup sigfillset(&set); @@ -1070,10 +1059,7 @@ static void jl_new_fiber(jl_task_t *t) static void jl_init_basefiber(size_t ssize) { #ifdef COPY_STACKS - size_t oldssize = jl_root_task->ssize; - jl_root_task->ssize = ssize; - jl_new_fiber(jl_root_task); - jl_root_task->ssize = oldssize; + jl_new_fiber(jl_root_task, ssize); memcpy(jl_basectx, jl_current_task->ctx, sizeof(jl_basectx)); #endif } @@ -1085,13 +1071,9 @@ void jl_init_root_task(void *stack, size_t ssize) jl_current_task = (jl_task_t*)jl_gc_allocobj(sizeof(jl_task_t)); jl_set_typeof(jl_current_task, jl_task_type); -#ifdef COPY_STACKS - jl_current_task->stkbuf = NULL; // address of stack save location - jl_current_task->ssize = 0; // size of saved piece -#else - jl_current_task->stkbuf = stack - ssize; // address of bottom of stack - jl_current_task->ssize = ssize; // sizeof stack -#endif + jl_current_task->copy_stack = 0; + jl_current_task->stkbuf = stack - ssize - 3000000; // guess at address of bottom of stack + jl_current_task->ssize = ssize + 3000000; // guess at sizeof stack jl_current_task->parent = jl_current_task; jl_current_task->current_module = jl_current_module; jl_current_task->tls = jl_nothing;