diff --git a/src/gc-stacks.c b/src/gc-stacks.c index 71668b3d4c2ff..934dac2d7d6c6 100644 --- a/src/gc-stacks.c +++ b/src/gc-stacks.c @@ -216,8 +216,12 @@ void sweep_stack_pools(void) continue; while (1) { jl_task_t *t = (jl_task_t*)lst[n]; + assert(jl_is_task(t)); if (gc_marked(jl_astaggedvalue(t)->bits.gc)) { - n++; + if (t->stkbuf == NULL) + ndel++; // jl_release_task_stack called + else + n++; } else { ndel++; @@ -243,3 +247,30 @@ void sweep_stack_pools(void) live_tasks->len -= ndel; } } + +JL_DLLEXPORT jl_array_t *jl_live_tasks(void) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + arraylist_t *live_tasks = &ptls->heap.live_tasks; + size_t i, j, l; + jl_array_t *a; + do { + l = live_tasks->len; + a = jl_alloc_vec_any(l + 1); // may gc + } while (l + 1 < live_tasks->len); + l = live_tasks->len; + void **lst = live_tasks->items; + j = 0; + ((void**)jl_array_data(a))[j++] = ptls->root_task; + for (i = 0; i < l; i++) { + if (((jl_task_t*)lst[i])->stkbuf != NULL) + ((void**)jl_array_data(a))[j++] = lst[i]; + } + l = jl_array_len(a); + if (j < l) { + JL_GC_PUSH1(&a); + jl_array_del_end(a, l - j); + JL_GC_POP(); + } + return a; +} diff --git a/src/julia.expmap b/src/julia.expmap index baf3220f147ed..d9d255cc8c547 100644 --- a/src/julia.expmap +++ b/src/julia.expmap @@ -32,6 +32,7 @@ add_library_mapping; utf8proc_*; jlbacktrace; + jlbacktracet; julia_type_to_llvm; _IO_stdin_used; __ZN4llvm23createLowerSimdLoopPassEv; diff --git a/src/julia.h b/src/julia.h index 66d32af6c4709..8cd070885c6fd 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1762,7 +1762,17 @@ typedef struct _jl_task_t { // current exception handler jl_handler_t *eh; - jl_ucontext_t ctx; // saved thread state + union { + jl_ucontext_t ctx; // saved thread state +#ifdef _OS_WINDOWS_ + jl_ucontext_t copy_stack_ctx; +#else + struct jl_stack_context_t copy_stack_ctx; +#endif + }; +#if defined(JL_TSAN_ENABLED) + void *tsan_state; +#endif void *stkbuf; // malloc'd memory (either copybuf or stack) size_t bufsz; // actual sizeof stkbuf unsigned int copy_stack:31; // sizeof stack for copybuf diff --git a/src/julia_threads.h b/src/julia_threads.h index 1eaf0e0035f52..4216b2012cedb 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -16,8 +16,9 @@ // Options for task switching algorithm (in order of preference): // JL_HAVE_ASM -- mostly setjmp -// JL_HAVE_ASYNCIFY -- task switching based on the binaryen asyncify transform -// JL_HAVE_UNW_CONTEXT -- hybrid of libunwind for start, setjmp for resume +// JL_HAVE_ASM && JL_HAVE_UNW_CONTEXT -- libunwind-based +// JL_HAVE_UNW_CONTEXT -- libunwind-based +// JL_HAVE_ASYNCIFY -- task switching based on the binary asyncify transform // JL_HAVE_UCONTEXT -- posix standard API, requires syscall for resume // JL_HAVE_SIGALTSTACK -- requires several syscall for start, setjmp for resume @@ -33,24 +34,25 @@ typedef win32_ucontext_t jl_ucontext_t; #if (defined(_CPU_X86_64_) || defined(_CPU_X86_) || defined(_CPU_AARCH64_) || \ defined(_CPU_ARM_) || defined(_CPU_PPC64_)) #define JL_HAVE_ASM -#elif defined(_OS_DARWIN_) +#endif +#if defined(_OS_DARWIN_) #define JL_HAVE_UNW_CONTEXT -#elif defined(_OS_LINUX_) -#define JL_HAVE_UCONTEXT +//#elif defined(_OS_LINUX_) +//#define JL_HAVE_UNW_CONTEXT // very slow, but more debugging #elif defined(_OS_EMSCRIPTEN_) #define JL_HAVE_ASYNCIFY -#else -#define JL_HAVE_UNW_CONTEXT +#elif !defined(JL_HAVE_ASM) +#define JL_HAVE_UNW_CONTEXT // optimistically? #endif #endif -#if defined(JL_HAVE_ASM) || defined(JL_HAVE_SIGALTSTACK) -typedef struct { + +struct jl_stack_context_t { jl_jmp_buf uc_mcontext; -#if defined(JL_TSAN_ENABLED) - void *tsan_state; -#endif -} jl_ucontext_t; +}; + +#if (!defined(JL_HAVE_UNW_CONTEXT) && defined(JL_HAVE_ASM)) || defined(JL_HAVE_SIGALTSTACK) +typedef struct jl_stack_context_t jl_ucontext_t; #endif #if defined(JL_HAVE_ASYNCIFY) #if defined(JL_TSAN_ENABLED) @@ -65,15 +67,14 @@ typedef struct { void *stacktop; } jl_ucontext_t; #endif -#if defined(JL_HAVE_UCONTEXT) || defined(JL_HAVE_UNW_CONTEXT) +#if defined(JL_HAVE_UNW_CONTEXT) #define UNW_LOCAL_ONLY #include -typedef struct { - ucontext_t ctx; -#if defined(JL_TSAN_ENABLED) - void *tsan_state; +typedef unw_context_t jl_ucontext_t; #endif -} jl_ucontext_t; +#if defined(JL_HAVE_UCONTEXT) +#include +typedef ucontext_t jl_ucontext_t; #endif #endif @@ -210,7 +211,15 @@ struct _jl_tls_states_t { struct _jl_timing_block_t *timing_stack; void *stackbase; size_t stacksize; - jl_ucontext_t base_ctx; // base context of stack + union { + jl_ucontext_t base_ctx; // base context of stack + // This hack is needed to support always_copy_stacks: +#ifdef _OS_WINDOWS_ + jl_ucontext_t copy_stack_ctx; +#else + struct jl_stack_context_t copy_stack_ctx; +#endif + }; jl_jmp_buf *safe_restore; // Temp storage for exception thrown in signal handler. Not rooted. struct _jl_value_t *sig_exception; diff --git a/src/partr.c b/src/partr.c index d12ee54207e59..d2f1d90b444b7 100644 --- a/src/partr.c +++ b/src/partr.c @@ -40,7 +40,7 @@ uint64_t io_wakeup_leave; JL_DLLEXPORT int jl_set_task_tid(jl_task_t *task, int tid) JL_NOTSAFEPOINT { // Try to acquire the lock on this task. - int16_t was = task->tid; + int16_t was = jl_atomic_load_relaxed(&task->tid); if (was == tid) return 1; if (was == -1) diff --git a/src/signals-unix.c b/src/signals-unix.c index 57ce2439fcb90..f89d32f09486a 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -43,6 +43,7 @@ #include "julia_assert.h" +// helper function for returning the unw_context_t inside a ucontext_t static bt_context_t *jl_to_bt_context(void *sigctx) { #ifdef __APPLE__ diff --git a/src/stackwalk.c b/src/stackwalk.c index 9150d48b29765..4b0e832664031 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -14,11 +14,11 @@ // returning from the callee function will invalidate the context #ifdef _OS_WINDOWS_ jl_mutex_t jl_in_stackwalk; -#define jl_unw_get(context) RtlCaptureContext(context) +#define jl_unw_get(context) (RtlCaptureContext(context), 0) #elif !defined(JL_DISABLE_LIBUNWIND) #define jl_unw_get(context) unw_getcontext(context) #else -void jl_unw_get(void *context) {}; +int jl_unw_get(void *context) { return -1; } #endif #ifdef __cplusplus @@ -204,7 +204,9 @@ NOINLINE size_t rec_backtrace(jl_bt_element_t *bt_data, size_t maxsize, int skip { bt_context_t context; memset(&context, 0, sizeof(context)); - jl_unw_get(&context); + int r = jl_unw_get(&context); + if (r < 0) + return 0; jl_gcframe_t *pgcstack = jl_pgcstack; bt_cursor_t cursor; if (!jl_unw_init(&cursor, &context)) @@ -239,9 +241,9 @@ JL_DLLEXPORT jl_value_t *jl_backtrace_from_here(int returnsp, int skip) bt_context_t context; bt_cursor_t cursor; memset(&context, 0, sizeof(context)); - jl_unw_get(&context); + int r = jl_unw_get(&context); jl_gcframe_t *pgcstack = jl_pgcstack; - if (jl_unw_init(&cursor, &context)) { + if (r == 0 && jl_unw_init(&cursor, &context)) { // Skip frame for jl_backtrace_from_here itself skip += 1; size_t offset = 0; @@ -688,8 +690,59 @@ void jl_print_bt_entry_codeloc(jl_bt_element_t *bt_entry) JL_NOTSAFEPOINT } } +extern bt_context_t *jl_to_bt_context(void *sigctx); + +void jl_rec_backtrace(jl_task_t *t) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + ptls->bt_size = 0; + if (t == ptls->current_task) { + ptls->bt_size = rec_backtrace(ptls->bt_data, JL_MAX_BT_SIZE, 0); + return; + } + if (t->copy_stack || !t->started || t->stkbuf == NULL) + return; + int old = jl_atomic_compare_exchange(&t->tid, -1, ptls->tid); + if (old != -1 && old != ptls->tid) + return; + bt_context_t *context = NULL; +#if defined(_OS_WINDOWS_) + bt_context_t c; + memset(&c, 0, sizeof(c)); + _JUMP_BUFFER *mctx = (_JUMP_BUFFER*)&t->ctx.uc_mcontext; +#if defined(_CPU_X86_64_) + c.Rbx = mctx->Rbx; + c.Rsp = mctx->Rsp; + c.Rbp = mctx->Rbp; + c.Rsi = mctx->Rsi; + c.Rdi = mctx->Rdi; + c.R12 = mctx->R12; + c.R13 = mctx->R13; + c.R14 = mctx->R14; + c.R15 = mctx->R15; + c.Rip = mctx->Rip; + memcpy(&c.Xmm6, &mctx->Xmm6, 10 * sizeof(mctx->Xmm6)); // Xmm6-Xmm15 +#else + c.Eip = mctx->Eip; + c.Esp = mctx->Esp; + c.Ebp = mctx->Ebp; +#endif + context = &c; +#elif defined(JL_HAVE_UNW_CONTEXT) + context = &t->ctx; +#elif defined(JL_HAVE_UCONTEXT) + context = jl_to_bt_context(&t->ctx); +#else +#endif + if (context) + ptls->bt_size = rec_backtrace_ctx(ptls->bt_data, JL_MAX_BT_SIZE, context, t->gcstack); + if (old == -1) + jl_atomic_store_relaxed(&t->tid, old); +} + //-------------------------------------------------- // Tools for interactive debugging in gdb + JL_DLLEXPORT void jl_gdblookup(void* ip) { jl_print_native_codeloc((uintptr_t)ip); @@ -701,9 +754,19 @@ JL_DLLEXPORT void jlbacktrace(void) JL_NOTSAFEPOINT jl_excstack_t *s = jl_get_ptls_states()->current_task->excstack; if (!s) return; - size_t bt_size = jl_excstack_bt_size(s, s->top); + size_t i, bt_size = jl_excstack_bt_size(s, s->top); jl_bt_element_t *bt_data = jl_excstack_bt_data(s, s->top); - for (size_t i = 0; i < bt_size; i += jl_bt_entry_size(bt_data + i)) { + for (i = 0; i < bt_size; i += jl_bt_entry_size(bt_data + i)) { + jl_print_bt_entry_codeloc(bt_data + i); + } +} +JL_DLLEXPORT void jlbacktracet(jl_task_t *t) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + jl_rec_backtrace(t); + size_t i, bt_size = ptls->bt_size; + jl_bt_element_t *bt_data = ptls->bt_data; + for (i = 0; i < bt_size; i += jl_bt_entry_size(bt_data + i)) { jl_print_bt_entry_codeloc(bt_data + i); } } diff --git a/src/task.c b/src/task.c index aa2e02c90e045..0927b685cb5ce 100644 --- a/src/task.c +++ b/src/task.c @@ -53,24 +53,15 @@ static inline void sanitizer_finish_switch_fiber(void) {} #endif #if defined(JL_TSAN_ENABLED) -static inline void tsan_destroy_ctx(jl_ptls_t ptls, jl_ucontext_t *ctx) { - if (ctx != &ptls->root_task->ctx) { - __tsan_destroy_fiber(ctx->tsan_state); +static inline void tsan_destroy_ctx(jl_ptls_t ptls, void *state) { + if (state != &ptls->root_task->state) { + __tsan_destroy_fiber(ctx->state); } - ctx->tsan_state = NULL; + ctx->state = NULL; } -static inline void tsan_switch_to_ctx(jl_ucontext_t *ctx) { - __tsan_switch_to_fiber(ctx->tsan_state, 0); +static inline void tsan_switch_to_ctx(void *state) { + __tsan_switch_to_fiber(state, 0); } -#else -static inline void tsan_destroy_ctx(jl_ptls_t ptls, jl_ucontext_t *ctx) {} -static inline void tsan_switch_to_ctx(jl_ucontext_t *ctx) {} -#endif - -#if !defined(_OS_WINDOWS_) -#ifdef JL_HAVE_UCONTEXT -#include -#endif #endif // empirically, jl_finish_task needs about 64k stack space to infer/run @@ -97,10 +88,6 @@ STATIC_OR_JS void jl_swap_fiber(jl_ucontext_t *lastt, jl_ucontext_t *t); STATIC_OR_JS void jl_start_fiber_swap(jl_ucontext_t *savet, jl_ucontext_t *t); STATIC_OR_JS void jl_start_fiber_set(jl_ucontext_t *t); -#ifdef JL_HAVE_UNW_CONTEXT -static JL_THREAD_LOCAL unw_cursor_t jl_basecursor; -#endif - #ifdef ALWAYS_COPY_STACKS # ifndef COPY_STACKS # error "ALWAYS_COPY_STACKS requires COPY_STACKS" @@ -111,7 +98,6 @@ static int always_copy_stacks = 0; #endif #ifdef COPY_STACKS - static void memcpy_a16(uint64_t *to, uint64_t *from, size_t nb) { memcpy((char*)jl_assume_aligned(to, 16), (char*)jl_assume_aligned(from, 16), nb); @@ -162,19 +148,41 @@ static void NOINLINE JL_NORETURN restore_stack(jl_task_t *t, jl_ptls_t ptls, cha memcpy_a16((uint64_t*)_x, (uint64_t*)_y, nb); // destroys all but the current stackframe sanitizer_start_switch_fiber(t->stkbuf, t->bufsz); - jl_set_fiber(&t->ctx); +#if defined(_OS_WINDOWS_) + jl_setcontext(&t->ctx); +#else + jl_longjmp(t->copy_stack_ctx.uc_mcontext, 1); +#endif abort(); // unreachable } + static void restore_stack2(jl_task_t *t, jl_ptls_t ptls, jl_task_t *lastt) { + assert(t->copy_stack && !lastt->copy_stack); size_t nb = t->copy_stack; char *_x = (char*)ptls->stackbase - nb; void *_y = t->stkbuf; assert(_x != NULL && _y != NULL); memcpy_a16((uint64_t*)_x, (uint64_t*)_y, nb); // destroys all but the current stackframe +#if defined(JL_HAVE_UNW_CONTEXT) + volatile int returns = 0; + int r = unw_getcontext(&lastt->ctx); + if (++returns == 2) // r is garbage after the first return + return; + if (r != 0 || returns != 1) + abort(); +#elif defined(JL_HAVE_ASM) || defined(JL_HAVE_SIGALTSTACK) || defined(_OS_WINDOWS_) + if (jl_setjmp(lastt->copy_stack_ctx.uc_mcontext, 0)) + return; +#else +#error COPY_STACKS is incompatible with this platform +#endif sanitizer_start_switch_fiber(t->stkbuf, t->bufsz); - jl_swap_fiber(&lastt->ctx, &t->ctx); - sanitizer_finish_switch_fiber(); +#if defined(_OS_WINDOWS_) + jl_setcontext(&t->ctx); +#else + jl_longjmp(t->copy_stack_ctx.uc_mcontext, 1); +#endif } #endif @@ -356,11 +364,10 @@ static void ctx_switch(jl_ptls_t ptls) t->copy_stack = 1; t->sticky = 1; t->bufsz = 0; -#ifdef JL_TSAN_ENABLED - memcpy(&t->ctx, &ptls->base_ctx, sizeof(t->ctx) - sizeof(t->ctx.tsan_state)); -#else - memcpy(&t->ctx, &ptls->base_ctx, sizeof(t->ctx)); -#endif + if (always_copy_stacks) + memcpy(&t->copy_stack_ctx, &ptls->copy_stack_ctx, sizeof(t->copy_stack_ctx)); + else + memcpy(&t->ctx, &ptls->base_ctx, sizeof(t->ctx)); #else jl_throw(jl_memory_exception); #endif @@ -380,7 +387,7 @@ static void ctx_switch(jl_ptls_t ptls) #ifdef COPY_STACKS if (lastt->copy_stack) { // save the old copy-stack save_stack(ptls, lastt, pt); // allocates (gc-safepoint, and can also fail) - if (jl_setjmp(lastt->ctx.uc_mcontext, 0)) { + if (jl_setjmp(lastt->copy_stack_ctx.uc_mcontext, 0)) { sanitizer_finish_switch_fiber(); // TODO: mutex unlock the thread we just switched from return; @@ -401,21 +408,20 @@ static void ctx_switch(jl_ptls_t ptls) #endif ptls->current_task = t; +#if defined(JL_TSAN_ENABLED) + tsan_switch_to_ctx(&t->tsan_state); + if (killed) + tsan_destroy_ctx(ptls, &lastt->tsan_state); +#endif if (t->started) { #ifdef COPY_STACKS if (t->copy_stack) { if (!killed && !lastt->copy_stack) restore_stack2(t, ptls, lastt); else if (lastt->copy_stack) { - tsan_switch_to_ctx(&t->ctx); - if (killed) - tsan_destroy_ctx(ptls, &lastt->ctx); - restore_stack(t, ptls, NULL); // (doesn't return) + restore_stack(t, ptls, NULL); // (doesn't return) } else { - tsan_switch_to_ctx(&t->ctx); - if (killed) - tsan_destroy_ctx(ptls, &lastt->ctx); restore_stack(t, ptls, (char*)1); // (doesn't return) } } @@ -424,8 +430,6 @@ static void ctx_switch(jl_ptls_t ptls) { sanitizer_start_switch_fiber(t->stkbuf, t->bufsz); if (killed) { - tsan_switch_to_ctx(&t->ctx); - tsan_destroy_ctx(ptls, &lastt->ctx); jl_set_fiber(&t->ctx); // (doesn't return) abort(); // unreachable } @@ -433,49 +437,42 @@ static void ctx_switch(jl_ptls_t ptls) if (lastt->copy_stack) { // Resume at the jl_setjmp earlier in this function, // don't do a full task swap - tsan_switch_to_ctx(&t->ctx); jl_set_fiber(&t->ctx); // (doesn't return) } else { jl_swap_fiber(&lastt->ctx, &t->ctx); - sanitizer_finish_switch_fiber(); } } } } else { sanitizer_start_switch_fiber(t->stkbuf, t->bufsz); - if (always_copy_stacks) { - tsan_switch_to_ctx(&t->ctx); - if (killed) { - tsan_destroy_ctx(ptls, &lastt->ctx); - } + if (t->copy_stack && always_copy_stacks) { #ifdef COPY_STACKS - jl_longjmp(ptls->base_ctx.uc_mcontext, 1); - abort(); // unreachable +#if defined(_OS_WINDOWS_) + jl_setcontext(&t->ctx); #else - abort(); // Should never happen + jl_longjmp(t->copy_stack_ctx.uc_mcontext, 1); +#endif #endif + abort(); // unreachable } else { if (killed) { - tsan_switch_to_ctx(&t->ctx); - tsan_destroy_ctx(ptls, &lastt->ctx); jl_start_fiber_set(&t->ctx); // (doesn't return) abort(); } else if (lastt->copy_stack) { // Resume at the jl_setjmp earlier in this function - tsan_switch_to_ctx(&t->ctx); jl_start_fiber_set(&t->ctx); // (doesn't return) abort(); } else { jl_start_fiber_swap(&lastt->ctx, &t->ctx); - sanitizer_finish_switch_fiber(); } } } + sanitizer_finish_switch_fiber(); } static jl_ptls_t NOINLINE refetch_ptls(void) @@ -714,11 +711,13 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, jl_value_t *completion memset(&t->ctx, 0, sizeof(t->ctx)); #endif #ifdef COPY_STACKS - if (t->copy_stack) + if (always_copy_stacks) + memcpy(&t->copy_stack_ctx, &ptls->copy_stack_ctx, sizeof(t->copy_stack_ctx)); + else if (t->copy_stack) memcpy(&t->ctx, &ptls->base_ctx, sizeof(t->ctx)); #endif #ifdef JL_TSAN_ENABLED - t->ctx.tsan_state = __tsan_create_fiber(0); + t->tsan_state = __tsan_create_fiber(0); #endif return t; } @@ -789,10 +788,16 @@ void jl_init_tasks(void) JL_GC_DISABLED else if (!strcmp(acs, "0") || !strcmp(acs, "no")) always_copy_stacks = 0; else { - jl_printf(JL_STDERR, "invalid JULIA_COPY_STACKS value: %s\n", acs); + jl_safe_printf("invalid JULIA_COPY_STACKS value: %s\n", acs); exit(1); } } +#ifndef COPY_STACKS + if (always_copy_stacks) { + jl_safe_printf("Julia built without COPY_STACKS support"); + exit(1); + } +#endif } STATIC_OR_JS void NOINLINE JL_NORETURN start_task(void) @@ -853,7 +858,6 @@ skip_pop_exception:; #if defined(JL_HAVE_UCONTEXT) #ifdef _OS_WINDOWS_ #define setcontext jl_setcontext -#define getcontext jl_getcontext #define swapcontext jl_swapcontext #define makecontext jl_makecontext #endif @@ -884,136 +888,155 @@ static void jl_start_fiber_set(jl_ucontext_t *t) static void jl_start_fiber_swap(jl_ucontext_t *lastt, jl_ucontext_t *t) { assert(lastt); - tsan_switch_to_ctx(t); swapcontext(lastt, t); } static void jl_swap_fiber(jl_ucontext_t *lastt, jl_ucontext_t *t) { - tsan_switch_to_ctx(t); swapcontext(lastt, t); } static void jl_set_fiber(jl_ucontext_t *t) { setcontext(t); } -static void jl_init_basefiber(size_t ssize) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - char *stkbuf = jl_alloc_fiber(&ptls->base_ctx, &ssize, NULL); - ptls->stackbase = stkbuf + ssize; - ptls->stacksize = ssize; -} #endif -#if defined(JL_HAVE_UNW_CONTEXT) -static void start_basefiber(void) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - if (jl_setjmp(ptls->base_ctx.uc_mcontext, 0)) - start_task(); // sanitizer_finish_switch_fiber is part of start_task - sanitizer_start_switch_fiber(jl_root_task->stkbuf, jl_root_task->bufsz); - tsan_switch_to_ctx(&jl_root_task->ctx); - jl_longjmp(jl_root_task->ctx.uc_mcontext, 1); - abort(); // unreachable -} -#if defined(_CPU_X86_) || defined(_CPU_X86_64_) -#define PUSH_RET(ctx, stk) \ - do { \ - stk -= sizeof(uintptr_t); \ - *(uintptr_t*)stk = 0; /* push null RIP/EIP onto the stack */ \ - } while (0) -#elif defined(_CPU_ARM_) -#define PUSH_RET(ctx, stk) \ - unw_set_reg(ctx, UNW_ARM_R14, 0) /* put NULL into the LR */ -#else -#error please define how to simulate a CALL on this platform -#endif +#if defined(JL_HAVE_UNW_CONTEXT) || defined(JL_HAVE_ASM) static char *jl_alloc_fiber(jl_ucontext_t *t, size_t *ssize, jl_task_t *owner) { char *stkbuf = (char*)jl_malloc_stack(ssize, owner); if (stkbuf == NULL) return NULL; - char *stk = stkbuf; - stk += *ssize; - PUSH_RET(&jl_basecursor, stk); - if (unw_set_reg(&jl_basecursor, UNW_REG_SP, (uintptr_t)stk) != 0) { - jl_free_stack((void*)stkbuf, *ssize); - jl_error("unw_set_reg UNW_REG_SP failed"); - } - uintptr_t fn; - if (t == &ptls->base_ctx) - fn = (uintptr_t)&start_basefiber; - else - fn = (uintptr_t)&start_task; - if (unw_set_reg(&jl_basecursor, UNW_REG_IP, fn) != 0) { - jl_free_stack((void*)stkbuf, *ssize); - jl_error("unw_set_reg UNW_REG_IP failed"); - } +#ifndef __clang_analyzer__ + ((char**)t)[0] = stkbuf; // stash the stack pointer somewhere for start_fiber + ((size_t*)t)[1] = *ssize; // stash the stack size somewhere for start_fiber +#endif return stkbuf; } +#endif -static void jl_start_fiber_set(jl_ucontext_t *t) +#if defined(JL_HAVE_UNW_CONTEXT) +static inline void jl_unw_swapcontext(unw_context_t *old, unw_cursor_t *c) { - unw_resume(&jl_basecursor); // (doesn't return) + volatile int returns = 0; + int r = unw_getcontext(old); + if (++returns == 2) // r is garbage after the first return + return; + if (r != 0 || returns != 1) + abort(); + unw_resume(c); } - -static void jl_start_fiber_swap(jl_ucontext_t *lastt, jl_ucontext_t *t) +static void jl_swap_fiber(jl_ucontext_t *lastt, jl_ucontext_t *t) { - assert(lastt); - if (jl_setjmp(lastt->uc_mcontext, 0)) - return; - tsan_switch_to_ctx(t); - jl_start_fiber_set(t); // doesn't return + unw_cursor_t c; + int r = unw_init_local(&c, t); + if (r < 0) + abort(); + jl_unw_swapcontext(lastt, &c); } - +static void jl_set_fiber(unw_context_t *t) +{ + unw_cursor_t c; + int r = unw_init_local(&c, t); + if (r < 0) + abort(); + unw_resume(&c); +} +#elif defined(JL_HAVE_ASM) static void jl_swap_fiber(jl_ucontext_t *lastt, jl_ucontext_t *t) { if (jl_setjmp(lastt->uc_mcontext, 0)) return; - tsan_switch_to_ctx(t); - jl_longjmp(t->uc_mcontext, 1); // (doesn't return) + jl_set_fiber(t); // doesn't return } static void jl_set_fiber(jl_ucontext_t *t) { - tsan_switch_to_ctx(t); jl_longjmp(t->uc_mcontext, 1); } -static void jl_init_basefiber(size_t ssize) +#endif + +#if defined(JL_HAVE_UNW_CONTEXT) && !defined(JL_HAVE_ASM) +#if defined(_CPU_X86_) || defined(_CPU_X86_64_) +#define PUSH_RET(ctx, stk) \ + do { \ + stk -= sizeof(uintptr_t); \ + *(uintptr_t*)stk = 0; /* push null RIP/EIP onto the stack */ \ + } while (0) +#elif defined(_CPU_ARM_) +#define PUSH_RET(ctx, stk) \ + if (unw_set_reg(ctx, UNW_ARM_R14, 0)) /* put NULL into the LR */ \ + abort(); +#else +#error please define how to simulate a CALL on this platform +#endif +static void jl_start_fiber_set(unw_context_t *t) { - int r = unw_getcontext(&ptls->base_ctx); - if (r != 0) - jl_error("unw_getcontext failed"); - r = unw_init_local(&jl_basecursor, &ptls->base_ctx); - if (r != 0) - jl_error("unw_init_local failed"); -#ifdef COPY_STACKS - jl_ptls_t ptls = jl_get_ptls_states(); - char *stkbuf = jl_alloc_fiber(&ptls->base_ctx, &ssize, NULL); - ptls->stackbase = stkbuf + ssize; - ptls->stacksize = ssize; - sanitizer_start_switch_fiber(stkbuf, sksize); - jl_start_fiber_swap(jl_root_task, &ptls->base_ctx); // finishes initializing jl_basectx - sanitizer_finish_switch_fiber(); + unw_cursor_t c; + char *stk = ((char**)t)[0]; + size_t ssize = ((size_t*)t)[1]; + uintptr_t fn = (uintptr_t)&start_task; + stk += ssize; + int r = unw_getcontext(t); + if (r) + abort(); + if (unw_init_local(&c, t)) + abort(); + PUSH_RET(&c, stk); +#if defined __linux__ +#error savannah nongnu libunwind is not capable of setting UNW_REG_SP, as required #endif + if (unw_set_reg(&c, UNW_REG_SP, (uintptr_t)stk)) + abort(); + if (unw_set_reg(&c, UNW_REG_IP, fn)) + abort(); + unw_resume(&c); // (doesn't return) +} +static void jl_start_fiber_swap(unw_context_t *lastt, unw_context_t *t) +{ + assert(lastt); + unw_cursor_t c; + char *stk = ((char**)t)[0]; + size_t ssize = ((size_t*)t)[1]; + uintptr_t fn = (uintptr_t)&start_task; + stk += ssize; + volatile int returns = 0; + int r = unw_getcontext(lastt); + if (++returns == 2) // r is garbage after the first return + return; + if (r != 0 || returns != 1) + abort(); + int r = unw_getcontext(t); + if (r != 0) + abort(); + if (unw_init_local(&c, t)) + abort(); + PUSH_RET(&c, stk); + if (unw_set_reg(&c, UNW_REG_SP, (uintptr_t)stk)) + abort(); + if (unw_set_reg(&c, UNW_REG_IP, fn)) + abort(); + jl_unw_swapcontext(lastt, &c); } #endif #if defined(JL_HAVE_ASM) -static char *jl_alloc_fiber(jl_ucontext_t *t, size_t *ssize, jl_task_t *owner) +static void jl_start_fiber_swap(jl_ucontext_t *lastt, jl_ucontext_t *t) { - char *stkbuf = (char*)jl_malloc_stack(ssize, owner); - if (stkbuf == NULL) - return NULL; -#ifndef __clang_analyzer__ - ((char**)t)[0] = stkbuf; // stash the stack pointer somewhere for start_fiber - ((size_t*)t)[1] = *ssize; // stash the stack size somewhere for start_fiber + assert(lastt); +#ifdef JL_HAVE_UNW_CONTEXT + volatile int returns = 0; + int r = unw_getcontext(lastt); + if (++returns == 2) // r is garbage after the first return + return; + if (r != 0 || returns != 1) + abort(); +#else + if (jl_setjmp(lastt->uc_mcontext, 0)) + return; #endif - return stkbuf; + jl_start_fiber_set(t); // doesn't return } - static void jl_start_fiber_set(jl_ucontext_t *t) { - char *stk = ((char**)t)[0]; size_t ssize = ((size_t*)t)[1]; uintptr_t fn = (uintptr_t)&start_task; @@ -1082,36 +1105,6 @@ static void jl_start_fiber_set(jl_ucontext_t *t) #endif __builtin_unreachable(); } - -static void jl_start_fiber_swap(jl_ucontext_t *lastt, jl_ucontext_t *t) -{ - assert(lastt); - if (jl_setjmp(lastt->uc_mcontext, 0)) - return; - tsan_switch_to_ctx(t); - jl_start_fiber_set(t); -} - -static void jl_swap_fiber(jl_ucontext_t *lastt, jl_ucontext_t *t) -{ - if (jl_setjmp(lastt->uc_mcontext, 0)) - return; - tsan_switch_to_ctx(t); - jl_longjmp(t->uc_mcontext, 1); // (doesn't return) -} -static void jl_set_fiber(jl_ucontext_t *t) -{ - jl_longjmp(t->uc_mcontext, 1); -} -static void jl_init_basefiber(size_t ssize) -{ -#ifdef COPY_STACKS - jl_ptls_t ptls = jl_get_ptls_states(); - char *stkbuf = jl_alloc_fiber(&ptls->base_ctx, &ssize, NULL); - ptls->stackbase = stkbuf + ssize; - ptls->stacksize = ssize; -#endif -} #endif #if defined(JL_HAVE_SIGALTSTACK) @@ -1119,7 +1112,7 @@ static void jl_init_basefiber(size_t ssize) #error TSAN support not currently implemented for this tasking model #endif -static void start_basefiber(void) +static void start_basefiber(int sig) { jl_ptls_t ptls = jl_get_ptls_states(); if (jl_setjmp(ptls->base_ctx.uc_mcontext, 0)) @@ -1134,6 +1127,7 @@ static char *jl_alloc_fiber(jl_ucontext_t *t, size_t *ssize, jl_task_t *owner) if (stk == NULL) return NULL; // setup + jl_ptls_t ptls = jl_get_ptls_states(); jl_ucontext_t base_ctx; memcpy(&base_ctx, &ptls->base_ctx, sizeof(ptls->base_ctx)); sigfillset(&set); @@ -1175,8 +1169,10 @@ static char *jl_alloc_fiber(jl_ucontext_t *t, size_t *ssize, jl_task_t *owner) jl_free_stack(stk, *ssize); jl_error("sigprocmask failed"); } - memcpy(&t, &ptls->base_ctx, sizeof(ptls->base_ctx)); - memcpy(&ptls->base_ctx, &base_ctx, sizeof(ptls->base_ctx)); + if (&ptls->base_ctx != t) { + memcpy(t, &ptls->base_ctx, sizeof(ptls->base_ctx)); + memcpy(&ptls->base_ctx, &base_ctx, sizeof(ptls->base_ctx)); // restore COPY_STACKS context + } return (char*)stk; } static void jl_start_fiber_set(jl_ucontext_t *t) { @@ -1184,6 +1180,7 @@ static void jl_start_fiber_set(jl_ucontext_t *t) { } static void jl_start_fiber_swap(jl_ucontext_t *lastt, jl_ucontext_t *t) { + assert(lastt); if (lastt && jl_setjmp(lastt->uc_mcontext, 0)) return; jl_start_fiber_set(t); @@ -1192,22 +1189,12 @@ static void jl_swap_fiber(jl_ucontext_t *lastt, jl_ucontext_t *t) { if (jl_setjmp(lastt->uc_mcontext, 0)) return; - jl_longjmp(t->uc_mcontext, 1); // (doesn't return) + jl_start_fiber_set(t); // doesn't return } static void jl_set_fiber(jl_ucontext_t *t) { jl_longjmp(t->uc_mcontext, 1); } -static void jl_init_basefiber(size_t ssize) -{ -#ifdef COPY_STACKS - jl_ptls_t ptls = jl_get_ptls_states(); - char *stkbuf = jl_alloc_fiber(jl_root_task, &ssize, NULL); - ptls->stackbase = stkbuf + ssize; - ptls->stacksize = ssize; - memcpy(&ptls->base_ctx, &jl_root_task->ctx, sizeof(ptls->base_ctx)); -#endif -} #endif #if defined(JL_HAVE_ASYNCIFY) @@ -1215,9 +1202,6 @@ static void jl_init_basefiber(size_t ssize) #error TSAN support not currently implemented for this tasking model #endif -static void jl_init_basefiber(size_t ssize) -{ -} static char *jl_alloc_fiber(jl_ucontext_t *t, size_t *ssize, jl_task_t *owner) JL_NOTSAFEPOINT { void *stk = jl_malloc_stack(ssize, owner); @@ -1274,20 +1258,29 @@ void jl_init_root_task(void *stack_lo, void *stack_hi) ptls->current_task->sticky = 1; #ifdef JL_TSAN_ENABLED - ptls->current_task->ctx.tsan_state = __tsan_get_current_fiber(); + ptls->current_task->tsan_state = __tsan_get_current_fiber(); #endif #ifdef COPY_STACKS + // initialize the base_ctx from which all future copy_stacks will be copies if (always_copy_stacks) { + // when this is set, we will attempt to corrupt the process stack to switch tasks, + // although this is unreliable, and thus not recommended ptls->stackbase = stack_hi; ptls->stacksize = ssize; - if (jl_setjmp(ptls->base_ctx.uc_mcontext, 0)) +#ifdef _OS_WINDOWS_ + ptls->copy_stack_ctx.uc_stack.ss_sp = stack_hi; + ptls->copy_stack_ctx.uc_stack.ss_size = ssize; +#endif + if (jl_setjmp(ptls->copy_stack_ctx.uc_mcontext, 0)) start_task(); // sanitizer_finish_switch_fiber is part of start_task return; } + ssize = JL_STACK_SIZE; + char *stkbuf = jl_alloc_fiber(&ptls->base_ctx, &ssize, NULL); + ptls->stackbase = stkbuf + ssize; + ptls->stacksize = ssize; #endif - - jl_init_basefiber(JL_STACK_SIZE); } JL_DLLEXPORT int jl_is_task_started(jl_task_t *t) JL_NOTSAFEPOINT @@ -1344,6 +1337,7 @@ JL_DLLEXPORT void jl_gdb_dump_threadinfo(void) } #endif + #ifdef __cplusplus } #endif