multi-threaded profiling

src/init.c

@@ -498,11 +498,6 @@ void _julia_init(JL_IMAGE_SEARCH rel)
 #endif
     uv_dlopen("ws2_32.dll", jl_winsock_handle);
     _jl_exe_handle.handle = GetModuleHandleA(NULL);
-    if (!DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
-                         GetCurrentProcess(), (PHANDLE)&hMainThread, 0,
-                         TRUE, DUPLICATE_SAME_ACCESS)) {
-        jl_printf(JL_STDERR, "WARNING: failed to access handle to main thread\n");
-    }
     SymSetOptions(SYMOPT_UNDNAME | SYMOPT_DEFERRED_LOADS | SYMOPT_LOAD_LINES);
     if (!SymInitialize(GetCurrentProcess(), NULL, 1)) {
         jl_printf(JL_STDERR, "WARNING: failed to initialize stack walk info\n");

src/julia.h

@@ -1364,7 +1364,6 @@ DLLEXPORT extern volatile sig_atomic_t jl_defer_signal;
     } while(0)
 
 DLLEXPORT void jl_sigint_action(void);
-DLLEXPORT void restore_signals(void);
 DLLEXPORT void jl_install_sigint_handler(void);
 DLLEXPORT void jl_sigatomic_begin(void);
 DLLEXPORT void jl_sigatomic_end(void);
@@ -1415,6 +1414,8 @@ typedef struct {
     jl_task_t **proot_task;
     jl_value_t **pexception_in_transit;
     jl_value_t * volatile *ptask_arg_in_transit;
+    uv_thread_t system_id;
+    void *signal_context; // bt_context_t
 } jl_thread_task_state_t;
 
 extern DLLEXPORT JL_THREAD jl_task_t * volatile jl_current_task;

src/julia_internal.h

@@ -81,6 +81,8 @@ JL_CALLABLE(jl_f_tuple);
 extern jl_function_t *jl_unprotect_stack_func;
 extern jl_function_t *jl_bottom_func;
 void jl_install_default_signal_handlers(void);
+void restore_signals(void);
+void jl_install_thread_signal_handler(void);
 
 extern jl_datatype_t *jl_box_type;
 extern jl_value_t *jl_box_any_type;
@@ -177,8 +179,8 @@ extern volatile int jl_in_stackwalk;
 typedef unw_context_t *bt_context_t;
 #endif
 #define MAX_BT_SIZE 80000
-extern ptrint_t bt_data[MAX_BT_SIZE+1];
-extern size_t bt_size;
+extern JL_THREAD ptrint_t bt_data[MAX_BT_SIZE+1];
+extern JL_THREAD size_t bt_size;
 DLLEXPORT size_t rec_backtrace(ptrint_t *data, size_t maxsize);
 DLLEXPORT size_t rec_backtrace_ctx(ptrint_t *data, size_t maxsize, bt_context_t ctx);
 #ifdef LIBOSXUNWIND

src/signal-handling.c

@@ -10,6 +10,8 @@
 extern "C" {
 #endif
 
+#include <threading.h>
+
 // Profiler control variables //
 static volatile ptrint_t *bt_data_prof = NULL;
 static volatile size_t bt_size_max = 0;

src/signals-apple.c

@@ -170,7 +170,6 @@ void attach_exception_port()
     if (retval!=KERN_SUCCESS) { mach_error(msg ":", (retval)); jl_exit(1); }
 
 static pthread_t profiler_thread;
-static mach_port_t main_thread;
 clock_serv_t clk;
 static int profile_started = 0;
 static mach_port_t profile_port = 0;
@@ -224,7 +223,8 @@ static kern_return_t profiler_segv_handler
 void *mach_profile_listener(void *arg)
 {
     (void)arg;
-    int max_size = 512;
+    int i;
+    const int max_size = 512;
     attach_exception_port();
     mach_profiler_thread = mach_thread_self();
     mig_reply_error_t *bufRequest = (mig_reply_error_t *) malloc(max_size);
@@ -233,24 +233,31 @@ void *mach_profile_listener(void *arg)
                                      0, max_size, profile_port,
                                      MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
         HANDLE_MACH_ERROR("mach_msg",ret);
-        if (bt_size_cur < bt_size_max) {
+        // sample each thread, round-robin style
+        for (i = 0; i < jl_n_threads; i++) {
+            // if there is no space left, break early
+            if (bt_size_cur >= bt_size_max - 1)
+                break;
+
+            mach_port_t tid_port = pthread_mach_thread_np(jl_all_task_states[i].system_id);
+
             kern_return_t ret;
             // Suspend the thread so we may safely sample it
-            ret = thread_suspend(main_thread);
-            HANDLE_MACH_ERROR("thread_suspend",ret);
+            ret = thread_suspend(tid_port);
+            HANDLE_MACH_ERROR("thread_suspend", ret);
 
             // Do the actual sampling
             unsigned int count = MACHINE_THREAD_STATE_COUNT;
             x86_thread_state64_t state;
 
             // Get the state of the suspended thread
-            ret = thread_get_state(main_thread,x86_THREAD_STATE64,(thread_state_t)&state,&count);
-            HANDLE_MACH_ERROR("thread_get_state",ret);
+            ret = thread_get_state(tid_port, x86_THREAD_STATE64, (thread_state_t)&state, &count);
+            HANDLE_MACH_ERROR("thread_get_state", ret);
 
             // Initialize the unwind context with the suspend thread's state
             unw_context_t uc;
-            memset(&uc,0,sizeof(unw_context_t));
-            memcpy(&uc,&state,sizeof(x86_thread_state64_t));
+            memset(&uc, 0, sizeof(unw_context_t));
+            memcpy(&uc, &state, sizeof(x86_thread_state64_t));
 
             /*
              *  Unfortunately compact unwind info is incorrectly generated for quite a number of
@@ -271,10 +278,10 @@ void *mach_profile_listener(void *arg)
 
             if (forceDwarf == 0) {
                 // Save the backtrace
-                bt_size_cur += rec_backtrace_ctx((ptrint_t*)bt_data_prof+bt_size_cur, bt_size_max-bt_size_cur-1, &uc);
+                bt_size_cur += rec_backtrace_ctx((ptrint_t*)bt_data_prof + bt_size_cur, bt_size_max - bt_size_cur - 1, &uc);
             }
             else if (forceDwarf == 1) {
-                bt_size_cur += rec_backtrace_ctx_dwarf((ptrint_t*)bt_data_prof+bt_size_cur, bt_size_max-bt_size_cur-1, &uc);
+                bt_size_cur += rec_backtrace_ctx_dwarf((ptrint_t*)bt_data_prof + bt_size_cur, bt_size_max - bt_size_cur - 1, &uc);
             }
             else if (forceDwarf == -1) {
                 jl_safe_printf("WARNING: profiler attempt to access an invalid memory location\n");
@@ -283,17 +290,16 @@ void *mach_profile_listener(void *arg)
             forceDwarf = -2;
 
             // Mark the end of this block with 0
-            bt_data_prof[bt_size_cur] = 0;
-            bt_size_cur++;
+            bt_data_prof[bt_size_cur++] = 0;
 
             // We're done! Resume the thread.
-            ret = thread_resume(main_thread);
-            HANDLE_MACH_ERROR("thread_resume",ret)
+            ret = thread_resume(tid_port);
+            HANDLE_MACH_ERROR("thread_resume", ret)
 
             if (running) {
                 // Reset the alarm
                 ret = clock_alarm(clk, TIME_RELATIVE, timerprof, profile_port);
-                HANDLE_MACH_ERROR("clock_alarm",ret)
+                HANDLE_MACH_ERROR("clock_alarm", ret)
             }
         }
     }
@@ -304,7 +310,6 @@ DLLEXPORT int jl_profile_start_timer(void)
     kern_return_t ret;
     if (!profile_started) {
         mach_port_t self = mach_task_self();
-        main_thread = mach_thread_self();
 
         ret = host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, (clock_serv_t *)&clk);
         HANDLE_MACH_ERROR("host_get_clock_service", ret);

src/signals-bsd.c

@@ -8,9 +8,6 @@ struct itimerval timerprof;
 
 DLLEXPORT int jl_profile_start_timer(void)
 {
-    struct sigaction sa;
-    sigset_t ss;
-
     timerprof.it_interval.tv_sec = nsecprof/GIGA;
     timerprof.it_interval.tv_usec = (nsecprof%GIGA)/1000;
     timerprof.it_value.tv_sec = nsecprof/GIGA;