Merge pull request #18169 from JuliaLang/tb/backport-asan

[release-0.5] additional ASAN-related backports

doc/devdocs/C.rst

@@ -12,3 +12,4 @@
    backtraces
    debuggingtips
    valgrind
+   sanitizers

doc/devdocs/sanitizers.rst

@@ -0,0 +1,44 @@
+*****************
+Sanitizer support
+*****************
+
+General considerations
+----------------------
+
+Using Clang's sanitizers obviously require you to use Clang (``USECLANG=1``), but there's
+another catch: most sanitizers require a run-time library, provided by the host compiler,
+while the instrumented code generated by Julia's JIT relies on functionality from that
+library. This implies that the LLVM version of your host compiler matches that of the LLVM
+library used within Julia.
+
+An easy solution is to have an dedicated build folder for providing a matching toolchain, by
+building with ``BUILD_LLVM_CLANG=1`` and overriding ``LLVM_USE_CMAKE=1`` (Autotool-based
+builds are incompatible with ASAN). You can then refer to this toolchain from another build
+folder by specifying ``USECLANG=1`` while overriding the ``CC`` and ``CXX`` variables.
+
+
+Address Sanitizer (ASAN)
+------------------------
+
+For detecting or debugging memory bugs, you can use Clang's `address sanitizer (ASAN)
+<http://clang.llvm.org/docs/AddressSanitizer.html>`_. By compiling with
+``SANITIZE=1`` you enable ASAN for the Julia compiler and its generated code. In addition,
+you can specify ``LLVM_SANITIZE=1`` to sanitize the LLVM library as well. Note that these
+options incur a high performance and memory cost. For example, using ASAN for Julia and LLVM
+makes ``testall1`` takes 8-10 times as long while using 20 times as much memory (this can
+be reduced to respectively a factor of 3 and 4 by using the options described below).
+
+By default, Julia sets the ``allow_user_segv_handler=1`` ASAN flag, which is required for
+signal delivery to work properly. You can define other options using the ``ASAN_OPTIONS``
+environment flag, in which case you'll need to repeat the default option mentioned before.
+For example, memory usage can be reduced by specifying ``fast_unwind_on_malloc=0`` and
+``malloc_context_size=2``, at the cost of backtrace accuracy. For now, Julia also sets
+``detect_leaks=0``, but this should be removed in the future.
+
+
+Memory Sanitizer (MSAN)
+-----------------------
+
+For detecting use of uninitialized memory, you can use Clang's `memory sanitizer (MSAN)
+<http://clang.llvm.org/docs/MemorySanitizer.html>`_ by compiling with
+``SANITIZE_MEMORY=1``.

src/codegen.cpp

@@ -4746,6 +4746,8 @@ static std::unique_ptr<Module> emit_function(jl_lambda_info_t *lam, jl_llvm_func
         }
     }
 
+    builder.ClearInsertionPoint();
+
     // step 14, Apply LLVM level inlining
     for(std::vector<CallInst*>::iterator it = ctx.to_inline.begin(); it != ctx.to_inline.end(); ++it) {
         Function *inlinef = (*it)->getCalledFunction();

src/init.c

@@ -46,6 +46,14 @@ extern BOOL (WINAPI *hSymRefreshModuleList)(HANDLE);
 #include <unistd.h>
 #endif
 
+#ifdef JL_ASAN_ENABLED
+JL_DLLEXPORT const char* __asan_default_options() {
+    return "allow_user_segv_handler=1:detect_leaks=0";
+    // FIXME: enable LSAN after fixing leaks & defining __lsan_default_suppressions(),
+    //        or defining __lsan_default_options = exitcode=0 once publicly available
+}
+#endif
+
 static const char system_image_path[256] = "\0" JL_SYSTEM_IMAGE_PATH;
 
 jl_options_t jl_options = { 0,    // quiet
@@ -628,17 +636,6 @@ void _julia_init(JL_IMAGE_SEARCH rel)
     }
 #endif
 
-
-#ifdef JL_ASAN_ENABLED
-    const char *asan_options = getenv("ASAN_OPTIONS");
-    if (!asan_options || !(strstr(asan_options, "allow_user_segv_handler=1") ||
-                           strstr(asan_options, "handle_segv=0"))) {
-        jl_printf(JL_STDERR,"WARNING: ASAN overrides Julia's SIGSEGV handler; "
-                            "disable SIGSEGV handling or allow custom handlers.\n");
-    }
-
-#endif
-
     jl_init_threading();
 
     jl_gc_init();

src/intrinsics.cpp

@@ -50,6 +50,14 @@ JL_DLLEXPORT uint32_t jl_get_LLVM_VERSION(void)
         ;
 }
 
+extern "C" JL_DLLEXPORT int8_t jl_is_memdebug() {
+#ifdef MEMDEBUG
+    return true;
+#else
+    return false;
+#endif
+}
+
 /*
   low-level intrinsics design: TODO: fix description below
   functions like add_int expect unboxed values of matching bit-length.

src/julia_internal.h

@@ -49,9 +49,6 @@ void jl_call_tracer(tracer_cb callback, jl_value_t *tracee);
 
 extern size_t jl_page_size;
 extern jl_function_t *jl_typeinf_func;
-#if defined(JL_USE_INTEL_JITEVENTS)
-extern unsigned sig_stack_size;
-#endif
 
 JL_DLLEXPORT extern int jl_lineno;
 JL_DLLEXPORT extern const char *jl_filename;

src/signals-unix.c

@@ -28,14 +28,10 @@
 #define HAVE_TIMER
 #endif
 
-#if defined(JL_USE_INTEL_JITEVENTS)
-unsigned sig_stack_size = SIGSTKSZ;
-#elif defined(_CPU_AARCH64_)
-// The default SIGSTKSZ causes stack overflow in libunwind.
-#define sig_stack_size (1 << 16)
-#else
-#define sig_stack_size SIGSTKSZ
-#endif
+// 8M signal stack, same as default stack size and enough
+// for reasonable finalizers.
+// Should also be enough for parallel GC when we have it =)
+#define sig_stack_size (8 * 1024 * 1024)
 
 static bt_context_t *jl_to_bt_context(void *sigctx)
 {

src/threading.c

@@ -778,13 +778,6 @@ void jl_init_threading(void)
     jl_all_tls_states = &_jl_all_tls_states;
     jl_n_threads = 1;
 
-#if defined(__linux__) && defined(JL_USE_INTEL_JITEVENTS)
-    if (jl_using_intel_jitevents)
-        // Intel VTune Amplifier needs at least 64k for alternate stack.
-        if (SIGSTKSZ < 1<<16)
-            sig_stack_size = 1<<16;
-#endif
-
     ti_init_master_thread();
 }
 

test/core.jl

@@ -4470,6 +4470,13 @@ let
     @test k(1) == 1
 end
 
+# PR #18054: compilation of cfunction leaves IRBuilder in bad state,
+#            causing heap-use-after-free when compiling f18054
+function f18054()
+    return Cint(0)
+end
+cfunction(f18054, Cint, ())
+
 # issue #18085
 f18085(a,x...) = (0,)
 for (f,g) in ((:asin,:sin), (:acos,:cos))

test/misc.jl

@@ -230,7 +230,8 @@ module Tmp14173
     A = randn(2000, 2000)
 end
 whos(IOBuffer(), Tmp14173) # warm up
-@test @allocated(whos(IOBuffer(), Tmp14173)) < 10000
+const MEMDEBUG = ccall(:jl_is_memdebug, Bool, ())
+@test @allocated(whos(IOBuffer(), Tmp14173)) < (MEMDEBUG ? 20000 : 8000)
 
 ## test conversion from UTF-8 to UTF-16 (for Windows APIs)