Merge pull request #15154 from JuliaLang/jn/compileall_travis

CI-test the compile-all mode

.travis.yml

@@ -54,6 +54,7 @@ notifications:
           - http://julia.mit.edu:8000/travis-hook
 before_install:
     - make check-whitespace
+    - JULIA_SYSIMG_BUILD_FLAGS="--output-ji ../usr/lib/julia/sys.ji"
     - if [ `uname` = "Linux" ]; then
         contrib/travis_fastfail.sh || exit 1;
         mkdir -p $HOME/bin;
@@ -62,7 +63,7 @@ before_install:
         ln -s /usr/bin/gfortran-5 $HOME/bin/gfortran;
         gcc --version;
         BUILDOPTS="-j3 VERBOSE=1 FORCE_ASSERTIONS=1";
-        export TESTSTORUN="all";
+        TESTSTORUN="all";
       elif [ `uname` = "Darwin" ]; then
         brew update;
         brew install -v jq bar;
@@ -80,13 +81,15 @@ before_install:
         export LDFLAGS="-L$(brew --prefix openblas-julia)/lib -L$(brew --prefix suite-sparse-julia)/lib";
         export DYLD_FALLBACK_LIBRARY_PATH="/usr/local/lib:/lib:/usr/lib:$(brew --prefix openblas-julia)/lib:$(brew --prefix suite-sparse-julia)/lib:$(brew --prefix arpack-julia)/lib";
         make $BUILDOPTS -C contrib -f repackage_system_suitesparse4.make;
-        export TESTSTORUN="core ccall mmap llvmcall threads"; fi # TODO: turn this back to "all" once it's fast enough to not time out
+        JULIA_SYSIMG_BUILD_FLAGS="$JULIA_SYSIMG_BUILD_FLAGS --compile=all";
+        COMPILE_MODE="--compile=min";
+        TESTSTORUN="core ccall mmap llvmcall threads"; fi # TODO: turn this back to "all" once it's fast enough to not time out
     - git clone -q git://git.kitenet.net/moreutils
 script:
     - make -C moreutils mispipe
     - make $BUILDOPTS -C base version_git.jl.phony
     - moreutils/mispipe "make $BUILDOPTS NO_GIT=1 -C deps" bar > deps.log || cat deps.log
-    - make $BUILDOPTS NO_GIT=1 JULIA_SYSIMG_BUILD_FLAGS="--output-ji ../usr/lib/julia/sys.ji" prefix=/tmp/julia install | moreutils/ts -s "%.s"
+    - make $BUILDOPTS NO_GIT=1 JULIA_SYSIMG_BUILD_FLAGS="$JULIA_SYSIMG_BUILD_FLAGS" prefix=/tmp/julia install | moreutils/ts -s "%.s"
     - make $BUILDOPTS NO_GIT=1 build-stats
     - du -sk /tmp/julia/*
     - if [ `uname` = "Darwin" ]; then
@@ -99,8 +102,8 @@ script:
         /tmp/julia/bin/julia-debug -J local.ji -e 'true' && rm local.ji
     - /tmp/julia/bin/julia -e 'versioninfo()'
     - export JULIA_CPU_CORES=2 && export JULIA_TEST_MAXRSS_MB=600 && cd /tmp/julia/share/julia/test &&
-        /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN &&
-        /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online pkg
+        /tmp/julia/bin/julia --check-bounds=yes $COMPILE_MODE runtests.jl $TESTSTORUN &&
+        /tmp/julia/bin/julia --check-bounds=yes $COMPILE_MODE runtests.jl libgit2-online pkg
     - cd `dirname $TRAVIS_BUILD_DIR` && mv julia2 julia && rm -rf julia/deps/build/julia-env
 # uncomment the following if failures are suspected to be due to the out-of-memory killer
 #    - dmesg

base/pkg/entry.jl

@@ -636,7 +636,7 @@ function test!(pkg::AbstractString,
             try
                 color = Base.have_color? "--color=yes" : "--color=no"
                 codecov = coverage? ["--code-coverage=user", "--inline=no"] : ["--code-coverage=none"]
-                julia_exe = joinpath(JULIA_HOME, Base.julia_exename())
+                julia_exe = Base.julia_cmd()
                 run(`$julia_exe --check-bounds=yes $codecov $color $test_path`)
                 info("$pkg tests passed")
             catch err

base/util.jl

@@ -377,7 +377,16 @@ function julia_cmd(julia=joinpath(JULIA_HOME, julia_exename()))
     opts = JLOptions()
     cpu_target = bytestring(opts.cpu_target)
     image_file = bytestring(opts.image_file)
-    `$julia -C$cpu_target -J$image_file`
+    compile = if opts.compile_enabled == 0
+                  "no"
+              elseif opts.compile_enabled == 2
+                  "all"
+              elseif opts.compile_enabled == 3
+                  "min"
+              else
+                  "yes"
+              end
+    `$julia -C$cpu_target -J$image_file --compile=$compile`
 end
 
 julia_exename() = ccall(:jl_is_debugbuild,Cint,())==0 ? "julia" : "julia-debug"

deps/Makefile

@@ -728,6 +728,10 @@ $(eval $(call LLVM_PATCH,win64-int128.llvm-3.3))
 else ifeq ($(LLVM_VER),3.7.0)
 $(eval $(call LLVM_PATCH,llvm-3.7.0))
 $(eval $(call LLVM_PATCH,llvm-3.7.1))
+$(eval $(call LLVM_PATCH,llvm-3.7.1_2))
+$(eval $(call LLVM_PATCH,llvm-3.7.1_3))
+$(eval $(call LLVM_PATCH,llvm-D14260))
+$(LLVM_SRC_DIR)/llvm-3.7.1_2.patch-applied: $(LLVM_SRC_DIR)/llvm-3.7.1.patch-applied
 else ifeq ($(LLVM_VER),3.7.1)
 $(eval $(call LLVM_PATCH,llvm-3.7.1))
 $(eval $(call LLVM_PATCH,llvm-3.7.1_2))

src/dump.c

@@ -410,9 +410,9 @@ static void jl_delayed_fptrs(jl_lambda_info_t *li, int32_t func, int32_t cfunc)
         delayed_fptrs[delayed_fptrs_n].func = func;
         delayed_fptrs[delayed_fptrs_n].cfunc = cfunc;
         delayed_fptrs_n++;
-        if (func > sysimg_fvars_max)
+        if (func > 0 && func > sysimg_fvars_max)
             sysimg_fvars_max = func;
-        if (cfunc > sysimg_fvars_max)
+        if (cfunc > 0 && cfunc > sysimg_fvars_max)
             sysimg_fvars_max = cfunc;
     }
 }

src/gf.c

@@ -283,7 +283,6 @@ static jl_lambda_info_t *jl_add_static_parameters(jl_lambda_info_t *l, jl_svec_t
     assert(l->sparam_vals == jl_emptysvec);
     assert(l->specTypes == NULL);
     jl_lambda_info_t *nli = jl_copy_lambda_info(l);
-    nli->unspecialized = l;
     nli->sparam_vals = sp; // no gc_wb needed
     nli->tfunc = jl_nothing;
     nli->specializations = NULL;
@@ -302,7 +301,7 @@ static jl_lambda_info_t *jl_add_static_parameters(jl_lambda_info_t *l, jl_svec_t
             nli->functionObjects.cFunctionList == NULL &&
             nli->functionID == 0 &&
             nli->specFunctionID == 0));
-    if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF) {
+    if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF || jl_options.compile_enabled == JL_OPTIONS_COMPILE_MIN) {
         // copy fptr from the unspecialized method definition
         jl_lambda_info_t *unspec = l->unspecialized;
         if (unspec != NULL) {
@@ -313,7 +312,7 @@ static jl_lambda_info_t *jl_add_static_parameters(jl_lambda_info_t *l, jl_svec_t
             nli->functionID = unspec->functionID;
             nli->specFunctionID = unspec->functionID;
         }
-        if (nli->fptr == NULL) {
+        if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_OFF && nli->fptr == NULL) {
             jl_printf(JL_STDERR,"code missing for ");
             jl_static_show(JL_STDERR, (jl_value_t*)nli);
             jl_printf(JL_STDERR, "  sysimg may not have been built with --compile=all\n");
@@ -326,24 +325,21 @@ static jl_lambda_info_t *jl_add_static_parameters(jl_lambda_info_t *l, jl_svec_t
 jl_lambda_info_t *jl_get_unspecialized(jl_lambda_info_t *method)
 {
     // one unspecialized version of a function can be shared among all cached specializations
-    jl_lambda_info_t *def = method;
-    if (def->specTypes) {
+    jl_lambda_info_t *def = method->def;
+    if (method->unspecialized != NULL)
+        return method->unspecialized;
+    if (method->specTypes && def->needs_sparam_vals_ducttape) {
         // a method is a specialization iff it has specTypes
         // but method->unspecialized points to the definition in that case
         // and definition->unspecialized points to the thing to call
-        def = def->unspecialized;
+        method->unspecialized = jl_add_static_parameters(def, method->sparam_vals, method->specTypes);
+        jl_gc_wb(method, method->unspecialized);
+        method->unspecialized->unspecialized = method->unspecialized;
+        def = method;
     }
-    if (__unlikely(def->unspecialized == NULL)) {
-        if (method->def->needs_sparam_vals_ducttape) {
-            method->unspecialized = jl_add_static_parameters(def, method->sparam_vals, method->specTypes);
-            jl_gc_wb(method, method->unspecialized);
-            method->unspecialized->unspecialized = method->unspecialized;
-            def = method;
-        }
-        else {
-            def->unspecialized = jl_add_static_parameters(def, jl_emptysvec, jl_anytuple_type);
-            jl_gc_wb(def, def->unspecialized);
-        }
+    if (def->unspecialized == NULL) {
+        def->unspecialized = jl_add_static_parameters(def, jl_emptysvec, jl_anytuple_type);
+        jl_gc_wb(def, def->unspecialized);
     }
     return def->unspecialized;
 }
@@ -1701,12 +1697,13 @@ static void _compile_all_deq(jl_array_t *found)
             linfo = jl_get_unspecialized(linfo);
         if (!linfo->inferred) {
             // force this function to be recompiled
-            jl_type_infer(linfo, linfo);
+            jl_type_infer(linfo, linfo->def);
             linfo->functionObjects.functionObject = NULL;
             linfo->functionObjects.specFunctionObject = NULL;
             linfo->functionObjects.cFunctionList = NULL;
             linfo->functionID = 0;
             linfo->specFunctionID = 0;
+            linfo->jlcall_api = 0;
         }
 
         // keep track of whether all possible signatures have been cached (and thus whether it can skip trying to compile the unspecialized function)
@@ -1738,7 +1735,7 @@ static void _compile_all_enq_ml(jl_methlist_t *ml, jl_array_t *found)
                     return; // builtin function
                 linfo = jl_get_unspecialized(linfo);
             }
-            if (!linfo->functionID || !linfo->inferred) {
+            if (!linfo->functionID) {
                 // and it still needs to be compiled
                 if (!ml->isstaged)
                     jl_cell_1d_push(found, (jl_value_t*)ml);

src/julia.h

@@ -1556,6 +1556,7 @@ JL_DLLEXPORT int jl_generating_output(void);
 #define JL_OPTIONS_COMPILE_OFF 0
 #define JL_OPTIONS_COMPILE_ON  1
 #define JL_OPTIONS_COMPILE_ALL 2
+#define JL_OPTIONS_COMPILE_MIN 3
 
 #define JL_OPTIONS_COLOR_ON 1
 #define JL_OPTIONS_COLOR_OFF 2

test/cmdlineargs.jl

@@ -1,6 +1,6 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
-let exename = `$(joinpath(JULIA_HOME, Base.julia_exename())) --precompiled=yes`
+let exename = `$(Base.julia_cmd()) --precompiled=yes`
     # --version
     let v = split(readstring(`$exename -v`), "julia version ")[end]
         @test Base.VERSION_STRING == chomp(v)

test/compile.jl

@@ -119,7 +119,7 @@ let dir = mktempdir(),
             Base.compilecache(:Time4b3a94a1a081a8cb)
         end)
 
-        exename = `$(joinpath(JULIA_HOME, Base.julia_exename())) --precompiled=yes`
+        exename = `$(Base.julia_cmd()) --precompiled=yes`
 
         testcode = """
             insert!(LOAD_PATH, 1, $(repr(dir)))

test/misc.jl

@@ -129,7 +129,7 @@ func4union(::Union{Type4Union,Int}) = ()
 # Disable on windows because of issue (missing flush) when redirecting STDERR.
 let
     redir_err = "redirect_stderr(STDOUT)"
-    exename = joinpath(JULIA_HOME, Base.julia_exename())
+    exename = Base.julia_cmd()
     script = "$redir_err; f(a::Number, b...) = 1;f(a, b::Number) = 1"
     warning_str = readstring(`$exename -f -e $script`)
     @test contains(warning_str, "f(Any, Number)")

test/parallel_exec.jl

@@ -623,7 +623,7 @@ if DoFullTest
     # time can only support a single topology and the current session
     # is already running in parallel under the default topology.
     script = joinpath(dirname(@__FILE__), "topology.jl")
-    cmd = `$(joinpath(JULIA_HOME,Base.julia_exename())) $script`
+    cmd = `$(Base.julia_cmd()) $script`
 
     (strm, proc) = open(pipeline(cmd, stderr=STDERR))
     wait(proc)

test/repl.jl

@@ -403,7 +403,7 @@ end
 
 ccall(:jl_exit_on_sigint, Void, (Cint,), 1)
 
-let exename = joinpath(JULIA_HOME, Base.julia_exename())
+let exename = Base.julia_cmd()
 
 # Test REPL in dumb mode
 @unix_only begin

test/workspace.jl

@@ -13,5 +13,5 @@ show(io, Pair)
 @assert takebuf_string(io) == "Pair{A,B}"
 @assert !Base.inbase(LastMain)
 """
-exename = joinpath(JULIA_HOME, Base.julia_exename())
+exename = Base.julia_cmd()
 run(`$exename -f -e $script`)

ui/repl.c

@@ -85,7 +85,7 @@ static const char opts[]  =
     " --no-history-file         Don't load history file (deprecated, use --history-file=no)\n\n"
 
     // code generation options
-    " --compile={yes|no|all}    Enable or disable compiler, or request exhaustive compilation\n"
+    " --compile={yes|no|all|min}Enable or disable JIT compiler, or request exhaustive compilation\n"
     " -C, --cpu-target <target> Limit usage of cpu features up to <target>\n"
     " -O, --optimize={0,1,2,3}  Set the optimization level (default 2 if unspecified or 3 if specified as -O)\n"
     " --inline={yes|no}         Control whether inlining is permitted (overrides functions declared as @inline)\n"
@@ -319,6 +319,8 @@ void parse_opts(int *argcp, char ***argvp)
                 jl_options.compile_enabled = JL_OPTIONS_COMPILE_OFF;
             else if (!strcmp(optarg,"all"))
                 jl_options.compile_enabled = JL_OPTIONS_COMPILE_ALL;
+            else if (!strcmp(optarg,"min"))
+                jl_options.compile_enabled = JL_OPTIONS_COMPILE_MIN;
             else
                 jl_errorf("julia: invalid argument to --compile (%s)", optarg);
             break;