wip: generate precompile as part of build process

Make.inc

@@ -134,6 +134,9 @@ else
 NO_GIT := 1
 endif
 
+# Setting for local precompile
+JULIA_LOCAL_PRECOMPILE ?= 1
+
 # Julia's Semantic Versioning system labels the three decimal places in a version number as
 # the major, minor and patch versions.  Typically the major version would be incremented
 # whenever a backwards-incompatible change is made, the minor version would be incremented

Makefile

@@ -74,13 +74,20 @@ julia-ui-release julia-ui-debug : julia-ui-% : julia-src-%
 julia-sysimg : julia-base julia-ui-$(JULIA_BUILD_MODE)
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) $(build_private_libdir)/sys.ji JULIA_EXECUTABLE='$(JULIA_EXECUTABLE)'
 
+julia-sysimg-precompile-release julia-sysimg-precompile-debug : julia-sysimg-precompile-% : julia-sysimg-%
+ifeq ($(JULIA_LOCAL_PRECOMPILE), 1)
+	@echo Generating precompile statements for the local system, disable with JULIA_LOCAL_PRECOMPILE=0 in Make.user
+	@$(JULIA_EXECUTABLE) $(JULIAHOME)/contrib/generate_precompile.jl
+	@$(MAKE) julia-sysimg-$*
+endif
+
 julia-sysimg-release : julia-sysimg julia-ui-release
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) $(build_private_libdir)/sys.$(SHLIB_EXT)
 
 julia-sysimg-debug : julia-sysimg julia-ui-debug
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) $(build_private_libdir)/sys-debug.$(SHLIB_EXT)
 
-julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest julia-base-cache
+julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-precompile-% julia-symlink julia-libccalltest julia-base-cache
 
 debug release : % : julia-%
 
@@ -567,6 +574,7 @@ distcleanall: cleanall
 	julia-debug julia-release julia-deps \
 	julia-ui-release julia-ui-debug julia-src-release julia-src-debug \
 	julia-symlink julia-base julia-sysimg julia-sysimg-release julia-sysimg-debug \
+	julia-sysimg-precompile-release julia-sysimg-precompile-debug \
 	test testall testall1 test clean distcleanall cleanall clean-* \
 	run-julia run-julia-debug run-julia-release run \
 	install binary-dist light-source-dist.tmp light-source-dist \

base/.gitignore

@@ -7,3 +7,4 @@
 /version_git.jl
 /version_git.jl.phony
 /userimg.jl
+/precompile_local.jl

base/Makefile

@@ -224,4 +224,5 @@ clean:
 	-rm -f $(BUILDDIR)/file_constants.jl
 	-rm -f $(BUILDDIR)/version_git.jl
 	-rm -f $(BUILDDIR)/version_git.jl.phony
+	-rm -f $(BUILDDIR)/precompile_local.jl
 	-rm -f $(build_private_libdir)/lib*.$(SHLIB_EXT)*

base/options.jl

@@ -26,6 +26,7 @@ struct JLOptions
     warn_overwrite::Int8
     can_inline::Int8
     polly::Int8
+    trace_compile::Int8
     fast_math::Int8
     worker::Int8
     cookie::Ptr{UInt8}

base/precompile.jl

@@ -16,6 +16,8 @@ for (_pkgid, _mod) in Base.loaded_modules
         @eval PrecompileStagingArea $(Symbol(_mod)) = $_mod
     end
 end
+f = joinpath(@__DIR__, "precompile_local.jl")
+isfile(f) && Base.include(PrecompileStagingArea, f)
 @eval PrecompileStagingArea begin
 precompile(Tuple{Type{Array{Base.StackTraces.StackFrame, 1}}, UndefInitializer, Int64})
 precompile(Tuple{Type{Array{Union{Nothing, String}, 1}}, UndefInitializer, Int64})

contrib/fixup_precompile.jl

@@ -11,60 +11,81 @@ const HEADER = """
 # Steps to regenerate this file:
 # 1. Remove all `precompile` calls
 # 2. Rebuild system image
-# 3. Enable TRACE_COMPILE in options.h and rebuild
-# 4. Run `./julia 2> precompiles.txt` and do various things.
+# 4. Run `./julia --trace-compile 2> precompiles.txt` and do various things.
 # 5. Run `./julia contrib/fixup_precompile.jl precompiles.txt to overwrite `precompile.jl`
 #    or ./julia contrib/fixup_precompile.jl --merge precompiles.txt to merge into existing
 #    `precompile.jl`
 """
 
-function fixup_precompile(new_precompile_file; merge=false)
-    old_precompile_file = joinpath(Sys.BINDIR, "..", "..", "base", "precompile.jl")
+function fixup_precompile(new_precompile_file; merge, keep_anonymous, header, output)
     precompile_statements = Set{String}()
 
-    for file in [new_precompile_file; merge ? old_precompile_file : []]
+    isfile(output) || touch(output)
+    for file in [new_precompile_file; merge ? output : []]
         for line in eachline(file)
             line = strip(line)
             # filter out closures, which might have different generated names in different environments
-            occursin(r"#[0-9]", line) && continue
+            if !keep_anonymous && occursin(r"#[0-9]", line)
+                continue
+            end
+
             # Other stuff than precompile statements might have been written to STDERR
             startswith(line, "precompile(Tuple{") || continue
             # Ok, add the line
             push!(precompile_statements, line)
         end
     end
 
-    open(old_precompile_file, "w") do f
-        println(f, HEADER)
-        println(f, """
-        let
-        PrecompileStagingArea = Module()
-        for (_pkgid, _mod) in Base.loaded_modules
-            if !(_pkgid.name in ("Main", "Core", "Base"))
-                @eval PrecompileStagingArea \$(Symbol(_mod)) = \$_mod
+    open(output, "w") do f
+        if header
+            println(f, HEADER)
+            println(f, """
+            let
+            PrecompileStagingArea = Module()
+            for (_pkgid, _mod) in Base.loaded_modules
+                if !(_pkgid.name in ("Main", "Core", "Base"))
+                    @eval PrecompileStagingArea \$(Symbol(_mod)) = \$_mod
+                end
             end
+            f = joinpath(@__DIR__, "precompile_local.jl")
+            isfile(f) && include(PrecompileStagingArea, f)
+            @eval PrecompileStagingArea begin
+            """)
         end
-        @eval PrecompileStagingArea begin""")
         for statement in sort(collect(precompile_statements))
             isgpl = needs_USE_GPL_LIBS(statement)
             isgpl && print(f, "if Base.USE_GPL_LIBS\n    ")
             println(f, statement)
             isgpl && println(f, "end")
         end
-        println(f, "end\nend")
-    end
-    if merge
-        "Merged $new_precompile_file into $old_precompile_file"
-    else
-        "Overwrite $old_precompile_file with $new_precompile_file"
+        if header
+            println(f, "end\nend")
+        end
     end
 end
 
-if length(ARGS) == 1
-    fixup_precompile(joinpath(pwd(), ARGS[1]))
-elseif length(ARGS) == 2
-    @assert ARGS[1] == "--merge"
-    fixup_precompile(joinpath(pwd(), ARGS[2]); merge = true)
-else
-    error("invalid arguments")
+function runit()
+    output = joinpath(Sys.BINDIR, "..", "..", "base", "precompile.jl")
+    merge = false
+    keep_anonymous = false
+    header=true
+    for arg in ARGS[1:end-1]
+        if arg == "--merge"
+            merge = true
+        elseif arg == "--keep-anonymous"
+            keep_anonymous = true
+        elseif arg == "--no-header"
+            header = false
+        elseif startswith(arg, "--output")
+            output = split(arg, "=")[2]
+        else
+            error("unknown argument $arg")
+        end
+    end
+    fixup_precompile(joinpath(pwd(), ARGS[end]); merge=merge, keep_anonymous=keep_anonymous, header=header, output=output)
 end
+
+running_as_script = abspath(PROGRAM_FILE) == @__FILE__
+if running_as_script
+    runit()
+end

contrib/generate_precompile.jl

@@ -0,0 +1,40 @@
+
+tmp = tempname()
+if haskey(Base.loaded_modules, Base.PkgId(Base.UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL"))
+    # Record precompile statements when starting a julia session with a repl
+    run(pipeline(`$(Base.julia_cmd()) --trace-compile=yes -e '
+        @async while true
+            sleep(0.01)
+            isdefined(Base, :active_repl) && exit(0)
+        end' -i`; stderr = tmp))
+    # Replay a REPL script
+    repl_replay = joinpath(@__DIR__, "precompile_replay.jl")
+    run(pipeline(`$(Base.julia_cmd()) --trace-compile=yes $repl_replay`; stderr=tmp, append=true))
+else
+    # No REPL, just record the startup
+    run(pipeline(`$(Base.julia_cmd()) --trace-compile=yes -e 'exit(0)'`; stderr=tmp))
+end
+
+# Replace the fake terminal with the real terminal and filter out everything we compiled in Main
+precompiles = readlines(tmp)
+new_precompiles = String[]
+for line in precompiles
+    line = replace(line, "FakeTerminals.FakeTerminal" => "REPL.Terminals.TTYTerminal")
+    (occursin(r"Main.", line) || occursin(r"FakeTerminals.", line)) && continue
+    push!(new_precompiles, line)
+end
+write(tmp, join(new_precompiles, '\n'))
+
+# Only write the precompile in case it is different
+
+include("fixup_precompile.jl")
+precompile_local = joinpath(@__DIR__, "..", "base/precompile_local.jl")
+tmp2 = tempname()
+isfile(precompile_local) && cp(precompile_local, tmp2)
+fixup_precompile(tmp; merge=true, keep_anonymous=true, header=false, output=tmp2)
+# Only update timestamp if different
+if !isfile(precompile_local) || (isfile(precompile_local) && (read(tmp2, String) != read(precompile_local, String)))
+    println("Updatingf...")
+    mv(tmp2, precompile_local; force=true)
+end
+rm(tmp)

contrib/precompile_replay.jl

@@ -0,0 +1,53 @@
+import REPL
+include(joinpath(Sys.STDLIB, "REPL", "test", "FakeTerminals.jl"))
+import .FakeTerminals.FakeTerminal
+include(joinpath(Sys.STDLIB, "REPL", "test", "fake_repl.jl"))
+
+const CTRL_C = '\x03'
+const UP_ARROW = "\e[A"
+const DOWN_ARROW = "\e[B"
+
+# TODO: Have a utility to generate this from a real REPL session?
+precompile_script = """
+2+2
+println("Hello")
+@time 1+1
+?reinterpret
+;ls
+using Ra\t$CTRL_C
+\\alpha\t$CTRL_C
+\e[200~paste here ;)\e[201~"$CTRL_C
+$UP_ARROW$DOWN_ARROW
+"""
+
+# Writing ^C to the repl will cause sigint, so let's not die on that
+ccall(:jl_exit_on_sigint, Cvoid, (Cint,), 0)
+
+fake_repl() do stdin_write, stdout_read, repl
+    repl.specialdisplay = REPL.REPLDisplay(repl)
+    repl.history_file = false
+
+    repltask = @async begin
+        REPL.run_repl(repl)
+    end
+
+    global inc = false
+    global b = Condition()
+    global c = Condition()
+    let cmd = "\"Hello REPL\""
+        write(stdin_write, "Main.inc || wait(Main.b); r = $cmd; notify(Main.c); r\r")
+    end
+    inc = true
+    notify(b)
+    wait(c)
+
+    write(stdin_write, precompile_script)
+
+    s = readavailable(stdout_read)
+
+    # Close REPL ^D
+    write(stdin_write, '\x04')
+    Base._wait(repltask)
+
+    nothing
+end

src/gf.c

@@ -1097,13 +1097,13 @@ static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype
     if (entry != NULL) {
         jl_method_t *m = entry->func.method;
         if (!jl_has_call_ambiguities((jl_value_t*)tt, m)) {
-#ifdef TRACE_COMPILE
-            if (!jl_has_free_typevars((jl_value_t*)tt)) {
-                jl_printf(JL_STDERR, "precompile(");
-                jl_static_show(JL_STDERR, (jl_value_t*)tt);
-                jl_printf(JL_STDERR, ")\n");
+            if (jl_options.trace_compile) {
+                if (!jl_has_free_typevars((jl_value_t*)tt)) {
+                    jl_printf(JL_STDERR, "precompile(");
+                    jl_static_show(JL_STDERR, (jl_value_t*)tt);
+                    jl_printf(JL_STDERR, ")\n");
+                }
             }
-#endif
             if (!mt_cache) {
                 intptr_t nspec = (mt == jl_type_type_mt ? m->nargs + 1 : mt->max_args + 2);
                 jl_compilation_sig(tt, env, m, nspec, &newparams);

src/jloptions.c

@@ -55,6 +55,11 @@ jl_options_t jl_options = { 0,    // quiet
                             0,    // method overwrite warning
                             1,    // can_inline
                             JL_OPTIONS_POLLY_ON, // polly
+#ifdef TRACE_COMPILE
+                            1, // trace_compile
+#else
+                            0, // trace_compile
+#endif
                             JL_OPTIONS_FAST_MATH_DEFAULT,
                             0,    // worker
                             NULL, // cookie
@@ -159,6 +164,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
            opt_warn_overwrite,
            opt_inline,
            opt_polly,
+           opt_trace_compile,
            opt_math_mode,
            opt_worker,
            opt_bind_to,
@@ -215,6 +221,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
         { "warn-overwrite",  required_argument, 0, opt_warn_overwrite },
         { "inline",          required_argument, 0, opt_inline },
         { "polly",           required_argument, 0, opt_polly },
+        { "trace-compile",   required_argument, 0, opt_trace_compile },
         { "math-mode",       required_argument, 0, opt_math_mode },
         { "handle-signals",  required_argument, 0, opt_handle_signals },
         // hidden command line options
@@ -567,6 +574,15 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
                 jl_errorf("julia: invalid argument to --polly (%s)", optarg);
             }
             break;
+         case opt_trace_compile:
+            if (!strcmp(optarg,"yes"))
+                jl_options.trace_compile = 1;
+            else if (!strcmp(optarg,"no"))
+                jl_options.trace_compile = 0;
+            else {
+                jl_errorf("julia: invalid argument to --trace-compile (%s)", optarg);
+            }
+            break;
         case opt_math_mode:
             if (!strcmp(optarg,"ieee"))
                 jl_options.fast_math = JL_OPTIONS_FAST_MATH_OFF;

src/julia.h

@@ -1794,6 +1794,7 @@ typedef struct {
     int8_t warn_overwrite;
     int8_t can_inline;
     int8_t polly;
+    int8_t trace_compile;
     int8_t fast_math;
     int8_t worker;
     const char *cookie;

stdlib/REPL/test/fake_repl.jl

@@ -0,0 +1,49 @@
+
+using Test
+
+function kill_timer(delay)
+    # Give ourselves a generous timer here, just to prevent
+    # this causing e.g. a CI hang when there's something unexpected in the output.
+    # This is really messy and leaves the process in an undefined state.
+    # the proper and correct way to do this in real code would be to destroy the
+    # IO handles: `close(stdout_read); close(stdin_write)`
+    test_task = current_task()
+    function kill_test(t)
+        # **DON'T COPY ME.**
+        # The correct way to handle timeouts is to close the handle:
+        # e.g. `close(stdout_read); close(stdin_write)`
+        schedule(test_task, "hard kill repl test"; error=true)
+        print(stderr, "WARNING: attempting hard kill of repl test after exceeding timeout\n")
+    end
+    return Timer(kill_test, delay)
+end
+
+# REPL tests
+function fake_repl(@nospecialize(f); options::REPL.Options=REPL.Options(confirm_exit=false))
+    # Use pipes so we can easily do blocking reads
+    # In the future if we want we can add a test that the right object
+    # gets displayed by intercepting the display
+    input = Pipe()
+    output = Pipe()
+    err = Pipe()
+    Base.link_pipe!(input, reader_supports_async=true, writer_supports_async=true)
+    Base.link_pipe!(output, reader_supports_async=true, writer_supports_async=true)
+    Base.link_pipe!(err, reader_supports_async=true, writer_supports_async=true)
+
+    repl = REPL.LineEditREPL(FakeTerminal(input.out, output.in, err.in), true)
+    repl.options = options
+
+    hard_kill = kill_timer(900) # Your debugging session starts now. You have 15 minutes. Go.
+    f(input.in, output.out, repl)
+    t = @async begin
+        close(input.in)
+        close(output.in)
+        close(err.in)
+    end
+    @test read(err.out, String) == ""
+    #display(read(output.out, String))
+    #print(read(output.out, String))
+    Base._wait(t)
+    close(hard_kill)
+    nothing
+end