Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Backport: precompile generation fixes #29071

Merged
merged 4 commits into from
Sep 8, 2018
Merged

Backport: precompile generation fixes #29071

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b4b82f9
implement the REPL replayer on Windows (#28608)
KristofferC Aug 14, 2018
e839657
work around precompile emitter failing to escape certain strings
KristofferC Aug 22, 2018
30192fa
make sure that the read task is done when precompiling (#29024)
KristofferC Sep 6, 2018
d83739a
comment out assertion
KristofferC Sep 7, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ define sysimg_builder
$$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(build_private_libdir)/sys$1-%.a : $$(build_private_libdir)/sys.ji
@$$(call PRINT_JULIA, cd $$(JULIAHOME)/base && \
if ! $$(call spawn,$3) $2 -C "$$(JULIA_CPU_TARGET)" --output-$$* $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \
--startup-file=no --warn-overwrite=yes --sysimage $$(call cygpath_w,$$<) $$(call cygpath_w,$$(JULIAHOME)/contrib/generate_precompile.jl) $$(call cygpath_w,$$<); then \
--startup-file=no --warn-overwrite=yes --sysimage $$(call cygpath_w,$$<) $$(call cygpath_w,$$(JULIAHOME)/contrib/generate_precompile.jl); then \
echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \
false; \
fi )
Expand Down
162 changes: 98 additions & 64 deletions contrib/generate_precompile.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ if !isdefined(Base, :uv_eventloop)
Base.reinit_stdio()
end
Base.include(@__MODULE__, joinpath(Sys.BINDIR, "..", "share", "julia", "test", "testhelpers", "FakePTYs.jl"))
import .FakePTYs: with_fake_pty
import .FakePTYs: open_fake_pty

CTRL_C = '\x03'
UP_ARROW = "\e[A"
Expand Down Expand Up @@ -43,6 +43,12 @@ if Pkg !== nothing
precompile_script *= Pkg.precompile_script
end

push!(LOAD_PATH, Sys.STDLIB)
using Sockets
Sockets.__init__()
using Libdl
empty!(LOAD_PATH)

function generate_precompile_statements()
start_time = time()

Expand All @@ -62,82 +68,110 @@ function generate_precompile_statements()
empty!(DEPOT_PATH)
end

# Create a staging area where all the loaded packages are available
PrecompileStagingArea = Module()
for (_pkgid, _mod) in Base.loaded_modules
if !(_pkgid.name in ("Main", "Core", "Base"))
eval(PrecompileStagingArea, :($(Symbol(_mod)) = $_mod))
print("Generating precompile statements...")
sysimg = Base.unsafe_string(Base.JLOptions().image_file)
mktemp() do precompile_file, _
# Run a repl process and replay our script
repl_output_buffer = IOBuffer()
@static if Sys.iswindows()
# Fake being cygwin
pipename = """\\\\?\\pipe\\cygwin-$("0"^16)-pty10-abcdef"""
server = listen(pipename)
slave = connect(pipename)
@assert ccall(:jl_ispty, Cint, (Ptr{Cvoid},), slave.handle) == 1
master = accept(server)
else
slave, master = open_fake_pty()
end
end

# TODO: Implement REPL replayer for Windows
@static if !Sys.iswindows()
print("Generating precompile statements...")
sysimg = isempty(ARGS) ? joinpath(dirname(Sys.BINDIR), "lib", "julia", "sys.ji") : ARGS[1]

mktemp() do precompile_file, _
# Run a repl process and replay our script
stdout_accumulator, stderr_accumulator = IOBuffer(), IOBuffer()
with_fake_pty() do slave, master
with_fake_pty() do slave_err, master_err
done = false
withenv("JULIA_HISTORY" => tempname(), "JULIA_PROJECT" => nothing,
"TERM" => "") do
p = run(`$(julia_cmd()) -O0 --trace-compile=$precompile_file --sysimage $sysimg
--startup-file=no --color=yes`,
slave, slave, slave_err; wait=false)
readuntil(master, "julia>", keep=true)
for (tty, accumulator) in (master => stdout_accumulator,
master_err => stderr_accumulator)
@async begin
while true
done && break
write(accumulator, readavailable(tty))
end
end
end
if have_repl
for l in split(precompile_script, '\n'; keepempty=false)
write(master, l, '\n')
end
done = false
withenv("JULIA_HISTORY" => tempname(), "JULIA_PROJECT" => nothing,
"TERM" => "") do
if have_repl
p = run(`$(julia_cmd()) -O0 --trace-compile=$precompile_file --sysimage $sysimg
--compile=all --startup-file=no --color=yes
-e 'import REPL; REPL.Terminals.is_precompiling[] = true'
-i`,
slave, slave, slave; wait=false)
readuntil(master, "julia>", keep=true)
t = @async begin
while true
sleep(0.5)
s = String(readavailable(master))
write(repl_output_buffer, s)
if occursin("__PRECOMPILE_END__", s)
break
end
write(master, "exit()\n")
wait(p)
done = true
end
end
if have_repl
for l in split(precompile_script, '\n'; keepempty=false)
write(master, l, '\n')
end
end
write(master, "print(\"__PRECOMPILE\", \"_END__\")", '\n')
wait(t)

# TODO Figure out why exit() on Windows doesn't exit the process
if Sys.iswindows()
print(master, "ccall(:_exit, Cvoid, (Cint,), 0)\n")
else
write(master, "exit()\n")
readuntil(master, "exit()\r\e[13C\r\n")
# @assert bytesavailable(master) == 0
end
wait(p)
else
# Is this even needed or is this already recorded just from starting this process?
p = run(`$(julia_cmd()) -O0 --trace-compile=$precompile_file --sysimage $sysimg
--compile=all --startup-file=no
-e0`)
end
end
close(master)

# Check what the REPL displayed
# stdout_output = String(take!(stdout_accumulator))
# println(stdout_output)
# Check what the REPL displayed
# repl_output = String(take!(repl_output_buffer))
# println(repl_output)

# Extract the precompile statements from stderr
statements = Set{String}()
for statement in split(read(precompile_file, String), '\n')
occursin("Main.", statement) && continue
push!(statements, statement)
end
# Extract the precompile statements from stderr
statements = Set{String}()
for statement in split(read(precompile_file, String), '\n')
occursin("Main.", statement) && continue
push!(statements, statement)
end

# Load the precompile statements
statements_ordered = join(sort(collect(statements)), '\n')
# println(statements_ordered)
if have_repl
# Seems like a reasonable number right now, adjust as needed
@assert length(statements) > 700
if have_repl
# Seems like a reasonable number right now, adjust as needed
# comment out if debugging script
@assert length(statements) > 700
end

# Create a staging area where all the loaded packages are available
PrecompileStagingArea = Module()
for (_pkgid, _mod) in Base.loaded_modules
if !(_pkgid.name in ("Main", "Core", "Base"))
eval(PrecompileStagingArea, :($(Symbol(_mod)) = $_mod))
end
end

Base.include_string(PrecompileStagingArea, statements_ordered)
print(" $(length(statements)) generated in ")
Base.time_print((time() - start_time) * 10^9)
println()
# Execute the collected precompile statements
include_time = @elapsed for statement in sort(collect(statements))
# println(statement)
# Work around #28808
occursin("\"YYYY-mm-dd\\THH:MM:SS\"", statement) && continue
try
Base.include_string(PrecompileStagingArea, statement)
catch ex
@error "Failed to precompile $statement"
rethrow(ex)
end
end
print(" $(length(statements)) generated in ")
tot_time = time() - start_time
Base.time_print(tot_time * 10^9)
print(" (overhead "); Base.time_print((tot_time - include_time) * 10^9); println(")")
end

# Fall back to explicit list on Windows, might as well include them
# for everyone though
Base.include(PrecompileStagingArea, "precompile_explicit.jl")

return
end

Expand Down
Loading