Skip to content

Commit

Permalink
implement the REPL replayer on Windows (#28608)
Browse files Browse the repository at this point in the history 
(cherry picked from commit 3fe2d08)
  • Loading branch information
@KristofferC
KristofferC committed Feb 11, 2019
1 parent ec9ea36 commit 9ad5424
Show file tree
Hide file tree
Showing 5 changed files with 87 additions and 1,239 deletions.
150 changes: 84 additions & 66 deletions contrib/generate_precompile.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ if !isdefined(Base, :uv_eventloop)
Base.reinit_stdio()
end
Base.include(@__MODULE__, joinpath(Sys.BINDIR, "..", "share", "julia", "test", "testhelpers", "FakePTYs.jl"))
import .FakePTYs: with_fake_pty
import .FakePTYs: open_fake_pty

CTRL_C = '\x03'
UP_ARROW = "\e[A"
Expand Down Expand Up @@ -43,6 +43,12 @@ if Pkg !== nothing
precompile_script *= Pkg.precompile_script
end

push!(LOAD_PATH, Sys.STDLIB)
using Sockets
Sockets.__init__()
using Libdl
empty!(LOAD_PATH)

function generate_precompile_statements()
start_time = time()

Expand All @@ -62,82 +68,94 @@ function generate_precompile_statements()
empty!(DEPOT_PATH)
end

# Create a staging area where all the loaded packages are available
PrecompileStagingArea = Module()
for (_pkgid, _mod) in Base.loaded_modules
if !(_pkgid.name in ("Main", "Core", "Base"))
eval(PrecompileStagingArea, :($(Symbol(_mod)) = $_mod))
print("Generating precompile statements...")
sysimg = isempty(ARGS) ? joinpath(dirname(Sys.BINDIR), "lib", "julia", "sys." * Libdl.dlext) : ARGS[1]

mktemp() do precompile_file, _
# Run a repl process and replay our script
repl_output_buffer = IOBuffer()
@static if Sys.iswindows()
# Fake being cygwin
pipename = """\\\\?\\pipe\\cygwin-$("0"^16)-pty10-abcdef"""
server = listen(pipename)
slave = connect(pipename)
@assert ccall(:jl_ispty, Cint, (Ptr{Cvoid},), slave.handle) == 1
master = accept(server)
else
slave, master = open_fake_pty()
end
end

# TODO: Implement REPL replayer for Windows
@static if !Sys.iswindows()
print("Generating precompile statements...")
sysimg = isempty(ARGS) ? joinpath(dirname(Sys.BINDIR), "lib", "julia", "sys.ji") : ARGS[1]

mktemp() do precompile_file, _
# Run a repl process and replay our script
stdout_accumulator, stderr_accumulator = IOBuffer(), IOBuffer()
with_fake_pty() do slave, master
with_fake_pty() do slave_err, master_err
done = false
withenv("JULIA_HISTORY" => tempname(), "JULIA_PROJECT" => nothing,
"TERM" => "") do
p = run(`$(julia_cmd()) -O0 --trace-compile=$precompile_file --sysimage $sysimg
--startup-file=no --color=yes`,
slave, slave, slave_err; wait=false)
readuntil(master, "julia>", keep=true)
for (tty, accumulator) in (master => stdout_accumulator,
master_err => stderr_accumulator)
@async begin
while true
done && break
write(accumulator, readavailable(tty))
end
end
end
if have_repl
for l in split(precompile_script, '\n'; keepempty=false)
write(master, l, '\n')
end
end
write(master, "exit()\n")
wait(p)
done = true
end
done = false
withenv("JULIA_HISTORY" => tempname(), "JULIA_PROJECT" => nothing,
"TERM" => "") do
p = run(`$(julia_cmd()) -O0 --trace-compile=$precompile_file --sysimage $sysimg
--compile=all --startup-file=no --color=yes
-e 'import REPL; REPL.Terminals.is_precompiling[] = true'
-i`,
slave, slave, slave; wait=false)
readuntil(master, "julia>", keep=true)
@async begin
while true
done && break
write(repl_output_buffer, readavailable(master))
end
end
if have_repl
for l in split(precompile_script, '\n'; keepempty=false)
write(master, l, '\n')
end
end
# TODO Figure out why exit() on Windows doesn't exit the process
if Sys.iswindows()
print(master, "ccall(:_exit, Cvoid, (Cint,), 0)\n")
else
write(master, "exit()\n")
end
wait(p)
done = true
end
close(master)

# Check what the REPL displayed
# stdout_output = String(take!(stdout_accumulator))
# println(stdout_output)
# Check what the REPL displayed
# repl_output = String(take!(repl_output_buffer))
# println(repl_output)

# Extract the precompile statements from stderr
statements = Set{String}()
for statement in split(read(precompile_file, String), '\n')
occursin("Main.", statement) && continue
push!(statements, statement)
end
# Extract the precompile statements from stderr
statements = Set{String}()
for statement in split(read(precompile_file, String), '\n')
occursin("Main.", statement) && continue
push!(statements, statement)
end

# Load the precompile statements
statements_ordered = join(sort(collect(statements)), '\n')
# println(statements_ordered)
if have_repl
# Seems like a reasonable number right now, adjust as needed
@assert length(statements) > 700
if have_repl
# Seems like a reasonable number right now, adjust as needed
# comment out if debugging script
@assert length(statements) > 700
end

# Create a staging area where all the loaded packages are available
PrecompileStagingArea = Module()
for (_pkgid, _mod) in Base.loaded_modules
if !(_pkgid.name in ("Main", "Core", "Base"))
eval(PrecompileStagingArea, :($(Symbol(_mod)) = $_mod))
end
end

Base.include_string(PrecompileStagingArea, statements_ordered)
print(" $(length(statements)) generated in ")
Base.time_print((time() - start_time) * 10^9)
println()
# Execute the collected precompile statements
include_time = @elapsed for statement in sort(collect(statements))
# println(statement)
try
Base.include_string(PrecompileStagingArea, statement)
catch ex
@error "Failed to precompile $statement"
rethrow(ex)
end
end
print(" $(length(statements)) generated in ")
tot_time = time() - start_time
Base.time_print(tot_time * 10^9)
print(" (overhead "); Base.time_print((tot_time - include_time) * 10^9); println(")")
end

# Fall back to explicit list on Windows, might as well include them
# for everyone though
Base.include(PrecompileStagingArea, "precompile_explicit.jl")

return
end

Expand Down
Loading

0 comments on commit 9ad5424

Please sign in to comment.