Skip to content

Commit

Permalink
[ext/HSG]: Enable generating historical stdlibs on macOS (JuliaLang#2417
Browse files Browse the repository at this point in the history 
)

Also, update the historical stdlib list
  • Loading branch information
@staticfloat
staticfloat authored Mar 1, 2021
1 parent 5d49619 commit 7b87092
Show file tree
Hide file tree
Showing 3 changed files with 110 additions and 59 deletions.
6 changes: 5 additions & 1 deletion ext/HistoricaStdlibGenerator/Manifest.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "1.0.15"

[[Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs"]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
path = "../.."
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78e"
version = "1.5.0"
Expand Down Expand Up @@ -134,3 +134,7 @@ uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
[[nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"

[[p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
122 changes: 85 additions & 37 deletions ext/HistoricaStdlibGenerator/generate_historical_stdlibs.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ function generate_nightly_url(major, minor, host = HostPlatform())
# Map arch
arch_str = Dict("x86_64" => "x64", "i686" => "x86", "aarch64" => "aarch64", "armv7l" => "armv7l", "ppc64le" => "ppc64le")[arch(host)]
# Map OS name
os_str = Dict("linux" => "linux", "windows" => "winnt", "macos" => "macos", "freebsd" => "freebsd")[os(host)]
os_str = Dict("linux" => "linux", "windows" => "winnt", "macos" => "mac", "freebsd" => "freebsd")[os(host)]
# Map wordsize tag
wordsize_str = Dict("x86_64" => "64", "i686" => "32", "aarch64" => "aarch64", "armv7l" => "armv7l", "ppc64le" => "ppc64")[arch(host)]

Expand All @@ -48,7 +48,7 @@ function generate_nightly_url(major, minor, host = HostPlatform())
string(major), ".", string(minor), "/",
"julia-latest-",
# linux64
os(host), wordsize_str,
os_str, wordsize_str,
".tar.gz",
)
end
Expand All @@ -68,20 +68,56 @@ function get_stdlibs(scratch_dir, julia_installer_name)
installer_path = joinpath(scratch_dir, julia_installer_name)
mktempdir() do dir
@info("Extracting $(julia_installer_name)")
run(`tar -C $(dir) --strip-components=1 -zxf $(installer_path)`)

jlexe = joinpath(dir, "bin", @static Sys.iswindows() ? "julia.exe" : "julia")
jlflags = ["--startup-file=no", "-O0"]
jlvers = VersionNumber(readchomp(`$(jlexe) $(jlflags) -e 'print(VERSION)'`))
jlvers = VersionNumber(jlvers.major, jlvers.minor, jlvers.patch)
@info("Auto-detected Julia version $(jlvers)")

if jlvers < v"1.1"
stdlibs_str = readchomp(`$(jlexe) $(jlflags) -e 'import Pkg; println(repr(Pkg.Types.gather_stdlib_uuids()))'`)
else
stdlibs_str = readchomp(`$(jlexe) $(jlflags) -e 'import Pkg; println(repr(Pkg.Types.load_stdlib()))'`)
mount_dir = joinpath(dir, "mount_dir")
try
if endswith(installer_path, ".dmg")
mkdir(mount_dir)
# Try to mount many times, as this seems to fail randomly
mount_cmd = `hdiutil mount $(installer_path) -mountpoint $(mount_dir)`
tries = 0
while !success(mount_cmd)
if tries > 10
error("Unable to mount via hdiutil!")
end
sleep(0.1)
tries += 1
end
#@show readdir(mount_dir; join=true)
app_dir = first(filter(d -> startswith(basename(d), "Julia-"), readdir(mount_dir; join=true)))
symlink(joinpath(app_dir, "Contents", "Resources", "julia", "bin"), joinpath(dir, "bin"))
elseif endswith(installer_path, ".exe")
error("This script doesn't work with `.exe` downloads")
else
run(`tar -C $(dir) --strip-components=1 -zxf $(installer_path)`)
end

jlexe = joinpath(dir, "bin", @static Sys.iswindows() ? "julia.exe" : "julia")
jlflags = ["--startup-file=no", "-O0"]
jlvers = VersionNumber(readchomp(`$(jlexe) $(jlflags) -e 'print(VERSION)'`))
jlvers = VersionNumber(jlvers.major, jlvers.minor, jlvers.patch)
@info("Auto-detected Julia version $(jlvers)")

if jlvers < v"1.1"
stdlibs_str = readchomp(`$(jlexe) $(jlflags) -e 'import Pkg; println(repr(Pkg.Types.gather_stdlib_uuids()))'`)
else
stdlibs_str = readchomp(`$(jlexe) $(jlflags) -e 'import Pkg; println(repr(Pkg.Types.load_stdlib()))'`)
end

return (jlvers, stdlibs_str)
finally
# Clean up mounted directories
if isdir(mount_dir)
unmount_cmd = `hdiutil detach $(mount_dir)`
tries = 0
while !success(unmount_cmd)
if tries > 10
error("Unable to unmount $(mount_dir)")
end
sleep(0.1)
tries += 1
end
end
end
return (jlvers, stdlibs_str)
end
end

Expand All @@ -101,21 +137,35 @@ versions_dict = Dict()
for _ in 1:num_concurrent_downloads
@async begin
for (url, hash) in jobs
fname = joinpath(scratch_dir, basename(url))
if !isfile(fname)
@info("Downloading $(url)")
Downloads.download(url, fname)
end
try
fname = joinpath(scratch_dir, basename(url))
if !isfile(fname)
@info("Downloading $(url)")
Downloads.download(url, fname)
end

if !isempty(hash)
calc_hash = bytes2hex(open(io -> sha256(io), fname, "r"))
if calc_hash != hash
@error("Hash mismatch on $(fname)")
if !isempty(hash)
calc_hash = bytes2hex(open(io -> sha256(io), fname, "r"))
if calc_hash != hash
@error("Hash mismatch on $(fname); deleting and re-downloading")
rm(fname; force=true)
Downloads.download(url, fname)
calc_hash = bytes2hex(open(io -> sha256(io), fname, "r"))
if calc_hash != hash
@error("Hash mismatch on $(fname); re-download failed!")
continue
end
end
end
end

version, stdlibs = get_stdlibs(scratch_dir, basename(url))
versions_dict[version] = eval(Meta.parse(stdlibs))
version, stdlibs = get_stdlibs(scratch_dir, basename(url))
versions_dict[version] = eval(Meta.parse(stdlibs))
catch e
if isa(e, InterruptException)
rethrow()
end
@error(e, exception=(e, catch_backtrace()))
end
end
end
end
Expand Down Expand Up @@ -145,12 +195,12 @@ unregistered_stdlibs = filter(all_stdlibs) do (uuid, name)
end

# Helper function for getting these printed out in a nicely-sorted order
function print_sorted(io::IO, d::Dict)
function print_sorted(io::IO, d::Dict; indent::Int=0)
println(io, "Dict(")
for pair in sort(collect(d), by = kv-> kv[2])
println(io, " ", pair, ",")
println(io, " "^indent, pair, ",")
end
println(io, " ),")
print(io, " "^(max(indent - 4, 0)), ")")
end

output_fname = joinpath(dirname(dirname(@__DIR__)), "src", "HistoricalStdlibs.jl")
Expand All @@ -168,18 +218,16 @@ open(output_fname, "w") do io
""")
for v in sorted_versions
print(io, " $(repr(v)) => ")
print_sorted(io, versions_dict[v])
print_sorted(io, versions_dict[v]; indent=8)
println(io, ",")
end
println(io, "]")

print(io, """
# Next, we also embed a list of stdlibs that must _always_ be treated as stdlibs,
# because they cannot be resolved in the registry; they have only ever existed within
# the Julia stdlib source tree, and because of that, trying to resolve them will fail.
const UNREGISTERED_STDLIBS = Dict(
""")
for (uuid, name) in unregistered_stdlibs
println(io, " $(repr(uuid)) => $(repr(name)),")
end
println(io, ")")
const UNREGISTERED_STDLIBS = """)
print_sorted(io, unregistered_stdlibs; indent=4)
println(io)
end
41 changes: 20 additions & 21 deletions src/HistoricalStdlibs.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,12 @@ const STDLIBS_BY_VERSION = [
UUID("deac9b47-8bc7-5906-a0fe-35ac56dc84c0") => "LibCURL_jll",
UUID("76f85450-5226-5b5a-8eaa-529ad045b433") => "LibGit2",
UUID("e37daf67-58a4-590a-8e99-b0245dd2ffc5") => "LibGit2_jll",
UUID("a83860b7-747b-57cf-bf1f-3e79990d037f") => "LibOSXUnwind_jll",
UUID("29816b5a-b9ab-546f-933c-edad1886dfa8") => "LibSSH2_jll",
UUID("183b4373-6708-53ba-ad28-60e28bb38547") => "LibUV_jll",
UUID("745a5e78-f969-53e9-954f-d19f2f74f4e3") => "LibUnwind_jll",
UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb") => "Libdl",
UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e") => "LinearAlgebra",
UUID("47c5dbc3-30ba-59ef-96a6-123e260183d9") => "LLVMLibUnwind_jll",
UUID("56ddb016-857b-54e1-b83d-db4d58db5568") => "Logging",
UUID("3a97d323-0669-5f0c-9066-3539efd106a3") => "MPFR_jll",
UUID("d6f4376e-aef5-505a-96c1-9c027394607a") => "Markdown",
Expand Down Expand Up @@ -100,32 +100,31 @@ const STDLIBS_BY_VERSION = [
# because they cannot be resolved in the registry; they have only ever existed within
# the Julia stdlib source tree, and because of that, trying to resolve them will fail.
const UNREGISTERED_STDLIBS = Dict(
UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c") => "Random",
UUID("10745b16-79ce-11e8-11f9-7d13ad32a3b2") => "Statistics",
UUID("8bb1440f-4735-579b-a4ab-409b98df4dab") => "DelimitedFiles",
UUID("a63ad114-7e13-5084-954f-fe012c677804") => "Mmap",
UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb") => "Libdl",
UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f") => "Base64",
UUID("8bf52ea8-c179-5cab-976a-9e18b702a9bc") => "CRC32c",
UUID("ade2ca70-3891-5945-98fb-dc099432e06a") => "Dates",
UUID("8bb1440f-4735-579b-a4ab-409b98df4dab") => "DelimitedFiles",
UUID("8ba89e20-285c-5b6f-9357-94700520ee1b") => "Distributed",
UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee") => "FileWatching",
UUID("9fa8497b-333b-5362-9e8d-4d0656e87820") => "Future",
UUID("b77e0a4c-d291-57a0-90e8-8db25a27a240") => "InteractiveUtils",
UUID("de0858da-6303-5e67-8744-51eddeeeb8d7") => "Printf",
UUID("76f85450-5226-5b5a-8eaa-529ad045b433") => "LibGit2",
UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb") => "Libdl",
UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e") => "LinearAlgebra",
UUID("56ddb016-857b-54e1-b83d-db4d58db5568") => "Logging",
UUID("d6f4376e-aef5-505a-96c1-9c027394607a") => "Markdown",
UUID("4af54fe1-eca0-43a8-85a7-787d91b784e3") => "LazyArtifacts",
UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb") => "REPL",
UUID("6462fe0b-24de-5631-8697-dd941f90decc") => "Sockets",
UUID("a63ad114-7e13-5084-954f-fe012c677804") => "Mmap",
UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") => "Pkg",
UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e") => "LinearAlgebra",
UUID("de0858da-6303-5e67-8744-51eddeeeb8d7") => "Printf",
UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79") => "Profile",
UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f") => "Base64",
UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee") => "FileWatching",
UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb") => "REPL",
UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c") => "Random",
UUID("9e88b42a-f829-5b0c-bbe9-9e923198166b") => "Serialization",
UUID("56ddb016-857b-54e1-b83d-db4d58db5568") => "Logging",
UUID("1a1011a3-84de-559e-8e89-a11a2f7dc383") => "SharedArrays",
UUID("6462fe0b-24de-5631-8697-dd941f90decc") => "Sockets",
UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf") => "SparseArrays",
UUID("10745b16-79ce-11e8-11f9-7d13ad32a3b2") => "Statistics",
UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40") => "Test",
UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4") => "UUIDs",
UUID("ade2ca70-3891-5945-98fb-dc099432e06a") => "Dates",
UUID("4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5") => "Unicode",
UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40") => "Test",
UUID("9fa8497b-333b-5362-9e8d-4d0656e87820") => "Future",
UUID("76f85450-5226-5b5a-8eaa-529ad045b433") => "LibGit2",
UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf") => "SparseArrays",
UUID("1a1011a3-84de-559e-8e89-a11a2f7dc383") => "SharedArrays",
UUID("8ba89e20-285c-5b6f-9357-94700520ee1b") => "Distributed",
)

0 comments on commit 7b87092

Please sign in to comment.