From 6059601d2ba8dc779c1414d47ebfdf16da3343f0 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Mon, 1 Mar 2021 13:11:24 -0800 Subject: [PATCH] [ext/HSG]: Enable generating historical stdlibs on macOS (#2417) Also, update the historical stdlib list (cherry picked from commit 7b870924a15b40e4462cb7b354af39050c79fbb4) --- ext/HistoricaStdlibGenerator/Manifest.toml | 6 +- .../generate_historical_stdlibs.jl | 122 ++++++++++++------ src/HistoricalStdlibs.jl | 39 +++--- 3 files changed, 109 insertions(+), 58 deletions(-) diff --git a/ext/HistoricaStdlibGenerator/Manifest.toml b/ext/HistoricaStdlibGenerator/Manifest.toml index d49be715e6..aed2488356 100644 --- a/ext/HistoricaStdlibGenerator/Manifest.toml +++ b/ext/HistoricaStdlibGenerator/Manifest.toml @@ -74,7 +74,7 @@ uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" version = "1.0.15" [[Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs"] +deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] path = "../.." uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78e" version = "1.5.0" @@ -134,3 +134,7 @@ uuid = "83775a58-1f1d-513f-b197-d71354ab007a" [[nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" + +[[p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" diff --git a/ext/HistoricaStdlibGenerator/generate_historical_stdlibs.jl b/ext/HistoricaStdlibGenerator/generate_historical_stdlibs.jl index 01f8dd1e02..dce1e9db24 100644 --- a/ext/HistoricaStdlibGenerator/generate_historical_stdlibs.jl +++ b/ext/HistoricaStdlibGenerator/generate_historical_stdlibs.jl @@ -34,7 +34,7 @@ function generate_nightly_url(major, minor, host = HostPlatform()) # Map arch arch_str = Dict("x86_64" => "x64", "i686" => "x86", "aarch64" => "aarch64", "armv7l" => "armv7l", "ppc64le" => "ppc64le")[arch(host)] # Map OS name - os_str = Dict("linux" => "linux", "windows" => "winnt", "macos" => "macos", "freebsd" => "freebsd")[os(host)] + os_str = Dict("linux" => "linux", "windows" => "winnt", "macos" => "mac", "freebsd" => "freebsd")[os(host)] # Map wordsize tag wordsize_str = Dict("x86_64" => "64", "i686" => "32", "aarch64" => "aarch64", "armv7l" => "armv7l", "ppc64le" => "ppc64")[arch(host)] @@ -48,7 +48,7 @@ function generate_nightly_url(major, minor, host = HostPlatform()) string(major), ".", string(minor), "/", "julia-latest-", # linux64 - os(host), wordsize_str, + os_str, wordsize_str, ".tar.gz", ) end @@ -68,20 +68,56 @@ function get_stdlibs(scratch_dir, julia_installer_name) installer_path = joinpath(scratch_dir, julia_installer_name) mktempdir() do dir @info("Extracting $(julia_installer_name)") - run(`tar -C $(dir) --strip-components=1 -zxf $(installer_path)`) - - jlexe = joinpath(dir, "bin", @static Sys.iswindows() ? "julia.exe" : "julia") - jlflags = ["--startup-file=no", "-O0"] - jlvers = VersionNumber(readchomp(`$(jlexe) $(jlflags) -e 'print(VERSION)'`)) - jlvers = VersionNumber(jlvers.major, jlvers.minor, jlvers.patch) - @info("Auto-detected Julia version $(jlvers)") - - if jlvers < v"1.1" - stdlibs_str = readchomp(`$(jlexe) $(jlflags) -e 'import Pkg; println(repr(Pkg.Types.gather_stdlib_uuids()))'`) - else - stdlibs_str = readchomp(`$(jlexe) $(jlflags) -e 'import Pkg; println(repr(Pkg.Types.load_stdlib()))'`) + mount_dir = joinpath(dir, "mount_dir") + try + if endswith(installer_path, ".dmg") + mkdir(mount_dir) + # Try to mount many times, as this seems to fail randomly + mount_cmd = `hdiutil mount $(installer_path) -mountpoint $(mount_dir)` + tries = 0 + while !success(mount_cmd) + if tries > 10 + error("Unable to mount via hdiutil!") + end + sleep(0.1) + tries += 1 + end + #@show readdir(mount_dir; join=true) + app_dir = first(filter(d -> startswith(basename(d), "Julia-"), readdir(mount_dir; join=true))) + symlink(joinpath(app_dir, "Contents", "Resources", "julia", "bin"), joinpath(dir, "bin")) + elseif endswith(installer_path, ".exe") + error("This script doesn't work with `.exe` downloads") + else + run(`tar -C $(dir) --strip-components=1 -zxf $(installer_path)`) + end + + jlexe = joinpath(dir, "bin", @static Sys.iswindows() ? "julia.exe" : "julia") + jlflags = ["--startup-file=no", "-O0"] + jlvers = VersionNumber(readchomp(`$(jlexe) $(jlflags) -e 'print(VERSION)'`)) + jlvers = VersionNumber(jlvers.major, jlvers.minor, jlvers.patch) + @info("Auto-detected Julia version $(jlvers)") + + if jlvers < v"1.1" + stdlibs_str = readchomp(`$(jlexe) $(jlflags) -e 'import Pkg; println(repr(Pkg.Types.gather_stdlib_uuids()))'`) + else + stdlibs_str = readchomp(`$(jlexe) $(jlflags) -e 'import Pkg; println(repr(Pkg.Types.load_stdlib()))'`) + end + + return (jlvers, stdlibs_str) + finally + # Clean up mounted directories + if isdir(mount_dir) + unmount_cmd = `hdiutil detach $(mount_dir)` + tries = 0 + while !success(unmount_cmd) + if tries > 10 + error("Unable to unmount $(mount_dir)") + end + sleep(0.1) + tries += 1 + end + end end - return (jlvers, stdlibs_str) end end @@ -101,21 +137,35 @@ versions_dict = Dict() for _ in 1:num_concurrent_downloads @async begin for (url, hash) in jobs - fname = joinpath(scratch_dir, basename(url)) - if !isfile(fname) - @info("Downloading $(url)") - Downloads.download(url, fname) - end + try + fname = joinpath(scratch_dir, basename(url)) + if !isfile(fname) + @info("Downloading $(url)") + Downloads.download(url, fname) + end - if !isempty(hash) - calc_hash = bytes2hex(open(io -> sha256(io), fname, "r")) - if calc_hash != hash - @error("Hash mismatch on $(fname)") + if !isempty(hash) + calc_hash = bytes2hex(open(io -> sha256(io), fname, "r")) + if calc_hash != hash + @error("Hash mismatch on $(fname); deleting and re-downloading") + rm(fname; force=true) + Downloads.download(url, fname) + calc_hash = bytes2hex(open(io -> sha256(io), fname, "r")) + if calc_hash != hash + @error("Hash mismatch on $(fname); re-download failed!") + continue + end + end end - end - version, stdlibs = get_stdlibs(scratch_dir, basename(url)) - versions_dict[version] = eval(Meta.parse(stdlibs)) + version, stdlibs = get_stdlibs(scratch_dir, basename(url)) + versions_dict[version] = eval(Meta.parse(stdlibs)) + catch e + if isa(e, InterruptException) + rethrow() + end + @error(e, exception=(e, catch_backtrace())) + end end end end @@ -145,12 +195,12 @@ unregistered_stdlibs = filter(all_stdlibs) do (uuid, name) end # Helper function for getting these printed out in a nicely-sorted order -function print_sorted(io::IO, d::Dict) +function print_sorted(io::IO, d::Dict; indent::Int=0) println(io, "Dict(") for pair in sort(collect(d), by = kv-> kv[2]) - println(io, " ", pair, ",") + println(io, " "^indent, pair, ",") end - println(io, " ),") + print(io, " "^(max(indent - 4, 0)), ")") end output_fname = joinpath(dirname(dirname(@__DIR__)), "src", "HistoricalStdlibs.jl") @@ -168,7 +218,8 @@ open(output_fname, "w") do io """) for v in sorted_versions print(io, " $(repr(v)) => ") - print_sorted(io, versions_dict[v]) + print_sorted(io, versions_dict[v]; indent=8) + println(io, ",") end println(io, "]") @@ -176,10 +227,7 @@ open(output_fname, "w") do io # Next, we also embed a list of stdlibs that must _always_ be treated as stdlibs, # because they cannot be resolved in the registry; they have only ever existed within # the Julia stdlib source tree, and because of that, trying to resolve them will fail. - const UNREGISTERED_STDLIBS = Dict( - """) - for (uuid, name) in unregistered_stdlibs - println(io, " $(repr(uuid)) => $(repr(name)),") - end - println(io, ")") + const UNREGISTERED_STDLIBS = """) + print_sorted(io, unregistered_stdlibs; indent=4) + println(io) end diff --git a/src/HistoricalStdlibs.jl b/src/HistoricalStdlibs.jl index 5f58bd7325..880f36fe54 100644 --- a/src/HistoricalStdlibs.jl +++ b/src/HistoricalStdlibs.jl @@ -100,32 +100,31 @@ const STDLIBS_BY_VERSION = [ # because they cannot be resolved in the registry; they have only ever existed within # the Julia stdlib source tree, and because of that, trying to resolve them will fail. const UNREGISTERED_STDLIBS = Dict( - UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c") => "Random", - UUID("10745b16-79ce-11e8-11f9-7d13ad32a3b2") => "Statistics", - UUID("8bb1440f-4735-579b-a4ab-409b98df4dab") => "DelimitedFiles", - UUID("a63ad114-7e13-5084-954f-fe012c677804") => "Mmap", - UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb") => "Libdl", + UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f") => "Base64", UUID("8bf52ea8-c179-5cab-976a-9e18b702a9bc") => "CRC32c", + UUID("ade2ca70-3891-5945-98fb-dc099432e06a") => "Dates", + UUID("8bb1440f-4735-579b-a4ab-409b98df4dab") => "DelimitedFiles", + UUID("8ba89e20-285c-5b6f-9357-94700520ee1b") => "Distributed", + UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee") => "FileWatching", + UUID("9fa8497b-333b-5362-9e8d-4d0656e87820") => "Future", UUID("b77e0a4c-d291-57a0-90e8-8db25a27a240") => "InteractiveUtils", - UUID("de0858da-6303-5e67-8744-51eddeeeb8d7") => "Printf", + UUID("76f85450-5226-5b5a-8eaa-529ad045b433") => "LibGit2", + UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb") => "Libdl", + UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e") => "LinearAlgebra", + UUID("56ddb016-857b-54e1-b83d-db4d58db5568") => "Logging", UUID("d6f4376e-aef5-505a-96c1-9c027394607a") => "Markdown", - UUID("4af54fe1-eca0-43a8-85a7-787d91b784e3") => "LazyArtifacts", - UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb") => "REPL", - UUID("6462fe0b-24de-5631-8697-dd941f90decc") => "Sockets", + UUID("a63ad114-7e13-5084-954f-fe012c677804") => "Mmap", UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") => "Pkg", - UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e") => "LinearAlgebra", + UUID("de0858da-6303-5e67-8744-51eddeeeb8d7") => "Printf", UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79") => "Profile", - UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f") => "Base64", - UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee") => "FileWatching", + UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb") => "REPL", + UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c") => "Random", UUID("9e88b42a-f829-5b0c-bbe9-9e923198166b") => "Serialization", - UUID("56ddb016-857b-54e1-b83d-db4d58db5568") => "Logging", + UUID("1a1011a3-84de-559e-8e89-a11a2f7dc383") => "SharedArrays", + UUID("6462fe0b-24de-5631-8697-dd941f90decc") => "Sockets", + UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf") => "SparseArrays", + UUID("10745b16-79ce-11e8-11f9-7d13ad32a3b2") => "Statistics", + UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40") => "Test", UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4") => "UUIDs", - UUID("ade2ca70-3891-5945-98fb-dc099432e06a") => "Dates", UUID("4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5") => "Unicode", - UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40") => "Test", - UUID("9fa8497b-333b-5362-9e8d-4d0656e87820") => "Future", - UUID("76f85450-5226-5b5a-8eaa-529ad045b433") => "LibGit2", - UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf") => "SparseArrays", - UUID("1a1011a3-84de-559e-8e89-a11a2f7dc383") => "SharedArrays", - UUID("8ba89e20-285c-5b6f-9357-94700520ee1b") => "Distributed", )