Skip to content

Commit

Permalink
[ext/HSG]: Store next release _and_ latest nightly (#2418)
Browse files Browse the repository at this point in the history 
In the event that we are in an RC phase where we have three "latest"
branch tips (e.g. latest stable: 1.5.3, latest RC: 1.6.0-rc1 and latest
`master`: 1.7.0-DEV) each with their own set of standard libraries, we
must store this information in Pkg so that we pass tests when running on
latest `master`. It is harmless for this information to be baked into an
older Julia, as we make no guarantees that Julia 1.6.0 final should have
perfect information about what standard libraries 1.7.0 will have.

(cherry picked from commit 7a9d965)
  • Loading branch information
@staticfloat @KristofferC
staticfloat authored and KristofferC committed Mar 2, 2021
1 parent 707c443 commit e33538f
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 7 deletions.
23 changes: 16 additions & 7 deletions ext/HistoricaStdlibGenerator/generate_historical_stdlibs.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,31 +30,38 @@ function select_url_hash(data, host = HostPlatform())
end
version_urls = sort(select_url_hash.(values(versions)), by = pair -> pair[1])

function generate_nightly_url(major, minor, host = HostPlatform())
function generate_nightly_url(jlver, host = HostPlatform())
# Map arch
arch_str = Dict("x86_64" => "x64", "i686" => "x86", "aarch64" => "aarch64", "armv7l" => "armv7l", "ppc64le" => "ppc64le")[arch(host)]
# Map OS name
os_str = Dict("linux" => "linux", "windows" => "winnt", "macos" => "mac", "freebsd" => "freebsd")[os(host)]
# Map wordsize tag
wordsize_str = Dict("x86_64" => "64", "i686" => "32", "aarch64" => "aarch64", "armv7l" => "armv7l", "ppc64le" => "ppc64")[arch(host)]

# If `jlver` is nothing, we don't namespace by version and just get the absolute latest version
ver_str = ""
if jlver !== nothing
ver_str = string(jlver.major, ".", jlver.minor, "/")
end

return string(
"https://julialangnightlies-s3.julialang.org/bin/",
# linux/
os_str, "/",
# x64/
arch_str, "/",
# 1.6/
string(major), ".", string(minor), "/",
# 1.6/ (or nothing, if `jlver === nothing`)
ver_str,
"julia-latest-",
# linux64
os_str, wordsize_str,
".tar.gz",
)
end
highest_release = maximum(VersionNumber.(string.(keys(versions))))
next_release_url = generate_nightly_url(highest_release.major, highest_release.minor + 1)
push!(version_urls, (next_release_url, ""))
next_release = VersionNumber(highest_release.major, highest_release.minor + 1, 0)
push!(version_urls, (generate_nightly_url(next_release), ""))
push!(version_urls, (generate_nightly_url(nothing), ""))
@info("Identified $(length(version_urls)) versions to try...")

# Next, we're going to download each of these to a scratch space
Expand Down Expand Up @@ -138,7 +145,9 @@ versions_dict = Dict()
@async begin
for (url, hash) in jobs
try
fname = joinpath(scratch_dir, basename(url))
# We might try to download two files that have the same basename
url_tag = bytes2hex(sha256(url))
fname = joinpath(scratch_dir, string(url_tag, "-", basename(url)))
if !isfile(fname)
@info("Downloading $(url)")
Downloads.download(url, fname)
Expand All @@ -158,7 +167,7 @@ versions_dict = Dict()
end
end

version, stdlibs = get_stdlibs(scratch_dir, basename(url))
version, stdlibs = get_stdlibs(scratch_dir, basename(fname))
versions_dict[version] = eval(Meta.parse(stdlibs))
catch e
if isa(e, InterruptException)
Expand Down
60 changes: 60 additions & 0 deletions src/HistoricalStdlibs.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,66 @@ const STDLIBS_BY_VERSION = [
UUID("8e850ede-7688-5339-a07c-302acd2aaf8d") => "nghttp2_jll",
UUID("3f19e933-33d8-53b3-aaab-bd5110c3b7a0") => "p7zip_jll",
),
v"1.7.0" => Dict(
UUID("0dad84c5-d112-42e6-8d28-ef12dabb789f") => "ArgTools",
UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33") => "Artifacts",
UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f") => "Base64",
UUID("8bf52ea8-c179-5cab-976a-9e18b702a9bc") => "CRC32c",
UUID("e66e0078-7015-5450-92f7-15fbd957f2ae") => "CompilerSupportLibraries_jll",
UUID("ade2ca70-3891-5945-98fb-dc099432e06a") => "Dates",
UUID("8bb1440f-4735-579b-a4ab-409b98df4dab") => "DelimitedFiles",
UUID("8ba89e20-285c-5b6f-9357-94700520ee1b") => "Distributed",
UUID("f43a241f-c20a-4ad4-852c-f6b1247861c6") => "Downloads",
UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee") => "FileWatching",
UUID("9fa8497b-333b-5362-9e8d-4d0656e87820") => "Future",
UUID("781609d7-10c4-51f6-84f2-b8444358ff6d") => "GMP_jll",
UUID("b77e0a4c-d291-57a0-90e8-8db25a27a240") => "InteractiveUtils",
UUID("47c5dbc3-30ba-59ef-96a6-123e260183d9") => "LLVMLibUnwind_jll",
UUID("4af54fe1-eca0-43a8-85a7-787d91b784e3") => "LazyArtifacts",
UUID("b27032c2-a3e7-50c8-80cd-2d36dbcbfd21") => "LibCURL",
UUID("deac9b47-8bc7-5906-a0fe-35ac56dc84c0") => "LibCURL_jll",
UUID("76f85450-5226-5b5a-8eaa-529ad045b433") => "LibGit2",
UUID("e37daf67-58a4-590a-8e99-b0245dd2ffc5") => "LibGit2_jll",
UUID("29816b5a-b9ab-546f-933c-edad1886dfa8") => "LibSSH2_jll",
UUID("183b4373-6708-53ba-ad28-60e28bb38547") => "LibUV_jll",
UUID("745a5e78-f969-53e9-954f-d19f2f74f4e3") => "LibUnwind_jll",
UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb") => "Libdl",
UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e") => "LinearAlgebra",
UUID("56ddb016-857b-54e1-b83d-db4d58db5568") => "Logging",
UUID("3a97d323-0669-5f0c-9066-3539efd106a3") => "MPFR_jll",
UUID("d6f4376e-aef5-505a-96c1-9c027394607a") => "Markdown",
UUID("c8ffd9c3-330d-5841-b78e-0817d7145fa1") => "MbedTLS_jll",
UUID("a63ad114-7e13-5084-954f-fe012c677804") => "Mmap",
UUID("14a3606d-f60d-562e-9121-12d972cd8159") => "MozillaCACerts_jll",
UUID("ca575930-c2e3-43a9-ace4-1e988b2c1908") => "NetworkOptions",
UUID("4536629a-c528-5b80-bd46-f80d51c5b363") => "OpenBLAS_jll",
UUID("05823500-19ac-5b8b-9628-191a04bc5112") => "OpenLibm_jll",
UUID("efcefdf7-47ab-520b-bdef-62a2eaa19f15") => "PCRE2_jll",
UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") => "Pkg",
UUID("de0858da-6303-5e67-8744-51eddeeeb8d7") => "Printf",
UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79") => "Profile",
UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb") => "REPL",
UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c") => "Random",
UUID("ea8e919c-243c-51af-8825-aaa63cd721ce") => "SHA",
UUID("9e88b42a-f829-5b0c-bbe9-9e923198166b") => "Serialization",
UUID("1a1011a3-84de-559e-8e89-a11a2f7dc383") => "SharedArrays",
UUID("6462fe0b-24de-5631-8697-dd941f90decc") => "Sockets",
UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf") => "SparseArrays",
UUID("10745b16-79ce-11e8-11f9-7d13ad32a3b2") => "Statistics",
UUID("4607b0f0-06f3-5cda-b6b1-a6196a1729e9") => "SuiteSparse",
UUID("bea87d4a-7f5b-5778-9afe-8cc45184846c") => "SuiteSparse_jll",
UUID("fa267f1f-6049-4f14-aa54-33bafae1ed76") => "TOML",
UUID("a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e") => "Tar",
UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40") => "Test",
UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4") => "UUIDs",
UUID("4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5") => "Unicode",
UUID("83775a58-1f1d-513f-b197-d71354ab007a") => "Zlib_jll",
UUID("05ff407c-b0c1-5878-9df8-858cc2e60c36") => "dSFMT_jll",
UUID("8f36deef-c2a5-5394-99ed-8e07531fb29a") => "libLLVM_jll",
UUID("8e850b90-86db-534c-a0d3-1478176c7d93") => "libblastrampoline_jll",
UUID("8e850ede-7688-5339-a07c-302acd2aaf8d") => "nghttp2_jll",
UUID("3f19e933-33d8-53b3-aaab-bd5110c3b7a0") => "p7zip_jll",
),
]
# Next, we also embed a list of stdlibs that must _always_ be treated as stdlibs,
# because they cannot be resolved in the registry; they have only ever existed within
Expand Down

0 comments on commit e33538f

Please sign in to comment.