Skip to content

Commit

Permalink
[ext/HSG]: Store next release _and_ latest nightly (JuliaLang#2418)
Browse files Browse the repository at this point in the history 
In the event that we are in an RC phase where we have three "latest"
branch tips (e.g. latest stable: 1.5.3, latest RC: 1.6.0-rc1 and latest
`master`: 1.7.0-DEV) each with their own set of standard libraries, we
must store this information in Pkg so that we pass tests when running on
latest `master`. It is harmless for this information to be baked into an
older Julia, as we make no guarantees that Julia 1.6.0 final should have
perfect information about what standard libraries 1.7.0 will have.
  • Loading branch information
@staticfloat
staticfloat authored Mar 1, 2021
1 parent 7b87092 commit 7a9d965
Show file tree
Hide file tree
Showing 3 changed files with 77 additions and 8 deletions.
23 changes: 16 additions & 7 deletions ext/HistoricaStdlibGenerator/generate_historical_stdlibs.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,31 +30,38 @@ function select_url_hash(data, host = HostPlatform())
end
version_urls = sort(select_url_hash.(values(versions)), by = pair -> pair[1])

function generate_nightly_url(major, minor, host = HostPlatform())
function generate_nightly_url(jlver, host = HostPlatform())
# Map arch
arch_str = Dict("x86_64" => "x64", "i686" => "x86", "aarch64" => "aarch64", "armv7l" => "armv7l", "ppc64le" => "ppc64le")[arch(host)]
# Map OS name
os_str = Dict("linux" => "linux", "windows" => "winnt", "macos" => "mac", "freebsd" => "freebsd")[os(host)]
# Map wordsize tag
wordsize_str = Dict("x86_64" => "64", "i686" => "32", "aarch64" => "aarch64", "armv7l" => "armv7l", "ppc64le" => "ppc64")[arch(host)]

# If `jlver` is nothing, we don't namespace by version and just get the absolute latest version
ver_str = ""
if jlver !== nothing
ver_str = string(jlver.major, ".", jlver.minor, "/")
end

return string(
"https://julialangnightlies-s3.julialang.org/bin/",
# linux/
os_str, "/",
# x64/
arch_str, "/",
# 1.6/
string(major), ".", string(minor), "/",
# 1.6/ (or nothing, if `jlver === nothing`)
ver_str,
"julia-latest-",
# linux64
os_str, wordsize_str,
".tar.gz",
)
end
highest_release = maximum(VersionNumber.(string.(keys(versions))))
next_release_url = generate_nightly_url(highest_release.major, highest_release.minor + 1)
push!(version_urls, (next_release_url, ""))
next_release = VersionNumber(highest_release.major, highest_release.minor + 1, 0)
push!(version_urls, (generate_nightly_url(next_release), ""))
push!(version_urls, (generate_nightly_url(nothing), ""))
@info("Identified $(length(version_urls)) versions to try...")

# Next, we're going to download each of these to a scratch space
Expand Down Expand Up @@ -138,7 +145,9 @@ versions_dict = Dict()
@async begin
for (url, hash) in jobs
try
fname = joinpath(scratch_dir, basename(url))
# We might try to download two files that have the same basename
url_tag = bytes2hex(sha256(url))
fname = joinpath(scratch_dir, string(url_tag, "-", basename(url)))
if !isfile(fname)
@info("Downloading $(url)")
Downloads.download(url, fname)
Expand All @@ -158,7 +167,7 @@ versions_dict = Dict()
end
end

version, stdlibs = get_stdlibs(scratch_dir, basename(url))
version, stdlibs = get_stdlibs(scratch_dir, basename(fname))
versions_dict[version] = eval(Meta.parse(stdlibs))
catch e
if isa(e, InterruptException)
Expand Down
60 changes: 60 additions & 0 deletions src/HistoricalStdlibs.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,66 @@ const STDLIBS_BY_VERSION = [
UUID("8e850ede-7688-5339-a07c-302acd2aaf8d") => "nghttp2_jll",
UUID("3f19e933-33d8-53b3-aaab-bd5110c3b7a0") => "p7zip_jll",
),
v"1.7.0" => Dict(
UUID("0dad84c5-d112-42e6-8d28-ef12dabb789f") => "ArgTools",
UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33") => "Artifacts",
UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f") => "Base64",
UUID("8bf52ea8-c179-5cab-976a-9e18b702a9bc") => "CRC32c",
UUID("e66e0078-7015-5450-92f7-15fbd957f2ae") => "CompilerSupportLibraries_jll",
UUID("ade2ca70-3891-5945-98fb-dc099432e06a") => "Dates",
UUID("8bb1440f-4735-579b-a4ab-409b98df4dab") => "DelimitedFiles",
UUID("8ba89e20-285c-5b6f-9357-94700520ee1b") => "Distributed",
UUID("f43a241f-c20a-4ad4-852c-f6b1247861c6") => "Downloads",
UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee") => "FileWatching",
UUID("9fa8497b-333b-5362-9e8d-4d0656e87820") => "Future",
UUID("781609d7-10c4-51f6-84f2-b8444358ff6d") => "GMP_jll",
UUID("b77e0a4c-d291-57a0-90e8-8db25a27a240") => "InteractiveUtils",
UUID("47c5dbc3-30ba-59ef-96a6-123e260183d9") => "LLVMLibUnwind_jll",
UUID("4af54fe1-eca0-43a8-85a7-787d91b784e3") => "LazyArtifacts",
UUID("b27032c2-a3e7-50c8-80cd-2d36dbcbfd21") => "LibCURL",
UUID("deac9b47-8bc7-5906-a0fe-35ac56dc84c0") => "LibCURL_jll",
UUID("76f85450-5226-5b5a-8eaa-529ad045b433") => "LibGit2",
UUID("e37daf67-58a4-590a-8e99-b0245dd2ffc5") => "LibGit2_jll",
UUID("29816b5a-b9ab-546f-933c-edad1886dfa8") => "LibSSH2_jll",
UUID("183b4373-6708-53ba-ad28-60e28bb38547") => "LibUV_jll",
UUID("745a5e78-f969-53e9-954f-d19f2f74f4e3") => "LibUnwind_jll",
UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb") => "Libdl",
UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e") => "LinearAlgebra",
UUID("56ddb016-857b-54e1-b83d-db4d58db5568") => "Logging",
UUID("3a97d323-0669-5f0c-9066-3539efd106a3") => "MPFR_jll",
UUID("d6f4376e-aef5-505a-96c1-9c027394607a") => "Markdown",
UUID("c8ffd9c3-330d-5841-b78e-0817d7145fa1") => "MbedTLS_jll",
UUID("a63ad114-7e13-5084-954f-fe012c677804") => "Mmap",
UUID("14a3606d-f60d-562e-9121-12d972cd8159") => "MozillaCACerts_jll",
UUID("ca575930-c2e3-43a9-ace4-1e988b2c1908") => "NetworkOptions",
UUID("4536629a-c528-5b80-bd46-f80d51c5b363") => "OpenBLAS_jll",
UUID("05823500-19ac-5b8b-9628-191a04bc5112") => "OpenLibm_jll",
UUID("efcefdf7-47ab-520b-bdef-62a2eaa19f15") => "PCRE2_jll",
UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") => "Pkg",
UUID("de0858da-6303-5e67-8744-51eddeeeb8d7") => "Printf",
UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79") => "Profile",
UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb") => "REPL",
UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c") => "Random",
UUID("ea8e919c-243c-51af-8825-aaa63cd721ce") => "SHA",
UUID("9e88b42a-f829-5b0c-bbe9-9e923198166b") => "Serialization",
UUID("1a1011a3-84de-559e-8e89-a11a2f7dc383") => "SharedArrays",
UUID("6462fe0b-24de-5631-8697-dd941f90decc") => "Sockets",
UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf") => "SparseArrays",
UUID("10745b16-79ce-11e8-11f9-7d13ad32a3b2") => "Statistics",
UUID("4607b0f0-06f3-5cda-b6b1-a6196a1729e9") => "SuiteSparse",
UUID("bea87d4a-7f5b-5778-9afe-8cc45184846c") => "SuiteSparse_jll",
UUID("fa267f1f-6049-4f14-aa54-33bafae1ed76") => "TOML",
UUID("a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e") => "Tar",
UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40") => "Test",
UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4") => "UUIDs",
UUID("4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5") => "Unicode",
UUID("83775a58-1f1d-513f-b197-d71354ab007a") => "Zlib_jll",
UUID("05ff407c-b0c1-5878-9df8-858cc2e60c36") => "dSFMT_jll",
UUID("8f36deef-c2a5-5394-99ed-8e07531fb29a") => "libLLVM_jll",
UUID("8e850b90-86db-534c-a0d3-1478176c7d93") => "libblastrampoline_jll",
UUID("8e850ede-7688-5339-a07c-302acd2aaf8d") => "nghttp2_jll",
UUID("3f19e933-33d8-53b3-aaab-bd5110c3b7a0") => "p7zip_jll",
),
]
# Next, we also embed a list of stdlibs that must _always_ be treated as stdlibs,
# because they cannot be resolved in the registry; they have only ever existed within
Expand Down
2 changes: 1 addition & 1 deletion test/new.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2505,7 +2505,7 @@ end
@show setdiff(Pkg.Types.STDLIBS_BY_VERSION[end][2], Pkg.Types.load_stdlib())
@show setdiff(Pkg.Types.load_stdlib(), Pkg.Types.STDLIBS_BY_VERSION[end][2])
end
@test_broken test_result # TODO: fix this test
@test test_result
end

@testset "Pkg.add() with julia_version" begin
Expand Down

0 comments on commit 7a9d965

Please sign in to comment.