From f4aed5dc84f6f62b9022fef5432c36727acf5933 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Mon, 1 Mar 2021 20:47:59 +0000 Subject: [PATCH] [ext/HSG]: Store next release _and_ latest nightly In the event that we are in an RC phase where we have three "latest" branch tips (e.g. latest stable: 1.5.3, latest RC: 1.6.0-rc1 and latest `master`: 1.7.0-DEV) each with their own set of standard libraries, we must store this information in Pkg so that we pass tests when running on latest `master`. It is harmless for this information to be baked into an older Julia, as we make no guarantees that Julia 1.6.0 final should have perfect information about what standard libraries 1.7.0 will have. --- .../generate_historical_stdlibs.jl | 23 ++++--- src/HistoricalStdlibs.jl | 60 +++++++++++++++++++ test/new.jl | 2 +- 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/ext/HistoricaStdlibGenerator/generate_historical_stdlibs.jl b/ext/HistoricaStdlibGenerator/generate_historical_stdlibs.jl index dce1e9db24..3b8e62a71d 100644 --- a/ext/HistoricaStdlibGenerator/generate_historical_stdlibs.jl +++ b/ext/HistoricaStdlibGenerator/generate_historical_stdlibs.jl @@ -30,7 +30,7 @@ function select_url_hash(data, host = HostPlatform()) end version_urls = sort(select_url_hash.(values(versions)), by = pair -> pair[1]) -function generate_nightly_url(major, minor, host = HostPlatform()) +function generate_nightly_url(jlver, host = HostPlatform()) # Map arch arch_str = Dict("x86_64" => "x64", "i686" => "x86", "aarch64" => "aarch64", "armv7l" => "armv7l", "ppc64le" => "ppc64le")[arch(host)] # Map OS name @@ -38,14 +38,20 @@ function generate_nightly_url(major, minor, host = HostPlatform()) # Map wordsize tag wordsize_str = Dict("x86_64" => "64", "i686" => "32", "aarch64" => "aarch64", "armv7l" => "armv7l", "ppc64le" => "ppc64")[arch(host)] + # If `jlver` is nothing, we don't namespace by version and just get the absolute latest version + ver_str = "" + if jlver !== nothing + ver_str = string(jlver.major, ".", jlver.minor, "/") + end + return string( "https://julialangnightlies-s3.julialang.org/bin/", # linux/ os_str, "/", # x64/ arch_str, "/", - # 1.6/ - string(major), ".", string(minor), "/", + # 1.6/ (or nothing, if `jlver === nothing`) + ver_str, "julia-latest-", # linux64 os_str, wordsize_str, @@ -53,8 +59,9 @@ function generate_nightly_url(major, minor, host = HostPlatform()) ) end highest_release = maximum(VersionNumber.(string.(keys(versions)))) -next_release_url = generate_nightly_url(highest_release.major, highest_release.minor + 1) -push!(version_urls, (next_release_url, "")) +next_release = VersionNumber(highest_release.major, highest_release.minor + 1, 0) +push!(version_urls, (generate_nightly_url(next_release), "")) +push!(version_urls, (generate_nightly_url(nothing), "")) @info("Identified $(length(version_urls)) versions to try...") # Next, we're going to download each of these to a scratch space @@ -138,7 +145,9 @@ versions_dict = Dict() @async begin for (url, hash) in jobs try - fname = joinpath(scratch_dir, basename(url)) + # We might try to download two files that have the same basename + url_tag = bytes2hex(sha256(url)) + fname = joinpath(scratch_dir, string(url_tag, "-", basename(url))) if !isfile(fname) @info("Downloading $(url)") Downloads.download(url, fname) @@ -158,7 +167,7 @@ versions_dict = Dict() end end - version, stdlibs = get_stdlibs(scratch_dir, basename(url)) + version, stdlibs = get_stdlibs(scratch_dir, basename(fname)) versions_dict[version] = eval(Meta.parse(stdlibs)) catch e if isa(e, InterruptException) diff --git a/src/HistoricalStdlibs.jl b/src/HistoricalStdlibs.jl index 880f36fe54..f9fb57d52c 100644 --- a/src/HistoricalStdlibs.jl +++ b/src/HistoricalStdlibs.jl @@ -95,6 +95,66 @@ const STDLIBS_BY_VERSION = [ UUID("8e850ede-7688-5339-a07c-302acd2aaf8d") => "nghttp2_jll", UUID("3f19e933-33d8-53b3-aaab-bd5110c3b7a0") => "p7zip_jll", ), + v"1.7.0" => Dict( + UUID("0dad84c5-d112-42e6-8d28-ef12dabb789f") => "ArgTools", + UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33") => "Artifacts", + UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f") => "Base64", + UUID("8bf52ea8-c179-5cab-976a-9e18b702a9bc") => "CRC32c", + UUID("e66e0078-7015-5450-92f7-15fbd957f2ae") => "CompilerSupportLibraries_jll", + UUID("ade2ca70-3891-5945-98fb-dc099432e06a") => "Dates", + UUID("8bb1440f-4735-579b-a4ab-409b98df4dab") => "DelimitedFiles", + UUID("8ba89e20-285c-5b6f-9357-94700520ee1b") => "Distributed", + UUID("f43a241f-c20a-4ad4-852c-f6b1247861c6") => "Downloads", + UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee") => "FileWatching", + UUID("9fa8497b-333b-5362-9e8d-4d0656e87820") => "Future", + UUID("781609d7-10c4-51f6-84f2-b8444358ff6d") => "GMP_jll", + UUID("b77e0a4c-d291-57a0-90e8-8db25a27a240") => "InteractiveUtils", + UUID("47c5dbc3-30ba-59ef-96a6-123e260183d9") => "LLVMLibUnwind_jll", + UUID("4af54fe1-eca0-43a8-85a7-787d91b784e3") => "LazyArtifacts", + UUID("b27032c2-a3e7-50c8-80cd-2d36dbcbfd21") => "LibCURL", + UUID("deac9b47-8bc7-5906-a0fe-35ac56dc84c0") => "LibCURL_jll", + UUID("76f85450-5226-5b5a-8eaa-529ad045b433") => "LibGit2", + UUID("e37daf67-58a4-590a-8e99-b0245dd2ffc5") => "LibGit2_jll", + UUID("29816b5a-b9ab-546f-933c-edad1886dfa8") => "LibSSH2_jll", + UUID("183b4373-6708-53ba-ad28-60e28bb38547") => "LibUV_jll", + UUID("745a5e78-f969-53e9-954f-d19f2f74f4e3") => "LibUnwind_jll", + UUID("8f399da3-3557-5675-b5ff-fb832c97cbdb") => "Libdl", + UUID("37e2e46d-f89d-539d-b4ee-838fcccc9c8e") => "LinearAlgebra", + UUID("56ddb016-857b-54e1-b83d-db4d58db5568") => "Logging", + UUID("3a97d323-0669-5f0c-9066-3539efd106a3") => "MPFR_jll", + UUID("d6f4376e-aef5-505a-96c1-9c027394607a") => "Markdown", + UUID("c8ffd9c3-330d-5841-b78e-0817d7145fa1") => "MbedTLS_jll", + UUID("a63ad114-7e13-5084-954f-fe012c677804") => "Mmap", + UUID("14a3606d-f60d-562e-9121-12d972cd8159") => "MozillaCACerts_jll", + UUID("ca575930-c2e3-43a9-ace4-1e988b2c1908") => "NetworkOptions", + UUID("4536629a-c528-5b80-bd46-f80d51c5b363") => "OpenBLAS_jll", + UUID("05823500-19ac-5b8b-9628-191a04bc5112") => "OpenLibm_jll", + UUID("efcefdf7-47ab-520b-bdef-62a2eaa19f15") => "PCRE2_jll", + UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f") => "Pkg", + UUID("de0858da-6303-5e67-8744-51eddeeeb8d7") => "Printf", + UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79") => "Profile", + UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb") => "REPL", + UUID("9a3f8284-a2c9-5f02-9a11-845980a1fd5c") => "Random", + UUID("ea8e919c-243c-51af-8825-aaa63cd721ce") => "SHA", + UUID("9e88b42a-f829-5b0c-bbe9-9e923198166b") => "Serialization", + UUID("1a1011a3-84de-559e-8e89-a11a2f7dc383") => "SharedArrays", + UUID("6462fe0b-24de-5631-8697-dd941f90decc") => "Sockets", + UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf") => "SparseArrays", + UUID("10745b16-79ce-11e8-11f9-7d13ad32a3b2") => "Statistics", + UUID("4607b0f0-06f3-5cda-b6b1-a6196a1729e9") => "SuiteSparse", + UUID("bea87d4a-7f5b-5778-9afe-8cc45184846c") => "SuiteSparse_jll", + UUID("fa267f1f-6049-4f14-aa54-33bafae1ed76") => "TOML", + UUID("a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e") => "Tar", + UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40") => "Test", + UUID("cf7118a7-6976-5b1a-9a39-7adc72f591a4") => "UUIDs", + UUID("4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5") => "Unicode", + UUID("83775a58-1f1d-513f-b197-d71354ab007a") => "Zlib_jll", + UUID("05ff407c-b0c1-5878-9df8-858cc2e60c36") => "dSFMT_jll", + UUID("8f36deef-c2a5-5394-99ed-8e07531fb29a") => "libLLVM_jll", + UUID("8e850b90-86db-534c-a0d3-1478176c7d93") => "libblastrampoline_jll", + UUID("8e850ede-7688-5339-a07c-302acd2aaf8d") => "nghttp2_jll", + UUID("3f19e933-33d8-53b3-aaab-bd5110c3b7a0") => "p7zip_jll", + ), ] # Next, we also embed a list of stdlibs that must _always_ be treated as stdlibs, # because they cannot be resolved in the registry; they have only ever existed within diff --git a/test/new.jl b/test/new.jl index 221e399408..7b5b62c261 100644 --- a/test/new.jl +++ b/test/new.jl @@ -2505,7 +2505,7 @@ end @show setdiff(Pkg.Types.STDLIBS_BY_VERSION[end][2], Pkg.Types.load_stdlib()) @show setdiff(Pkg.Types.load_stdlib(), Pkg.Types.STDLIBS_BY_VERSION[end][2]) end - @test_broken test_result # TODO: fix this test + @test test_result end @testset "Pkg.add() with julia_version" begin