Skip to content

Commit

Permalink
upgradable stdlibs
Browse files Browse the repository at this point in the history 
Fixes #53983

This breaks one test, which relied on it being undefined behavior to
rely on the execution of `__init__` having occurred before `using` is
allowed to return. That usability issue is now (partially) eliminated.
  • Loading branch information
@vtjnash
vtjnash committed Jun 20, 2024
1 parent 4d0149d commit 2b8201a
Show file tree
Hide file tree
Showing 3 changed files with 96 additions and 66 deletions.
4 changes: 4 additions & 0 deletions base/Base.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -606,6 +606,10 @@ function __init__()
init_active_project()
append!(empty!(_sysimage_modules), keys(loaded_modules))
empty!(explicit_loaded_modules)
@assert isempty(loaded_precompiles)
for (mod, key) in module_keys
loaded_precompiles[key => module_build_id(mod)] = mod
end
if haskey(ENV, "JULIA_MAX_NUM_PRECOMPILE_FILES")
MAX_NUM_PRECOMPILE_FILES[] = parse(Int, ENV["JULIA_MAX_NUM_PRECOMPILE_FILES"])
end
Expand Down
148 changes: 85 additions & 63 deletions base/loading.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1218,8 +1218,8 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No
dep = depmods[i]
dep isa Module && continue
_, depkey, depbuild_id = dep::Tuple{String, PkgId, UInt128}
@assert root_module_exists(depkey)
dep = root_module(depkey)
dep = loaded_precompiles[depkey => depbuild_id]
@assert PkgId(dep) == depkey && module_build_id(dep) === depbuild_id
depmods[i] = dep
end

Expand Down Expand Up @@ -1276,7 +1276,8 @@ function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String)
push!(Base.Docs.modules, M)
end
if parentmodule(M) === M
register_root_module(M)
push!(loaded_modules_order, M)
loaded_precompiles[pkg => module_build_id(M)] = M
end
end

Expand Down Expand Up @@ -1703,7 +1704,7 @@ function compilecache_path(pkg::PkgId;
if staledeps === true
continue
end
staledeps, _ = staledeps::Tuple{Vector{Any}, Union{Nothing, String}}
staledeps, _, _ = staledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
# finish checking staledeps module graph
for i in 1:length(staledeps)
dep = staledeps[i]
Expand Down Expand Up @@ -1791,23 +1792,23 @@ end
# search for a precompile cache file to load, after some various checks
function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt128)
assert_havelock(require_lock)
if root_module_exists(modkey)
loaded = root_module(modkey)
else
loaded = maybe_root_module(modkey)
if loaded === nothing
loaded = start_loading(modkey)
if loaded === nothing
try
modpath = locate_package(modkey)
modpath === nothing && return nothing
set_pkgorigin_version_path(modkey, String(modpath))
loaded = _require_search_from_serialized(modkey, String(modpath), build_id, true)
finally
end_loading(modkey, loaded)
end
if loaded isa Module
insert_extension_triggers(modkey)
run_package_callbacks(modkey)
end
end
if loaded === nothing
try
modpath = locate_package(modkey)
isnothing(modpath) && error("Cannot locate source for $(repr("text/plain", modkey))")
modpath = String(modpath)::String
set_pkgorigin_version_path(modkey, modpath)
loaded = _require_search_from_serialized(modkey, modpath, build_id, true)
finally
end_loading(modkey, loaded)
end
if loaded isa Module
insert_extension_triggers(modkey)
run_package_callbacks(modkey)
end
end
if loaded isa Module && PkgId(loaded) == modkey && module_build_id(loaded) === build_id
Expand Down Expand Up @@ -1880,10 +1881,12 @@ function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union
depmods[i] = dep
end
# then load the file
return _include_from_serialized(pkg, path, ocachepath, depmods, ignore_native)
loaded = _include_from_serialized(pkg, path, ocachepath, depmods, ignore_native)
loaded isa Module && register_root_module(loaded)
return loaded
end

# returns `nothing` if require found a precompile cache for this sourcepath, but couldn't load it
# returns `nothing` if require found a precompile cache for this sourcepath, but couldn't load it or it was stale
# returns the set of modules restored if the cache load succeeded
@constprop :none function _require_search_from_serialized(pkg::PkgId, sourcepath::String, build_id::UInt128, stalecheck::Bool; reasons=nothing, DEPOT_PATH::typeof(DEPOT_PATH)=DEPOT_PATH)
assert_havelock(require_lock)
Expand All @@ -1895,7 +1898,7 @@ end
continue
end
try
staledeps, ocachefile = staledeps::Tuple{Vector{Any}, Union{Nothing, String}}
staledeps, ocachefile, build_id = staledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
# finish checking staledeps module graph
for i in 1:length(staledeps)
dep = staledeps[i]
Expand All @@ -1907,14 +1910,19 @@ end
if modstaledeps === true
continue
end
modstaledeps, modocachepath = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}}
modstaledeps, modocachepath, _ = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
staledeps[i] = (modpath, modkey, modbuild_id, modpath_to_try, modstaledeps, modocachepath)
@goto check_next_dep
end
@debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache."
@goto check_next_path
@label check_next_dep
end
M = get(loaded_precompiles, pkg => build_id, nothing)
if isa(M, Module)
stalecheck && register_root_module(M)
return M
end
if stalecheck
try
touch(path_to_try) # update timestamp of precompilation file
Expand All @@ -1927,26 +1935,25 @@ end
dep = staledeps[i]
dep isa Module && continue
modpath, modkey, modbuild_id, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, UInt128, String, Vector{Any}, Union{Nothing, String}}
dep = nothing
if root_module_exists(modkey)
dep = root_module(modkey)
dep = get(loaded_precompiles, modkey => modbuild_id, nothing)
if dep === nothing
dep = maybe_root_module(modkey)
end
while true
if dep isa Module
if PkgId(dep) == modkey && module_build_id(dep) === modbuild_id
break
else
if stalecheck
@debug "Rejecting cache file $path_to_try because module $modkey is already loaded and incompatible."
@goto check_next_path
end
@debug "Rejecting cache file $path_to_try because module $modkey got loaded at a different version than expected."
@goto check_next_path
end
end
dep = start_loading(modkey)
if dep === nothing
try
set_pkgorigin_version_path(modkey, modpath)
dep = _include_from_serialized(modkey, modcachepath, modocachepath, modstaledeps)
dep isa Module && stalecheck && register_root_module(dep)
finally
end_loading(modkey, dep)
end
Expand All @@ -1960,7 +1967,11 @@ end
end
staledeps[i] = dep
end
restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps)
restored = get(loaded_precompiles, pkg => build_id, nothing)
if !isa(restored, Module)
restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps)
end
isa(restored, Module) && stalecheck && register_root_module(restored)
isa(restored, Module) && return restored
@debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored
@label check_next_path
Expand Down Expand Up @@ -2046,7 +2057,7 @@ const package_callbacks = Any[]
const include_callbacks = Any[]

# used to optionally track dependencies when requiring a module:
const _concrete_dependencies = Pair{PkgId,UInt128}[] # these dependency versions are "set in stone", and the process should try to avoid invalidating them
const _concrete_dependencies = Pair{PkgId,UInt128}[] # these dependency versions are "set in stone", because they are explicitly loaded, and the process should try to avoid invalidating them
const _require_dependencies = Any[] # a list of (mod, abspath, fsize, hash, mtime) tuples that are the file dependencies of the module currently being precompiled
const _track_dependencies = Ref(false) # set this to true to track the list of file dependencies
function _include_dependency(mod::Module, _path::AbstractString; track_content=true)
Expand Down Expand Up @@ -2289,14 +2300,19 @@ end
PkgOrigin() = PkgOrigin(nothing, nothing, nothing)
const pkgorigins = Dict{PkgId,PkgOrigin}()

const loaded_modules = Dict{PkgId,Module}()
# Emptied on Julia start
const explicit_loaded_modules = Dict{PkgId,Module}()
const explicit_loaded_modules = Dict{PkgId,Module}() # Emptied on Julia start
const loaded_modules = Dict{PkgId,Module}() # available to be explicitly loaded
const loaded_precompiles = Dict{Pair{PkgId,UInt128},Module}() # extended (complete) list of modules, available to be loaded
const loaded_modules_order = Vector{Module}()
const module_keys = IdDict{Module,PkgId}() # the reverse
const module_keys = IdDict{Module,PkgId}() # the reverse of loaded_modules

root_module_key(m::Module) = @lock require_lock module_keys[m]

function module_build_id(m::Module)
hi, lo = ccall(:jl_module_build_id, NTuple{2,UInt64}, (Any,), m)
return (UInt128(hi) << 64) | lo
end

@constprop :none function register_root_module(m::Module)
# n.b. This is called from C after creating a new module in `Base.__toplevel__`,
# instead of adding them to the binding table there.
Expand All @@ -2312,7 +2328,7 @@ root_module_key(m::Module) = @lock require_lock module_keys[m]
end
end
end
push!(loaded_modules_order, m)
haskey(loaded_precompiles, key => module_build_id(m)) || push!(loaded_modules_order, m)
loaded_modules[key] = m
explicit_loaded_modules[key] = m
module_keys[m] = key
Expand Down Expand Up @@ -2344,6 +2360,9 @@ root_module_exists(key::PkgId) = @lock require_lock haskey(loaded_modules, key)
loaded_modules_array() = @lock require_lock copy(loaded_modules_order)

function unreference_module(key::PkgId)
if haskey(explicit_loaded_modules, key)
m = pop!(explicit_loaded_modules, key)
end
if haskey(loaded_modules, key)
m = pop!(loaded_modules, key)
# need to ensure all modules are GC rooted; will still be referenced
Expand Down Expand Up @@ -2488,7 +2507,7 @@ function _require(pkg::PkgId, env=nothing)
return loaded
end

# load a serialized file directly
# load a serialized file directly, including dependencies (without checking staleness except for immediate conflicts)
function _require_from_serialized(uuidkey::PkgId, path::String, ocachepath::Union{String, Nothing}, sourcepath::String)
@lock require_lock begin
set_pkgorigin_version_path(uuidkey, sourcepath)
Expand Down Expand Up @@ -2922,13 +2941,15 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in
cachepath = compilecache_dir(pkg)

# build up the list of modules that we want the precompile process to preserve
concrete_deps = copy(_concrete_dependencies)
if keep_loaded_modules
for mod in loaded_modules_array()
if !(mod === Main || mod === Core || mod === Base)
push!(concrete_deps, PkgId(mod) => module_build_id(mod))
concrete_deps = copy(_concrete_dependencies)
for (pkgreq, modreq) in loaded_modules # TODO: convert all relevant staleness heuristics to use explicit_loaded_modules instead
if !(pkgreq === Main || pkgreq === Core || pkgreq === Base)
push!(concrete_deps, pkgreq => module_build_id(modreq))
end
end
else
concrete_deps = empty(_concrete_dependencies)
end
# run the expression and cache the result
verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug
Expand Down Expand Up @@ -3056,11 +3077,6 @@ function rename_unique_ocachefile(tmppath_so::String, ocachefile_orig::String, o
return ocachefile
end

function module_build_id(m::Module)
hi, lo = ccall(:jl_module_build_id, NTuple{2,UInt64}, (Any,), m)
return (UInt128(hi) << 64) | lo
end

function object_build_id(obj)
mod = ccall(:jl_object_top_module, Any, (Any,), obj)
if mod === nothing
Expand Down Expand Up @@ -3639,7 +3655,7 @@ end
@constprop :none function stale_cachefile(modkey::PkgId, build_id::UInt128, modpath::String, cachefile::String;
ignore_loaded::Bool=false, requested_flags::CacheFlags=CacheFlags(),
reasons::Union{Dict{String,Int},Nothing}=nothing, stalecheck::Bool=true)
# XXX: this function appears to dl all of the file validation, not just those checks related to stale
# n.b.: this function does nearly all of the file validation, not just those checks related to stale, so the name is potentially unclear
io = open(cachefile, "r")
try
checksum = isvalid_cache_header(io)
Expand Down Expand Up @@ -3693,8 +3709,8 @@ end
record_reason(reasons, "for different pkgid")
return true
end
id_build = (UInt128(checksum) << 64) | id.second
if build_id != UInt128(0)
id_build = (UInt128(checksum) << 64) | id.second
if id_build != build_id
@debug "Ignoring cache file $cachefile for $modkey ($((UUID(id_build)))) since it does not provide desired build_id ($((UUID(build_id))))"
record_reason(reasons, "for different buildid")
Expand All @@ -3709,8 +3725,12 @@ end
depmods = Vector{Any}(undef, ndeps)
for i in 1:ndeps
req_key, req_build_id = required_modules[i]
# Module is already loaded
if root_module_exists(req_key)
# Check if module is already loaded
if !stalecheck && haskey(loaded_precompiles, req_key => req_build_id)
M = loaded_precompiles[req_key => req_build_id]
@assert PkgId(M) == req_key && module_build_id(M) === req_build_id
depmods[i] = M
elseif root_module_exists(req_key)
M = root_module(req_key)
if PkgId(M) == req_key && module_build_id(M) === req_build_id
depmods[i] = M
Expand Down Expand Up @@ -3741,17 +3761,19 @@ end
# check if this file is going to provide one of our concrete dependencies
# or if it provides a version that conflicts with our concrete dependencies
# or neither
for (req_key, req_build_id) in _concrete_dependencies
build_id = get(modules, req_key, UInt64(0))
if build_id !== UInt64(0)
build_id |= UInt128(checksum) << 64
if build_id === req_build_id
stalecheck = false
break
if stalecheck
for (req_key, req_build_id) in _concrete_dependencies
build_id = get(modules, req_key, UInt64(0))
if build_id !== UInt64(0)
build_id |= UInt128(checksum) << 64
if build_id === req_build_id
stalecheck = false
break
end
@debug "Rejecting cache file $cachefile because it provides the wrong build_id (got $((UUID(build_id)))) for $req_key (want $(UUID(req_build_id)))"
record_reason(reasons, "wrong dep buildid")
return true # cachefile doesn't provide the required version of the dependency
end
@debug "Rejecting cache file $cachefile because it provides the wrong build_id (got $((UUID(build_id)))) for $req_key (want $(UUID(req_build_id)))"
record_reason(reasons, "wrong dep buildid")
return true # cachefile doesn't provide the required version of the dependency
end
end

Expand Down Expand Up @@ -3839,7 +3861,7 @@ end
return true
end

return depmods, ocachefile # fresh cachefile
return depmods, ocachefile, id_build # fresh cachefile
finally
close(io)
end
Expand Down
10 changes: 7 additions & 3 deletions test/precompile.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -612,13 +612,13 @@ precompile_test_harness(false) do dir
empty_prefs_hash = Base.get_preferences_hash(nothing, String[])
@test cachefile == Base.compilecache_path(Base.PkgId("FooBar"), empty_prefs_hash)
@test isfile(joinpath(cachedir, "FooBar.ji"))
Tsc = Bool(Base.JLOptions().use_pkgimages) ? Tuple{<:Vector, String} : Tuple{<:Vector, Nothing}
Tsc = Bool(Base.JLOptions().use_pkgimages) ? Tuple{<:Vector, String, UInt128} : Tuple{<:Vector, Nothing, UInt128}
@test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tsc
@test !isdefined(Main, :FooBar)
@test !isdefined(Main, :FooBar1)

relFooBar_file = joinpath(dir, "subfolder", "..", "FooBar.jl")
@test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa (Sys.iswindows() ? Tuple{<:Vector, String} : Bool) # `..` is not a symlink on Windows
@test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa (Sys.iswindows() ? Tuple{<:Vector, String, UInt128} : Bool) # `..` is not a symlink on Windows
mkdir(joinpath(dir, "subfolder"))
@test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tsc

Expand Down Expand Up @@ -1519,6 +1519,7 @@ precompile_test_harness("Issue #26028") do load_path
module Foo26028
module Bar26028
x = 0
y = 0
end
function __init__()
include(joinpath(@__DIR__, "Baz26028.jl"))
Expand All @@ -1528,7 +1529,10 @@ precompile_test_harness("Issue #26028") do load_path
write(joinpath(load_path, "Baz26028.jl"),
"""
module Baz26028
import Foo26028.Bar26028.x
using Test
@test_throws(ConcurrencyViolationError("deadlock detected in loading Foo26028 -> Foo26028"),
@eval import Foo26028.Bar26028.x)
import ..Foo26028.Bar26028.y
end
""")
Base.compilecache(Base.PkgId("Foo26028"))
Expand Down

0 comments on commit 2b8201a

Please sign in to comment.