Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

finish implementation of upgradable stdlibs #54739

Merged
merged 1 commit into from
Jun 21, 2024
Merged

finish implementation of upgradable stdlibs #54739

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions base/Base.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -606,6 +606,10 @@ function __init__()
init_active_project()
append!(empty!(_sysimage_modules), keys(loaded_modules))
empty!(explicit_loaded_modules)
@assert isempty(loaded_precompiles)
for (mod, key) in module_keys
loaded_precompiles[key => module_build_id(mod)] = mod
end
if haskey(ENV, "JULIA_MAX_NUM_PRECOMPILE_FILES")
MAX_NUM_PRECOMPILE_FILES[] = parse(Int, ENV["JULIA_MAX_NUM_PRECOMPILE_FILES"])
end
Expand Down
148 changes: 85 additions & 63 deletions base/loading.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1218,8 +1218,8 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No
dep = depmods[i]
dep isa Module && continue
_, depkey, depbuild_id = dep::Tuple{String, PkgId, UInt128}
@assert root_module_exists(depkey)
dep = root_module(depkey)
dep = loaded_precompiles[depkey => depbuild_id]
@assert PkgId(dep) == depkey && module_build_id(dep) === depbuild_id
depmods[i] = dep
end

Expand Down Expand Up @@ -1276,7 +1276,8 @@ function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String)
push!(Base.Docs.modules, M)
end
if parentmodule(M) === M
register_root_module(M)
push!(loaded_modules_order, M)
loaded_precompiles[pkg => module_build_id(M)] = M
end
end

Expand Down Expand Up @@ -1703,7 +1704,7 @@ function compilecache_path(pkg::PkgId;
if staledeps === true
continue
end
staledeps, _ = staledeps::Tuple{Vector{Any}, Union{Nothing, String}}
staledeps, _, _ = staledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
# finish checking staledeps module graph
for i in 1:length(staledeps)
dep = staledeps[i]
Expand Down Expand Up @@ -1791,23 +1792,23 @@ end
# search for a precompile cache file to load, after some various checks
function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt128)
assert_havelock(require_lock)
if root_module_exists(modkey)
loaded = root_module(modkey)
else
loaded = maybe_root_module(modkey)
if loaded === nothing
loaded = start_loading(modkey)
if loaded === nothing
try
modpath = locate_package(modkey)
modpath === nothing && return nothing
set_pkgorigin_version_path(modkey, String(modpath))
loaded = _require_search_from_serialized(modkey, String(modpath), build_id, true)
finally
end_loading(modkey, loaded)
end
if loaded isa Module
insert_extension_triggers(modkey)
run_package_callbacks(modkey)
end
end
if loaded === nothing
try
modpath = locate_package(modkey)
isnothing(modpath) && error("Cannot locate source for $(repr("text/plain", modkey))")
modpath = String(modpath)::String
set_pkgorigin_version_path(modkey, modpath)
loaded = _require_search_from_serialized(modkey, modpath, build_id, true)
finally
end_loading(modkey, loaded)
end
if loaded isa Module
insert_extension_triggers(modkey)
run_package_callbacks(modkey)
end
end
if loaded isa Module && PkgId(loaded) == modkey && module_build_id(loaded) === build_id
Expand Down Expand Up @@ -1880,10 +1881,12 @@ function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union
depmods[i] = dep
end
# then load the file
return _include_from_serialized(pkg, path, ocachepath, depmods, ignore_native)
loaded = _include_from_serialized(pkg, path, ocachepath, depmods, ignore_native)
loaded isa Module && register_root_module(loaded)
return loaded
end

# returns `nothing` if require found a precompile cache for this sourcepath, but couldn't load it
# returns `nothing` if require found a precompile cache for this sourcepath, but couldn't load it or it was stale
# returns the set of modules restored if the cache load succeeded
@constprop :none function _require_search_from_serialized(pkg::PkgId, sourcepath::String, build_id::UInt128, stalecheck::Bool; reasons=nothing, DEPOT_PATH::typeof(DEPOT_PATH)=DEPOT_PATH)
assert_havelock(require_lock)
Expand All @@ -1895,7 +1898,7 @@ end
continue
end
try
staledeps, ocachefile = staledeps::Tuple{Vector{Any}, Union{Nothing, String}}
staledeps, ocachefile, build_id = staledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
# finish checking staledeps module graph
for i in 1:length(staledeps)
dep = staledeps[i]
Expand All @@ -1907,14 +1910,19 @@ end
if modstaledeps === true
continue
end
modstaledeps, modocachepath = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}}
modstaledeps, modocachepath, _ = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
staledeps[i] = (modpath, modkey, modbuild_id, modpath_to_try, modstaledeps, modocachepath)
@goto check_next_dep
end
@debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache."
@goto check_next_path
@label check_next_dep
end
M = get(loaded_precompiles, pkg => build_id, nothing)
if isa(M, Module)
stalecheck && register_root_module(M)
return M
end
if stalecheck
try
touch(path_to_try) # update timestamp of precompilation file
Expand All @@ -1927,26 +1935,25 @@ end
dep = staledeps[i]
dep isa Module && continue
modpath, modkey, modbuild_id, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, UInt128, String, Vector{Any}, Union{Nothing, String}}
dep = nothing
if root_module_exists(modkey)
dep = root_module(modkey)
dep = get(loaded_precompiles, modkey => modbuild_id, nothing)
if dep === nothing
dep = maybe_root_module(modkey)
end
while true
if dep isa Module
if PkgId(dep) == modkey && module_build_id(dep) === modbuild_id
break
else
if stalecheck
@debug "Rejecting cache file $path_to_try because module $modkey is already loaded and incompatible."
@goto check_next_path
end
@debug "Rejecting cache file $path_to_try because module $modkey got loaded at a different version than expected."
@goto check_next_path
end
end
dep = start_loading(modkey)
if dep === nothing
try
set_pkgorigin_version_path(modkey, modpath)
dep = _include_from_serialized(modkey, modcachepath, modocachepath, modstaledeps)
dep isa Module && stalecheck && register_root_module(dep)
finally
end_loading(modkey, dep)
end
Expand All @@ -1960,7 +1967,11 @@ end
end
staledeps[i] = dep
end
restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps)
restored = get(loaded_precompiles, pkg => build_id, nothing)
if !isa(restored, Module)
restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps)
end
isa(restored, Module) && stalecheck && register_root_module(restored)
isa(restored, Module) && return restored
@debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored
@label check_next_path
Expand Down Expand Up @@ -2046,7 +2057,7 @@ const package_callbacks = Any[]
const include_callbacks = Any[]

# used to optionally track dependencies when requiring a module:
const _concrete_dependencies = Pair{PkgId,UInt128}[] # these dependency versions are "set in stone", and the process should try to avoid invalidating them
const _concrete_dependencies = Pair{PkgId,UInt128}[] # these dependency versions are "set in stone", because they are explicitly loaded, and the process should try to avoid invalidating them
const _require_dependencies = Any[] # a list of (mod, abspath, fsize, hash, mtime) tuples that are the file dependencies of the module currently being precompiled
const _track_dependencies = Ref(false) # set this to true to track the list of file dependencies
function _include_dependency(mod::Module, _path::AbstractString; track_content=true)
Expand Down Expand Up @@ -2289,14 +2300,19 @@ end
PkgOrigin() = PkgOrigin(nothing, nothing, nothing)
const pkgorigins = Dict{PkgId,PkgOrigin}()

const loaded_modules = Dict{PkgId,Module}()
# Emptied on Julia start
const explicit_loaded_modules = Dict{PkgId,Module}()
const explicit_loaded_modules = Dict{PkgId,Module}() # Emptied on Julia start
const loaded_modules = Dict{PkgId,Module}() # available to be explicitly loaded
const loaded_precompiles = Dict{Pair{PkgId,UInt128},Module}() # extended (complete) list of modules, available to be loaded
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we be clearer about the differences between these three dicts? Is the first one modules loaded during this session? Is the second one currently visible modules, and the third one all modules? I'm not sure.

const loaded_modules_order = Vector{Module}()
const module_keys = IdDict{Module,PkgId}() # the reverse
const module_keys = IdDict{Module,PkgId}() # the reverse of loaded_modules

root_module_key(m::Module) = @lock require_lock module_keys[m]

function module_build_id(m::Module)
hi, lo = ccall(:jl_module_build_id, NTuple{2,UInt64}, (Any,), m)
return (UInt128(hi) << 64) | lo
end

@constprop :none function register_root_module(m::Module)
# n.b. This is called from C after creating a new module in `Base.__toplevel__`,
# instead of adding them to the binding table there.
Expand All @@ -2312,7 +2328,7 @@ root_module_key(m::Module) = @lock require_lock module_keys[m]
end
end
end
push!(loaded_modules_order, m)
haskey(loaded_precompiles, key => module_build_id(m)) || push!(loaded_modules_order, m)
loaded_modules[key] = m
explicit_loaded_modules[key] = m
module_keys[m] = key
Expand Down Expand Up @@ -2344,6 +2360,9 @@ root_module_exists(key::PkgId) = @lock require_lock haskey(loaded_modules, key)
loaded_modules_array() = @lock require_lock copy(loaded_modules_order)

function unreference_module(key::PkgId)
if haskey(explicit_loaded_modules, key)
m = pop!(explicit_loaded_modules, key)
end
if haskey(loaded_modules, key)
m = pop!(loaded_modules, key)
# need to ensure all modules are GC rooted; will still be referenced
Expand Down Expand Up @@ -2488,7 +2507,7 @@ function _require(pkg::PkgId, env=nothing)
return loaded
end

# load a serialized file directly
# load a serialized file directly, including dependencies (without checking staleness except for immediate conflicts)
function _require_from_serialized(uuidkey::PkgId, path::String, ocachepath::Union{String, Nothing}, sourcepath::String)
@lock require_lock begin
set_pkgorigin_version_path(uuidkey, sourcepath)
Expand Down Expand Up @@ -2922,13 +2941,15 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in
cachepath = compilecache_dir(pkg)

# build up the list of modules that we want the precompile process to preserve
concrete_deps = copy(_concrete_dependencies)
if keep_loaded_modules
for mod in loaded_modules_array()
if !(mod === Main || mod === Core || mod === Base)
push!(concrete_deps, PkgId(mod) => module_build_id(mod))
concrete_deps = copy(_concrete_dependencies)
for (pkgreq, modreq) in loaded_modules # TODO: convert all relevant staleness heuristics to use explicit_loaded_modules instead
if !(pkgreq === Main || pkgreq === Core || pkgreq === Base)
push!(concrete_deps, pkgreq => module_build_id(modreq))
end
end
else
concrete_deps = empty(_concrete_dependencies)
end
# run the expression and cache the result
verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug
Expand Down Expand Up @@ -3056,11 +3077,6 @@ function rename_unique_ocachefile(tmppath_so::String, ocachefile_orig::String, o
return ocachefile
end

function module_build_id(m::Module)
hi, lo = ccall(:jl_module_build_id, NTuple{2,UInt64}, (Any,), m)
return (UInt128(hi) << 64) | lo
end

function object_build_id(obj)
mod = ccall(:jl_object_top_module, Any, (Any,), obj)
if mod === nothing
Expand Down Expand Up @@ -3639,7 +3655,7 @@ end
@constprop :none function stale_cachefile(modkey::PkgId, build_id::UInt128, modpath::String, cachefile::String;
ignore_loaded::Bool=false, requested_flags::CacheFlags=CacheFlags(),
reasons::Union{Dict{String,Int},Nothing}=nothing, stalecheck::Bool=true)
# XXX: this function appears to dl all of the file validation, not just those checks related to stale
# n.b.: this function does nearly all of the file validation, not just those checks related to stale, so the name is potentially unclear
io = open(cachefile, "r")
try
checksum = isvalid_cache_header(io)
Expand Down Expand Up @@ -3693,8 +3709,8 @@ end
record_reason(reasons, "for different pkgid")
return true
end
id_build = (UInt128(checksum) << 64) | id.second
if build_id != UInt128(0)
id_build = (UInt128(checksum) << 64) | id.second
if id_build != build_id
@debug "Ignoring cache file $cachefile for $modkey ($((UUID(id_build)))) since it does not provide desired build_id ($((UUID(build_id))))"
record_reason(reasons, "for different buildid")
Expand All @@ -3709,8 +3725,12 @@ end
depmods = Vector{Any}(undef, ndeps)
for i in 1:ndeps
req_key, req_build_id = required_modules[i]
# Module is already loaded
if root_module_exists(req_key)
# Check if module is already loaded
if !stalecheck && haskey(loaded_precompiles, req_key => req_build_id)
M = loaded_precompiles[req_key => req_build_id]
@assert PkgId(M) == req_key && module_build_id(M) === req_build_id
depmods[i] = M
elseif root_module_exists(req_key)
M = root_module(req_key)
if PkgId(M) == req_key && module_build_id(M) === req_build_id
depmods[i] = M
Expand Down Expand Up @@ -3741,17 +3761,19 @@ end
# check if this file is going to provide one of our concrete dependencies
# or if it provides a version that conflicts with our concrete dependencies
# or neither
for (req_key, req_build_id) in _concrete_dependencies
build_id = get(modules, req_key, UInt64(0))
if build_id !== UInt64(0)
build_id |= UInt128(checksum) << 64
if build_id === req_build_id
stalecheck = false
break
if stalecheck
for (req_key, req_build_id) in _concrete_dependencies
build_id = get(modules, req_key, UInt64(0))
if build_id !== UInt64(0)
build_id |= UInt128(checksum) << 64
if build_id === req_build_id
stalecheck = false
break
end
@debug "Rejecting cache file $cachefile because it provides the wrong build_id (got $((UUID(build_id)))) for $req_key (want $(UUID(req_build_id)))"
record_reason(reasons, "wrong dep buildid")
return true # cachefile doesn't provide the required version of the dependency
end
@debug "Rejecting cache file $cachefile because it provides the wrong build_id (got $((UUID(build_id)))) for $req_key (want $(UUID(req_build_id)))"
record_reason(reasons, "wrong dep buildid")
return true # cachefile doesn't provide the required version of the dependency
end
end

Expand Down Expand Up @@ -3839,7 +3861,7 @@ end
return true
end

return depmods, ocachefile # fresh cachefile
return depmods, ocachefile, id_build # fresh cachefile
finally
close(io)
end
Expand Down
10 changes: 7 additions & 3 deletions test/precompile.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -612,13 +612,13 @@ precompile_test_harness(false) do dir
empty_prefs_hash = Base.get_preferences_hash(nothing, String[])
@test cachefile == Base.compilecache_path(Base.PkgId("FooBar"), empty_prefs_hash)
@test isfile(joinpath(cachedir, "FooBar.ji"))
Tsc = Bool(Base.JLOptions().use_pkgimages) ? Tuple{<:Vector, String} : Tuple{<:Vector, Nothing}
Tsc = Bool(Base.JLOptions().use_pkgimages) ? Tuple{<:Vector, String, UInt128} : Tuple{<:Vector, Nothing, UInt128}
@test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tsc
@test !isdefined(Main, :FooBar)
@test !isdefined(Main, :FooBar1)

relFooBar_file = joinpath(dir, "subfolder", "..", "FooBar.jl")
@test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa (Sys.iswindows() ? Tuple{<:Vector, String} : Bool) # `..` is not a symlink on Windows
@test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa (Sys.iswindows() ? Tuple{<:Vector, String, UInt128} : Bool) # `..` is not a symlink on Windows
mkdir(joinpath(dir, "subfolder"))
@test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tsc

Expand Down Expand Up @@ -1519,6 +1519,7 @@ precompile_test_harness("Issue #26028") do load_path
module Foo26028
module Bar26028
x = 0
y = 0
end
function __init__()
include(joinpath(@__DIR__, "Baz26028.jl"))
Expand All @@ -1528,7 +1529,10 @@ precompile_test_harness("Issue #26028") do load_path
write(joinpath(load_path, "Baz26028.jl"),
"""
module Baz26028
import Foo26028.Bar26028.x
using Test
@test_throws(ConcurrencyViolationError("deadlock detected in loading Foo26028 -> Foo26028"),
@eval import Foo26028.Bar26028.x)
import ..Foo26028.Bar26028.y
end
""")
Base.compilecache(Base.PkgId("Foo26028"))
Expand Down