From a684d22b29668fc50872cceb312b24aa5ba5243a Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Tue, 14 Feb 2023 21:49:23 +0100 Subject: [PATCH 1/3] delay loading of extensions as much as possible --- base/loading.jl | 86 ++++++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 8b104af41aec5..68373bc53ba31 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1080,7 +1080,6 @@ function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String) end function run_package_callbacks(modkey::PkgId) - run_extension_callbacks(modkey) assert_havelock(require_lock) unlock(require_lock) try @@ -1204,53 +1203,56 @@ function run_extension_callbacks(extid::ExtensionId) return succeeded end -function run_extension_callbacks(pkgid::PkgId) +function run_extension_callbacks() assert_havelock(require_lock) - # take ownership of extids that depend on this pkgid - extids = pop!(EXT_DORMITORY, pkgid, nothing) - extids === nothing && return - for extid in extids - if extid.ntriggers > 0 - # It is possible that pkgid was loaded in an environment - # below the one of the parent. This will cause a load failure when the - # pkg ext tries to load the triggers. Therefore, check this first - # before loading the pkg ext. - pkgenv = Base.identify_package_env(extid.id, pkgid.name) - ext_not_allowed_load = false - if pkgenv === nothing - ext_not_allowed_load = true - else - pkg, env = pkgenv - path = Base.locate_package(pkg, env) - if path === nothing + for pkgid in copy(keys(EXT_DORMITORY)) + haskey(Base.loaded_modules, pkgid) || continue + # take ownership of extids that depend on this pkgid + extids = pop!(EXT_DORMITORY, pkgid, nothing) + extids === nothing && continue + for extid in extids + if extid.ntriggers > 0 + # It is possible that pkgid was loaded in an environment + # below the one of the parent. This will cause a load failure when the + # pkg ext tries to load the triggers. Therefore, check this first + # before loading the pkg ext. + pkgenv = Base.identify_package_env(extid.id, pkgid.name) + ext_not_allowed_load = false + if pkgenv === nothing ext_not_allowed_load = true + else + pkg, env = pkgenv + path = Base.locate_package(pkg, env) + if path === nothing + ext_not_allowed_load = true + end + end + if ext_not_allowed_load + @debug "Extension $(extid.id.name) of $(extid.parentid.name) will not be loaded \ + since $(pkgid.name) loaded in environment lower in load path" + # indicate extid is expected to fail + extid.ntriggers *= -1 + else + # indicate pkgid is loaded + extid.ntriggers -= 1 end end - if ext_not_allowed_load - @debug "Extension $(extid.id.name) of $(extid.parentid.name) will not be loaded \ - since $(pkgid.name) loaded in environment lower in load path" - # indicate extid is expected to fail - extid.ntriggers *= -1 - else + if extid.ntriggers < 0 # indicate pkgid is loaded - extid.ntriggers -= 1 + extid.ntriggers += 1 + succeeded = false + else + succeeded = true + end + if extid.ntriggers == 0 + # actually load extid, now that all dependencies are met, + # and record the result + succeeded = succeeded && run_extension_callbacks(extid) + succeeded || push!(EXT_DORMITORY_FAILED, extid) end - end - if extid.ntriggers < 0 - # indicate pkgid is loaded - extid.ntriggers += 1 - succeeded = false - else - succeeded = true - end - if extid.ntriggers == 0 - # actually load extid, now that all dependencies are met, - # and record the result - succeeded = succeeded && run_extension_callbacks(extid) - succeeded || push!(EXT_DORMITORY_FAILED, extid) end end - nothing + return end """ @@ -1637,7 +1639,9 @@ function require(into::Module, mod::Symbol) if _track_dependencies[] push!(_require_dependencies, (into, binpack(uuidkey), 0.0)) end - return _require_prelocked(uuidkey, env) + mod = _require_prelocked(uuidkey, env) + into === Main && run_extension_callbacks() + return mod finally LOADING_CACHE[] = nothing end From 0fd670f1dc5c3d85e7d1f43543d1e91d48c671ff Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Sun, 12 Mar 2023 21:26:29 +0100 Subject: [PATCH 2/3] change to look at `package_locks` and do not load when precompiling --- base/loading.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 68373bc53ba31..a6fec95db89fd 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1639,9 +1639,7 @@ function require(into::Module, mod::Symbol) if _track_dependencies[] push!(_require_dependencies, (into, binpack(uuidkey), 0.0)) end - mod = _require_prelocked(uuidkey, env) - into === Main && run_extension_callbacks() - return mod + return _require_prelocked(uuidkey, env) finally LOADING_CACHE[] = nothing end @@ -1669,6 +1667,10 @@ function _require_prelocked(uuidkey::PkgId, env=nothing) else newm = root_module(uuidkey) end + # Load extensions when not precompiling and not in a nested package load + if JLOptions().incremental == 0 && isempty(package_locks) + run_extension_callbacks() + end return newm end From 8718be093129dbd8c107d1c7656654164bbf1418 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Wed, 15 Mar 2023 14:10:50 +0100 Subject: [PATCH 3/3] deterministic order for iterating over triggers --- base/loading.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index a6fec95db89fd..b7acda85fea0d 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1205,8 +1205,9 @@ end function run_extension_callbacks() assert_havelock(require_lock) - for pkgid in copy(keys(EXT_DORMITORY)) - haskey(Base.loaded_modules, pkgid) || continue + loaded_triggers = collect(intersect(keys(Base.loaded_modules), keys(Base.EXT_DORMITORY))) + sort!(loaded_triggers; by=x->x.uuid) + for pkgid in loaded_triggers # take ownership of extids that depend on this pkgid extids = pop!(EXT_DORMITORY, pkgid, nothing) extids === nothing && continue