From 78351b5c478b65295dfa143d33ddc41f546baceb Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 8 Mar 2024 10:21:33 -0500 Subject: [PATCH] Use Base parallel precompilation to build stdlibs (#53598) Follow-on from https://github.com/JuliaLang/julia/pull/53403 This extends `Base.Precompilation.precompilepkgs` to take a list of configurations to precompile each package with, while parallelizing across all packages and configurations, and uses it to build the stdlib pkgimages. It simplifies the stdlib pkgimage build process but is (currently) dependent on having an accurately resolved Manifest.toml (Project.toml included to make the manifest easier to make). Any new/removed stdlibs or changes their dependencies will require updating the Manifest.toml. It's a bit chicken and egg, but should be manageable with manual editing of the Manifest.toml. In terms of speed improvement: MacOS aarch64 CI runner 6m19s before, 5m19 with this Note that CI builds will show the basic print with timing of each package, whereas local build will be the tidier fancy print without timings. Co-authored-by: Valentin Churavy --- Makefile | 3 +- base/loading.jl | 33 ++++- base/precompilation.jl | 318 +++++++++++++++++++++++------------------ base/util.jl | 1 + pkgimage.mk | 141 ++---------------- stdlib/Manifest.toml | 300 ++++++++++++++++++++++++++++++++++++++ stdlib/Project.toml | 61 ++++++++ stdlib/stdlib.mk | 21 ++- sysimage.mk | 2 +- 9 files changed, 589 insertions(+), 291 deletions(-) create mode 100644 stdlib/Manifest.toml create mode 100644 stdlib/Project.toml diff --git a/Makefile b/Makefile index 5ce3003dba231..d7d25998688c5 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ julia-debug julia-release : julia-% : julia-sysimg-% julia-src-% julia-symlink j julia-libccalllazyfoo julia-libccalllazybar julia-libllvmcalltest julia-base-cache stdlibs-cache-release stdlibs-cache-debug : stdlibs-cache-% : julia-% - @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f pkgimage.mk all-$* + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f pkgimage.mk $* debug release : % : julia-% stdlibs-cache-% @@ -588,6 +588,7 @@ clean: | $(CLEAN_TARGETS) @-$(MAKE) -C $(BUILDROOT)/cli clean @-$(MAKE) -C $(BUILDROOT)/test clean @-$(MAKE) -C $(BUILDROOT)/stdlib clean + @-$(MAKE) -C $(BUILDROOT) -f pkgimage.mk clean -rm -f $(BUILDROOT)/julia -rm -f $(BUILDROOT)/*.tar.gz -rm -f $(build_depsbindir)/stringreplace \ diff --git a/base/loading.jl b/base/loading.jl index 5d56819159e83..774aa22bdacbd 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1524,7 +1524,21 @@ function CacheFlags(f::UInt8) CacheFlags(use_pkgimages, debug_level, check_bounds, inline, opt_level) end CacheFlags(f::Int) = CacheFlags(UInt8(f)) -CacheFlags() = CacheFlags(ccall(:jl_cache_flags, UInt8, ())) +function CacheFlags(cf::CacheFlags=CacheFlags(ccall(:jl_cache_flags, UInt8, ())); + use_pkgimages::Union{Nothing,Bool}=nothing, + debug_level::Union{Nothing,Int}=nothing, + check_bounds::Union{Nothing,Int}=nothing, + inline::Union{Nothing,Bool}=nothing, + opt_level::Union{Nothing,Int}=nothing + ) + return CacheFlags( + use_pkgimages === nothing ? cf.use_pkgimages : use_pkgimages, + debug_level === nothing ? cf.debug_level : debug_level, + check_bounds === nothing ? cf.check_bounds : check_bounds, + inline === nothing ? cf.inline : inline, + opt_level === nothing ? cf.opt_level : opt_level + ) +end function _cacheflag_to_uint8(cf::CacheFlags)::UInt8 f = UInt8(0) @@ -2768,7 +2782,7 @@ function compilecache_dir(pkg::PkgId) return joinpath(DEPOT_PATH[1], entrypath) end -function compilecache_path(pkg::PkgId, prefs_hash::UInt64; project::String=something(Base.active_project(), ""))::String +function compilecache_path(pkg::PkgId, prefs_hash::UInt64; flags::CacheFlags=CacheFlags(), project::String=something(Base.active_project(), ""))::String entrypath, entryfile = cache_file_entry(pkg) cachepath = joinpath(DEPOT_PATH[1], entrypath) isdir(cachepath) || mkpath(cachepath) @@ -2778,7 +2792,7 @@ function compilecache_path(pkg::PkgId, prefs_hash::UInt64; project::String=somet crc = _crc32c(project) crc = _crc32c(unsafe_string(JLOptions().image_file), crc) crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc) - crc = _crc32c(ccall(:jl_cache_flags, UInt8, ()), crc) + crc = _crc32c(_cacheflag_to_uint8(flags), crc) cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) if cpu_target === nothing @@ -2810,7 +2824,8 @@ end const MAX_NUM_PRECOMPILE_FILES = Ref(10) function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, internal_stdout::IO = stdout, - keep_loaded_modules::Bool = true; flags::Cmd=``, reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}()) + keep_loaded_modules::Bool = true; flags::Cmd=``, cacheflags::CacheFlags=CacheFlags(), + reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}()) @nospecialize internal_stderr internal_stdout # decide where to put the resulting cache file @@ -2859,7 +2874,7 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in # Read preferences hash back from .ji file (we can't precompute because # we don't actually know what the list of compile-time preferences are without compiling) prefs_hash = preferences_hash(tmppath) - cachefile = compilecache_path(pkg, prefs_hash) + cachefile = compilecache_path(pkg, prefs_hash; flags=cacheflags) ocachefile = cache_objects ? ocachefile_from_cachefile(cachefile) : nothing # append checksum for so to the end of the .ji file: @@ -3408,7 +3423,7 @@ global parse_pidfile_hook # The preferences hash is only known after precompilation so just assume no preferences. # Also ignore the active project, which means that if all other conditions are equal, # the same package cannot be precompiled from different projects and/or different preferences at the same time. -compilecache_pidfile_path(pkg::PkgId) = compilecache_path(pkg, UInt64(0); project="") * ".pidfile" +compilecache_pidfile_path(pkg::PkgId; flags::CacheFlags=CacheFlags()) = compilecache_path(pkg, UInt64(0); project="", flags) * ".pidfile" const compilecache_pidlock_stale_age = 10 @@ -3536,12 +3551,16 @@ end M = root_module(req_key) if PkgId(M) == req_key && module_build_id(M) === req_build_id depmods[i] = M + elseif M == Core + @debug "Rejecting cache file $cachefile because it was made with a different julia version" + record_reason(reasons, "wrong julia version") + return true # Won't be able to fulfill dependency elseif ignore_loaded || !stalecheck # Used by Pkg.precompile given that there it's ok to precompile different versions of loaded packages @goto locate_branch else @debug "Rejecting cache file $cachefile because module $req_key is already loaded and incompatible." - record_reason(reasons, req_key == PkgId(Core) ? "wrong julia version" : "wrong dep version loaded") + record_reason(reasons, "wrong dep version loaded") return true # Won't be able to fulfill dependency end else diff --git a/base/precompilation.jl b/base/precompilation.jl index 9eaa0539f522e..3259b6a7e063b 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -335,16 +335,29 @@ function printpkgstyle(io, header, msg; color=:light_green) println(io, " ", msg) end +const Config = Pair{Cmd, Base.CacheFlags} +const PkgConfig = Tuple{Base.PkgId,Config} + +function precompilepkgs(pkgs::Vector{String}=String[]; + internal_call::Bool=false, + strict::Bool = false, + warn_loaded::Bool = true, + timing::Bool = false, + _from_loading::Bool=false, + configs::Union{Config,Vector{Config}}=(``=>Base.CacheFlags()), + io::IO=stderr, + flags_cacheflags=nothing) + + if flags_cacheflags !== nothing + # TODO: Remove `flags_cacheflags` once Pkg moves to the renamed kwarg `configs` + configs = flags_cacheflags + end + configs = configs isa Config ? [configs] : configs -function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false, - strict::Bool=false, warn_loaded = true, timing::Bool = false, - _from_loading::Bool=false, flags_cacheflags::Pair{Cmd, Base.CacheFlags}=(``=>Base.CacheFlags()), io::IO=stderr) time_start = time_ns() env = ExplicitEnv() - flags, cacheflags = flags_cacheflags - # Windows sometimes hits a ReadOnlyMemoryError, so we halve the default number of tasks. Issue #2323 # TODO: Investigate why this happens in windows and restore the full task limit default_num_tasks = Sys.iswindows() ? div(Sys.CPU_THREADS::Int, 2) + 1 : Sys.CPU_THREADS::Int + 1 @@ -356,7 +369,8 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false # asking for timing disables fancy mode, as timing is shown in non-fancy mode fancyprint = can_fancyprint(io) && !timing - if _from_loading && !Sys.isinteractive() + if _from_loading && !Sys.isinteractive() && Base.get_bool_env("JULIA_TESTS", false) + # suppress passive loading printing in julia test suite. `JULIA_TESTS` is set in Base.runtests io = devnull end @@ -441,13 +455,16 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false end # initialize signalling - started = Dict{Base.PkgId,Bool}() - was_processed = Dict{Base.PkgId,Base.Event}() - was_recompiled = Dict{Base.PkgId,Bool}() - for pkgid in keys(depsmap) - started[pkgid] = false - was_processed[pkgid] = Base.Event() - was_recompiled[pkgid] = false + started = Dict{PkgConfig,Bool}() + was_processed = Dict{PkgConfig,Base.Event}() + was_recompiled = Dict{PkgConfig,Bool}() + for config in configs + for pkgid in keys(depsmap) + dep_config = (pkgid, config) + started[dep_config] = false + was_processed[dep_config] = Base.Event() + was_recompiled[dep_config] = false + end end @debug "precompile: signalling initialized" _group=:precompile @@ -481,7 +498,10 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false for pkg in keys(depsmap) if scan_pkg!(pkg, depsmap) push!(circular_deps, pkg) - notify(was_processed[pkg]) + for dep_config in keys(was_processed) + # notify all to allow skipping + dep_config[1] == pkg && notify(was_processed[dep_config]) + end end end if !isempty(circular_deps) @@ -526,14 +546,18 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false end target = join(pkgs, ", ") else - target = "project..." + target = "project" + end + if length(configs) > 1 || !isempty(only(configs)[1]) # if multiple configs or only one is not default + target *= " for $(length(configs)) compilation configurations..." + else + target *= "..." end @debug "precompile: packages filtered" _group=:precompile - pkg_queue = Base.PkgId[] - failed_deps = Dict{Base.PkgId, String}() - skipped_deps = Base.PkgId[] - precomperr_deps = Base.PkgId[] # packages that may succeed after a restart (i.e. loaded packages with no cache file) + pkg_queue = PkgConfig[] + failed_deps = Dict{PkgConfig, String}() + precomperr_deps = PkgConfig[] # packages that may succeed after a restart (i.e. loaded packages with no cache file) print_lock = io isa Base.LibuvStream ? io.lock::ReentrantLock : ReentrantLock() first_started = Base.Event() @@ -556,7 +580,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false in_printloop || wait(t_print) # wait to let the print loop cease first if err isa InterruptException lock(print_lock) do - println(io, " Interrupted: Exiting precompilation...") + println(io, " Interrupted: Exiting precompilation...", ansi_cleartoendofline) end interrupted = true return true @@ -564,13 +588,13 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false return false end end - - std_outputs = Dict{Base.PkgId,String}() - taskwaiting = Set{Base.PkgId}() - pkgspidlocked = Dict{Base.PkgId,String}() + std_outputs = Dict{PkgConfig,String}() + taskwaiting = Set{PkgConfig}() + pkgspidlocked = Dict{PkgConfig,String}() pkg_liveprinted = nothing - function monitor_std(pkg, pipe; single_requested_pkg=false) + function monitor_std(pkg_config, pipe; single_requested_pkg=false) + pkg, config = pkg_config try liveprinting = false while !eof(pipe) @@ -586,14 +610,14 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false print(io, ansi_cleartoendofline, str) end end - std_outputs[pkg] = string(get(std_outputs, pkg, ""), str) - if !in(pkg, taskwaiting) && occursin("waiting for IO to finish", str) + std_outputs[pkg_config] = string(get(std_outputs, pkg_config, ""), str) + if !in(pkg_config, taskwaiting) && occursin("waiting for IO to finish", str) !fancyprint && lock(print_lock) do println(io, pkg.name, color_string(" Waiting for background task / IO / timer.", Base.warn_color())) end - push!(taskwaiting, pkg) + push!(taskwaiting, pkg_config) end - if !fancyprint && in(pkg, taskwaiting) + if !fancyprint && in(pkg_config, taskwaiting) lock(print_lock) do print(io, str) end @@ -618,7 +642,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false i = 1 last_length = 0 bar = MiniProgressBar(; indent=2, header = "Progress", color = Base.info_color(), percentage=false, always_reprint=true) - n_total = length(depsmap) + n_total = length(depsmap) * length(configs) bar.max = n_total - n_already_precomp final_loop = false n_print_rows = 0 @@ -644,30 +668,35 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false str = sprint(io -> show_progress(io, bar; termwidth, carriagereturn=false); context=io) print(iostr, Base._truncate_at_width_or_chars(true, str, termwidth), "\n") end - for dep in pkg_queue_show + for dep_config in pkg_queue_show + dep, config = dep_config loaded = warn_loaded && haskey(Base.loaded_modules, dep) _name = haskey(exts, dep) ? string(exts[dep], " → ", dep.name) : dep.name name = dep in direct_deps ? _name : string(color_string(_name, :light_black)) - line = if dep in precomperr_deps + if length(configs) > 1 + config_str = isempty(config[1]) ? "" : "$(join(config[1], " "))" + name *= color_string(" $(config_str)", :light_black) + end + line = if dep_config in precomperr_deps string(color_string(" ? ", Base.warn_color()), name) - elseif haskey(failed_deps, dep) + elseif haskey(failed_deps, dep_config) string(color_string(" ✗ ", Base.error_color()), name) - elseif was_recompiled[dep] + elseif was_recompiled[dep_config] !loaded && interrupted_or_done.set && continue loaded || @async begin # keep successful deps visible for short period sleep(1); - filter!(!isequal(dep), pkg_queue) + filter!(!isequal(dep_config), pkg_queue) end string(color_string(" ✓ ", loaded ? Base.warn_color() : :green), name) - elseif started[dep] + elseif started[dep_config] # Offset each spinner animation using the first character in the package name as the seed. # If not offset, on larger terminal fonts it looks odd that they all sync-up anim_char = anim_chars[(i + Int(dep.name[1])) % length(anim_chars) + 1] anim_char_colored = dep in direct_deps ? anim_char : color_string(anim_char, :light_black) - waiting = if haskey(pkgspidlocked, dep) - who_has_lock = pkgspidlocked[dep] + waiting = if haskey(pkgspidlocked, dep_config) + who_has_lock = pkgspidlocked[dep_config] color_string(" Being precompiled by $(who_has_lock)", Base.info_color()) - elseif dep in taskwaiting + elseif dep_config in taskwaiting color_string(" Waiting for background task / IO / timer. Interrupt to inspect", Base.warn_color()) else "" @@ -704,103 +733,108 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false for (pkg, deps) in depsmap cachepaths = Base.find_all_in_cache_path(pkg) sourcepath = Base.locate_package(pkg) - if sourcepath === nothing - failed_deps[pkg] = "Error: Missing source file for $(pkg)" - notify(was_processed[pkg]) - continue - end - # Heuristic for when precompilation is disabled - if occursin(r"\b__precompile__\(\s*false\s*\)", read(sourcepath, String)) - notify(was_processed[pkg]) - continue - end - single_requested_pkg = length(pkgs) == 1 && only(pkgs) == pkg.name - - task = @async begin - try - loaded = haskey(Base.loaded_modules, pkg) - for dep in deps # wait for deps to finish - wait(was_processed[dep]) - end - - circular = pkg in circular_deps - is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, stale_cache, cachepaths, sourcepath, flags=cacheflags) - if !circular && is_stale - Base.acquire(parallel_limiter) - is_direct_dep = pkg in direct_deps - - # std monitoring - std_pipe = Base.link_pipe!(Pipe(); reader_supports_async=true, writer_supports_async=true) - t_monitor = @async monitor_std(pkg, std_pipe; single_requested_pkg) - - _name = haskey(exts, pkg) ? string(exts[pkg], " → ", pkg.name) : pkg.name - name = is_direct_dep ? _name : string(color_string(_name, :light_black)) - !fancyprint && lock(print_lock) do - isempty(pkg_queue) && printpkgstyle(io, :Precompiling, target) - end - push!(pkg_queue, pkg) - started[pkg] = true - fancyprint && notify(first_started) - if interrupted_or_done.set - notify(was_processed[pkg]) - Base.release(parallel_limiter) - return + for config in configs + pkg_config = (pkg, config) + if sourcepath === nothing + failed_deps[pkg_config] = "Error: Missing source file for $(pkg)" + notify(was_processed[pkg_config]) + continue + end + # Heuristic for when precompilation is disabled + if occursin(r"\b__precompile__\(\s*false\s*\)", read(sourcepath, String)) + notify(was_processed[pkg_config]) + continue + end + flags, cacheflags = config + task = @async begin + try + loaded = haskey(Base.loaded_modules, pkg) + for dep in deps # wait for deps to finish + wait(was_processed[(dep,config)]) end - try - # allows processes to wait if another process is precompiling a given package to - # a functionally identical package cache (except for preferences, which may differ) - t = @elapsed ret = precompile_pkgs_maybe_cachefile_lock(io, print_lock, fancyprint, pkg, pkgspidlocked, hascolor, cacheflags) do - Base.with_logger(Base.NullLogger()) do - # The false here means we ignore loaded modules, so precompile for a fresh session - Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, false; flags) - end + circular = pkg in circular_deps + is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, stale_cache, cachepaths, sourcepath, flags=cacheflags) + if !circular && is_stale + Base.acquire(parallel_limiter) + is_direct_dep = pkg in direct_deps + + # std monitoring + std_pipe = Base.link_pipe!(Pipe(); reader_supports_async=true, writer_supports_async=true) + t_monitor = @async monitor_std(pkg_config, std_pipe; single_requested_pkg) + + _name = haskey(exts, pkg) ? string(exts[pkg], " → ", pkg.name) : pkg.name + name = is_direct_dep ? _name : string(color_string(_name, :light_black)) + if length(configs) > 1 + config_str = isempty(config[1]) ? "" : "$(join(config[1], " "))" + name *= color_string(" $(config_str)", :light_black) + end + !fancyprint && lock(print_lock) do + isempty(pkg_queue) && printpkgstyle(io, :Precompiling, target) end - if ret isa Base.PrecompilableError - push!(precomperr_deps, pkg) - !fancyprint && lock(print_lock) do - println(io, _timing_string(t), color_string(" ? ", Base.warn_color()), name) + push!(pkg_queue, pkg_config) + started[pkg_config] = true + fancyprint && notify(first_started) + if interrupted_or_done.set + notify(was_processed[pkg_config]) + Base.release(parallel_limiter) + return + end + try + # allows processes to wait if another process is precompiling a given package to + # a functionally identical package cache (except for preferences, which may differ) + t = @elapsed ret = precompile_pkgs_maybe_cachefile_lock(io, print_lock, fancyprint, pkg_config, pkgspidlocked, hascolor) do + Base.with_logger(Base.NullLogger()) do + # The false here means we ignore loaded modules, so precompile for a fresh session + Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, false; flags, cacheflags) + end end - else - !fancyprint && lock(print_lock) do - println(io, _timing_string(t), color_string(" ✓ ", loaded ? Base.warn_color() : :green), name) + if ret isa Base.PrecompilableError + push!(precomperr_deps, pkg_config) + !fancyprint && lock(print_lock) do + println(io, _timing_string(t), color_string(" ? ", Base.warn_color()), name) + end + else + !fancyprint && lock(print_lock) do + println(io, _timing_string(t), color_string(" ✓ ", loaded ? Base.warn_color() : :green), name) + end + was_recompiled[pkg_config] = true end - was_recompiled[pkg] = true - end - loaded && (n_loaded += 1) - catch err - close(std_pipe.in) # close pipe to end the std output monitor - wait(t_monitor) - if err isa ErrorException || (err isa ArgumentError && startswith(err.msg, "Invalid header in cache file")) - failed_deps[pkg] = (strict || is_direct_dep) ? string(sprint(showerror, err), "\n", strip(get(std_outputs, pkg, ""))) : "" - delete!(std_outputs, pkg) # so it's not shown as warnings, given error report - !fancyprint && lock(print_lock) do - println(io, " "^9, color_string(" ✗ ", Base.error_color()), name) + loaded && (n_loaded += 1) + catch err + close(std_pipe.in) # close pipe to end the std output monitor + wait(t_monitor) + if err isa ErrorException || (err isa ArgumentError && startswith(err.msg, "Invalid header in cache file")) + failed_deps[dep_config] = (strict || is_direct_dep) ? string(sprint(showerror, err), "\n", strip(get(std_outputs, pkg, ""))) : "" + delete!(std_outputs, pkg_config) # so it's not shown as warnings, given error report + !fancyprint && lock(print_lock) do + println(io, " "^9, color_string(" ✗ ", Base.error_color()), name) + end + else + rethrow() end - else - rethrow() + finally + isopen(std_pipe.in) && close(std_pipe.in) # close pipe to end the std output monitor + wait(t_monitor) + Base.release(parallel_limiter) end - finally - isopen(std_pipe.in) && close(std_pipe.in) # close pipe to end the std output monitor - wait(t_monitor) - Base.release(parallel_limiter) + else + is_stale || (n_already_precomp += 1) end - else - is_stale || (n_already_precomp += 1) + n_done += 1 + notify(was_processed[pkg_config]) + catch err_outer + # For debugging: + # println("Task failed $err_outer") # logging doesn't show here + handle_interrupt(err_outer) || rethrow() + notify(was_processed[pkg_config]) + finally + filter!(!istaskdone, tasks) + length(tasks) == 1 && notify(interrupted_or_done) end - n_done += 1 - notify(was_processed[pkg]) - catch err_outer - # For debugging: - # @error "Task failed" exception=(err_outer, catch_backtrace()) - handle_interrupt(err_outer) || rethrow() - notify(was_processed[pkg]) - finally - filter!(!istaskdone, tasks) - length(tasks) == 1 && notify(interrupted_or_done) end + push!(tasks, task) end - push!(tasks, task) end isempty(tasks) && notify(interrupted_or_done) try @@ -818,28 +852,27 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false if ndeps > 0 || !isempty(failed_deps) || (quick_exit && !isempty(std_outputs)) str = sprint() do iostr if !quick_exit - plural = ndeps == 1 ? "y" : "ies" - print(iostr, " $(ndeps) dependenc$(plural) successfully precompiled in $(seconds_elapsed) seconds") - if n_already_precomp > 0 || !isempty(circular_deps) || !isempty(skipped_deps) + plural = length(configs) > 1 ? "dependency configurations" : ndeps == 1 ? "dependency" : "dependencies" + print(iostr, " $(ndeps) $(plural) successfully precompiled in $(seconds_elapsed) seconds") + if n_already_precomp > 0 || !isempty(circular_deps) n_already_precomp > 0 && (print(iostr, ". $n_already_precomp already precompiled")) !isempty(circular_deps) && (print(iostr, ". $(length(circular_deps)) skipped due to circular dependency")) - !isempty(skipped_deps) && (print(iostr, ". $(length(skipped_deps)) skipped during auto due to previous errors")) print(iostr, ".") end if n_loaded > 0 - plural1 = n_loaded == 1 ? "y" : "ies" + plural1 = length(configs) > 1 ? "dependency configurations" : n_loaded == 1 ? "dependency" : "dependencies" plural2 = n_loaded == 1 ? "a different version is" : "different versions are" plural3 = n_loaded == 1 ? "" : "s" print(iostr, "\n ", color_string(string(n_loaded), Base.warn_color()), - " dependenc$(plural1) precompiled but $(plural2) currently loaded. Restart julia to access the new version$(plural3)" + " $(plural1) precompiled but $(plural2) currently loaded. Restart julia to access the new version$(plural3)" ) end if !isempty(precomperr_deps) - pluralpc = length(precomperr_deps) == 1 ? "y" : "ies" + pluralpc = length(configs) > 1 ? "dependency configurations" : precomperr_deps == 1 ? "dependency" : "dependencies" print(iostr, "\n ", color_string(string(length(precomperr_deps)), Base.warn_color()), - " dependenc$(pluralpc) failed but may be precompilable after restarting julia" + " $(pluralpc) failed but may be precompilable after restarting julia" ) end end @@ -850,13 +883,14 @@ function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false plural1 = length(std_outputs) == 1 ? "y" : "ies" plural2 = length(std_outputs) == 1 ? "" : "s" print(iostr, "\n ", color_string("$(length(std_outputs))", Base.warn_color()), " dependenc$(plural1) had output during precompilation:") - for (pkgid, err) in std_outputs - err = if pkgid == pkg_liveprinted + for (pkg_config, err) in std_outputs + pkg, config = pkg_config + err = if pkg == pkg_liveprinted "[Output was shown above]" else join(split(strip(err), "\n"), color_string("\n│ ", Base.warn_color())) end - name = haskey(exts, pkgid) ? string(exts[pkgid], " → ", pkgid.name) : pkgid.name + name = haskey(exts, pkg) ? string(exts[pkg], " → ", pkg.name) : pkg.name print(iostr, color_string("\n┌ ", Base.warn_color()), name, color_string("\n│ ", Base.warn_color()), err, color_string("\n└ ", Base.warn_color())) end end @@ -912,14 +946,16 @@ function _color_string(cstr::String, col::Union{Int64, Symbol}, hascolor) end # Can be merged with `maybe_cachefile_lock` in loading? -function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint::Bool, pkg::Base.PkgId, pkgspidlocked::Dict{Base.PkgId,String}, hascolor, cacheflags) +function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint::Bool, pkg_config, pkgspidlocked, hascolor) + pkg, config = pkg_config + flags, cacheflags = config FileWatching = Base.loaded_modules[Base.PkgId(Base.UUID("7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"), "FileWatching")] stale_age = Base.compilecache_pidlock_stale_age - pidfile = Base.compilecache_pidfile_path(pkg) + pidfile = Base.compilecache_pidfile_path(pkg, flags=cacheflags) cachefile = FileWatching.trymkpidlock(f, pidfile; stale_age) if cachefile === false pid, hostname, age = FileWatching.Pidfile.parse_pidfile(pidfile) - pkgspidlocked[pkg] = if isempty(hostname) || hostname == gethostname() + pkgspidlocked[pkg_config] = if isempty(hostname) || hostname == gethostname() if pid == getpid() "an async task in this process (pidfile: $pidfile)" else @@ -929,7 +965,7 @@ function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLo "another machine (hostname: $hostname, pid: $pid, pidfile: $pidfile)" end !fancyprint && lock(print_lock) do - println(io, " ", pkg.name, _color_string(" Being precompiled by $(pkgspidlocked[pkg])", Base.info_color(), hascolor)) + println(io, " ", pkg.name, _color_string(" Being precompiled by $(pkgspidlocked[pkg_config])", Base.info_color(), hascolor)) end # wait until the lock is available FileWatching.mkpidlock(pidfile; stale_age) do @@ -937,7 +973,7 @@ function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLo if Base.isprecompiled(pkg; ignore_loaded=true, flags=cacheflags) # don't use caches for this as the env state will have changed return nothing # returning nothing indicates a process waited for another else - delete!(pkgspidlocked, pkg) + delete!(pkgspidlocked, pkg_config) return f() # precompile end end diff --git a/base/util.jl b/base/util.jl index c91f04e8baf96..5e86f026f8f9a 100644 --- a/base/util.jl +++ b/base/util.jl @@ -687,6 +687,7 @@ function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS / 2), pathsep = Sys.iswindows() ? ";" : ":" ENV2["JULIA_DEPOT_PATH"] = string(mktempdir(; cleanup = true), pathsep) # make sure the default depots can be loaded ENV2["JULIA_LOAD_PATH"] = string("@", pathsep, "@stdlib") + ENV2["JULIA_TESTS"] = "true" delete!(ENV2, "JULIA_PROJECT") try run(setenv(`$(julia_cmd()) $(joinpath(Sys.BINDIR, diff --git a/pkgimage.mk b/pkgimage.mk index c9de49d2f8421..485e27996bb44 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -7,146 +7,31 @@ include $(JULIAHOME)/stdlib/stdlib.mk # set some influential environment variables export JULIA_DEPOT_PATH := $(build_prefix)/share/julia -export JULIA_LOAD_PATH := @stdlib +export JULIA_LOAD_PATH := @stdlib:$(JULIAHOME)/stdlib unexport JULIA_PROJECT := unexport JULIA_BINDIR := export JULIA_FALLBACK_REPL := true default: release -release: all-release -debug: all-debug +release: $(BUILDDIR)/stdlib/release.image +debug: $(BUILDDIR)/stdlib/debug.image all: release debug -$(JULIA_DEPOT_PATH): +$(JULIA_DEPOT_PATH)/compiled: mkdir -p $@ print-depot-path: @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e '@show Base.DEPOT_PATH') -all-release: $(addprefix cache-release-, $(STDLIBS)) -all-debug: $(addprefix cache-debug-, $(STDLIBS)) +$(BUILDDIR)/stdlib/%.image: $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib/Manifest.toml $(INDEPENDENT_STDLIBS_SRCS) $(JULIA_DEPOT_PATH)/compiled + export JULIA_CPU_TARGET="$(JULIA_CPU_TARGET)" + @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.Precompilation.precompilepkgs(;configs=[``=>Base.CacheFlags(), `--check-bounds=yes`=>Base.CacheFlags(;check_bounds=1)])') + touch $@ -define stdlib_builder -ifneq ($(filter $(1),$(INDEPENDENT_STDLIBS) $(PKG_EXTS)),) -# Define target-specific export for `JULIA_CPU_TARGET` -$$(BUILDDIR)/stdlib/$1.release.image: export JULIA_CPU_TARGET=$(JULIA_CPU_TARGET) -$$(BUILDDIR)/stdlib/$1.debug.image: export JULIA_CPU_TARGET=$(JULIA_CPU_TARGET) -ifneq ($(filter $(1),$(INDEPENDENT_STDLIBS)),) -$$(BUILDDIR)/stdlib/$1.release.image: $$($1_SRCS) $$(addsuffix .release.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) $(build_private_libdir)/sys.$(SHLIB_EXT) - @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))') - @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.compilecache(Base.identify_package("$1"))') - touch $$@ -$$(BUILDDIR)/stdlib/$1.debug.image: $$($1_SRCS) $$(addsuffix .debug.image,$$(addprefix $$(BUILDDIR)/stdlib/,$2)) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) - @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'Base.compilecache(Base.identify_package("$1"))') - @$$(call PRINT_JULIA, $$(call spawn,$$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.compilecache(Base.identify_package("$1"))') - touch $$@ -endif -ifneq ($(filter $(1),$(PKG_EXTS)),) -# This is weird. It set up for multiple Pkg exts because that suits the structure here better -# but it hard codes the deps and `import REPL, Pkg` -$$(BUILDDIR)/stdlib/REPLExt.release.image: $$(REPLExt_SRCS) $$(BUILDDIR)/stdlib/Pkg.release.image $$(BUILDDIR)/stdlib/REPL.release.image - @$$(call PRINT_JULIA, $$(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'using REPL; using Pkg') - @$$(call PRINT_JULIA, $$(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e 'using REPL; using Pkg') - touch $$@ -$$(BUILDDIR)/stdlib/REPLExt.debug.image: $$(REPLExt_SRCS) $(BUILDDIR)/stdlib/Pkg.debug.image $$(BUILDDIR)/stdlib/REPL.debug.image - @$$(call PRINT_JULIA, $$(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes -e 'using REPL; using Pkg') - @$$(call PRINT_JULIA, $$(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e 'using REPL; using Pkg') - touch $$@ -endif -else -ifneq ($(filter $(1),$(STDLIBS_WITHIN_SYSIMG)),) -$$(BUILDDIR)/stdlib/$1.release.image: - touch $$@ -$$(BUILDDIR)/stdlib/$1.debug.image: - touch $$@ -else -$$(error $(1) neither in STDLIBS_WITHIN_SYSIMG nor INDEPENDENT_STDLIBS) -endif -endif -cache-release-$1: $$(BUILDDIR)/stdlib/$1.release.image -cache-debug-$1: $$(BUILDDIR)/stdlib/$1.debug.image -.SECONDARY: $$(BUILDDIR)/stdlib/$1.release.image $$(BUILDDIR)/stdlib/$1.debug.image -endef +$(BUILDDIR)/stdlib/release.image: $(build_private_libdir)/sys.$(SHLIB_EXT) +$(BUILDDIR)/stdlib/debug.image: $(build_private_libdir)/sys-debug.$(SHLIB_EXT) -# Note: you can check for the correctness of this tree by running `JULIA_DEBUG=nested_precomp make` and looking -# out for `Debug: Nested precompilation` logs. - -# no dependencies -$(eval $(call stdlib_builder,MozillaCACerts_jll,)) -$(eval $(call stdlib_builder,ArgTools,)) -$(eval $(call stdlib_builder,Artifacts,)) -$(eval $(call stdlib_builder,Base64,)) -$(eval $(call stdlib_builder,CRC32c,)) -$(eval $(call stdlib_builder,FileWatching,)) -$(eval $(call stdlib_builder,Libdl,)) -$(eval $(call stdlib_builder,Mmap,)) -$(eval $(call stdlib_builder,NetworkOptions,)) -$(eval $(call stdlib_builder,SHA,)) -$(eval $(call stdlib_builder,Serialization,)) -$(eval $(call stdlib_builder,Sockets,)) -$(eval $(call stdlib_builder,Unicode,)) -$(eval $(call stdlib_builder,Profile,)) -$(eval $(call stdlib_builder,StyledStrings,)) -$(eval $(call stdlib_builder,SuiteSparse_jll,)) - -# 1-depth packages -$(eval $(call stdlib_builder,GMP_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,LLVMLibUnwind_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,LibUV_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,LibUnwind_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,MbedTLS_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,nghttp2_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,OpenLibm_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,PCRE2_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,Zlib_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,dSFMT_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,libLLVM_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,libblastrampoline_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,p7zip_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,OpenBLAS_jll,Artifacts Libdl)) -$(eval $(call stdlib_builder,Markdown,Base64)) -$(eval $(call stdlib_builder,Printf,Unicode)) -$(eval $(call stdlib_builder,Random,SHA)) -$(eval $(call stdlib_builder,Logging,StyledStrings)) -$(eval $(call stdlib_builder,Tar,ArgTools,SHA)) -$(eval $(call stdlib_builder,DelimitedFiles,Mmap)) -$(eval $(call stdlib_builder,JuliaSyntaxHighlighting,StyledStrings)) - -# 2-depth packages -$(eval $(call stdlib_builder,LLD_jll,Zlib_jll libLLVM_jll Artifacts Libdl)) -$(eval $(call stdlib_builder,LibSSH2_jll,Artifacts Libdl MbedTLS_jll)) -$(eval $(call stdlib_builder,MPFR_jll,Artifacts Libdl GMP_jll)) -$(eval $(call stdlib_builder,LinearAlgebra,Libdl libblastrampoline_jll OpenBLAS_jll)) -$(eval $(call stdlib_builder,Dates,Printf)) -$(eval $(call stdlib_builder,Distributed,Random Serialization Sockets)) -$(eval $(call stdlib_builder,Future,Random)) -$(eval $(call stdlib_builder,UUIDs,Random SHA)) -$(eval $(call stdlib_builder,InteractiveUtils,Markdown)) - - # 3-depth packages -$(eval $(call stdlib_builder,LibGit2_jll,MbedTLS_jll LibSSH2_jll Artifacts Libdl)) -$(eval $(call stdlib_builder,LibCURL_jll,LibSSH2_jll nghttp2_jll MbedTLS_jll Zlib_jll Artifacts Libdl)) -$(eval $(call stdlib_builder,REPL,InteractiveUtils Markdown Sockets StyledStrings Unicode)) -$(eval $(call stdlib_builder,SharedArrays,Distributed Mmap Random Serialization)) -$(eval $(call stdlib_builder,TOML,Dates)) -$(eval $(call stdlib_builder,Test,Logging Random Serialization InteractiveUtils)) - -# 4-depth packages -$(eval $(call stdlib_builder,LibGit2,LibGit2_jll NetworkOptions Printf SHA Base64)) -$(eval $(call stdlib_builder,LibCURL,LibCURL_jll MozillaCACerts_jll)) - -# 5-depth packages -$(eval $(call stdlib_builder,Downloads,ArgTools FileWatching LibCURL NetworkOptions)) - -# 6-depth packages -$(eval $(call stdlib_builder,Pkg, Artifacts Dates Downloads FileWatching LibGit2 Libdl\ - Logging Markdown Printf REPL Random SHA Serialization\ - TOML Tar UUIDs p7zip_jll)) - -# 7-depth packages -$(eval $(call stdlib_builder,LazyArtifacts,Artifacts Pkg)) -$(eval $(call stdlib_builder,REPLExt,Pkg REPL)) - -$(eval $(call stdlib_builder,SparseArrays,Libdl LinearAlgebra Random Serialization SuiteSparse_jll)) -$(eval $(call stdlib_builder,Statistics,LinearAlgebra SparseArrays)) +clean: + rm -rf $(JULIA_DEPOT_PATH)/compiled + rm -f $(BUILDDIR)/stdlib/*.image diff --git a/stdlib/Manifest.toml b/stdlib/Manifest.toml new file mode 100644 index 0000000000000..f2ae43f231db5 --- /dev/null +++ b/stdlib/Manifest.toml @@ -0,0 +1,300 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.12.0-DEV" +manifest_format = "2.0" +project_hash = "13f2dd600364a1e8b659dc5796bf185b37d1c95d" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.CRC32c]] +uuid = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" +version = "1.11.0" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.1.0+0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.DelimitedFiles]] +deps = ["Mmap"] +git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +version = "1.9.1" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" +version = "1.11.0" + +[[deps.GMP_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" +version = "6.3.0+0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.JuliaSyntaxHighlighting]] +deps = ["StyledStrings"] +uuid = "dc6e5ff7-fb65-4e79-a425-ec3bc9c03011" +version = "1.11.0" + +[[deps.LLD_jll]] +deps = ["Artifacts", "Libdl", "Zlib_jll", "libLLVM_jll"] +uuid = "d55e3150-da41-5e91-b323-ecfd1eec6109" +version = "16.0.6+4" + +[[deps.LLVMLibUnwind_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "47c5dbc3-30ba-59ef-96a6-123e260183d9" +version = "12.0.1+0" + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" +version = "1.11.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.6.0+0" + +[[deps.LibGit2]] +deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.7.2+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.0+1" + +[[deps.LibUV_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "183b4373-6708-53ba-ad28-60e28bb38547" +version = "2.0.1+15" + +[[deps.LibUnwind_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "745a5e78-f969-53e9-954f-d19f2f74f4e3" +version = "1.7.2+2" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +version = "1.11.0" + +[[deps.Logging]] +deps = ["StyledStrings"] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.MPFR_jll]] +deps = ["Artifacts", "GMP_jll", "Libdl"] +uuid = "3a97d323-0669-5f0c-9066-3539efd106a3" +version = "4.2.0+1" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.6+0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2023.12.12" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.26+2" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.1+2" + +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.42.0+1" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.11.0" +weakdeps = ["REPL"] + + [deps.Pkg.extensions] + REPLExt = "REPL" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.Profile]] +uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" +version = "1.11.0" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" +version = "1.11.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.11.0" + +[[deps.Statistics]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.11.1" +weakdeps = ["SparseArrays"] + + [deps.Statistics.extensions] + SparseArraysExt = ["SparseArrays"] + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "7.6.0+0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+1" + +[[deps.dSFMT_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05ff407c-b0c1-5878-9df8-858cc2e60c36" +version = "2.2.5+0" + +[[deps.libLLVM_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8f36deef-c2a5-5394-99ed-8e07531fb29a" +version = "16.0.6+4" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.8.0+1" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.59.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+2" diff --git a/stdlib/Project.toml b/stdlib/Project.toml new file mode 100644 index 0000000000000..cc7ba99dd4e4f --- /dev/null +++ b/stdlib/Project.toml @@ -0,0 +1,61 @@ +[deps] +ArgTools = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +Future = "9fa8497b-333b-5362-9e8d-4d0656e87820" +GMP_jll = "781609d7-10c4-51f6-84f2-b8444358ff6d" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +JuliaSyntaxHighlighting = "dc6e5ff7-fb65-4e79-a425-ec3bc9c03011" +LLD_jll = "d55e3150-da41-5e91-b323-ecfd1eec6109" +LLVMLibUnwind_jll = "47c5dbc3-30ba-59ef-96a6-123e260183d9" +LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3" +LibCURL = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +LibCURL_jll = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" +LibGit2_jll = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +LibSSH2_jll = "29816b5a-b9ab-546f-933c-edad1886dfa8" +LibUV_jll = "183b4373-6708-53ba-ad28-60e28bb38547" +LibUnwind_jll = "745a5e78-f969-53e9-954f-d19f2f74f4e3" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +MPFR_jll = "3a97d323-0669-5f0c-9066-3539efd106a3" +Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" +MbedTLS_jll = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +Mmap = "a63ad114-7e13-5084-954f-fe012c677804" +MozillaCACerts_jll = "14a3606d-f60d-562e-9121-12d972cd8159" +NetworkOptions = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +OpenBLAS_jll = "4536629a-c528-5b80-bd46-f80d51c5b363" +OpenLibm_jll = "05823500-19ac-5b8b-9628-191a04bc5112" +PCRE2_jll = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" +REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +SharedArrays = "1a1011a3-84de-559e-8e89-a11a2f7dc383" +Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +StyledStrings = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +SuiteSparse_jll = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" +dSFMT_jll = "05ff407c-b0c1-5878-9df8-858cc2e60c36" +libLLVM_jll = "8f36deef-c2a5-5394-99ed-8e07531fb29a" +libblastrampoline_jll = "8e850b90-86db-534c-a0d3-1478176c7d93" +nghttp2_jll = "8e850ede-7688-5339-a07c-302acd2aaf8d" +p7zip_jll = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" diff --git a/stdlib/stdlib.mk b/stdlib/stdlib.mk index b23754f3a3cac..b79059d3368b1 100644 --- a/stdlib/stdlib.mk +++ b/stdlib/stdlib.mk @@ -11,24 +11,19 @@ INDEPENDENT_STDLIBS := \ LibCURL_jll LibSSH2_jll LibGit2_jll nghttp2_jll MozillaCACerts_jll MbedTLS_jll \ MPFR_jll OpenLibm_jll PCRE2_jll p7zip_jll Zlib_jll -PKG_EXTS := \ - REPLExt - -STDLIBS := $(STDLIBS_WITHIN_SYSIMG) $(INDEPENDENT_STDLIBS) $(PKG_EXTS) +STDLIBS := $(STDLIBS_WITHIN_SYSIMG) $(INDEPENDENT_STDLIBS) VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) -SYSIMG_STDLIB_SRCS = +SYSIMG_STDLIBS_SRCS = +INDEPENDENT_STDLIBS_SRCS = define STDLIB_srcs -ifneq ($(filter $(1),$(PKG_EXTS)),) - $1_SRCS := $$(shell find $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/Pkg/ext/$1 -name \*.jl) \ - $$(wildcard $$(build_prefix)/manifest/$$(VERSDIR)/Pkg) $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/Pkg/Project.toml -else - $1_SRCS := $$(shell find $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/$1/src -name \*.jl) \ - $$(wildcard $$(build_prefix)/manifest/$$(VERSDIR)/$1) $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/$1/Project.toml -endif +$1_SRCS := $$(shell find $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/$1/src -name \*.jl) \ +$$(wildcard $$(build_prefix)/manifest/$$(VERSDIR)/$1) $$(build_datarootdir)/julia/stdlib/$$(VERSDIR)/$1/Project.toml ifneq ($(filter $(1),$(STDLIBS_WITHIN_SYSIMG)),) - SYSIMG_STDLIB_SRCS += $$($1_SRCS) + SYSIMG_STDLIBS_SRCS += $$($1_SRCS) +else + INDEPENDENT_STDLIBS_SRCS += $$($1_SRCS) endif endef diff --git a/sysimage.mk b/sysimage.mk index e6fa54be5f186..b5f47be0160a4 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -57,7 +57,7 @@ COMPILER_SRCS += $(shell find $(JULIAHOME)/base/compiler -name \*.jl) # sort these to remove duplicates BASE_SRCS := $(sort $(shell find $(JULIAHOME)/base -name \*.jl -and -not -name sysimg.jl) \ $(shell find $(BUILDROOT)/base -name \*.jl -and -not -name sysimg.jl)) -STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(SYSIMG_STDLIB_SRCS) +STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(SYSIMG_STDLIBS_SRCS) RELBUILDROOT := $(call rel_path,$(JULIAHOME)/base,$(BUILDROOT)/base)/ # <-- make sure this always has a trailing slash $(build_private_libdir)/corecompiler.ji: $(COMPILER_SRCS)