StatProfilerHTML.jl report
Generated on Thu, 21 Dec 2023 12:59:22
File source code
Line Exclusive Inclusive Code
1 # This file is a part of Julia. License is MIT: https://julialang.org/license
2
3 # Base.require is the implementation for the `import` statement
4 const require_lock = ReentrantLock()
5
6 # Cross-platform case-sensitive path canonicalization
7
8 if Sys.isunix() && !Sys.isapple()
9 # assume case-sensitive filesystems, don't have to do anything
10 isfile_casesensitive(path) = isaccessiblefile(path)
11 elseif Sys.iswindows()
12 # GetLongPathName Win32 function returns the case-preserved filename on NTFS.
13 function isfile_casesensitive(path)
14 isaccessiblefile(path) || return false # Fail fast
15 basename(Filesystem.longpath(path)) == basename(path)
16 end
17 elseif Sys.isapple()
18 # HFS+ filesystem is case-preserving. The getattrlist API returns
19 # a case-preserved filename. In the rare event that HFS+ is operating
20 # in case-sensitive mode, this will still work but will be redundant.
21
22 # Constants from <sys/attr.h>
23 const ATRATTR_BIT_MAP_COUNT = 5
24 const ATTR_CMN_NAME = 1
25 const BITMAPCOUNT = 1
26 const COMMONATTR = 5
27 const FSOPT_NOFOLLOW = 1 # Don't follow symbolic links
28
29 const attr_list = zeros(UInt8, 24)
30 attr_list[BITMAPCOUNT] = ATRATTR_BIT_MAP_COUNT
31 attr_list[COMMONATTR] = ATTR_CMN_NAME
32
33 # This essentially corresponds to the following C code:
34 # attrlist attr_list;
35 # memset(&attr_list, 0, sizeof(attr_list));
36 # attr_list.bitmapcount = ATTR_BIT_MAP_COUNT;
37 # attr_list.commonattr = ATTR_CMN_NAME;
38 # struct Buffer {
39 # u_int32_t total_length;
40 # u_int32_t filename_offset;
41 # u_int32_t filename_length;
42 # char filename[max_filename_length];
43 # };
44 # Buffer buf;
45 # getattrpath(path, &attr_list, &buf, sizeof(buf), FSOPT_NOFOLLOW);
46 function isfile_casesensitive(path)
47 isaccessiblefile(path) || return false
48 path_basename = String(basename(path))
49 local casepreserved_basename
50 header_size = 12
51 buf = Vector{UInt8}(undef, length(path_basename) + header_size + 1)
52 while true
53 ret = ccall(:getattrlist, Cint,
54 (Cstring, Ptr{Cvoid}, Ptr{Cvoid}, Csize_t, Culong),
55 path, attr_list, buf, sizeof(buf), FSOPT_NOFOLLOW)
56 systemerror(:getattrlist, ret ≠ 0)
57 filename_length = GC.@preserve buf unsafe_load(
58 convert(Ptr{UInt32}, pointer(buf) + 8))
59 if (filename_length + header_size) > length(buf)
60 resize!(buf, filename_length + header_size)
61 continue
62 end
63 casepreserved_basename =
64 view(buf, (header_size+1):(header_size+filename_length-1))
65 break
66 end
67 # Hack to compensate for inability to create a string from a subarray with no allocations.
68 codeunits(path_basename) == casepreserved_basename && return true
69
70 # If there is no match, it's possible that the file does exist but HFS+
71 # performed unicode normalization. See https://developer.apple.com/library/mac/qa/qa1235/_index.html.
72 isascii(path_basename) && return false
73 codeunits(Unicode.normalize(path_basename, :NFD)) == casepreserved_basename
74 end
75 else
76 # Generic fallback that performs a slow directory listing.
77 function isfile_casesensitive(path)
78 isaccessiblefile(path) || return false
79 dir, filename = splitdir(path)
80 any(readdir(dir) .== filename)
81 end
82 end
83
84 # Check if the file is accessible. If stat fails return `false`
85
86 function isaccessibledir(dir)
87 return try
88 isdir(dir)
89 catch err
90 err isa IOError || rethrow()
91 false
92 end
93 end
94
95 function isaccessiblefile(file)
96 return try
97 isfile(file)
98 catch err
99 err isa IOError || rethrow()
100 false
101 end
102 end
103
104 function isaccessiblepath(path)
105 return try
106 ispath(path)
107 catch err
108 err isa IOError || rethrow()
109 false
110 end
111 end
112
113 ## SHA1 ##
114
115 struct SHA1
116 bytes::NTuple{20, UInt8}
117 end
118 function SHA1(bytes::Vector{UInt8})
119 length(bytes) == 20 ||
120 throw(ArgumentError("wrong number of bytes for SHA1 hash: $(length(bytes))"))
121 return SHA1(ntuple(i->bytes[i], Val(20)))
122 end
123 SHA1(s::AbstractString) = SHA1(hex2bytes(s))
124 parse(::Type{SHA1}, s::AbstractString) = SHA1(s)
125 function tryparse(::Type{SHA1}, s::AbstractString)
126 try
127 return parse(SHA1, s)
128 catch e
129 if isa(e, ArgumentError)
130 return nothing
131 end
132 rethrow(e)
133 end
134 end
135
136 string(hash::SHA1) = bytes2hex(hash.bytes)
137 print(io::IO, hash::SHA1) = bytes2hex(io, hash.bytes)
138 show(io::IO, hash::SHA1) = print(io, "SHA1(\"", hash, "\")")
139
140 isless(a::SHA1, b::SHA1) = isless(a.bytes, b.bytes)
141 hash(a::SHA1, h::UInt) = hash((SHA1, a.bytes), h)
142 ==(a::SHA1, b::SHA1) = a.bytes == b.bytes
143
144 # fake uuid5 function (for self-assigned UUIDs)
145 # TODO: delete and use real uuid5 once it's in stdlib
146
147 function uuid5(namespace::UUID, key::String)
148 u::UInt128 = 0
149 h = hash(namespace)
150 for _ = 1:sizeof(u)÷sizeof(h)
151 u <<= sizeof(h) << 3
152 u |= (h = hash(key, h))
153 end
154 u &= 0xffffffffffff0fff3fffffffffffffff
155 u |= 0x00000000000050008000000000000000
156 return UUID(u)
157 end
158
159 const ns_dummy_uuid = UUID("fe0723d6-3a44-4c41-8065-ee0f42c8ceab")
160
161 function dummy_uuid(project_file::String)
162 @lock require_lock begin
163 cache = LOADING_CACHE[]
164 if cache !== nothing
165 uuid = get(cache.dummy_uuid, project_file, nothing)
166 uuid === nothing || return uuid
167 end
168 project_path = try
169 realpath(project_file)
170 catch ex
171 ex isa IOError || rethrow()
172 project_file
173 end
174 uuid = uuid5(ns_dummy_uuid, project_path)
175 if cache !== nothing
176 cache.dummy_uuid[project_file] = uuid
177 end
178 return uuid
179 end
180 end
181
182 ## package path slugs: turning UUID + SHA1 into a pair of 4-byte "slugs" ##
183
184 const slug_chars = String(['A':'Z'; 'a':'z'; '0':'9'])
185
186 function slug(x::UInt32, p::Int)
187 y::UInt32 = x
188 sprint(sizehint=p) do io
189 n = length(slug_chars)
190 for i = 1:p
191 y, d = divrem(y, n)
192 write(io, slug_chars[1+d])
193 end
194 end
195 end
196
197 function package_slug(uuid::UUID, p::Int=5)
198 crc = _crc32c(uuid)
199 return slug(crc, p)
200 end
201
202 function version_slug(uuid::UUID, sha1::SHA1, p::Int=5)
203 crc = _crc32c(uuid)
204 crc = _crc32c(sha1.bytes, crc)
205 return slug(crc, p)
206 end
207
208 mutable struct CachedTOMLDict
209 path::String
210 inode::UInt64
211 mtime::Float64
212 size::Int64
213 hash::UInt32
214 d::Dict{String, Any}
215 end
216
217 function CachedTOMLDict(p::TOML.Parser, path::String)
218 s = stat(path)
219 content = read(path)
220 crc32 = _crc32c(content)
221 TOML.reinit!(p, String(content); filepath=path)
222 d = TOML.parse(p)
223 return CachedTOMLDict(
224 path,
225 s.inode,
226 s.mtime,
227 s.size,
228 crc32,
229 d,
230 )
231 end
232
233 function get_updated_dict(p::TOML.Parser, f::CachedTOMLDict)
234 s = stat(f.path)
235 # note, this might miss very rapid in-place updates, such that mtime is
236 # identical but that is solvable by not doing in-place updates, and not
237 # rapidly changing these files
238 if s.inode != f.inode || s.mtime != f.mtime || f.size != s.size
239 content = read(f.path)
240 new_hash = _crc32c(content)
241 if new_hash != f.hash
242 f.inode = s.inode
243 f.mtime = s.mtime
244 f.size = s.size
245 f.hash = new_hash
246 TOML.reinit!(p, String(content); filepath=f.path)
247 return f.d = TOML.parse(p)
248 end
249 end
250 return f.d
251 end
252
253 struct LoadingCache
254 load_path::Vector{String}
255 dummy_uuid::Dict{String, UUID}
256 env_project_file::Dict{String, Union{Bool, String}}
257 project_file_manifest_path::Dict{String, Union{Nothing, String}}
258 require_parsed::Set{String}
259 identified_where::Dict{Tuple{PkgId, String}, Union{Nothing, Tuple{PkgId, Union{Nothing, String}}}}
260 identified::Dict{String, Union{Nothing, Tuple{PkgId, Union{Nothing, String}}}}
261 located::Dict{Tuple{PkgId, Union{String, Nothing}}, Union{Tuple{Union{String, Nothing}, Union{String, Nothing}}, Nothing}}
262 end
263 const LOADING_CACHE = Ref{Union{LoadingCache, Nothing}}(nothing)
264 LoadingCache() = LoadingCache(load_path(), Dict(), Dict(), Dict(), Set(), Dict(), Dict(), Dict())
265
266
267 struct TOMLCache
268 p::TOML.Parser
269 d::Dict{String, CachedTOMLDict}
270 end
271 const TOML_CACHE = TOMLCache(TOML.Parser(), Dict{String, Dict{String, Any}}())
272
273 parsed_toml(project_file::AbstractString) = parsed_toml(project_file, TOML_CACHE, require_lock)
274 function parsed_toml(project_file::AbstractString, toml_cache::TOMLCache, toml_lock::ReentrantLock)
275 lock(toml_lock) do
276 cache = LOADING_CACHE[]
277 dd = if !haskey(toml_cache.d, project_file)
278 d = CachedTOMLDict(toml_cache.p, project_file)
279 toml_cache.d[project_file] = d
280 d.d
281 else
282 d = toml_cache.d[project_file]
283 # We are in a require call and have already parsed this TOML file
284 # assume that it is unchanged to avoid hitting disk
285 if cache !== nothing && project_file in cache.require_parsed
286 d.d
287 else
288 get_updated_dict(toml_cache.p, d)
289 end
290 end
291 if cache !== nothing
292 push!(cache.require_parsed, project_file)
293 end
294 return dd
295 end
296 end
297
298 ## package identification: determine unique identity of package to be loaded ##
299
300 # Used by Pkg but not used in loading itself
301 function find_package(arg)
302 pkgenv = identify_package_env(arg)
303 pkgenv === nothing && return nothing
304 pkg, env = pkgenv
305 return locate_package(pkg, env)
306 end
307
308 """
309 Base.identify_package_env(name::String)::Union{Tuple{PkgId, String}, Nothing}
310 Base.identify_package_env(where::Union{Module,PkgId}, name::String)::Union{Tuple{PkgId, String} Nothing}
311
312 Same as [`Base.identify_package`](@ref) except that the path to the environment where the package is identified
313 is also returned.
314 """
315 identify_package_env(where::Module, name::String) = identify_package_env(PkgId(where), name)
316 function identify_package_env(where::PkgId, name::String)
317 cache = LOADING_CACHE[]
318 if cache !== nothing
319 pkg_env = get(cache.identified_where, (where, name), nothing)
320 pkg_env === nothing || return pkg_env
321 end
322 pkg_env = nothing
323 if where.name === name
324 pkg_env = where, nothing
325 elseif where.uuid === nothing
326 pkg_env = identify_package_env(name) # ignore `where`
327 else
328 for env in load_path()
329 pkgid = manifest_deps_get(env, where, name)
330 pkgid === nothing && continue # not found--keep looking
331 if pkgid.uuid !== nothing
332 pkg_env = pkgid, env # found in explicit environment--use it
333 end
334 break # found in implicit environment--return "not found"
335 end
336 end
337 if cache !== nothing
338 cache.identified_where[(where, name)] = pkg_env
339 end
340 return pkg_env
341 end
342 function identify_package_env(name::String)
343 cache = LOADING_CACHE[]
344 if cache !== nothing
345 pkg_env = get(cache.identified, name, nothing)
346 pkg_env === nothing || return pkg_env
347 end
348 pkg_env = nothing
349 for env in load_path()
350 pkg = project_deps_get(env, name)
351 if pkg !== nothing
352 pkg_env = pkg, env # found--return it
353 break
354 end
355 end
356 if cache !== nothing
357 cache.identified[name] = pkg_env
358 end
359 return pkg_env
360 end
361
362 _nothing_or_first(x) = x === nothing ? nothing : first(x)
363
364 """
365 Base.identify_package(name::String)::Union{PkgId, Nothing}
366 Base.identify_package(where::Union{Module,PkgId}, name::String)::Union{PkgId, Nothing}
367
368 Identify the package by its name from the current environment stack, returning
369 its `PkgId`, or `nothing` if it cannot be found.
370
371 If only the `name` argument is provided, it searches each environment in the
372 stack and its named direct dependencies.
373
374 There `where` argument provides the context from where to search for the
375 package: in this case it first checks if the name matches the context itself,
376 otherwise it searches all recursive dependencies (from the resolved manifest of
377 each environment) until it locates the context `where`, and from there
378 identifies the dependency with the corresponding name.
379
380 ```julia-repl
381 julia> Base.identify_package("Pkg") # Pkg is a dependency of the default environment
382 Pkg [44cfe95a-1eb2-52ea-b672-e2afdf69b78f]
383
384 julia> using LinearAlgebra
385
386 julia> Base.identify_package(LinearAlgebra, "Pkg") # Pkg is not a dependency of LinearAlgebra
387 ```
388 """
389 identify_package(where::Module, name::String) = _nothing_or_first(identify_package_env(where, name))
390 identify_package(where::PkgId, name::String) = _nothing_or_first(identify_package_env(where, name))
391 identify_package(name::String) = _nothing_or_first(identify_package_env(name))
392
393 function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing)
394 cache = LOADING_CACHE[]
395 if cache !== nothing
396 pathenv = get(cache.located, (pkg, stopenv), nothing)
397 pathenv === nothing || return pathenv
398 end
399 path = nothing
400 env′ = nothing
401 if pkg.uuid === nothing
402 for env in load_path()
403 env′ = env
404 # look for the toplevel pkg `pkg.name` in this entry
405 found = project_deps_get(env, pkg.name)
406 if found !== nothing
407 @assert found.name == pkg.name
408 if found.uuid === nothing
409 # pkg.name is present in this directory or project file,
410 # return the path the entry point for the code, if it could be found
411 # otherwise, signal failure
412 path = implicit_manifest_uuid_path(env, pkg)
413 @goto done
414 end
415 end
416 if !(loading_extension || precompiling_extension)
417 stopenv == env && @goto done
418 end
419 end
420 else
421 for env in load_path()
422 env′ = env
423 path = manifest_uuid_path(env, pkg)
424 # missing is used as a sentinel to stop looking further down in envs
425 if path === missing
426 path = nothing
427 @goto done
428 end
429 if path !== nothing
430 path = entry_path(path, pkg.name)
431 @goto done
432 end
433 if !(loading_extension || precompiling_extension)
434 stopenv == env && break
435 end
436 end
437 # Allow loading of stdlibs if the name/uuid are given
438 # e.g. if they have been explicitly added to the project/manifest
439 mbypath = manifest_uuid_path(Sys.STDLIB, pkg)
440 if mbypath isa String
441 path = entry_path(mbypath, pkg.name)
442 @goto done
443 end
444 end
445 @label done
446 if cache !== nothing
447 cache.located[(pkg, stopenv)] = path, env′
448 end
449 return path, env′
450 end
451
452 """
453 Base.locate_package(pkg::PkgId)::Union{String, Nothing}
454
455 The path to the entry-point file for the package corresponding to the identifier
456 `pkg`, or `nothing` if not found. See also [`identify_package`](@ref).
457
458 ```julia-repl
459 julia> pkg = Base.identify_package("Pkg")
460 Pkg [44cfe95a-1eb2-52ea-b672-e2afdf69b78f]
461
462 julia> Base.locate_package(pkg)
463 "/path/to/julia/stdlib/v$(VERSION.major).$(VERSION.minor)/Pkg/src/Pkg.jl"
464 ```
465 """
466 function locate_package(pkg::PkgId, stopenv::Union{String, Nothing}=nothing)::Union{Nothing,String}
467 _nothing_or_first(locate_package_env(pkg, stopenv))
468 end
469
470 """
471 pathof(m::Module)
472
473 Return the path of the `m.jl` file that was used to `import` module `m`,
474 or `nothing` if `m` was not imported from a package.
475
476 Use [`dirname`](@ref) to get the directory part and [`basename`](@ref)
477 to get the file name part of the path.
478 """
479 function pathof(m::Module)
480 @lock require_lock begin
481 pkgid = get(module_keys, m, nothing)
482 pkgid === nothing && return nothing
483 origin = get(pkgorigins, pkgid, nothing)
484 origin === nothing && return nothing
485 path = origin.path
486 path === nothing && return nothing
487 return fixup_stdlib_path(path)
488 end
489 end
490
491 """
492 pkgdir(m::Module[, paths::String...])
493
494 Return the root directory of the package that declared module `m`,
495 or `nothing` if `m` was not declared in a package. Optionally further
496 path component strings can be provided to construct a path within the
497 package root.
498
499 To get the root directory of the package that implements the current module
500 the form `pkgdir(@__MODULE__)` can be used.
501
502 ```julia-repl
503 julia> pkgdir(Foo)
504 "/path/to/Foo.jl"
505
506 julia> pkgdir(Foo, "src", "file.jl")
507 "/path/to/Foo.jl/src/file.jl"
508 ```
509
510 !!! compat "Julia 1.7"
511 The optional argument `paths` requires at least Julia 1.7.
512 """
513 function pkgdir(m::Module, paths::String...)
514 rootmodule = moduleroot(m)
515 path = pathof(rootmodule)
516 path === nothing && return nothing
517 return joinpath(dirname(dirname(path)), paths...)
518 end
519
520 function get_pkgversion_from_path(path)
521 project_file = locate_project_file(path)
522 if project_file isa String
523 d = parsed_toml(project_file)
524 v = get(d, "version", nothing)
525 if v !== nothing
526 return VersionNumber(v::String)
527 end
528 end
529 return nothing
530 end
531
532 """
533 pkgversion(m::Module)
534
535 Return the version of the package that imported module `m`,
536 or `nothing` if `m` was not imported from a package, or imported
537 from a package without a version field set.
538
539 The version is read from the package's Project.toml during package
540 load.
541
542 To get the version of the package that imported the current module
543 the form `pkgversion(@__MODULE__)` can be used.
544
545 !!! compat "Julia 1.9"
546 This function was introduced in Julia 1.9.
547 """
548 function pkgversion(m::Module)
549 path = pkgdir(m)
550 path === nothing && return nothing
551 @lock require_lock begin
552 v = get_pkgversion_from_path(path)
553 pkgorigin = get(pkgorigins, PkgId(moduleroot(m)), nothing)
554 # Cache the version
555 if pkgorigin !== nothing && pkgorigin.version === nothing
556 pkgorigin.version = v
557 end
558 return v
559 end
560 end
561
562 ## generic project & manifest API ##
563
564 const project_names = ("JuliaProject.toml", "Project.toml")
565 const manifest_names = ("JuliaManifest.toml", "Manifest.toml")
566 const preferences_names = ("JuliaLocalPreferences.toml", "LocalPreferences.toml")
567
568 function locate_project_file(env::String)
569 for proj in project_names
570 project_file = joinpath(env, proj)
571 if isfile_casesensitive(project_file)
572 return project_file
573 end
574 end
575 return true
576 end
577
578 # classify the LOAD_PATH entry to be one of:
579 # - `false`: nonexistent / nothing to see here
580 # - `true`: `env` is an implicit environment
581 # - `path`: the path of an explicit project file
582 function env_project_file(env::String)::Union{Bool,String}
583 @lock require_lock begin
584 cache = LOADING_CACHE[]
585 if cache !== nothing
586 project_file = get(cache.env_project_file, env, nothing)
587 project_file === nothing || return project_file
588 end
589 if isdir(env)
590 project_file = locate_project_file(env)
591 elseif basename(env) in project_names && isfile_casesensitive(env)
592 project_file = env
593 else
594 project_file = false
595 end
596 if cache !== nothing
597 cache.env_project_file[env] = project_file
598 end
599 return project_file
600 end
601 end
602
603 function project_deps_get(env::String, name::String)::Union{Nothing,PkgId}
604 project_file = env_project_file(env)
605 if project_file isa String
606 pkg_uuid = explicit_project_deps_get(project_file, name)
607 pkg_uuid === nothing || return PkgId(pkg_uuid, name)
608 elseif project_file
609 return implicit_project_deps_get(env, name)
610 end
611 return nothing
612 end
613
614 function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Nothing,PkgId}
615 uuid = where.uuid
616 @assert uuid !== nothing
617 project_file = env_project_file(env)
618 if project_file isa String
619 # first check if `where` names the Project itself
620 proj = project_file_name_uuid(project_file, where.name)
621 if proj == where
622 # if `where` matches the project, use [deps] section as manifest, and stop searching
623 pkg_uuid = explicit_project_deps_get(project_file, name)
624 return PkgId(pkg_uuid, name)
625 end
626 d = parsed_toml(project_file)
627 exts = get(d, "extensions", nothing)::Union{Dict{String, Any}, Nothing}
628 if exts !== nothing
629 # Check if `where` is an extension of the project
630 if where.name in keys(exts) && where.uuid == uuid5(proj.uuid::UUID, where.name)
631 # Extensions can load weak deps...
632 weakdeps = get(d, "weakdeps", nothing)::Union{Dict{String, Any}, Nothing}
633 if weakdeps !== nothing
634 wuuid = get(weakdeps, name, nothing)::Union{String, Nothing}
635 if wuuid !== nothing
636 return PkgId(UUID(wuuid), name)
637 end
638 end
639 # ... and they can load same deps as the project itself
640 mby_uuid = explicit_project_deps_get(project_file, name)
641 mby_uuid === nothing || return PkgId(mby_uuid, name)
642 end
643 end
644 # look for manifest file and `where` stanza
645 return explicit_manifest_deps_get(project_file, where, name)
646 elseif project_file
647 # if env names a directory, search it
648 return implicit_manifest_deps_get(env, where, name)
649 end
650 return nothing
651 end
652
653 function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String,Missing}
654 project_file = env_project_file(env)
655 if project_file isa String
656 proj = project_file_name_uuid(project_file, pkg.name)
657 if proj == pkg
658 # if `pkg` matches the project, return the project itself
659 return project_file_path(project_file)
660 end
661 mby_ext = project_file_ext_path(project_file, pkg.name)
662 mby_ext === nothing || return mby_ext
663 # look for manifest file and `where` stanza
664 return explicit_manifest_uuid_path(project_file, pkg)
665 elseif project_file
666 # if env names a directory, search it
667 return implicit_manifest_uuid_path(env, pkg)
668 end
669 return nothing
670 end
671
672
673 function find_ext_path(project_path::String, extname::String)
674 extfiledir = joinpath(project_path, "ext", extname, extname * ".jl")
675 isfile(extfiledir) && return extfiledir
676 return joinpath(project_path, "ext", extname * ".jl")
677 end
678
679 function project_file_ext_path(project_file::String, name::String)
680 d = parsed_toml(project_file)
681 p = project_file_path(project_file)
682 exts = get(d, "extensions", nothing)::Union{Dict{String, Any}, Nothing}
683 if exts !== nothing
684 if name in keys(exts)
685 return find_ext_path(p, name)
686 end
687 end
688 return nothing
689 end
690
691 # find project file's top-level UUID entry (or nothing)
692 function project_file_name_uuid(project_file::String, name::String)::PkgId
693 d = parsed_toml(project_file)
694 uuid′ = get(d, "uuid", nothing)::Union{String, Nothing}
695 uuid = uuid′ === nothing ? dummy_uuid(project_file) : UUID(uuid′)
696 name = get(d, "name", name)::String
697 return PkgId(uuid, name)
698 end
699
700 function project_file_path(project_file::String)
701 d = parsed_toml(project_file)
702 joinpath(dirname(project_file), get(d, "path", "")::String)
703 end
704
705 # find project file's corresponding manifest file
706 function project_file_manifest_path(project_file::String)::Union{Nothing,String}
707 @lock require_lock begin
708 cache = LOADING_CACHE[]
709 if cache !== nothing
710 manifest_path = get(cache.project_file_manifest_path, project_file, missing)
711 manifest_path === missing || return manifest_path
712 end
713 dir = abspath(dirname(project_file))
714 d = parsed_toml(project_file)
715 explicit_manifest = get(d, "manifest", nothing)::Union{String, Nothing}
716 manifest_path = nothing
717 if explicit_manifest !== nothing
718 manifest_file = normpath(joinpath(dir, explicit_manifest))
719 if isfile_casesensitive(manifest_file)
720 manifest_path = manifest_file
721 end
722 end
723 if manifest_path === nothing
724 for mfst in manifest_names
725 manifest_file = joinpath(dir, mfst)
726 if isfile_casesensitive(manifest_file)
727 manifest_path = manifest_file
728 break
729 end
730 end
731 end
732 if cache !== nothing
733 cache.project_file_manifest_path[project_file] = manifest_path
734 end
735 return manifest_path
736 end
737 end
738
739 # given a directory (implicit env from LOAD_PATH) and a name,
740 # check if it is an implicit package
741 function entry_point_and_project_file_inside(dir::String, name::String)::Union{Tuple{Nothing,Nothing},Tuple{String,Nothing},Tuple{String,String}}
742 path = normpath(joinpath(dir, "src", "$name.jl"))
743 isfile_casesensitive(path) || return nothing, nothing
744 for proj in project_names
745 project_file = normpath(joinpath(dir, proj))
746 isfile_casesensitive(project_file) || continue
747 return path, project_file
748 end
749 return path, nothing
750 end
751
752 # given a project directory (implicit env from LOAD_PATH) and a name,
753 # find an entry point for `name`, and see if it has an associated project file
754 function entry_point_and_project_file(dir::String, name::String)::Union{Tuple{Nothing,Nothing},Tuple{String,Nothing},Tuple{String,String}}
755 path = normpath(joinpath(dir, "$name.jl"))
756 isfile_casesensitive(path) && return path, nothing
757 dir = joinpath(dir, name)
758 path, project_file = entry_point_and_project_file_inside(dir, name)
759 path === nothing || return path, project_file
760 dir = dir * ".jl"
761 path, project_file = entry_point_and_project_file_inside(dir, name)
762 path === nothing || return path, project_file
763 return nothing, nothing
764 end
765
766 # given a path and a name, return the entry point
767 function entry_path(path::String, name::String)::Union{Nothing,String}
768 isfile_casesensitive(path) && return normpath(path)
769 path = normpath(joinpath(path, "src", "$name.jl"))
770 isfile_casesensitive(path) && return path
771 return nothing # source not found
772 end
773
774 ## explicit project & manifest API ##
775
776 # find project file root or deps `name => uuid` mapping
777 # return `nothing` if `name` is not found
778 function explicit_project_deps_get(project_file::String, name::String)::Union{Nothing,UUID}
779 d = parsed_toml(project_file)
780 root_uuid = dummy_uuid(project_file)
781 if get(d, "name", nothing)::Union{String, Nothing} === name
782 uuid = get(d, "uuid", nothing)::Union{String, Nothing}
783 return uuid === nothing ? root_uuid : UUID(uuid)
784 end
785 deps = get(d, "deps", nothing)::Union{Dict{String, Any}, Nothing}
786 if deps !== nothing
787 uuid = get(deps, name, nothing)::Union{String, Nothing}
788 uuid === nothing || return UUID(uuid)
789 end
790 return nothing
791 end
792
793 function is_v1_format_manifest(raw_manifest::Dict{String})
794 if haskey(raw_manifest, "manifest_format")
795 mf = raw_manifest["manifest_format"]
796 if mf isa Dict{String} && haskey(mf, "uuid")
797 # the off-chance where an old format manifest has a dep called "manifest_format"
798 return true
799 end
800 return false
801 else
802 return true
803 end
804 end
805
806 # returns a deps list for both old and new manifest formats
807 function get_deps(raw_manifest::Dict)
808 if is_v1_format_manifest(raw_manifest)
809 return raw_manifest
810 else
811 # if the manifest has no deps, there won't be a `deps` field
812 return get(Dict{String, Any}, raw_manifest, "deps")::Dict{String, Any}
813 end
814 end
815
816 # find `where` stanza and return the PkgId for `name`
817 # return `nothing` if it did not find `where` (indicating caller should continue searching)
818 function explicit_manifest_deps_get(project_file::String, where::PkgId, name::String)::Union{Nothing,PkgId}
819 manifest_file = project_file_manifest_path(project_file)
820 manifest_file === nothing && return nothing # manifest not found--keep searching LOAD_PATH
821 d = get_deps(parsed_toml(manifest_file))
822 found_where = false
823 found_name = false
824 for (dep_name, entries) in d
825 entries::Vector{Any}
826 for entry in entries
827 entry = entry::Dict{String, Any}
828 uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
829 uuid === nothing && continue
830 if UUID(uuid) === where.uuid
831 found_where = true
832 # deps is either a list of names (deps = ["DepA", "DepB"]) or
833 # a table of entries (deps = {"DepA" = "6ea...", "DepB" = "55d..."}
834 deps = get(entry, "deps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing}
835 if deps isa Vector{String}
836 found_name = name in deps
837 break
838 elseif deps isa Dict{String, Any}
839 deps = deps::Dict{String, Any}
840 for (dep, uuid) in deps
841 uuid::String
842 if dep === name
843 return PkgId(UUID(uuid), name)
844 end
845 end
846 end
847 else # Check for extensions
848 extensions = get(entry, "extensions", nothing)
849 if extensions !== nothing
850 if haskey(extensions, where.name) && where.uuid == uuid5(UUID(uuid), where.name)
851 found_where = true
852 if name == dep_name
853 return PkgId(UUID(uuid), name)
854 end
855 exts = extensions[where.name]::Union{String, Vector{String}}
856 if (exts isa String && name == exts) || (exts isa Vector{String} && name in exts)
857 weakdeps = get(entry, "weakdeps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing}
858 if weakdeps !== nothing
859 if weakdeps isa Vector{String}
860 found_name = name in weakdeps
861 break
862 elseif weakdeps isa Dict{String, Any}
863 weakdeps = weakdeps::Dict{String, Any}
864 for (dep, uuid) in weakdeps
865 uuid::String
866 if dep === name
867 return PkgId(UUID(uuid), name)
868 end
869 end
870 end
871 end
872 end
873 # `name` is not an ext, do standard lookup as if this was the parent
874 return identify_package(PkgId(UUID(uuid), dep_name), name)
875 end
876 end
877 end
878 end
879 end
880 found_where || return nothing
881 found_name || return PkgId(name)
882 # Only reach here if deps was not a dict which mean we have a unique name for the dep
883 name_deps = get(d, name, nothing)::Union{Nothing, Vector{Any}}
884 if name_deps === nothing || length(name_deps) != 1
885 error("expected a single entry for $(repr(name)) in $(repr(project_file))")
886 end
887 entry = first(name_deps::Vector{Any})::Dict{String, Any}
888 uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
889 uuid === nothing && return nothing
890 return PkgId(UUID(uuid), name)
891 end
892
893 # find `uuid` stanza, return the corresponding path
894 function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{Nothing,String,Missing}
895 manifest_file = project_file_manifest_path(project_file)
896 manifest_file === nothing && return nothing # no manifest, skip env
897
898 d = get_deps(parsed_toml(manifest_file))
899 entries = get(d, pkg.name, nothing)::Union{Nothing, Vector{Any}}
900 if entries !== nothing
901 for entry in entries
902 entry = entry::Dict{String, Any}
903 uuid = get(entry, "uuid", nothing)::Union{Nothing, String}
904 uuid === nothing && continue
905 if UUID(uuid) === pkg.uuid
906 return explicit_manifest_entry_path(manifest_file, pkg, entry)
907 end
908 end
909 end
910 # Extensions
911 for (name, entries) in d
912 entries = entries::Vector{Any}
913 for entry in entries
914 uuid = get(entry, "uuid", nothing)::Union{Nothing, String}
915 extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}}
916 if extensions !== nothing && haskey(extensions, pkg.name) && uuid !== nothing && uuid5(UUID(uuid), pkg.name) == pkg.uuid
917 parent_path = locate_package(PkgId(UUID(uuid), name))
918 if parent_path === nothing
919 error("failed to find source of parent package: \"$name\"")
920 end
921 p = normpath(dirname(parent_path), "..")
922 return find_ext_path(p, pkg.name)
923 end
924 end
925 end
926 return nothing
927 end
928
929 function explicit_manifest_entry_path(manifest_file::String, pkg::PkgId, entry::Dict{String,Any})
930 path = get(entry, "path", nothing)::Union{Nothing, String}
931 if path !== nothing
932 path = normpath(abspath(dirname(manifest_file), path))
933 return path
934 end
935 hash = get(entry, "git-tree-sha1", nothing)::Union{Nothing, String}
936 hash === nothing && return nothing
937 hash = SHA1(hash)
938 # Keep the 4 since it used to be the default
939 uuid = pkg.uuid::UUID # checked within `explicit_manifest_uuid_path`
940 for slug in (version_slug(uuid, hash), version_slug(uuid, hash, 4))
941 for depot in DEPOT_PATH
942 path = joinpath(depot, "packages", pkg.name, slug)
943 ispath(path) && return abspath(path)
944 end
945 end
946 # no depot contains the package, return missing to stop looking
947 return missing
948 end
949
950 ## implicit project & manifest API ##
951
952 # look for an entry point for `name` from a top-level package (no environment)
953 # otherwise return `nothing` to indicate the caller should keep searching
954 function implicit_project_deps_get(dir::String, name::String)::Union{Nothing,PkgId}
955 path, project_file = entry_point_and_project_file(dir, name)
956 if project_file === nothing
957 path === nothing && return nothing
958 return PkgId(name)
959 end
960 proj = project_file_name_uuid(project_file, name)
961 proj.name == name || return nothing
962 return proj
963 end
964
965 # look for an entry-point for `name`, check that UUID matches
966 # if there's a project file, look up `name` in its deps and return that
967 # otherwise return `nothing` to indicate the caller should keep searching
968 function implicit_manifest_deps_get(dir::String, where::PkgId, name::String)::Union{Nothing,PkgId}
969 @assert where.uuid !== nothing
970 project_file = entry_point_and_project_file(dir, where.name)[2]
971 project_file === nothing && return nothing # a project file is mandatory for a package with a uuid
972 proj = project_file_name_uuid(project_file, where.name)
973 proj == where || return nothing # verify that this is the correct project file
974 # this is the correct project, so stop searching here
975 pkg_uuid = explicit_project_deps_get(project_file, name)
976 return PkgId(pkg_uuid, name)
977 end
978
979 # look for an entry-point for `pkg` and return its path if UUID matches
980 function implicit_manifest_uuid_path(dir::String, pkg::PkgId)::Union{Nothing,String}
981 path, project_file = entry_point_and_project_file(dir, pkg.name)
982 if project_file === nothing
983 pkg.uuid === nothing || return nothing
984 return path
985 end
986 proj = project_file_name_uuid(project_file, pkg.name)
987 proj == pkg || return nothing
988 return path
989 end
990
991 ## other code loading functionality ##
992
993 function find_source_file(path::AbstractString)
994 (isabspath(path) || isfile(path)) && return path
995 base_path = joinpath(Sys.BINDIR, DATAROOTDIR, "julia", "base", path)
996 return isfile(base_path) ? normpath(base_path) : nothing
997 end
998
999 cache_file_entry(pkg::PkgId) = joinpath(
1000 "compiled",
1001 "v$(VERSION.major).$(VERSION.minor)",
1002 pkg.uuid === nothing ? "" : pkg.name),
1003 pkg.uuid === nothing ? pkg.name : package_slug(pkg.uuid)
1004
1005 function find_all_in_cache_path(pkg::PkgId)
1006 paths = String[]
1007 entrypath, entryfile = cache_file_entry(pkg)
1008 for path in joinpath.(DEPOT_PATH, entrypath)
1009 isdir(path) || continue
1010 for file in readdir(path, sort = false) # no sort given we sort later
1011 if !((pkg.uuid === nothing && file == entryfile * ".ji") ||
1012 (pkg.uuid !== nothing && startswith(file, entryfile * "_") &&
1013 endswith(file, ".ji")))
1014 continue
1015 end
1016 filepath = joinpath(path, file)
1017 isfile_casesensitive(filepath) && push!(paths, filepath)
1018 end
1019 end
1020 if length(paths) > 1
1021 # allocating the sort vector is less expensive than using sort!(.. by=mtime), which would
1022 # call the relatively slow mtime multiple times per path
1023 p = sortperm(mtime.(paths), rev = true)
1024 return paths[p]
1025 else
1026 return paths
1027 end
1028 end
1029
1030 ocachefile_from_cachefile(cachefile) = string(chopsuffix(cachefile, ".ji"), ".", Base.Libc.dlext)
1031 cachefile_from_ocachefile(cachefile) = string(chopsuffix(cachefile, ".$(Base.Libc.dlext)"), ".ji")
1032
1033
1034 # use an Int counter so that nested @time_imports calls all remain open
1035 const TIMING_IMPORTS = Threads.Atomic{Int}(0)
1036
1037 # these return either the array of modules loaded from the path / content given
1038 # or an Exception that describes why it couldn't be loaded
1039 # and it reconnects the Base.Docs.META
1040 function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}, depmods::Vector{Any})
1041 assert_havelock(require_lock)
1042 timing_imports = TIMING_IMPORTS[] > 0
1043 try
1044 if timing_imports
1045 t_before = time_ns()
1046 cumulative_compile_timing(true)
1047 t_comp_before = cumulative_compile_time_ns()
1048 end
1049
1050 if ocachepath !== nothing
1051 @debug "Loading object cache file $ocachepath for $pkg"
1052 sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any, Cint, Cstring), ocachepath, depmods, false, pkg.name)
1053 else
1054 @debug "Loading cache file $path for $pkg"
1055 sv = ccall(:jl_restore_incremental, Any, (Cstring, Any, Cint, Cstring), path, depmods, false, pkg.name)
1056 end
1057 if isa(sv, Exception)
1058 return sv
1059 end
1060
1061 restored = register_restored_modules(sv, pkg, path)
1062
1063 for M in restored
1064 M = M::Module
1065 if parentmodule(M) === M && PkgId(M) == pkg
1066 if timing_imports
1067 elapsed = round((time_ns() - t_before) / 1e6, digits = 1)
1068 comp_time, recomp_time = cumulative_compile_time_ns() .- t_comp_before
1069 print(lpad(elapsed, 9), " ms ")
1070 parentid = get(EXT_PRIMED, pkg, nothing)
1071 if parentid !== nothing
1072 print(parentid.name, " → ")
1073 end
1074 print(pkg.name)
1075 if comp_time > 0
1076 printstyled(" ", Ryu.writefixed(Float64(100 * comp_time / (elapsed * 1e6)), 2), "% compilation time", color = Base.info_color())
1077 end
1078 if recomp_time > 0
1079 perc = Float64(100 * recomp_time / comp_time)
1080 printstyled(" (", perc < 1 ? "<1" : Ryu.writefixed(perc, 0), "% recompilation)", color = Base.warn_color())
1081 end
1082 println()
1083 end
1084 return M
1085 end
1086 end
1087 return ErrorException("Required dependency $pkg failed to load from a cache file.")
1088
1089 finally
1090 timing_imports && cumulative_compile_timing(false)
1091 end
1092 end
1093
1094 function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String)
1095 # This function is also used by PkgCacheInspector.jl
1096 restored = sv[1]::Vector{Any}
1097 for M in restored
1098 M = M::Module
1099 if isdefined(M, Base.Docs.META) && getfield(M, Base.Docs.META) !== nothing
1100 push!(Base.Docs.modules, M)
1101 end
1102 if parentmodule(M) === M
1103 register_root_module(M)
1104 end
1105 end
1106
1107 # Register this cache path now - If Requires.jl is loaded, Revise may end
1108 # up looking at the cache path during the init callback.
1109 get!(PkgOrigin, pkgorigins, pkg).cachepath = path
1110
1111 inits = sv[2]::Vector{Any}
1112 if !isempty(inits)
1113 unlock(require_lock) # temporarily _unlock_ during these callbacks
1114 try
1115 for (i, mod) in pairs(inits)
1116 run_module_init(mod, i)
1117 end
1118 finally
1119 lock(require_lock)
1120 end
1121 end
1122 return restored
1123 end
1124
1125 function run_module_init(mod::Module, i::Int=1)
1126 # `i` informs ordering for the `@time_imports` report formatting
1127 if TIMING_IMPORTS[] == 0
1128 ccall(:jl_init_restored_module, Cvoid, (Any,), mod)
1129 else
1130 if isdefined(mod, :__init__)
1131 connector = i > 1 ? "├" : "┌"
1132 printstyled(" $connector ", color = :light_black)
1133
1134 elapsedtime = time_ns()
1135 cumulative_compile_timing(true)
1136 compile_elapsedtimes = cumulative_compile_time_ns()
1137
1138 ccall(:jl_init_restored_module, Cvoid, (Any,), mod)
1139
1140 elapsedtime = (time_ns() - elapsedtime) / 1e6
1141 cumulative_compile_timing(false);
1142 comp_time, recomp_time = (cumulative_compile_time_ns() .- compile_elapsedtimes) ./ 1e6
1143
1144 print(round(elapsedtime, digits=1), " ms $mod.__init__() ")
1145 if comp_time > 0
1146 printstyled(Ryu.writefixed(Float64(100 * comp_time / elapsedtime), 2), "% compilation time", color = Base.info_color())
1147 end
1148 if recomp_time > 0
1149 perc = Float64(100 * recomp_time / comp_time)
1150 printstyled(" (", perc < 1 ? "<1" : Ryu.writefixed(perc, 0), "% recompilation)", color = Base.warn_color())
1151 end
1152 println()
1153 end
1154 end
1155 end
1156
1157 function run_package_callbacks(modkey::PkgId)
1158 run_extension_callbacks(modkey)
1159 assert_havelock(require_lock)
1160 unlock(require_lock)
1161 try
1162 for callback in package_callbacks
1163 invokelatest(callback, modkey)
1164 end
1165 catch
1166 # Try to continue loading if a callback errors
1167 errs = current_exceptions()
1168 @error "Error during package callback" exception=errs
1169 finally
1170 lock(require_lock)
1171 end
1172 nothing
1173 end
1174
1175
1176 ##############
1177 # Extensions #
1178 ##############
1179
1180 mutable struct ExtensionId
1181 const id::PkgId
1182 const parentid::PkgId # just need the name, for printing
1183 ntriggers::Int # how many more packages must be defined until this is loaded
1184 end
1185
1186 const EXT_PRIMED = Dict{PkgId, PkgId}() # Extension -> Parent
1187 const EXT_DORMITORY = Dict{PkgId,Vector{ExtensionId}}() # Trigger -> Extensions that can be triggered by it
1188 const EXT_DORMITORY_FAILED = ExtensionId[]
1189
1190 function insert_extension_triggers(pkg::PkgId)
1191 pkg.uuid === nothing && return
1192 path_env_loc = locate_package_env(pkg)
1193 path_env_loc === nothing && return
1194 path, env_loc = path_env_loc
1195 if path === nothing || env_loc === nothing
1196 return
1197 end
1198 insert_extension_triggers(env_loc, pkg)
1199 end
1200
1201 function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missing}
1202 project_file = env_project_file(env)
1203 if project_file isa String
1204 # Look in project for extensions to insert
1205 proj_pkg = project_file_name_uuid(project_file, pkg.name)
1206 if pkg == proj_pkg
1207 d_proj = parsed_toml(project_file)
1208 weakdeps = get(d_proj, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}}
1209 extensions = get(d_proj, "extensions", nothing)::Union{Nothing, Dict{String, Any}}
1210 extensions === nothing && return
1211 weakdeps === nothing && return
1212 if weakdeps isa Dict{String, Any}
1213 return _insert_extension_triggers(pkg, extensions, weakdeps)
1214 end
1215 end
1216
1217 # Now look in manifest
1218 manifest_file = project_file_manifest_path(project_file)
1219 manifest_file === nothing && return
1220 d = get_deps(parsed_toml(manifest_file))
1221 for (dep_name, entries) in d
1222 entries::Vector{Any}
1223 for entry in entries
1224 entry = entry::Dict{String, Any}
1225 uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
1226 uuid === nothing && continue
1227 if UUID(uuid) == pkg.uuid
1228 weakdeps = get(entry, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}}
1229 extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}}
1230 extensions === nothing && return
1231 weakdeps === nothing && return
1232 if weakdeps isa Dict{String, Any}
1233 return _insert_extension_triggers(pkg, extensions, weakdeps)
1234 end
1235
1236 d_weakdeps = Dict{String, Any}()
1237 for (dep_name, entries) in d
1238 dep_name in weakdeps || continue
1239 entries::Vector{Any}
1240 if length(entries) != 1
1241 error("expected a single entry for $(repr(dep_name)) in $(repr(project_file))")
1242 end
1243 entry = first(entries)::Dict{String, Any}
1244 uuid = entry["uuid"]::String
1245 d_weakdeps[dep_name] = uuid
1246 end
1247 @assert length(d_weakdeps) == length(weakdeps)
1248 return _insert_extension_triggers(pkg, extensions, d_weakdeps)
1249 end
1250 end
1251 end
1252 end
1253 return nothing
1254 end
1255
1256 function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}, weakdeps::Dict{String, Any})
1257 for (ext, triggers) in extensions
1258 triggers = triggers::Union{String, Vector{String}}
1259 triggers isa String && (triggers = [triggers])
1260 id = PkgId(uuid5(parent.uuid, ext), ext)
1261 if id in keys(EXT_PRIMED) || haskey(Base.loaded_modules, id)
1262 continue # extension is already primed or loaded, don't add it again
1263 end
1264 EXT_PRIMED[id] = parent
1265 gid = ExtensionId(id, parent, 1 + length(triggers))
1266 trigger1 = get!(Vector{ExtensionId}, EXT_DORMITORY, parent)
1267 push!(trigger1, gid)
1268 for trigger in triggers
1269 # TODO: Better error message if this lookup fails?
1270 uuid_trigger = UUID(weakdeps[trigger]::String)
1271 trigger_id = PkgId(uuid_trigger, trigger)
1272 if !haskey(Base.loaded_modules, trigger_id) || haskey(package_locks, trigger_id)
1273 trigger1 = get!(Vector{ExtensionId}, EXT_DORMITORY, trigger_id)
1274 push!(trigger1, gid)
1275 else
1276 gid.ntriggers -= 1
1277 end
1278 end
1279 end
1280 end
1281
1282 loading_extension::Bool = false
1283 precompiling_extension::Bool = false
1284 function run_extension_callbacks(extid::ExtensionId)
1285 assert_havelock(require_lock)
1286 succeeded = try
1287 # Used by Distributed to now load extensions in the package callback
1288 global loading_extension = true
1289 _require_prelocked(extid.id)
1290 @debug "Extension $(extid.id.name) of $(extid.parentid.name) loaded"
1291 true
1292 catch
1293 # Try to continue loading if loading an extension errors
1294 errs = current_exceptions()
1295 @error "Error during loading of extension $(extid.id.name) of $(extid.parentid.name), \
1296 use `Base.retry_load_extensions()` to retry." exception=errs
1297 false
1298 finally
1299 global loading_extension = false
1300 end
1301 return succeeded
1302 end
1303
1304 function run_extension_callbacks(pkgid::PkgId)
1305 assert_havelock(require_lock)
1306 # take ownership of extids that depend on this pkgid
1307 extids = pop!(EXT_DORMITORY, pkgid, nothing)
1308 extids === nothing && return
1309 for extid in extids
1310 if extid.ntriggers > 0
1311 # indicate pkgid is loaded
1312 extid.ntriggers -= 1
1313 end
1314 if extid.ntriggers < 0
1315 # indicate pkgid is loaded
1316 extid.ntriggers += 1
1317 succeeded = false
1318 else
1319 succeeded = true
1320 end
1321 if extid.ntriggers == 0
1322 # actually load extid, now that all dependencies are met,
1323 # and record the result
1324 succeeded = succeeded && run_extension_callbacks(extid)
1325 succeeded || push!(EXT_DORMITORY_FAILED, extid)
1326 end
1327 end
1328 return
1329 end
1330
1331 """
1332 retry_load_extensions()
1333
1334 Loads all the (not yet loaded) extensions that have their extension-dependencies loaded.
1335 This is used in cases where the automatic loading of an extension failed
1336 due to some problem with the extension. Instead of restarting the Julia session,
1337 the extension can be fixed, and this function run.
1338 """
1339 function retry_load_extensions()
1340 @lock require_lock begin
1341 # this copy is desired since run_extension_callbacks will release this lock
1342 # so this can still mutate the list to drop successful ones
1343 failed = copy(EXT_DORMITORY_FAILED)
1344 empty!(EXT_DORMITORY_FAILED)
1345 filter!(failed) do extid
1346 return !run_extension_callbacks(extid)
1347 end
1348 prepend!(EXT_DORMITORY_FAILED, failed)
1349 end
1350 return
1351 end
1352
1353 """
1354 get_extension(parent::Module, extension::Symbol)
1355
1356 Return the module for `extension` of `parent` or return `nothing` if the extension is not loaded.
1357 """
1358 get_extension(parent::Module, ext::Symbol) = get_extension(PkgId(parent), ext)
1359 function get_extension(parentid::PkgId, ext::Symbol)
1360 parentid.uuid === nothing && return nothing
1361 extid = PkgId(uuid5(parentid.uuid, string(ext)), string(ext))
1362 return get(loaded_modules, extid, nothing)
1363 end
1364
1365 # End extensions
1366
1367 # should sync with the types of arguments of `stale_cachefile`
1368 const StaleCacheKey = Tuple{Base.PkgId, UInt128, String, String}
1369
1370 """
1371 Base.isprecompiled(pkg::PkgId; ignore_loaded::Bool=false)
1372
1373 Returns whether a given PkgId within the active project is precompiled.
1374
1375 By default this check observes the same approach that code loading takes
1376 with respect to when different versions of dependencies are currently loaded
1377 to that which is expected. To ignore loaded modules and answer as if in a
1378 fresh julia session specify `ignore_loaded=true`.
1379
1380 !!! compat "Julia 1.10"
1381 This function requires at least Julia 1.10.
1382 """
1383 function isprecompiled(pkg::PkgId;
1384 ignore_loaded::Bool=false,
1385 stale_cache::Dict{StaleCacheKey,Bool}=Dict{StaleCacheKey, Bool}(),
1386 cachepaths::Vector{String}=Base.find_all_in_cache_path(pkg),
1387 sourcepath::Union{String,Nothing}=Base.locate_package(pkg)
1388 )
1389 isnothing(sourcepath) && error("Cannot locate source for $(repr(pkg))")
1390 for path_to_try in cachepaths
1391 staledeps = stale_cachefile(sourcepath, path_to_try, ignore_loaded = true)
1392 if staledeps === true
1393 continue
1394 end
1395 staledeps, _ = staledeps::Tuple{Vector{Any}, Union{Nothing, String}}
1396 # finish checking staledeps module graph
1397 for i in 1:length(staledeps)
1398 dep = staledeps[i]
1399 dep isa Module && continue
1400 modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128}
1401 modpaths = find_all_in_cache_path(modkey)
1402 for modpath_to_try in modpaths::Vector{String}
1403 stale_cache_key = (modkey, modbuild_id, modpath, modpath_to_try)::StaleCacheKey
1404 if get!(() -> stale_cachefile(stale_cache_key...; ignore_loaded) === true,
1405 stale_cache, stale_cache_key)
1406 continue
1407 end
1408 @goto check_next_dep
1409 end
1410 @goto check_next_path
1411 @label check_next_dep
1412 end
1413 try
1414 # update timestamp of precompilation file so that it is the first to be tried by code loading
1415 touch(path_to_try)
1416 catch ex
1417 # file might be read-only and then we fail to update timestamp, which is fine
1418 ex isa IOError || rethrow()
1419 end
1420 return true
1421 @label check_next_path
1422 end
1423 return false
1424 end
1425
1426 # loads a precompile cache file, after checking stale_cachefile tests
1427 function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt128)
1428 assert_havelock(require_lock)
1429 loaded = nothing
1430 if root_module_exists(modkey)
1431 loaded = root_module(modkey)
1432 else
1433 loaded = start_loading(modkey)
1434 if loaded === nothing
1435 try
1436 modpath = locate_package(modkey)
1437 modpath === nothing && return nothing
1438 set_pkgorigin_version_path(modkey, String(modpath))
1439 loaded = _require_search_from_serialized(modkey, String(modpath), build_id)
1440 finally
1441 end_loading(modkey, loaded)
1442 end
1443 if loaded isa Module
1444 insert_extension_triggers(modkey)
1445 run_package_callbacks(modkey)
1446 end
1447 end
1448 end
1449 if !(loaded isa Module) || PkgId(loaded) != modkey
1450 return ErrorException("Required dependency $modkey failed to load from a cache file.")
1451 end
1452 return loaded
1453 end
1454
1455 # loads a precompile cache file, ignoring stale_cachefile tests
1456 # assuming all depmods are already loaded and everything is valid
1457 function _tryrequire_from_serialized(modkey::PkgId, path::String, ocachepath::Union{Nothing, String}, sourcepath::String, depmods::Vector{Any})
1458 assert_havelock(require_lock)
1459 loaded = nothing
1460 if root_module_exists(modkey)
1461 loaded = root_module(modkey)
1462 else
1463 loaded = start_loading(modkey)
1464 if loaded === nothing
1465 try
1466 for i in 1:length(depmods)
1467 dep = depmods[i]
1468 dep isa Module && continue
1469 _, depkey, depbuild_id = dep::Tuple{String, PkgId, UInt128}
1470 @assert root_module_exists(depkey)
1471 dep = root_module(depkey)
1472 depmods[i] = dep
1473 end
1474 set_pkgorigin_version_path(modkey, sourcepath)
1475 loaded = _include_from_serialized(modkey, path, ocachepath, depmods)
1476 finally
1477 end_loading(modkey, loaded)
1478 end
1479 if loaded isa Module
1480 insert_extension_triggers(modkey)
1481 run_package_callbacks(modkey)
1482 end
1483 end
1484 end
1485 if !(loaded isa Module) || PkgId(loaded) != modkey
1486 return ErrorException("Required dependency $modkey failed to load from a cache file.")
1487 end
1488 return loaded
1489 end
1490
1491 # loads a precompile cache file, ignoring stale_cachefile tests
1492 # load the best available (non-stale) version of all dependent modules first
1493 function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String})
1494 assert_havelock(require_lock)
1495 local depmodnames
1496 io = open(path, "r")
1497 try
1498 iszero(isvalid_cache_header(io)) && return ArgumentError("Invalid header in cache file $path.")
1499 _, _, depmodnames, _, _, _, clone_targets, _ = parse_cache_header(io)
1500 pkgimage = !isempty(clone_targets)
1501 if pkgimage
1502 ocachepath !== nothing || return ArgumentError("Expected ocachepath to be provided")
1503 isfile(ocachepath) || return ArgumentError("Ocachepath $ocachepath is not a file.")
1504 ocachepath == ocachefile_from_cachefile(path) || return ArgumentError("$ocachepath is not the expected ocachefile")
1505 # TODO: Check for valid clone_targets?
1506 isvalid_pkgimage_crc(io, ocachepath) || return ArgumentError("Invalid checksum in cache file $ocachepath.")
1507 else
1508 @assert ocachepath === nothing
1509 end
1510 isvalid_file_crc(io) || return ArgumentError("Invalid checksum in cache file $path.")
1511 finally
1512 close(io)
1513 end
1514 ndeps = length(depmodnames)
1515 depmods = Vector{Any}(undef, ndeps)
1516 for i in 1:ndeps
1517 modkey, build_id = depmodnames[i]
1518 dep = _tryrequire_from_serialized(modkey, build_id)
1519 if !isa(dep, Module)
1520 return dep
1521 end
1522 depmods[i] = dep
1523 end
1524 # then load the file
1525 return _include_from_serialized(pkg, path, ocachepath, depmods)
1526 end
1527
1528 # returns `nothing` if require found a precompile cache for this sourcepath, but couldn't load it
1529 # returns the set of modules restored if the cache load succeeded
1530 @constprop :none function _require_search_from_serialized(pkg::PkgId, sourcepath::String, build_id::UInt128)
1531 assert_havelock(require_lock)
1532 paths = find_all_in_cache_path(pkg)
1533 for path_to_try in paths::Vector{String}
1534 staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try)
1535 if staledeps === true
1536 continue
1537 end
1538 staledeps, ocachefile = staledeps::Tuple{Vector{Any}, Union{Nothing, String}}
1539 # finish checking staledeps module graph
1540 for i in 1:length(staledeps)
1541 dep = staledeps[i]
1542 dep isa Module && continue
1543 modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128}
1544 modpaths = find_all_in_cache_path(modkey)
1545 for modpath_to_try in modpaths
1546 modstaledeps = stale_cachefile(modkey, modbuild_id, modpath, modpath_to_try)
1547 if modstaledeps === true
1548 continue
1549 end
1550 modstaledeps, modocachepath = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}}
1551 staledeps[i] = (modpath, modkey, modpath_to_try, modstaledeps, modocachepath)
1552 @goto check_next_dep
1553 end
1554 @debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache."
1555 @goto check_next_path
1556 @label check_next_dep
1557 end
1558 try
1559 touch(path_to_try) # update timestamp of precompilation file
1560 catch ex # file might be read-only and then we fail to update timestamp, which is fine
1561 ex isa IOError || rethrow()
1562 end
1563 # finish loading module graph into staledeps
1564 for i in 1:length(staledeps)
1565 dep = staledeps[i]
1566 dep isa Module && continue
1567 modpath, modkey, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, String, Vector{Any}, Union{Nothing, String}}
1568 dep = _tryrequire_from_serialized(modkey, modcachepath, modocachepath, modpath, modstaledeps)
1569 if !isa(dep, Module)
1570 @debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep
1571 @goto check_next_path
1572 end
1573 staledeps[i] = dep
1574 end
1575 restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps)
1576 isa(restored, Module) && return restored
1577 @debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored
1578 continue
1579 @label check_next_path
1580 end
1581 return nothing
1582 end
1583
1584 # to synchronize multiple tasks trying to import/using something
1585 const package_locks = Dict{PkgId,Pair{Task,Threads.Condition}}()
1586
1587 debug_loading_deadlocks::Bool = true # Enable a slightly more expensive, but more complete algorithm that can handle simultaneous tasks.
1588 # This only triggers if you have multiple tasks trying to load the same package at the same time,
1589 # so it is unlikely to make a difference normally.
1590 function start_loading(modkey::PkgId)
1591 # handle recursive calls to require
1592 assert_havelock(require_lock)
1593 loading = get(package_locks, modkey, nothing)
1594 if loading !== nothing
1595 # load already in progress for this module on the task
1596 task, cond = loading
1597 deps = String[modkey.name]
1598 pkgid = modkey
1599 assert_havelock(cond.lock)
1600 if debug_loading_deadlocks && current_task() !== task
1601 waiters = Dict{Task,Pair{Task,PkgId}}() # invert to track waiting tasks => loading tasks
1602 for each in package_locks
1603 cond2 = each[2][2]
1604 assert_havelock(cond2.lock)
1605 for waiting in cond2.waitq
1606 push!(waiters, waiting => (each[2][1] => each[1]))
1607 end
1608 end
1609 while true
1610 running = get(waiters, task, nothing)
1611 running === nothing && break
1612 task, pkgid = running
1613 push!(deps, pkgid.name)
1614 task === current_task() && break
1615 end
1616 end
1617 if current_task() === task
1618 others = String[modkey.name] # repeat this to emphasize the cycle here
1619 for each in package_locks # list the rest of the packages being loaded too
1620 if each[2][1] === task
1621 other = each[1].name
1622 other == modkey.name || other == pkgid.name || push!(others, other)
1623 end
1624 end
1625 msg = sprint(deps, others) do io, deps, others
1626 print(io, "deadlock detected in loading ")
1627 join(io, deps, " -> ")
1628 print(io, " -> ")
1629 join(io, others, " && ")
1630 end
1631 throw(ConcurrencyViolationError(msg))
1632 end
1633 return wait(cond)
1634 end
1635 package_locks[modkey] = current_task() => Threads.Condition(require_lock)
1636 return
1637 end
1638
1639 function end_loading(modkey::PkgId, @nospecialize loaded)
1640 loading = pop!(package_locks, modkey)
1641 notify(loading[2], loaded, all=true)
1642 nothing
1643 end
1644
1645 # to notify downstream consumers that a module was successfully loaded
1646 # Callbacks take the form (mod::Base.PkgId) -> nothing.
1647 # WARNING: This is an experimental feature and might change later, without deprecation.
1648 const package_callbacks = Any[]
1649 # to notify downstream consumers that a file has been included into a particular module
1650 # Callbacks take the form (mod::Module, filename::String) -> nothing
1651 # WARNING: This is an experimental feature and might change later, without deprecation.
1652 const include_callbacks = Any[]
1653
1654 # used to optionally track dependencies when requiring a module:
1655 const _concrete_dependencies = Pair{PkgId,UInt128}[] # these dependency versions are "set in stone", and the process should try to avoid invalidating them
1656 const _require_dependencies = Any[] # a list of (mod, path, mtime) tuples that are the file dependencies of the module currently being precompiled
1657 const _track_dependencies = Ref(false) # set this to true to track the list of file dependencies
1658 function _include_dependency(mod::Module, _path::AbstractString)
1659 prev = source_path(nothing)
1660 if prev === nothing
1661 path = abspath(_path)
1662 else
1663 path = normpath(joinpath(dirname(prev), _path))
1664 end
1665 if _track_dependencies[]
1666 @lock require_lock begin
1667 push!(_require_dependencies, (mod, path, mtime(path)))
1668 end
1669 end
1670 return path, prev
1671 end
1672
1673 """
1674 include_dependency(path::AbstractString)
1675
1676 In a module, declare that the file, directory, or symbolic link specified by `path`
1677 (relative or absolute) is a dependency for precompilation; that is, the module will need
1678 to be recompiled if the modification time of `path` changes.
1679
1680 This is only needed if your module depends on a path that is not used via [`include`](@ref). It has
1681 no effect outside of compilation.
1682 """
1683 function include_dependency(path::AbstractString)
1684 _include_dependency(Main, path)
1685 return nothing
1686 end
1687
1688 # we throw PrecompilableError when a module doesn't want to be precompiled
1689 import Core: PrecompilableError
1690 function show(io::IO, ex::PrecompilableError)
1691 print(io, "Declaring __precompile__(false) is not allowed in files that are being precompiled.")
1692 end
1693 precompilableerror(ex::PrecompilableError) = true
1694 precompilableerror(ex::WrappedException) = precompilableerror(ex.error)
1695 precompilableerror(@nospecialize ex) = false
1696
1697 # Call __precompile__(false) at the top of a tile prevent it from being precompiled (false)
1698 """
1699 __precompile__(isprecompilable::Bool)
1700
1701 Specify whether the file calling this function is precompilable, defaulting to `true`.
1702 If a module or file is *not* safely precompilable, it should call `__precompile__(false)` in
1703 order to throw an error if Julia attempts to precompile it.
1704 """
1705 @noinline function __precompile__(isprecompilable::Bool=true)
1706 if !isprecompilable && ccall(:jl_generating_output, Cint, ()) != 0
1707 throw(PrecompilableError())
1708 end
1709 nothing
1710 end
1711
1712 # require always works in Main scope and loads files from node 1
1713 const toplevel_load = Ref(true)
1714
1715 const _require_world_age = Ref{UInt}(typemax(UInt))
1716
1717 """
1718 require(into::Module, module::Symbol)
1719
1720 This function is part of the implementation of [`using`](@ref) / [`import`](@ref), if a module is not
1721 already defined in `Main`. It can also be called directly to force reloading a module,
1722 regardless of whether it has been loaded before (for example, when interactively developing
1723 libraries).
1724
1725 Loads a source file, in the context of the `Main` module, on every active node, searching
1726 standard locations for files. `require` is considered a top-level operation, so it sets the
1727 current `include` path but does not use it to search for files (see help for [`include`](@ref)).
1728 This function is typically used to load library code, and is implicitly called by `using` to
1729 load packages.
1730
1731 When searching for files, `require` first looks for package code in the global array
1732 [`LOAD_PATH`](@ref). `require` is case-sensitive on all platforms, including those with
1733 case-insensitive filesystems like macOS and Windows.
1734
1735 For more details regarding code loading, see the manual sections on [modules](@ref modules) and
1736 [parallel computing](@ref code-availability).
1737 """
1738 function require(into::Module, mod::Symbol)
1739 if _require_world_age[] != typemax(UInt)
1740 Base.invoke_in_world(_require_world_age[], __require, into, mod)
1741 else
1742 @invokelatest __require(into, mod)
1743 end
1744 end
1745
1746 function __require(into::Module, mod::Symbol)
1747 @lock require_lock begin
1748 LOADING_CACHE[] = LoadingCache()
1749 try
1750 uuidkey_env = identify_package_env(into, String(mod))
1751 # Core.println("require($(PkgId(into)), $mod) -> $uuidkey_env")
1752 if uuidkey_env === nothing
1753 where = PkgId(into)
1754 if where.uuid === nothing
1755 hint, dots = begin
1756 if isdefined(into, mod) && getfield(into, mod) isa Module
1757 true, "."
1758 elseif isdefined(parentmodule(into), mod) && getfield(parentmodule(into), mod) isa Module
1759 true, ".."
1760 else
1761 false, ""
1762 end
1763 end
1764 hint_message = hint ? ", maybe you meant `import/using $(dots)$(mod)`" : ""
1765 start_sentence = hint ? "Otherwise, run" : "Run"
1766 throw(ArgumentError("""
1767 Package $mod not found in current path$hint_message.
1768 - $start_sentence `import Pkg; Pkg.add($(repr(String(mod))))` to install the $mod package."""))
1769 else
1770 throw(ArgumentError("""
1771 Package $(where.name) does not have $mod in its dependencies:
1772 - You may have a partially installed environment. Try `Pkg.instantiate()`
1773 to ensure all packages in the environment are installed.
1774 - Or, if you have $(where.name) checked out for development and have
1775 added $mod as a dependency but haven't updated your primary
1776 environment's manifest file, try `Pkg.resolve()`.
1777 - Otherwise you may need to report an issue with $(where.name)"""))
1778 end
1779 end
1780 uuidkey, env = uuidkey_env
1781 if _track_dependencies[]
1782 push!(_require_dependencies, (into, binpack(uuidkey), 0.0))
1783 end
1784 return _require_prelocked(uuidkey, env)
1785 finally
1786 LOADING_CACHE[] = nothing
1787 end
1788 end
1789 end
1790
1791 require(uuidkey::PkgId) = @lock require_lock _require_prelocked(uuidkey)
1792
1793 const REPL_PKGID = PkgId(UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL")
1794
1795 function _require_prelocked(uuidkey::PkgId, env=nothing)
1796 if _require_world_age[] != typemax(UInt)
1797 Base.invoke_in_world(_require_world_age[], __require_prelocked, uuidkey, env)
1798 else
1799 @invokelatest __require_prelocked(uuidkey, env)
1800 end
1801 end
1802
1803 function __require_prelocked(uuidkey::PkgId, env=nothing)
1804 assert_havelock(require_lock)
1805 if !root_module_exists(uuidkey)
1806 newm = _require(uuidkey, env)
1807 if newm === nothing
1808 error("package `$(uuidkey.name)` did not define the expected \
1809 module `$(uuidkey.name)`, check for typos in package module name")
1810 end
1811 insert_extension_triggers(uuidkey)
1812 # After successfully loading, notify downstream consumers
1813 run_package_callbacks(uuidkey)
1814 if uuidkey == REPL_PKGID
1815 REPL_MODULE_REF[] = newm
1816 end
1817 else
1818 newm = root_module(uuidkey)
1819 end
1820 return newm
1821 end
1822
1823 mutable struct PkgOrigin
1824 path::Union{String,Nothing}
1825 cachepath::Union{String,Nothing}
1826 version::Union{VersionNumber,Nothing}
1827 end
1828 PkgOrigin() = PkgOrigin(nothing, nothing, nothing)
1829 const pkgorigins = Dict{PkgId,PkgOrigin}()
1830
1831 const loaded_modules = Dict{PkgId,Module}()
1832 const loaded_modules_order = Vector{Module}()
1833 const module_keys = IdDict{Module,PkgId}() # the reverse
1834
1835 is_root_module(m::Module) = @lock require_lock haskey(module_keys, m)
1836 root_module_key(m::Module) = @lock require_lock module_keys[m]
1837
1838 @constprop :none function register_root_module(m::Module)
1839 # n.b. This is called from C after creating a new module in `Base.__toplevel__`,
1840 # instead of adding them to the binding table there.
1841 @lock require_lock begin
1842 key = PkgId(m, String(nameof(m)))
1843 if haskey(loaded_modules, key)
1844 oldm = loaded_modules[key]
1845 if oldm !== m
1846 if (0 != ccall(:jl_generating_output, Cint, ())) && (JLOptions().incremental != 0)
1847 error("Replacing module `$(key.name)`")
1848 else
1849 @warn "Replacing module `$(key.name)`"
1850 end
1851 end
1852 end
1853 push!(loaded_modules_order, m)
1854 loaded_modules[key] = m
1855 module_keys[m] = key
1856 end
1857 nothing
1858 end
1859
1860 register_root_module(Core)
1861 register_root_module(Base)
1862 register_root_module(Main)
1863
1864 # This is used as the current module when loading top-level modules.
1865 # It has the special behavior that modules evaluated in it get added
1866 # to the loaded_modules table instead of getting bindings.
1867 baremodule __toplevel__
1868 using Base
1869 end
1870
1871 # get a top-level Module from the given key
1872 root_module(key::PkgId) = @lock require_lock loaded_modules[key]
1873 function root_module(where::Module, name::Symbol)
1874 key = identify_package(where, String(name))
1875 key isa PkgId || throw(KeyError(name))
1876 return root_module(key)
1877 end
1878 maybe_root_module(key::PkgId) = @lock require_lock get(loaded_modules, key, nothing)
1879
1880 root_module_exists(key::PkgId) = @lock require_lock haskey(loaded_modules, key)
1881 loaded_modules_array() = @lock require_lock copy(loaded_modules_order)
1882
1883 function unreference_module(key::PkgId)
1884 if haskey(loaded_modules, key)
1885 m = pop!(loaded_modules, key)
1886 # need to ensure all modules are GC rooted; will still be referenced
1887 # in module_keys
1888 end
1889 end
1890
1891 # whoever takes the package_locks[pkg] must call this function immediately
1892 function set_pkgorigin_version_path(pkg::PkgId, path::Union{String,Nothing})
1893 assert_havelock(require_lock)
1894 pkgorigin = get!(PkgOrigin, pkgorigins, pkg)
1895 if path !== nothing
1896 # Pkg needs access to the version of packages in the sysimage.
1897 if Core.Compiler.generating_sysimg()
1898 pkgorigin.version = get_pkgversion_from_path(joinpath(dirname(path), ".."))
1899 end
1900 end
1901 pkgorigin.path = path
1902 nothing
1903 end
1904
1905 # A hook to allow code load to use Pkg.precompile
1906 const PKG_PRECOMPILE_HOOK = Ref{Function}()
1907
1908 # Returns `nothing` or the new(ish) module
1909 function _require(pkg::PkgId, env=nothing)
1910 assert_havelock(require_lock)
1911 loaded = start_loading(pkg)
1912 loaded === nothing || return loaded
1913
1914 last = toplevel_load[]
1915 try
1916 toplevel_load[] = false
1917 # perform the search operation to select the module file require intends to load
1918 path = locate_package(pkg, env)
1919 if path === nothing
1920 throw(ArgumentError("""
1921 Package $pkg is required but does not seem to be installed:
1922 - Run `Pkg.instantiate()` to install all recorded dependencies.
1923 """))
1924 end
1925 set_pkgorigin_version_path(pkg, path)
1926
1927 pkg_precompile_attempted = false # being safe to avoid getting stuck in a Pkg.precompile loop
1928
1929 # attempt to load the module file via the precompile cache locations
1930 if JLOptions().use_compiled_modules != 0
1931 @label load_from_cache
1932 m = _require_search_from_serialized(pkg, path, UInt128(0))
1933 if m isa Module
1934 return m
1935 end
1936 end
1937
1938 # if the module being required was supposed to have a particular version
1939 # but it was not handled by the precompile loader, complain
1940 for (concrete_pkg, concrete_build_id) in _concrete_dependencies
1941 if pkg == concrete_pkg
1942 @warn """Module $(pkg.name) with build ID $((UUID(concrete_build_id))) is missing from the cache.
1943 This may mean $pkg does not support precompilation but is imported by a module that does."""
1944 if JLOptions().incremental != 0
1945 # during incremental precompilation, this should be fail-fast
1946 throw(PrecompilableError())
1947 end
1948 end
1949 end
1950
1951 if JLOptions().use_compiled_modules != 0
1952 if (0 == ccall(:jl_generating_output, Cint, ())) || (JLOptions().incremental != 0)
1953 if !pkg_precompile_attempted && isinteractive() && isassigned(PKG_PRECOMPILE_HOOK)
1954 pkg_precompile_attempted = true
1955 unlock(require_lock)
1956 try
1957 @invokelatest PKG_PRECOMPILE_HOOK[](pkg.name, _from_loading = true)
1958 finally
1959 lock(require_lock)
1960 end
1961 @goto load_from_cache
1962 end
1963 # spawn off a new incremental pre-compile task for recursive `require` calls
1964 cachefile_or_module = maybe_cachefile_lock(pkg, path) do
1965 # double-check now that we have lock
1966 m = _require_search_from_serialized(pkg, path, UInt128(0))
1967 m isa Module && return m
1968 compilecache(pkg, path)
1969 end
1970 cachefile_or_module isa Module && return cachefile_or_module::Module
1971 cachefile = cachefile_or_module
1972 if isnothing(cachefile) # maybe_cachefile_lock returns nothing if it had to wait for another process
1973 @goto load_from_cache # the new cachefile will have the newest mtime so will come first in the search
1974 elseif isa(cachefile, Exception)
1975 if precompilableerror(cachefile)
1976 verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug
1977 @logmsg verbosity "Skipping precompilation since __precompile__(false). Importing $pkg."
1978 else
1979 @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m
1980 end
1981 # fall-through to loading the file locally if not incremental
1982 else
1983 cachefile, ocachefile = cachefile::Tuple{String, Union{Nothing, String}}
1984 m = _tryrequire_from_serialized(pkg, cachefile, ocachefile)
1985 if !isa(m, Module)
1986 @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m
1987 else
1988 return m
1989 end
1990 end
1991 if JLOptions().incremental != 0
1992 # during incremental precompilation, this should be fail-fast
1993 throw(PrecompilableError())
1994 end
1995 end
1996 end
1997
1998 # just load the file normally via include
1999 # for unknown dependencies
2000 uuid = pkg.uuid
2001 uuid = (uuid === nothing ? (UInt64(0), UInt64(0)) : convert(NTuple{2, UInt64}, uuid))
2002 old_uuid = ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), __toplevel__)
2003 if uuid !== old_uuid
2004 ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), __toplevel__, uuid)
2005 end
2006 unlock(require_lock)
2007 try
2008 include(__toplevel__, path)
2009 loaded = get(loaded_modules, pkg, nothing)
2010 finally
2011 lock(require_lock)
2012 if uuid !== old_uuid
2013 ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), __toplevel__, old_uuid)
2014 end
2015 end
2016 finally
2017 toplevel_load[] = last
2018 end_loading(pkg, loaded)
2019 end
2020 return loaded
2021 end
2022
2023 # Only used from test/precompile.jl
2024 function _require_from_serialized(uuidkey::PkgId, path::String, ocachepath::Union{String, Nothing})
2025 @lock require_lock begin
2026 set_pkgorigin_version_path(uuidkey, nothing)
2027 newm = _tryrequire_from_serialized(uuidkey, path, ocachepath)
2028 newm isa Module || throw(newm)
2029 insert_extension_triggers(uuidkey)
2030 # After successfully loading, notify downstream consumers
2031 run_package_callbacks(uuidkey)
2032 return newm
2033 end
2034 end
2035
2036
2037
2038 # relative-path load
2039
2040 """
2041 include_string([mapexpr::Function,] m::Module, code::AbstractString, filename::AbstractString="string")
2042
2043 Like [`include`](@ref), except reads code from the given string rather than from a file.
2044
2045 The optional first argument `mapexpr` can be used to transform the included code before
2046 it is evaluated: for each parsed expression `expr` in `code`, the `include_string` function
2047 actually evaluates `mapexpr(expr)`. If it is omitted, `mapexpr` defaults to [`identity`](@ref).
2048
2049 !!! compat "Julia 1.5"
2050 Julia 1.5 is required for passing the `mapexpr` argument.
2051 """
2052
288 (100 %) samples spent in include_string
288 (100 %) (incl.) when called from #invokelatest#2 line 887
function include_string(mapexpr::Function, mod::Module, code::AbstractString,
2053 filename::AbstractString="string")
2054 loc = LineNumberNode(1, Symbol(filename))
2055 try
2056 ast = Meta.parseall(code, filename=filename)
2057 @assert Meta.isexpr(ast, :toplevel)
2058 result = nothing
2059 line_and_ex = Expr(:toplevel, loc, nothing)
2060 for ex in ast.args
2061 if ex isa LineNumberNode
2062 loc = ex
2063 line_and_ex.args[1] = ex
2064 continue
2065 end
2066 ex = mapexpr(ex)
2067 # Wrap things to be eval'd in a :toplevel expr to carry line
2068 # information as part of the expr.
2069 line_and_ex.args[2] = ex
2070 288 (100 %)
288 (100 %) samples spent calling eval
result = Core.eval(mod, line_and_ex)
2071 end
2072 return result
2073 catch exc
2074 # TODO: Now that stacktraces are more reliable we should remove
2075 # LoadError and expose the real error type directly.
2076 rethrow(LoadError(filename, loc.line, exc))
2077 end
2078 end
2079
2080 include_string(m::Module, txt::AbstractString, fname::AbstractString="string") =
2081 include_string(identity, m, txt, fname)
2082
2083 function source_path(default::Union{AbstractString,Nothing}="")
2084 s = current_task().storage
2085 if s !== nothing
2086 s = s::IdDict{Any,Any}
2087 if haskey(s, :SOURCE_PATH)
2088 return s[:SOURCE_PATH]::Union{Nothing,String}
2089 end
2090 end
2091 return default
2092 end
2093
2094 function source_dir()
2095 p = source_path(nothing)
2096 return p === nothing ? pwd() : dirname(p)
2097 end
2098
2099 """
2100 Base.include([mapexpr::Function,] m::Module, path::AbstractString)
2101
2102 Evaluate the contents of the input source file in the global scope of module `m`.
2103 Every module (except those defined with [`baremodule`](@ref)) has its own
2104 definition of `include` omitting the `m` argument, which evaluates the file in that module.
2105 Returns the result of the last evaluated expression of the input file. During including,
2106 a task-local include path is set to the directory containing the file. Nested calls to
2107 `include` will search relative to that path. This function is typically used to load source
2108 interactively, or to combine files in packages that are broken into multiple source files.
2109
2110 The optional first argument `mapexpr` can be used to transform the included code before
2111 it is evaluated: for each parsed expression `expr` in `path`, the `include` function
2112 actually evaluates `mapexpr(expr)`. If it is omitted, `mapexpr` defaults to [`identity`](@ref).
2113
2114 !!! compat "Julia 1.5"
2115 Julia 1.5 is required for passing the `mapexpr` argument.
2116 """
2117 Base.include # defined in Base.jl
2118
2119 # Full include() implementation which is used after bootstrap
2120 function _include(mapexpr::Function, mod::Module, _path::AbstractString)
2121 @noinline # Workaround for module availability in _simplify_include_frames
2122 path, prev = _include_dependency(mod, _path)
2123 for callback in include_callbacks # to preserve order, must come before eval in include_string
2124 invokelatest(callback, mod, path)
2125 end
2126 code = read(path, String)
2127 tls = task_local_storage()
2128 tls[:SOURCE_PATH] = path
2129 try
2130 return include_string(mapexpr, mod, code, path)
2131 finally
2132 if prev === nothing
2133 delete!(tls, :SOURCE_PATH)
2134 else
2135 tls[:SOURCE_PATH] = prev
2136 end
2137 end
2138 end
2139
2140 """
2141 evalfile(path::AbstractString, args::Vector{String}=String[])
2142
2143 Load the file into an anonymous module using [`include`](@ref), evaluate all expressions,
2144 and return the value of the last expression.
2145 The optional `args` argument can be used to set the input arguments of the script (i.e. the global `ARGS` variable).
2146 Note that definitions (e.g. methods, globals) are evaluated in the anonymous module and do not affect the current module.
2147
2148 # Example
2149
2150 ```jldoctest
2151 julia> write("testfile.jl", \"\"\"
2152 @show ARGS
2153 1 + 1
2154 \"\"\");
2155
2156 julia> x = evalfile("testfile.jl", ["ARG1", "ARG2"]);
2157 ARGS = ["ARG1", "ARG2"]
2158
2159 julia> x
2160 2
2161
2162 julia> rm("testfile.jl")
2163 ```
2164 """
2165 function evalfile(path::AbstractString, args::Vector{String}=String[])
2166 return Core.eval(Module(:__anon__),
2167 Expr(:toplevel,
2168 :(const ARGS = $args),
2169 :(eval(x) = $(Expr(:core, :eval))(__anon__, x)),
2170 :(include(x) = $(Expr(:top, :include))(__anon__, x)),
2171 :(include(mapexpr::Function, x) = $(Expr(:top, :include))(mapexpr, __anon__, x)),
2172 :(include($path))))
2173 end
2174 evalfile(path::AbstractString, args::Vector) = evalfile(path, String[args...])
2175
2176 function load_path_setup_code(load_path::Bool=true)
2177 code = """
2178 append!(empty!(Base.DEPOT_PATH), $(repr(map(abspath, DEPOT_PATH))))
2179 append!(empty!(Base.DL_LOAD_PATH), $(repr(map(abspath, DL_LOAD_PATH))))
2180 """
2181 if load_path
2182 load_path = map(abspath, Base.load_path())
2183 path_sep = Sys.iswindows() ? ';' : ':'
2184 any(path -> path_sep in path, load_path) &&
2185 error("LOAD_PATH entries cannot contain $(repr(path_sep))")
2186 code *= """
2187 append!(empty!(Base.LOAD_PATH), $(repr(load_path)))
2188 ENV["JULIA_LOAD_PATH"] = $(repr(join(load_path, Sys.iswindows() ? ';' : ':')))
2189 Base.set_active_project(nothing)
2190 """
2191 end
2192 return code
2193 end
2194
2195 # this is called in the external process that generates precompiled package files
2196 function include_package_for_output(pkg::PkgId, input::String, depot_path::Vector{String}, dl_load_path::Vector{String}, load_path::Vector{String},
2197 concrete_deps::typeof(_concrete_dependencies), source::Union{Nothing,String})
2198 append!(empty!(Base.DEPOT_PATH), depot_path)
2199 append!(empty!(Base.DL_LOAD_PATH), dl_load_path)
2200 append!(empty!(Base.LOAD_PATH), load_path)
2201 ENV["JULIA_LOAD_PATH"] = join(load_path, Sys.iswindows() ? ';' : ':')
2202 set_active_project(nothing)
2203 Base._track_dependencies[] = true
2204 get!(Base.PkgOrigin, Base.pkgorigins, pkg).path = input
2205 append!(empty!(Base._concrete_dependencies), concrete_deps)
2206 uuid_tuple = pkg.uuid === nothing ? (UInt64(0), UInt64(0)) : convert(NTuple{2, UInt64}, pkg.uuid)
2207
2208 ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, uuid_tuple)
2209 if source !== nothing
2210 task_local_storage()[:SOURCE_PATH] = source
2211 end
2212
2213 ccall(:jl_set_newly_inferred, Cvoid, (Any,), Core.Compiler.newly_inferred)
2214 Core.Compiler.track_newly_inferred.x = true
2215 try
2216 Base.include(Base.__toplevel__, input)
2217 catch ex
2218 precompilableerror(ex) || rethrow()
2219 @debug "Aborting `create_expr_cache'" exception=(ErrorException("Declaration of __precompile__(false) not allowed"), catch_backtrace())
2220 exit(125) # we define status = 125 means PrecompileableError
2221 finally
2222 Core.Compiler.track_newly_inferred.x = false
2223 end
2224 end
2225
2226 const PRECOMPILE_TRACE_COMPILE = Ref{String}()
2227 function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::Union{Nothing, String},
2228 concrete_deps::typeof(_concrete_dependencies), internal_stderr::IO = stderr, internal_stdout::IO = stdout)
2229 @nospecialize internal_stderr internal_stdout
2230 rm(output, force=true) # Remove file if it exists
2231 output_o === nothing || rm(output_o, force=true)
2232 depot_path = map(abspath, DEPOT_PATH)
2233 dl_load_path = map(abspath, DL_LOAD_PATH)
2234 load_path = map(abspath, Base.load_path())
2235 path_sep = Sys.iswindows() ? ';' : ':'
2236 any(path -> path_sep in path, load_path) &&
2237 error("LOAD_PATH entries cannot contain $(repr(path_sep))")
2238
2239 deps_strs = String[]
2240 function pkg_str(_pkg::PkgId)
2241 if _pkg.uuid === nothing
2242 "Base.PkgId($(repr(_pkg.name)))"
2243 else
2244 "Base.PkgId(Base.UUID(\"$(_pkg.uuid)\"), $(repr(_pkg.name)))"
2245 end
2246 end
2247 for (pkg, build_id) in concrete_deps
2248 push!(deps_strs, "$(pkg_str(pkg)) => $(repr(build_id))")
2249 end
2250
2251 if output_o !== nothing
2252 cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing)
2253 opt_level = Base.JLOptions().opt_level
2254 opts = `-O$(opt_level) --output-o $(output_o) --output-ji $(output) --output-incremental=yes`
2255 else
2256 cpu_target = nothing
2257 opts = `-O0 --output-ji $(output) --output-incremental=yes`
2258 end
2259
2260 deps_eltype = sprint(show, eltype(concrete_deps); context = :module=>nothing)
2261 deps = deps_eltype * "[" * join(deps_strs, ",") * "]"
2262 trace = isassigned(PRECOMPILE_TRACE_COMPILE) ? `--trace-compile=$(PRECOMPILE_TRACE_COMPILE[])` : ``
2263 io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd) $(opts)
2264 --startup-file=no --history-file=no --warn-overwrite=yes
2265 --color=$(have_color === nothing ? "auto" : have_color ? "yes" : "no")
2266 $trace
2267 -`,
2268 "OPENBLAS_NUM_THREADS" => 1,
2269 "JULIA_NUM_THREADS" => 1),
2270 stderr = internal_stderr, stdout = internal_stdout),
2271 "w", stdout)
2272 # write data over stdin to avoid the (unlikely) case of exceeding max command line size
2273 write(io.in, """
2274 empty!(Base.EXT_DORMITORY) # If we have a custom sysimage with `EXT_DORMITORY` prepopulated
2275 Base.precompiling_extension = $(loading_extension)
2276 Base.include_package_for_output($(pkg_str(pkg)), $(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)),
2277 $(repr(load_path)), $deps, $(repr(source_path(nothing))))
2278 """)
2279 close(io.in)
2280 return io
2281 end
2282
2283 function compilecache_dir(pkg::PkgId)
2284 entrypath, entryfile = cache_file_entry(pkg)
2285 return joinpath(DEPOT_PATH[1], entrypath)
2286 end
2287
2288 function compilecache_path(pkg::PkgId, prefs_hash::UInt64; project::String=something(Base.active_project(), ""))::String
2289 entrypath, entryfile = cache_file_entry(pkg)
2290 cachepath = joinpath(DEPOT_PATH[1], entrypath)
2291 isdir(cachepath) || mkpath(cachepath)
2292 if pkg.uuid === nothing
2293 abspath(cachepath, entryfile) * ".ji"
2294 else
2295 crc = _crc32c(project)
2296 crc = _crc32c(unsafe_string(JLOptions().image_file), crc)
2297 crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc)
2298 crc = _crc32c(ccall(:jl_cache_flags, UInt8, ()), crc)
2299
2300 cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing)
2301 if cpu_target === nothing
2302 cpu_target = unsafe_string(JLOptions().cpu_target)
2303 end
2304 crc = _crc32c(cpu_target, crc)
2305
2306 crc = _crc32c(prefs_hash, crc)
2307 project_precompile_slug = slug(crc, 5)
2308 abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji"))
2309 end
2310 end
2311
2312 """
2313 Base.compilecache(module::PkgId)
2314
2315 Creates a precompiled cache file for a module and all of its dependencies.
2316 This can be used to reduce package load times. Cache files are stored in
2317 `DEPOT_PATH[1]/compiled`. See [Module initialization and precompilation](@ref)
2318 for important notes.
2319 """
2320 function compilecache(pkg::PkgId, internal_stderr::IO = stderr, internal_stdout::IO = stdout)
2321 @nospecialize internal_stderr internal_stdout
2322 path = locate_package(pkg)
2323 path === nothing && throw(ArgumentError("$pkg not found during precompilation"))
2324 return compilecache(pkg, path, internal_stderr, internal_stdout)
2325 end
2326
2327 const MAX_NUM_PRECOMPILE_FILES = Ref(10)
2328
2329 function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, internal_stdout::IO = stdout,
2330 keep_loaded_modules::Bool = true)
2331
2332 @nospecialize internal_stderr internal_stdout
2333 # decide where to put the resulting cache file
2334 cachepath = compilecache_dir(pkg)
2335
2336 # build up the list of modules that we want the precompile process to preserve
2337 concrete_deps = copy(_concrete_dependencies)
2338 if keep_loaded_modules
2339 for mod in loaded_modules_array()
2340 if !(mod === Main || mod === Core || mod === Base)
2341 push!(concrete_deps, PkgId(mod) => module_build_id(mod))
2342 end
2343 end
2344 end
2345 # run the expression and cache the result
2346 verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug
2347 @logmsg verbosity "Precompiling $pkg"
2348
2349 # create a temporary file in `cachepath` directory, write the cache in it,
2350 # write the checksum, _and then_ atomically move the file to `cachefile`.
2351 mkpath(cachepath)
2352 cache_objects = JLOptions().use_pkgimages != 0
2353 tmppath, tmpio = mktemp(cachepath)
2354
2355 if cache_objects
2356 tmppath_o, tmpio_o = mktemp(cachepath)
2357 tmppath_so, tmpio_so = mktemp(cachepath)
2358 else
2359 tmppath_o = nothing
2360 end
2361 local p
2362 try
2363 close(tmpio)
2364 if cache_objects
2365 close(tmpio_o)
2366 close(tmpio_so)
2367 end
2368 p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, internal_stderr, internal_stdout)
2369
2370 if success(p)
2371 if cache_objects
2372 # Run linker over tmppath_o
2373 Linking.link_image(tmppath_o, tmppath_so)
2374 end
2375
2376 # Read preferences hash back from .ji file (we can't precompute because
2377 # we don't actually know what the list of compile-time preferences are without compiling)
2378 prefs_hash = preferences_hash(tmppath)
2379 cachefile = compilecache_path(pkg, prefs_hash)
2380 ocachefile = cache_objects ? ocachefile_from_cachefile(cachefile) : nothing
2381
2382 # append checksum for so to the end of the .ji file:
2383 crc_so = UInt32(0)
2384 if cache_objects
2385 crc_so = open(_crc32c, tmppath_so, "r")
2386 end
2387
2388 # append extra crc to the end of the .ji file:
2389 open(tmppath, "r+") do f
2390 if iszero(isvalid_cache_header(f))
2391 error("Invalid header for $pkg in new cache file $(repr(tmppath)).")
2392 end
2393 seekend(f)
2394 write(f, crc_so)
2395 seekstart(f)
2396 write(f, _crc32c(f))
2397 end
2398
2399 # inherit permission from the source file (and make them writable)
2400 chmod(tmppath, filemode(path) & 0o777 | 0o200)
2401
2402 # prune the directory with cache files
2403 if pkg.uuid !== nothing
2404 entrypath, entryfile = cache_file_entry(pkg)
2405 cachefiles = filter!(x -> startswith(x, entryfile * "_") && endswith(x, ".ji"), readdir(cachepath))
2406 if length(cachefiles) >= MAX_NUM_PRECOMPILE_FILES[]
2407 idx = findmin(mtime.(joinpath.(cachepath, cachefiles)))[2]
2408 evicted_cachefile = joinpath(cachepath, cachefiles[idx])
2409 @debug "Evicting file from cache" evicted_cachefile
2410 rm(evicted_cachefile; force=true)
2411 try
2412 rm(ocachefile_from_cachefile(evicted_cachefile); force=true)
2413 @static if Sys.isapple()
2414 rm(ocachefile_from_cachefile(evicted_cachefile) * ".dSYM"; force=true, recursive=true)
2415 end
2416 catch e
2417 e isa IOError || rethrow()
2418 end
2419 end
2420 end
2421
2422 if cache_objects
2423 try
2424 rename(tmppath_so, ocachefile::String; force=true)
2425 catch e
2426 e isa IOError || rethrow()
2427 isfile(ocachefile::String) || rethrow()
2428 # Windows prevents renaming a file that is in use so if there is a Julia session started
2429 # with a package image loaded, we cannot rename that file.
2430 # The code belows append a `_i` to the name of the cache file where `i` is the smallest number such that
2431 # that cache file does not exist.
2432 ocachename, ocacheext = splitext(ocachefile::String)
2433 old_cachefiles = Set(readdir(cachepath))
2434 num = 1
2435 while true
2436 ocachefile = ocachename * "_$num" * ocacheext
2437 in(basename(ocachefile), old_cachefiles) || break
2438 num += 1
2439 end
2440 # TODO: Risk for a race here if some other process grabs this name before us
2441 cachefile = cachefile_from_ocachefile(ocachefile)
2442 rename(tmppath_so, ocachefile::String; force=true)
2443 end
2444 @static if Sys.isapple()
2445 run(`$(Linking.dsymutil()) $ocachefile`, Base.DevNull(), Base.DevNull(), Base.DevNull())
2446 end
2447 end
2448 # this is atomic according to POSIX (not Win32):
2449 rename(tmppath, cachefile; force=true)
2450 return cachefile, ocachefile
2451 end
2452 finally
2453 rm(tmppath, force=true)
2454 if cache_objects
2455 rm(tmppath_o::String, force=true)
2456 rm(tmppath_so, force=true)
2457 end
2458 end
2459 if p.exitcode == 125
2460 return PrecompilableError()
2461 else
2462 error("Failed to precompile $pkg to $(repr(tmppath)).")
2463 end
2464 end
2465
2466 function module_build_id(m::Module)
2467 hi, lo = ccall(:jl_module_build_id, NTuple{2,UInt64}, (Any,), m)
2468 return (UInt128(hi) << 64) | lo
2469 end
2470
2471 function isvalid_cache_header(f::IOStream)
2472 pkgimage = Ref{UInt8}()
2473 checksum = ccall(:jl_read_verify_header, UInt64, (Ptr{Cvoid}, Ptr{UInt8}, Ptr{Int64}, Ptr{Int64}), f.ios, pkgimage, Ref{Int64}(), Ref{Int64}()) # returns checksum id or zero
2474
2475 if !iszero(checksum) && pkgimage[] != 0
2476 @debug "Cache header was for pkgimage"
2477 return UInt64(0) # We somehow read the header for a pkgimage and not a ji
2478 end
2479 return checksum
2480 end
2481 isvalid_file_crc(f::IOStream) = (_crc32c(seekstart(f), filesize(f) - 4) == read(f, UInt32))
2482
2483 function isvalid_pkgimage_crc(f::IOStream, ocachefile::String)
2484 seekstart(f) # TODO necessary
2485 seek(f, filesize(f) - 8)
2486 expected_crc_so = read(f, UInt32)
2487 crc_so = open(_crc32c, ocachefile, "r")
2488 expected_crc_so == crc_so
2489 end
2490
2491 struct CacheHeaderIncludes
2492 id::PkgId
2493 filename::String
2494 mtime::Float64
2495 modpath::Vector{String} # seemingly not needed in Base, but used by Revise
2496 end
2497
2498 function parse_cache_header(f::IO)
2499 flags = read(f, UInt8)
2500 modules = Vector{Pair{PkgId, UInt64}}()
2501 while true
2502 n = read(f, Int32)
2503 n == 0 && break
2504 sym = String(read(f, n)) # module name
2505 uuid = UUID((read(f, UInt64), read(f, UInt64))) # pkg UUID
2506 build_id = read(f, UInt64) # build UUID (mostly just a timestamp)
2507 push!(modules, PkgId(uuid, sym) => build_id)
2508 end
2509 totbytes = read(f, Int64) # total bytes for file dependencies + preferences
2510 # read the list of requirements
2511 # and split the list into include and requires statements
2512 includes = CacheHeaderIncludes[]
2513 requires = Pair{PkgId, PkgId}[]
2514 while true
2515 n2 = read(f, Int32)
2516 totbytes -= 4
2517 if n2 == 0
2518 break
2519 end
2520 depname = String(read(f, n2))
2521 totbytes -= n2
2522 mtime = read(f, Float64)
2523 totbytes -= 8
2524 n1 = read(f, Int32)
2525 totbytes -= 4
2526 # map ids to keys
2527 modkey = (n1 == 0) ? PkgId("") : modules[n1].first
2528 modpath = String[]
2529 if n1 != 0
2530 # determine the complete module path
2531 while true
2532 n1 = read(f, Int32)
2533 totbytes -= 4
2534 if n1 == 0
2535 break
2536 end
2537 push!(modpath, String(read(f, n1)))
2538 totbytes -= n1
2539 end
2540 end
2541 if depname[1] == '\0'
2542 push!(requires, modkey => binunpack(depname))
2543 else
2544 push!(includes, CacheHeaderIncludes(modkey, depname, mtime, modpath))
2545 end
2546 end
2547 prefs = String[]
2548 while true
2549 n2 = read(f, Int32)
2550 totbytes -= 4
2551 if n2 == 0
2552 break
2553 end
2554 push!(prefs, String(read(f, n2)))
2555 totbytes -= n2
2556 end
2557 prefs_hash = read(f, UInt64)
2558 totbytes -= 8
2559 srctextpos = read(f, Int64)
2560 totbytes -= 8
2561 @assert totbytes == 0 "header of cache file appears to be corrupt (totbytes == $(totbytes))"
2562 # read the list of modules that are required to be present during loading
2563 required_modules = Vector{Pair{PkgId, UInt128}}()
2564 while true
2565 n = read(f, Int32)
2566 n == 0 && break
2567 sym = String(read(f, n)) # module name
2568 uuid = UUID((read(f, UInt64), read(f, UInt64))) # pkg UUID
2569 build_id = UInt128(read(f, UInt64)) << 64
2570 build_id |= read(f, UInt64)
2571 push!(required_modules, PkgId(uuid, sym) => build_id)
2572 end
2573 l = read(f, Int32)
2574 clone_targets = read(f, l)
2575
2576 return modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags
2577 end
2578
2579 function parse_cache_header(cachefile::String; srcfiles_only::Bool=false)
2580 io = open(cachefile, "r")
2581 try
2582 iszero(isvalid_cache_header(io)) && throw(ArgumentError("Invalid header in cache file $cachefile."))
2583 ret = parse_cache_header(io)
2584 srcfiles_only || return ret
2585 _, (includes, _), _, srctextpos, _... = ret
2586 srcfiles = srctext_files(io, srctextpos)
2587 delidx = Int[]
2588 for (i, chi) in enumerate(includes)
2589 chi.filename ∈ srcfiles || push!(delidx, i)
2590 end
2591 deleteat!(includes, delidx)
2592 return ret
2593 finally
2594 close(io)
2595 end
2596 end
2597
2598 preferences_hash(f::IO) = parse_cache_header(f)[6]
2599 function preferences_hash(cachefile::String)
2600 io = open(cachefile, "r")
2601 try
2602 if iszero(isvalid_cache_header(io))
2603 throw(ArgumentError("Invalid header in cache file $cachefile."))
2604 end
2605 return preferences_hash(io)
2606 finally
2607 close(io)
2608 end
2609 end
2610
2611 function cache_dependencies(f::IO)
2612 _, (includes, _), modules, _... = parse_cache_header(f)
2613 return modules, map(chi -> (chi.filename, chi.mtime), includes) # return just filename and mtime
2614 end
2615
2616 function cache_dependencies(cachefile::String)
2617 io = open(cachefile, "r")
2618 try
2619 iszero(isvalid_cache_header(io)) && throw(ArgumentError("Invalid header in cache file $cachefile."))
2620 return cache_dependencies(io)
2621 finally
2622 close(io)
2623 end
2624 end
2625
2626 function read_dependency_src(io::IO, filename::AbstractString)
2627 srctextpos = parse_cache_header(io)[4]
2628 srctextpos == 0 && error("no source-text stored in cache file")
2629 seek(io, srctextpos)
2630 return _read_dependency_src(io, filename)
2631 end
2632
2633 function _read_dependency_src(io::IO, filename::AbstractString)
2634 while !eof(io)
2635 filenamelen = read(io, Int32)
2636 filenamelen == 0 && break
2637 fn = String(read(io, filenamelen))
2638 len = read(io, UInt64)
2639 if fn == filename
2640 return String(read(io, len))
2641 end
2642 seek(io, position(io) + len)
2643 end
2644 error(filename, " is not stored in the source-text cache")
2645 end
2646
2647 function read_dependency_src(cachefile::String, filename::AbstractString)
2648 io = open(cachefile, "r")
2649 try
2650 iszero(isvalid_cache_header(io)) && throw(ArgumentError("Invalid header in cache file $cachefile."))
2651 return read_dependency_src(io, filename)
2652 finally
2653 close(io)
2654 end
2655 end
2656
2657 function srctext_files(f::IO, srctextpos::Int64)
2658 files = Set{String}()
2659 srctextpos == 0 && return files
2660 seek(f, srctextpos)
2661 while !eof(f)
2662 filenamelen = read(f, Int32)
2663 filenamelen == 0 && break
2664 fn = String(read(f, filenamelen))
2665 len = read(f, UInt64)
2666 push!(files, fn)
2667 seek(f, position(f) + len)
2668 end
2669 return files
2670 end
2671
2672 # Test to see if this UUID is mentioned in this `Project.toml`; either as
2673 # the top-level UUID (e.g. that of the project itself), as a dependency,
2674 # or as an extra/weakdep for Preferences.
2675 function get_uuid_name(project::Dict{String, Any}, uuid::UUID)
2676 uuid_p = get(project, "uuid", nothing)::Union{Nothing, String}
2677 name = get(project, "name", nothing)::Union{Nothing, String}
2678 if name !== nothing && uuid_p !== nothing && UUID(uuid_p) == uuid
2679 return name
2680 end
2681 deps = get(project, "deps", nothing)::Union{Nothing, Dict{String, Any}}
2682 if deps !== nothing
2683 for (k, v) in deps
2684 if uuid == UUID(v::String)
2685 return k
2686 end
2687 end
2688 end
2689 for subkey in ("deps", "extras", "weakdeps")
2690 subsection = get(project, subkey, nothing)::Union{Nothing, Dict{String, Any}}
2691 if subsection !== nothing
2692 for (k, v) in subsection
2693 if uuid == UUID(v::String)
2694 return k
2695 end
2696 end
2697 end
2698 end
2699 return nothing
2700 end
2701
2702 function get_uuid_name(project_toml::String, uuid::UUID)
2703 project = parsed_toml(project_toml)
2704 return get_uuid_name(project, uuid)
2705 end
2706
2707 # If we've asked for a specific UUID, this function will extract the prefs
2708 # for that particular UUID. Otherwise, it returns all preferences.
2709 function filter_preferences(prefs::Dict{String, Any}, pkg_name)
2710 if pkg_name === nothing
2711 return prefs
2712 else
2713 return get(Dict{String, Any}, prefs, pkg_name)::Dict{String, Any}
2714 end
2715 end
2716
2717 function collect_preferences(project_toml::String, uuid::Union{UUID,Nothing})
2718 # We'll return a list of dicts to be merged
2719 dicts = Dict{String, Any}[]
2720
2721 project = parsed_toml(project_toml)
2722 pkg_name = nothing
2723 if uuid !== nothing
2724 # If we've been given a UUID, map that to the name of the package as
2725 # recorded in the preferences section. If we can't find that mapping,
2726 # exit out, as it means there's no way preferences can be set for that
2727 # UUID, as we only allow actual dependencies to have preferences set.
2728 pkg_name = get_uuid_name(project, uuid)
2729 if pkg_name === nothing
2730 return dicts
2731 end
2732 end
2733
2734 # Look first inside of `Project.toml` to see we have preferences embedded within there
2735 proj_preferences = get(Dict{String, Any}, project, "preferences")::Dict{String, Any}
2736 push!(dicts, filter_preferences(proj_preferences, pkg_name))
2737
2738 # Next, look for `(Julia)LocalPreferences.toml` files next to this `Project.toml`
2739 project_dir = dirname(project_toml)
2740 for name in preferences_names
2741 toml_path = joinpath(project_dir, name)
2742 if isfile(toml_path)
2743 prefs = parsed_toml(toml_path)
2744 push!(dicts, filter_preferences(prefs, pkg_name))
2745
2746 # If we find `JuliaLocalPreferences.toml`, don't look for `LocalPreferences.toml`
2747 break
2748 end
2749 end
2750
2751 return dicts
2752 end
2753
2754 """
2755 recursive_prefs_merge(base::Dict, overrides::Dict...)
2756
2757 Helper function to merge preference dicts recursively, honoring overrides in nested
2758 dictionaries properly.
2759 """
2760 function recursive_prefs_merge(base::Dict{String, Any}, overrides::Dict{String, Any}...)
2761 new_base = Base._typeddict(base, overrides...)
2762
2763 for override in overrides
2764 # Clear entries are keys that should be deleted from any previous setting.
2765 override_clear = get(override, "__clear__", nothing)
2766 if override_clear isa Vector{String}
2767 for k in override_clear
2768 delete!(new_base, k)
2769 end
2770 end
2771
2772 for (k, override_k) in override
2773 # Note that if `base` has a mapping that is _not_ a `Dict`, and `override`
2774 new_base_k = get(new_base, k, nothing)
2775 if new_base_k isa Dict{String, Any} && override_k isa Dict{String, Any}
2776 new_base[k] = recursive_prefs_merge(new_base_k, override_k)
2777 else
2778 new_base[k] = override_k
2779 end
2780 end
2781 end
2782 return new_base
2783 end
2784
2785 function get_preferences(uuid::Union{UUID,Nothing} = nothing)
2786 merged_prefs = Dict{String,Any}()
2787 for env in reverse(load_path())
2788 project_toml = env_project_file(env)
2789 if !isa(project_toml, String)
2790 continue
2791 end
2792
2793 # Collect all dictionaries from the current point in the load path, then merge them in
2794 dicts = collect_preferences(project_toml, uuid)
2795 merged_prefs = recursive_prefs_merge(merged_prefs, dicts...)
2796 end
2797 return merged_prefs
2798 end
2799
2800 function get_preferences_hash(uuid::Union{UUID, Nothing}, prefs_list::Vector{String})
2801 # Start from a predictable hash point to ensure that the same preferences always
2802 # hash to the same value, modulo changes in how Dictionaries are hashed.
2803 h = UInt(0)
2804 uuid === nothing && return UInt64(h)
2805
2806 # Load the preferences
2807 prefs = get_preferences(uuid)
2808
2809 # Walk through each name that's called out as a compile-time preference
2810 for name in prefs_list
2811 prefs_value = get(prefs, name, nothing)
2812 if prefs_value !== nothing
2813 h = hash(prefs_value, h)::UInt
2814 end
2815 end
2816 # We always return a `UInt64` so that our serialization format is stable
2817 return UInt64(h)
2818 end
2819
2820 get_preferences_hash(m::Module, prefs_list::Vector{String}) = get_preferences_hash(PkgId(m).uuid, prefs_list)
2821
2822 # This is how we keep track of who is using what preferences at compile-time
2823 const COMPILETIME_PREFERENCES = Dict{UUID,Set{String}}()
2824
2825 # In `Preferences.jl`, if someone calls `load_preference(@__MODULE__, key)` while we're precompiling,
2826 # we mark that usage as a usage at compile-time and call this method, so that at the end of `.ji` generation,
2827 # we can record the list of compile-time preferences and embed that into the `.ji` header
2828 function record_compiletime_preference(uuid::UUID, key::String)
2829 pref = get!(Set{String}, COMPILETIME_PREFERENCES, uuid)
2830 push!(pref, key)
2831 return nothing
2832 end
2833 get_compiletime_preferences(uuid::UUID) = collect(get(Vector{String}, COMPILETIME_PREFERENCES, uuid))
2834 get_compiletime_preferences(m::Module) = get_compiletime_preferences(PkgId(m).uuid)
2835 get_compiletime_preferences(::Nothing) = String[]
2836
2837 function check_clone_targets(clone_targets)
2838 rejection_reason = ccall(:jl_check_pkgimage_clones, Any, (Ptr{Cchar},), clone_targets)
2839 if rejection_reason !== nothing
2840 return rejection_reason
2841 end
2842 end
2843
2844 struct CacheFlags
2845 # OOICCDDP - see jl_cache_flags
2846 use_pkgimages::Bool
2847 debug_level::Int
2848 check_bounds::Int
2849 inline::Bool
2850 opt_level::Int
2851
2852 function CacheFlags(f::UInt8)
2853 use_pkgimages = Bool(f & 1)
2854 debug_level = Int((f >> 1) & 3)
2855 check_bounds = Int((f >> 3) & 3)
2856 inline = Bool((f >> 5) & 1)
2857 opt_level = Int((f >> 6) & 3) # define OPT_LEVEL in statiddata_utils
2858 new(use_pkgimages, debug_level, check_bounds, inline, opt_level)
2859 end
2860 end
2861 CacheFlags(f::Int) = CacheFlags(UInt8(f))
2862 CacheFlags() = CacheFlags(ccall(:jl_cache_flags, UInt8, ()))
2863
2864 function show(io::IO, cf::CacheFlags)
2865 print(io, "use_pkgimages = ", cf.use_pkgimages)
2866 print(io, ", debug_level = ", cf.debug_level)
2867 print(io, ", check_bounds = ", cf.check_bounds)
2868 print(io, ", inline = ", cf.inline)
2869 print(io, ", opt_level = ", cf.opt_level)
2870 end
2871
2872 struct ImageTarget
2873 name::String
2874 flags::Int32
2875 ext_features::String
2876 features_en::Vector{UInt8}
2877 features_dis::Vector{UInt8}
2878 end
2879
2880 function parse_image_target(io::IO)
2881 flags = read(io, Int32)
2882 nfeature = read(io, Int32)
2883 feature_en = read(io, 4*nfeature)
2884 feature_dis = read(io, 4*nfeature)
2885 name_len = read(io, Int32)
2886 name = String(read(io, name_len))
2887 ext_features_len = read(io, Int32)
2888 ext_features = String(read(io, ext_features_len))
2889 ImageTarget(name, flags, ext_features, feature_en, feature_dis)
2890 end
2891
2892 function parse_image_targets(targets::Vector{UInt8})
2893 io = IOBuffer(targets)
2894 ntargets = read(io, Int32)
2895 targets = Vector{ImageTarget}(undef, ntargets)
2896 for i in 1:ntargets
2897 targets[i] = parse_image_target(io)
2898 end
2899 return targets
2900 end
2901
2902 function current_image_targets()
2903 targets = @ccall jl_reflect_clone_targets()::Vector{UInt8}
2904 return parse_image_targets(targets)
2905 end
2906
2907 struct FeatureName
2908 name::Cstring
2909 bit::UInt32 # bit index into a `uint32_t` array;
2910 llvmver::UInt32 # 0 if it is available on the oldest LLVM version we support
2911 end
2912
2913 function feature_names()
2914 fnames = Ref{Ptr{FeatureName}}()
2915 nf = Ref{Csize_t}()
2916 @ccall jl_reflect_feature_names(fnames::Ptr{Ptr{FeatureName}}, nf::Ptr{Csize_t})::Cvoid
2917 if fnames[] == C_NULL
2918 @assert nf[] == 0
2919 return Vector{FeatureName}(undef, 0)
2920 end
2921 Base.unsafe_wrap(Array, fnames[], nf[], own=false)
2922 end
2923
2924 function test_feature(features::Vector{UInt8}, feat::FeatureName)
2925 bitidx = feat.bit
2926 u8idx = div(bitidx, 8) + 1
2927 bit = bitidx % 8
2928 return (features[u8idx] & (1 << bit)) != 0
2929 end
2930
2931 function show(io::IO, it::ImageTarget)
2932 print(io, it.name)
2933 if !isempty(it.ext_features)
2934 print(io, ",", it.ext_features)
2935 end
2936 print(io, "; flags=", it.flags)
2937 print(io, "; features_en=(")
2938 first = true
2939 for feat in feature_names()
2940 if test_feature(it.features_en, feat)
2941 name = Base.unsafe_string(feat.name)
2942 if first
2943 first = false
2944 print(io, name)
2945 else
2946 print(io, ", ", name)
2947 end
2948 end
2949 end
2950 print(io, ")")
2951 # Is feature_dis useful?
2952 end
2953
2954 # Set by FileWatching.__init__()
2955 global mkpidlock_hook
2956 global trymkpidlock_hook
2957 global parse_pidfile_hook
2958
2959 # The preferences hash is only known after precompilation so just assume no preferences.
2960 # Also ignore the active project, which means that if all other conditions are equal,
2961 # the same package cannot be precompiled from different projects and/or different preferences at the same time.
2962 compilecache_pidfile_path(pkg::PkgId) = compilecache_path(pkg, UInt64(0); project="") * ".pidfile"
2963
2964 const compilecache_pidlock_stale_age = 10
2965
2966 # Allows processes to wait if another process is precompiling a given source already.
2967 # The lock file mtime will be updated when held at most every `stale_age/2` seconds, with expected
2968 # variance of 10 seconds or more being infrequent but not unusual.
2969 # After `stale_age` seconds beyond the mtime of the lock file, the lock file is deleted and
2970 # precompilation will proceed if the locking process no longer exists or after `stale_age * 5`
2971 # seconds if the process does still exist.
2972 # If the lock is held by another host, it will conservatively wait `stale_age * 5`
2973 # seconds since processes cannot be checked remotely
2974 function maybe_cachefile_lock(f, pkg::PkgId, srcpath::String; stale_age=compilecache_pidlock_stale_age)
2975 if @isdefined(mkpidlock_hook) && @isdefined(trymkpidlock_hook) && @isdefined(parse_pidfile_hook)
2976 pidfile = compilecache_pidfile_path(pkg)
2977 cachefile = invokelatest(trymkpidlock_hook, f, pidfile; stale_age)
2978 if cachefile === false
2979 pid, hostname, age = invokelatest(parse_pidfile_hook, pidfile)
2980 verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug
2981 if isempty(hostname) || hostname == gethostname()
2982 @logmsg verbosity "Waiting for another process (pid: $pid) to finish precompiling $pkg. Pidfile: $pidfile"
2983 else
2984 @logmsg verbosity "Waiting for another machine (hostname: $hostname, pid: $pid) to finish precompiling $pkg. Pidfile: $pidfile"
2985 end
2986 # wait until the lock is available, but don't actually acquire it
2987 # returning nothing indicates a process waited for another
2988 return invokelatest(mkpidlock_hook, Returns(nothing), pidfile; stale_age)
2989 end
2990 return cachefile
2991 else
2992 # for packages loaded before FileWatching.__init__()
2993 f()
2994 end
2995 end
2996 # returns true if it "cachefile.ji" is stale relative to "modpath.jl" and build_id for modkey
2997 # otherwise returns the list of dependencies to also check
2998 @constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false)
2999 return stale_cachefile(PkgId(""), UInt128(0), modpath, cachefile; ignore_loaded)
3000 end
3001 @constprop :none function stale_cachefile(modkey::PkgId, build_id::UInt128, modpath::String, cachefile::String; ignore_loaded::Bool = false)
3002 io = open(cachefile, "r")
3003 try
3004 checksum = isvalid_cache_header(io)
3005 if iszero(checksum)
3006 @debug "Rejecting cache file $cachefile due to it containing an invalid cache header"
3007 return true # invalid cache file
3008 end
3009 modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags = parse_cache_header(io)
3010 if isempty(modules)
3011 return true # ignore empty file
3012 end
3013 if ccall(:jl_match_cache_flags, UInt8, (UInt8,), flags) == 0
3014 @debug """
3015 Rejecting cache file $cachefile for $modkey since the flags are mismatched
3016 current session: $(CacheFlags())
3017 cache file: $(CacheFlags(flags))
3018 """
3019 return true
3020 end
3021 pkgimage = !isempty(clone_targets)
3022 if pkgimage
3023 ocachefile = ocachefile_from_cachefile(cachefile)
3024 if JLOptions().use_pkgimages == 0
3025 # presence of clone_targets means native code cache
3026 @debug "Rejecting cache file $cachefile for $modkey since it would require usage of pkgimage"
3027 return true
3028 end
3029 rejection_reasons = check_clone_targets(clone_targets)
3030 if !isnothing(rejection_reasons)
3031 @debug("Rejecting cache file $cachefile for $modkey:",
3032 Reasons=rejection_reasons,
3033 var"Image Targets"=parse_image_targets(clone_targets),
3034 var"Current Targets"=current_image_targets())
3035 return true
3036 end
3037 if !isfile(ocachefile)
3038 @debug "Rejecting cache file $cachefile for $modkey since pkgimage $ocachefile was not found"
3039 return true
3040 end
3041 else
3042 ocachefile = nothing
3043 end
3044 id = first(modules)
3045 if id.first != modkey && modkey != PkgId("")
3046 @debug "Rejecting cache file $cachefile for $modkey since it is for $id instead"
3047 return true
3048 end
3049 if build_id != UInt128(0)
3050 id_build = (UInt128(checksum) << 64) | id.second
3051 if id_build != build_id
3052 @debug "Ignoring cache file $cachefile for $modkey ($((UUID(id_build)))) since it is does not provide desired build_id ($((UUID(build_id))))"
3053 return true
3054 end
3055 end
3056 id = id.first
3057 modules = Dict{PkgId, UInt64}(modules)
3058
3059 # Check if transitive dependencies can be fulfilled
3060 ndeps = length(required_modules)
3061 depmods = Vector{Any}(undef, ndeps)
3062 for i in 1:ndeps
3063 req_key, req_build_id = required_modules[i]
3064 # Module is already loaded
3065 if root_module_exists(req_key)
3066 M = root_module(req_key)
3067 if PkgId(M) == req_key && module_build_id(M) === req_build_id
3068 depmods[i] = M
3069 elseif ignore_loaded
3070 # Used by Pkg.precompile given that there it's ok to precompile different versions of loaded packages
3071 @goto locate_branch
3072 else
3073 @debug "Rejecting cache file $cachefile because module $req_key is already loaded and incompatible."
3074 return true # Won't be able to fulfill dependency
3075 end
3076 else
3077 @label locate_branch
3078 path = locate_package(req_key)
3079 if path === nothing
3080 @debug "Rejecting cache file $cachefile because dependency $req_key not found."
3081 return true # Won't be able to fulfill dependency
3082 end
3083 depmods[i] = (path, req_key, req_build_id)
3084 end
3085 end
3086
3087 # check if this file is going to provide one of our concrete dependencies
3088 # or if it provides a version that conflicts with our concrete dependencies
3089 # or neither
3090 skip_timecheck = false
3091 for (req_key, req_build_id) in _concrete_dependencies
3092 build_id = get(modules, req_key, UInt64(0))
3093 if build_id !== UInt64(0)
3094 build_id |= UInt128(checksum) << 64
3095 if build_id === req_build_id
3096 skip_timecheck = true
3097 break
3098 end
3099 @debug "Rejecting cache file $cachefile because it provides the wrong build_id (got $((UUID(build_id)))) for $req_key (want $(UUID(req_build_id)))"
3100 return true # cachefile doesn't provide the required version of the dependency
3101 end
3102 end
3103
3104 # now check if this file is fresh relative to its source files
3105 if !skip_timecheck
3106 if !samefile(includes[1].filename, modpath) && !samefile(fixup_stdlib_path(includes[1].filename), modpath)
3107 @debug "Rejecting cache file $cachefile because it is for file $(includes[1].filename) not file $modpath"
3108 return true # cache file was compiled from a different path
3109 end
3110 for (modkey, req_modkey) in requires
3111 # verify that `require(modkey, name(req_modkey))` ==> `req_modkey`
3112 if identify_package(modkey, req_modkey.name) != req_modkey
3113 @debug "Rejecting cache file $cachefile because uuid mapping for $modkey => $req_modkey has changed"
3114 return true
3115 end
3116 end
3117 for chi in includes
3118 f, ftime_req = chi.filename, chi.mtime
3119 if !ispath(f)
3120 _f = fixup_stdlib_path(f)
3121 if isfile(_f) && startswith(_f, Sys.STDLIB)
3122 # mtime is changed by extraction
3123 @debug "Skipping mtime check for file $f used by $cachefile, since it is a stdlib"
3124 continue
3125 end
3126 @debug "Rejecting stale cache file $cachefile because file $f does not exist"
3127 return true
3128 end
3129 ftime = mtime(f)
3130 is_stale = ( ftime != ftime_req ) &&
3131 ( ftime != floor(ftime_req) ) && # Issue #13606, PR #13613: compensate for Docker images rounding mtimes
3132 ( ftime != ceil(ftime_req) ) && # PR: #47433 Compensate for CirceCI's truncating of timestamps in its caching
3133 ( ftime != trunc(ftime_req, digits=6) ) && # Issue #20837, PR #20840: compensate for GlusterFS truncating mtimes to microseconds
3134 ( ftime != 1.0 ) && # PR #43090: provide compatibility with Nix mtime.
3135 !( 0 < (ftime_req - ftime) < 1e-6 ) # PR #45552: Compensate for Windows tar giving mtimes that may be incorrect by up to one microsecond
3136 if is_stale
3137 @debug "Rejecting stale cache file $cachefile (mtime $ftime_req) because file $f (mtime $ftime) has changed"
3138 return true
3139 end
3140 end
3141 end
3142
3143 if !isvalid_file_crc(io)
3144 @debug "Rejecting cache file $cachefile because it has an invalid checksum"
3145 return true
3146 end
3147
3148 if pkgimage
3149 if !isvalid_pkgimage_crc(io, ocachefile::String)
3150 @debug "Rejecting cache file $cachefile because $ocachefile has an invalid checksum"
3151 return true
3152 end
3153 end
3154
3155 curr_prefs_hash = get_preferences_hash(id.uuid, prefs)
3156 if prefs_hash != curr_prefs_hash
3157 @debug "Rejecting cache file $cachefile because preferences hash does not match 0x$(string(prefs_hash, base=16)) != 0x$(string(curr_prefs_hash, base=16))"
3158 return true
3159 end
3160
3161 return depmods, ocachefile # fresh cachefile
3162 finally
3163 close(io)
3164 end
3165 end
3166
3167 """
3168 @__FILE__ -> String
3169
3170 Expand to a string with the path to the file containing the
3171 macrocall, or an empty string if evaluated by `julia -e <expr>`.
3172 Return `nothing` if the macro was missing parser source information.
3173 Alternatively see [`PROGRAM_FILE`](@ref).
3174 """
3175 macro __FILE__()
3176 __source__.file === nothing && return nothing
3177 return String(__source__.file::Symbol)
3178 end
3179
3180 """
3181 @__DIR__ -> String
3182
3183 Expand to a string with the absolute path to the directory of the file
3184 containing the macrocall.
3185 Return the current working directory if run from a REPL or if evaluated by `julia -e <expr>`.
3186 """
3187 macro __DIR__()
3188 __source__.file === nothing && return nothing
3189 _dirname = dirname(String(__source__.file::Symbol))
3190 return isempty(_dirname) ? pwd() : abspath(_dirname)
3191 end
3192
3193 """
3194 precompile(f, argtypes::Tuple{Vararg{Any}})
3195
3196 Compile the given function `f` for the argument tuple (of types) `argtypes`, but do not execute it.
3197 """
3198 function precompile(@nospecialize(f), @nospecialize(argtypes::Tuple))
3199 precompile(Tuple{Core.Typeof(f), argtypes...})
3200 end
3201
3202 const ENABLE_PRECOMPILE_WARNINGS = Ref(false)
3203 function precompile(@nospecialize(argt::Type))
3204 ret = ccall(:jl_compile_hint, Int32, (Any,), argt) != 0
3205 if !ret && ENABLE_PRECOMPILE_WARNINGS[]
3206 @warn "Inactive precompile statement" maxlog=100 form=argt _module=nothing _file=nothing _line=0
3207 end
3208 return ret
3209 end
3210
3211 # Variants that work for `invoke`d calls for which the signature may not be sufficient
3212 precompile(mi::Core.MethodInstance, world::UInt=get_world_counter()) =
3213 (ccall(:jl_compile_method_instance, Cvoid, (Any, Any, UInt), mi, C_NULL, world); return true)
3214
3215 """
3216 precompile(f, argtypes::Tuple{Vararg{Any}}, m::Method)
3217
3218 Precompile a specific method for the given argument types. This may be used to precompile
3219 a different method than the one that would ordinarily be chosen by dispatch, thus
3220 mimicking `invoke`.
3221 """
3222 function precompile(@nospecialize(f), @nospecialize(argtypes::Tuple), m::Method)
3223 precompile(Tuple{Core.Typeof(f), argtypes...}, m)
3224 end
3225
3226 function precompile(@nospecialize(argt::Type), m::Method)
3227 atype, sparams = ccall(:jl_type_intersection_with_env, Any, (Any, Any), argt, m.sig)::SimpleVector
3228 mi = Core.Compiler.specialize_method(m, atype, sparams)
3229 return precompile(mi)
3230 end
3231
3232 precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), Nothing))
3233 precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String))
3234 precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), IO, IO))
3235 precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), IO, IO))