Line | Exclusive | Inclusive | Code |
---|---|---|---|
1 | # This file is a part of Julia. License is MIT: https://julialang.org/license | ||
2 | |||
3 | # Base.require is the implementation for the `import` statement | ||
4 | const require_lock = ReentrantLock() | ||
5 | |||
6 | # Cross-platform case-sensitive path canonicalization | ||
7 | |||
8 | if Sys.isunix() && !Sys.isapple() | ||
9 | # assume case-sensitive filesystems, don't have to do anything | ||
10 | isfile_casesensitive(path) = isaccessiblefile(path) | ||
11 | elseif Sys.iswindows() | ||
12 | # GetLongPathName Win32 function returns the case-preserved filename on NTFS. | ||
13 | function isfile_casesensitive(path) | ||
14 | isaccessiblefile(path) || return false # Fail fast | ||
15 | basename(Filesystem.longpath(path)) == basename(path) | ||
16 | end | ||
17 | elseif Sys.isapple() | ||
18 | # HFS+ filesystem is case-preserving. The getattrlist API returns | ||
19 | # a case-preserved filename. In the rare event that HFS+ is operating | ||
20 | # in case-sensitive mode, this will still work but will be redundant. | ||
21 | |||
22 | # Constants from <sys/attr.h> | ||
23 | const ATRATTR_BIT_MAP_COUNT = 5 | ||
24 | const ATTR_CMN_NAME = 1 | ||
25 | const BITMAPCOUNT = 1 | ||
26 | const COMMONATTR = 5 | ||
27 | const FSOPT_NOFOLLOW = 1 # Don't follow symbolic links | ||
28 | |||
29 | const attr_list = zeros(UInt8, 24) | ||
30 | attr_list[BITMAPCOUNT] = ATRATTR_BIT_MAP_COUNT | ||
31 | attr_list[COMMONATTR] = ATTR_CMN_NAME | ||
32 | |||
33 | # This essentially corresponds to the following C code: | ||
34 | # attrlist attr_list; | ||
35 | # memset(&attr_list, 0, sizeof(attr_list)); | ||
36 | # attr_list.bitmapcount = ATTR_BIT_MAP_COUNT; | ||
37 | # attr_list.commonattr = ATTR_CMN_NAME; | ||
38 | # struct Buffer { | ||
39 | # u_int32_t total_length; | ||
40 | # u_int32_t filename_offset; | ||
41 | # u_int32_t filename_length; | ||
42 | # char filename[max_filename_length]; | ||
43 | # }; | ||
44 | # Buffer buf; | ||
45 | # getattrpath(path, &attr_list, &buf, sizeof(buf), FSOPT_NOFOLLOW); | ||
46 | function isfile_casesensitive(path) | ||
47 | isaccessiblefile(path) || return false | ||
48 | path_basename = String(basename(path)) | ||
49 | local casepreserved_basename | ||
50 | header_size = 12 | ||
51 | buf = Vector{UInt8}(undef, length(path_basename) + header_size + 1) | ||
52 | while true | ||
53 | ret = ccall(:getattrlist, Cint, | ||
54 | (Cstring, Ptr{Cvoid}, Ptr{Cvoid}, Csize_t, Culong), | ||
55 | path, attr_list, buf, sizeof(buf), FSOPT_NOFOLLOW) | ||
56 | systemerror(:getattrlist, ret ≠ 0) | ||
57 | filename_length = GC.@preserve buf unsafe_load( | ||
58 | convert(Ptr{UInt32}, pointer(buf) + 8)) | ||
59 | if (filename_length + header_size) > length(buf) | ||
60 | resize!(buf, filename_length + header_size) | ||
61 | continue | ||
62 | end | ||
63 | casepreserved_basename = | ||
64 | view(buf, (header_size+1):(header_size+filename_length-1)) | ||
65 | break | ||
66 | end | ||
67 | # Hack to compensate for inability to create a string from a subarray with no allocations. | ||
68 | codeunits(path_basename) == casepreserved_basename && return true | ||
69 | |||
70 | # If there is no match, it's possible that the file does exist but HFS+ | ||
71 | # performed unicode normalization. See https://developer.apple.com/library/mac/qa/qa1235/_index.html. | ||
72 | isascii(path_basename) && return false | ||
73 | codeunits(Unicode.normalize(path_basename, :NFD)) == casepreserved_basename | ||
74 | end | ||
75 | else | ||
76 | # Generic fallback that performs a slow directory listing. | ||
77 | function isfile_casesensitive(path) | ||
78 | isaccessiblefile(path) || return false | ||
79 | dir, filename = splitdir(path) | ||
80 | any(readdir(dir) .== filename) | ||
81 | end | ||
82 | end | ||
83 | |||
84 | # Check if the file is accessible. If stat fails return `false` | ||
85 | |||
86 | function isaccessibledir(dir) | ||
87 | return try | ||
88 | isdir(dir) | ||
89 | catch err | ||
90 | err isa IOError || rethrow() | ||
91 | false | ||
92 | end | ||
93 | end | ||
94 | |||
95 | function isaccessiblefile(file) | ||
96 | return try | ||
97 | isfile(file) | ||
98 | catch err | ||
99 | err isa IOError || rethrow() | ||
100 | false | ||
101 | end | ||
102 | end | ||
103 | |||
104 | function isaccessiblepath(path) | ||
105 | return try | ||
106 | ispath(path) | ||
107 | catch err | ||
108 | err isa IOError || rethrow() | ||
109 | false | ||
110 | end | ||
111 | end | ||
112 | |||
113 | ## SHA1 ## | ||
114 | |||
115 | struct SHA1 | ||
116 | bytes::NTuple{20, UInt8} | ||
117 | end | ||
118 | function SHA1(bytes::Vector{UInt8}) | ||
119 | length(bytes) == 20 || | ||
120 | throw(ArgumentError("wrong number of bytes for SHA1 hash: $(length(bytes))")) | ||
121 | return SHA1(ntuple(i->bytes[i], Val(20))) | ||
122 | end | ||
123 | SHA1(s::AbstractString) = SHA1(hex2bytes(s)) | ||
124 | parse(::Type{SHA1}, s::AbstractString) = SHA1(s) | ||
125 | function tryparse(::Type{SHA1}, s::AbstractString) | ||
126 | try | ||
127 | return parse(SHA1, s) | ||
128 | catch e | ||
129 | if isa(e, ArgumentError) | ||
130 | return nothing | ||
131 | end | ||
132 | rethrow(e) | ||
133 | end | ||
134 | end | ||
135 | |||
136 | string(hash::SHA1) = bytes2hex(hash.bytes) | ||
137 | print(io::IO, hash::SHA1) = bytes2hex(io, hash.bytes) | ||
138 | show(io::IO, hash::SHA1) = print(io, "SHA1(\"", hash, "\")") | ||
139 | |||
140 | isless(a::SHA1, b::SHA1) = isless(a.bytes, b.bytes) | ||
141 | hash(a::SHA1, h::UInt) = hash((SHA1, a.bytes), h) | ||
142 | ==(a::SHA1, b::SHA1) = a.bytes == b.bytes | ||
143 | |||
144 | # fake uuid5 function (for self-assigned UUIDs) | ||
145 | # TODO: delete and use real uuid5 once it's in stdlib | ||
146 | |||
147 | function uuid5(namespace::UUID, key::String) | ||
148 | u::UInt128 = 0 | ||
149 | h = hash(namespace) | ||
150 | for _ = 1:sizeof(u)÷sizeof(h) | ||
151 | u <<= sizeof(h) << 3 | ||
152 | u |= (h = hash(key, h)) | ||
153 | end | ||
154 | u &= 0xffffffffffff0fff3fffffffffffffff | ||
155 | u |= 0x00000000000050008000000000000000 | ||
156 | return UUID(u) | ||
157 | end | ||
158 | |||
159 | const ns_dummy_uuid = UUID("fe0723d6-3a44-4c41-8065-ee0f42c8ceab") | ||
160 | |||
161 | function dummy_uuid(project_file::String) | ||
162 | @lock require_lock begin | ||
163 | cache = LOADING_CACHE[] | ||
164 | if cache !== nothing | ||
165 | uuid = get(cache.dummy_uuid, project_file, nothing) | ||
166 | uuid === nothing || return uuid | ||
167 | end | ||
168 | project_path = try | ||
169 | realpath(project_file) | ||
170 | catch ex | ||
171 | ex isa IOError || rethrow() | ||
172 | project_file | ||
173 | end | ||
174 | uuid = uuid5(ns_dummy_uuid, project_path) | ||
175 | if cache !== nothing | ||
176 | cache.dummy_uuid[project_file] = uuid | ||
177 | end | ||
178 | return uuid | ||
179 | end | ||
180 | end | ||
181 | |||
182 | ## package path slugs: turning UUID + SHA1 into a pair of 4-byte "slugs" ## | ||
183 | |||
184 | const slug_chars = String(['A':'Z'; 'a':'z'; '0':'9']) | ||
185 | |||
186 | function slug(x::UInt32, p::Int) | ||
187 | y::UInt32 = x | ||
188 | sprint(sizehint=p) do io | ||
189 | n = length(slug_chars) | ||
190 | for i = 1:p | ||
191 | y, d = divrem(y, n) | ||
192 | write(io, slug_chars[1+d]) | ||
193 | end | ||
194 | end | ||
195 | end | ||
196 | |||
197 | function package_slug(uuid::UUID, p::Int=5) | ||
198 | crc = _crc32c(uuid) | ||
199 | return slug(crc, p) | ||
200 | end | ||
201 | |||
202 | function version_slug(uuid::UUID, sha1::SHA1, p::Int=5) | ||
203 | crc = _crc32c(uuid) | ||
204 | crc = _crc32c(sha1.bytes, crc) | ||
205 | return slug(crc, p) | ||
206 | end | ||
207 | |||
208 | mutable struct CachedTOMLDict | ||
209 | path::String | ||
210 | inode::UInt64 | ||
211 | mtime::Float64 | ||
212 | size::Int64 | ||
213 | hash::UInt32 | ||
214 | d::Dict{String, Any} | ||
215 | end | ||
216 | |||
217 | function CachedTOMLDict(p::TOML.Parser, path::String) | ||
218 | s = stat(path) | ||
219 | content = read(path) | ||
220 | crc32 = _crc32c(content) | ||
221 | TOML.reinit!(p, String(content); filepath=path) | ||
222 | d = TOML.parse(p) | ||
223 | return CachedTOMLDict( | ||
224 | path, | ||
225 | s.inode, | ||
226 | s.mtime, | ||
227 | s.size, | ||
228 | crc32, | ||
229 | d, | ||
230 | ) | ||
231 | end | ||
232 | |||
233 | function get_updated_dict(p::TOML.Parser, f::CachedTOMLDict) | ||
234 | s = stat(f.path) | ||
235 | # note, this might miss very rapid in-place updates, such that mtime is | ||
236 | # identical but that is solvable by not doing in-place updates, and not | ||
237 | # rapidly changing these files | ||
238 | if s.inode != f.inode || s.mtime != f.mtime || f.size != s.size | ||
239 | content = read(f.path) | ||
240 | new_hash = _crc32c(content) | ||
241 | if new_hash != f.hash | ||
242 | f.inode = s.inode | ||
243 | f.mtime = s.mtime | ||
244 | f.size = s.size | ||
245 | f.hash = new_hash | ||
246 | TOML.reinit!(p, String(content); filepath=f.path) | ||
247 | return f.d = TOML.parse(p) | ||
248 | end | ||
249 | end | ||
250 | return f.d | ||
251 | end | ||
252 | |||
253 | struct LoadingCache | ||
254 | load_path::Vector{String} | ||
255 | dummy_uuid::Dict{String, UUID} | ||
256 | env_project_file::Dict{String, Union{Bool, String}} | ||
257 | project_file_manifest_path::Dict{String, Union{Nothing, String}} | ||
258 | require_parsed::Set{String} | ||
259 | identified_where::Dict{Tuple{PkgId, String}, Union{Nothing, Tuple{PkgId, Union{Nothing, String}}}} | ||
260 | identified::Dict{String, Union{Nothing, Tuple{PkgId, Union{Nothing, String}}}} | ||
261 | located::Dict{Tuple{PkgId, Union{String, Nothing}}, Union{Tuple{Union{String, Nothing}, Union{String, Nothing}}, Nothing}} | ||
262 | end | ||
263 | const LOADING_CACHE = Ref{Union{LoadingCache, Nothing}}(nothing) | ||
264 | LoadingCache() = LoadingCache(load_path(), Dict(), Dict(), Dict(), Set(), Dict(), Dict(), Dict()) | ||
265 | |||
266 | |||
267 | struct TOMLCache | ||
268 | p::TOML.Parser | ||
269 | d::Dict{String, CachedTOMLDict} | ||
270 | end | ||
271 | const TOML_CACHE = TOMLCache(TOML.Parser(), Dict{String, Dict{String, Any}}()) | ||
272 | |||
273 | parsed_toml(project_file::AbstractString) = parsed_toml(project_file, TOML_CACHE, require_lock) | ||
274 | function parsed_toml(project_file::AbstractString, toml_cache::TOMLCache, toml_lock::ReentrantLock) | ||
275 | lock(toml_lock) do | ||
276 | cache = LOADING_CACHE[] | ||
277 | dd = if !haskey(toml_cache.d, project_file) | ||
278 | d = CachedTOMLDict(toml_cache.p, project_file) | ||
279 | toml_cache.d[project_file] = d | ||
280 | d.d | ||
281 | else | ||
282 | d = toml_cache.d[project_file] | ||
283 | # We are in a require call and have already parsed this TOML file | ||
284 | # assume that it is unchanged to avoid hitting disk | ||
285 | if cache !== nothing && project_file in cache.require_parsed | ||
286 | d.d | ||
287 | else | ||
288 | get_updated_dict(toml_cache.p, d) | ||
289 | end | ||
290 | end | ||
291 | if cache !== nothing | ||
292 | push!(cache.require_parsed, project_file) | ||
293 | end | ||
294 | return dd | ||
295 | end | ||
296 | end | ||
297 | |||
298 | ## package identification: determine unique identity of package to be loaded ## | ||
299 | |||
300 | # Used by Pkg but not used in loading itself | ||
301 | function find_package(arg) | ||
302 | pkgenv = identify_package_env(arg) | ||
303 | pkgenv === nothing && return nothing | ||
304 | pkg, env = pkgenv | ||
305 | return locate_package(pkg, env) | ||
306 | end | ||
307 | |||
308 | """ | ||
309 | Base.identify_package_env(name::String)::Union{Tuple{PkgId, String}, Nothing} | ||
310 | Base.identify_package_env(where::Union{Module,PkgId}, name::String)::Union{Tuple{PkgId, String} Nothing} | ||
311 | |||
312 | Same as [`Base.identify_package`](@ref) except that the path to the environment where the package is identified | ||
313 | is also returned. | ||
314 | """ | ||
315 | identify_package_env(where::Module, name::String) = identify_package_env(PkgId(where), name) | ||
316 | function identify_package_env(where::PkgId, name::String) | ||
317 | cache = LOADING_CACHE[] | ||
318 | if cache !== nothing | ||
319 | pkg_env = get(cache.identified_where, (where, name), nothing) | ||
320 | pkg_env === nothing || return pkg_env | ||
321 | end | ||
322 | pkg_env = nothing | ||
323 | if where.name === name | ||
324 | pkg_env = where, nothing | ||
325 | elseif where.uuid === nothing | ||
326 | pkg_env = identify_package_env(name) # ignore `where` | ||
327 | else | ||
328 | for env in load_path() | ||
329 | pkgid = manifest_deps_get(env, where, name) | ||
330 | pkgid === nothing && continue # not found--keep looking | ||
331 | if pkgid.uuid !== nothing | ||
332 | pkg_env = pkgid, env # found in explicit environment--use it | ||
333 | end | ||
334 | break # found in implicit environment--return "not found" | ||
335 | end | ||
336 | end | ||
337 | if cache !== nothing | ||
338 | cache.identified_where[(where, name)] = pkg_env | ||
339 | end | ||
340 | return pkg_env | ||
341 | end | ||
342 | function identify_package_env(name::String) | ||
343 | cache = LOADING_CACHE[] | ||
344 | if cache !== nothing | ||
345 | pkg_env = get(cache.identified, name, nothing) | ||
346 | pkg_env === nothing || return pkg_env | ||
347 | end | ||
348 | pkg_env = nothing | ||
349 | for env in load_path() | ||
350 | pkg = project_deps_get(env, name) | ||
351 | if pkg !== nothing | ||
352 | pkg_env = pkg, env # found--return it | ||
353 | break | ||
354 | end | ||
355 | end | ||
356 | if cache !== nothing | ||
357 | cache.identified[name] = pkg_env | ||
358 | end | ||
359 | return pkg_env | ||
360 | end | ||
361 | |||
362 | _nothing_or_first(x) = x === nothing ? nothing : first(x) | ||
363 | |||
364 | """ | ||
365 | Base.identify_package(name::String)::Union{PkgId, Nothing} | ||
366 | Base.identify_package(where::Union{Module,PkgId}, name::String)::Union{PkgId, Nothing} | ||
367 | |||
368 | Identify the package by its name from the current environment stack, returning | ||
369 | its `PkgId`, or `nothing` if it cannot be found. | ||
370 | |||
371 | If only the `name` argument is provided, it searches each environment in the | ||
372 | stack and its named direct dependencies. | ||
373 | |||
374 | There `where` argument provides the context from where to search for the | ||
375 | package: in this case it first checks if the name matches the context itself, | ||
376 | otherwise it searches all recursive dependencies (from the resolved manifest of | ||
377 | each environment) until it locates the context `where`, and from there | ||
378 | identifies the dependency with the corresponding name. | ||
379 | |||
380 | ```julia-repl | ||
381 | julia> Base.identify_package("Pkg") # Pkg is a dependency of the default environment | ||
382 | Pkg [44cfe95a-1eb2-52ea-b672-e2afdf69b78f] | ||
383 | |||
384 | julia> using LinearAlgebra | ||
385 | |||
386 | julia> Base.identify_package(LinearAlgebra, "Pkg") # Pkg is not a dependency of LinearAlgebra | ||
387 | ``` | ||
388 | """ | ||
389 | identify_package(where::Module, name::String) = _nothing_or_first(identify_package_env(where, name)) | ||
390 | identify_package(where::PkgId, name::String) = _nothing_or_first(identify_package_env(where, name)) | ||
391 | identify_package(name::String) = _nothing_or_first(identify_package_env(name)) | ||
392 | |||
393 | function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing) | ||
394 | cache = LOADING_CACHE[] | ||
395 | if cache !== nothing | ||
396 | pathenv = get(cache.located, (pkg, stopenv), nothing) | ||
397 | pathenv === nothing || return pathenv | ||
398 | end | ||
399 | path = nothing | ||
400 | env′ = nothing | ||
401 | if pkg.uuid === nothing | ||
402 | for env in load_path() | ||
403 | env′ = env | ||
404 | # look for the toplevel pkg `pkg.name` in this entry | ||
405 | found = project_deps_get(env, pkg.name) | ||
406 | if found !== nothing | ||
407 | @assert found.name == pkg.name | ||
408 | if found.uuid === nothing | ||
409 | # pkg.name is present in this directory or project file, | ||
410 | # return the path the entry point for the code, if it could be found | ||
411 | # otherwise, signal failure | ||
412 | path = implicit_manifest_uuid_path(env, pkg) | ||
413 | @goto done | ||
414 | end | ||
415 | end | ||
416 | if !(loading_extension || precompiling_extension) | ||
417 | stopenv == env && @goto done | ||
418 | end | ||
419 | end | ||
420 | else | ||
421 | for env in load_path() | ||
422 | env′ = env | ||
423 | path = manifest_uuid_path(env, pkg) | ||
424 | # missing is used as a sentinel to stop looking further down in envs | ||
425 | if path === missing | ||
426 | path = nothing | ||
427 | @goto done | ||
428 | end | ||
429 | if path !== nothing | ||
430 | path = entry_path(path, pkg.name) | ||
431 | @goto done | ||
432 | end | ||
433 | if !(loading_extension || precompiling_extension) | ||
434 | stopenv == env && break | ||
435 | end | ||
436 | end | ||
437 | # Allow loading of stdlibs if the name/uuid are given | ||
438 | # e.g. if they have been explicitly added to the project/manifest | ||
439 | mbypath = manifest_uuid_path(Sys.STDLIB, pkg) | ||
440 | if mbypath isa String | ||
441 | path = entry_path(mbypath, pkg.name) | ||
442 | @goto done | ||
443 | end | ||
444 | end | ||
445 | @label done | ||
446 | if cache !== nothing | ||
447 | cache.located[(pkg, stopenv)] = path, env′ | ||
448 | end | ||
449 | return path, env′ | ||
450 | end | ||
451 | |||
452 | """ | ||
453 | Base.locate_package(pkg::PkgId)::Union{String, Nothing} | ||
454 | |||
455 | The path to the entry-point file for the package corresponding to the identifier | ||
456 | `pkg`, or `nothing` if not found. See also [`identify_package`](@ref). | ||
457 | |||
458 | ```julia-repl | ||
459 | julia> pkg = Base.identify_package("Pkg") | ||
460 | Pkg [44cfe95a-1eb2-52ea-b672-e2afdf69b78f] | ||
461 | |||
462 | julia> Base.locate_package(pkg) | ||
463 | "/path/to/julia/stdlib/v$(VERSION.major).$(VERSION.minor)/Pkg/src/Pkg.jl" | ||
464 | ``` | ||
465 | """ | ||
466 | function locate_package(pkg::PkgId, stopenv::Union{String, Nothing}=nothing)::Union{Nothing,String} | ||
467 | _nothing_or_first(locate_package_env(pkg, stopenv)) | ||
468 | end | ||
469 | |||
470 | """ | ||
471 | pathof(m::Module) | ||
472 | |||
473 | Return the path of the `m.jl` file that was used to `import` module `m`, | ||
474 | or `nothing` if `m` was not imported from a package. | ||
475 | |||
476 | Use [`dirname`](@ref) to get the directory part and [`basename`](@ref) | ||
477 | to get the file name part of the path. | ||
478 | """ | ||
479 | function pathof(m::Module) | ||
480 | @lock require_lock begin | ||
481 | pkgid = get(module_keys, m, nothing) | ||
482 | pkgid === nothing && return nothing | ||
483 | origin = get(pkgorigins, pkgid, nothing) | ||
484 | origin === nothing && return nothing | ||
485 | path = origin.path | ||
486 | path === nothing && return nothing | ||
487 | return fixup_stdlib_path(path) | ||
488 | end | ||
489 | end | ||
490 | |||
491 | """ | ||
492 | pkgdir(m::Module[, paths::String...]) | ||
493 | |||
494 | Return the root directory of the package that declared module `m`, | ||
495 | or `nothing` if `m` was not declared in a package. Optionally further | ||
496 | path component strings can be provided to construct a path within the | ||
497 | package root. | ||
498 | |||
499 | To get the root directory of the package that implements the current module | ||
500 | the form `pkgdir(@__MODULE__)` can be used. | ||
501 | |||
502 | ```julia-repl | ||
503 | julia> pkgdir(Foo) | ||
504 | "/path/to/Foo.jl" | ||
505 | |||
506 | julia> pkgdir(Foo, "src", "file.jl") | ||
507 | "/path/to/Foo.jl/src/file.jl" | ||
508 | ``` | ||
509 | |||
510 | !!! compat "Julia 1.7" | ||
511 | The optional argument `paths` requires at least Julia 1.7. | ||
512 | """ | ||
513 | function pkgdir(m::Module, paths::String...) | ||
514 | rootmodule = moduleroot(m) | ||
515 | path = pathof(rootmodule) | ||
516 | path === nothing && return nothing | ||
517 | return joinpath(dirname(dirname(path)), paths...) | ||
518 | end | ||
519 | |||
520 | function get_pkgversion_from_path(path) | ||
521 | project_file = locate_project_file(path) | ||
522 | if project_file isa String | ||
523 | d = parsed_toml(project_file) | ||
524 | v = get(d, "version", nothing) | ||
525 | if v !== nothing | ||
526 | return VersionNumber(v::String) | ||
527 | end | ||
528 | end | ||
529 | return nothing | ||
530 | end | ||
531 | |||
532 | """ | ||
533 | pkgversion(m::Module) | ||
534 | |||
535 | Return the version of the package that imported module `m`, | ||
536 | or `nothing` if `m` was not imported from a package, or imported | ||
537 | from a package without a version field set. | ||
538 | |||
539 | The version is read from the package's Project.toml during package | ||
540 | load. | ||
541 | |||
542 | To get the version of the package that imported the current module | ||
543 | the form `pkgversion(@__MODULE__)` can be used. | ||
544 | |||
545 | !!! compat "Julia 1.9" | ||
546 | This function was introduced in Julia 1.9. | ||
547 | """ | ||
548 | function pkgversion(m::Module) | ||
549 | path = pkgdir(m) | ||
550 | path === nothing && return nothing | ||
551 | @lock require_lock begin | ||
552 | v = get_pkgversion_from_path(path) | ||
553 | pkgorigin = get(pkgorigins, PkgId(moduleroot(m)), nothing) | ||
554 | # Cache the version | ||
555 | if pkgorigin !== nothing && pkgorigin.version === nothing | ||
556 | pkgorigin.version = v | ||
557 | end | ||
558 | return v | ||
559 | end | ||
560 | end | ||
561 | |||
562 | ## generic project & manifest API ## | ||
563 | |||
564 | const project_names = ("JuliaProject.toml", "Project.toml") | ||
565 | const manifest_names = ("JuliaManifest.toml", "Manifest.toml") | ||
566 | const preferences_names = ("JuliaLocalPreferences.toml", "LocalPreferences.toml") | ||
567 | |||
568 | function locate_project_file(env::String) | ||
569 | for proj in project_names | ||
570 | project_file = joinpath(env, proj) | ||
571 | if isfile_casesensitive(project_file) | ||
572 | return project_file | ||
573 | end | ||
574 | end | ||
575 | return true | ||
576 | end | ||
577 | |||
578 | # classify the LOAD_PATH entry to be one of: | ||
579 | # - `false`: nonexistent / nothing to see here | ||
580 | # - `true`: `env` is an implicit environment | ||
581 | # - `path`: the path of an explicit project file | ||
582 | function env_project_file(env::String)::Union{Bool,String} | ||
583 | @lock require_lock begin | ||
584 | cache = LOADING_CACHE[] | ||
585 | if cache !== nothing | ||
586 | project_file = get(cache.env_project_file, env, nothing) | ||
587 | project_file === nothing || return project_file | ||
588 | end | ||
589 | if isdir(env) | ||
590 | project_file = locate_project_file(env) | ||
591 | elseif basename(env) in project_names && isfile_casesensitive(env) | ||
592 | project_file = env | ||
593 | else | ||
594 | project_file = false | ||
595 | end | ||
596 | if cache !== nothing | ||
597 | cache.env_project_file[env] = project_file | ||
598 | end | ||
599 | return project_file | ||
600 | end | ||
601 | end | ||
602 | |||
603 | function project_deps_get(env::String, name::String)::Union{Nothing,PkgId} | ||
604 | project_file = env_project_file(env) | ||
605 | if project_file isa String | ||
606 | pkg_uuid = explicit_project_deps_get(project_file, name) | ||
607 | pkg_uuid === nothing || return PkgId(pkg_uuid, name) | ||
608 | elseif project_file | ||
609 | return implicit_project_deps_get(env, name) | ||
610 | end | ||
611 | return nothing | ||
612 | end | ||
613 | |||
614 | function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Nothing,PkgId} | ||
615 | uuid = where.uuid | ||
616 | @assert uuid !== nothing | ||
617 | project_file = env_project_file(env) | ||
618 | if project_file isa String | ||
619 | # first check if `where` names the Project itself | ||
620 | proj = project_file_name_uuid(project_file, where.name) | ||
621 | if proj == where | ||
622 | # if `where` matches the project, use [deps] section as manifest, and stop searching | ||
623 | pkg_uuid = explicit_project_deps_get(project_file, name) | ||
624 | return PkgId(pkg_uuid, name) | ||
625 | end | ||
626 | d = parsed_toml(project_file) | ||
627 | exts = get(d, "extensions", nothing)::Union{Dict{String, Any}, Nothing} | ||
628 | if exts !== nothing | ||
629 | # Check if `where` is an extension of the project | ||
630 | if where.name in keys(exts) && where.uuid == uuid5(proj.uuid::UUID, where.name) | ||
631 | # Extensions can load weak deps... | ||
632 | weakdeps = get(d, "weakdeps", nothing)::Union{Dict{String, Any}, Nothing} | ||
633 | if weakdeps !== nothing | ||
634 | wuuid = get(weakdeps, name, nothing)::Union{String, Nothing} | ||
635 | if wuuid !== nothing | ||
636 | return PkgId(UUID(wuuid), name) | ||
637 | end | ||
638 | end | ||
639 | # ... and they can load same deps as the project itself | ||
640 | mby_uuid = explicit_project_deps_get(project_file, name) | ||
641 | mby_uuid === nothing || return PkgId(mby_uuid, name) | ||
642 | end | ||
643 | end | ||
644 | # look for manifest file and `where` stanza | ||
645 | return explicit_manifest_deps_get(project_file, where, name) | ||
646 | elseif project_file | ||
647 | # if env names a directory, search it | ||
648 | return implicit_manifest_deps_get(env, where, name) | ||
649 | end | ||
650 | return nothing | ||
651 | end | ||
652 | |||
653 | function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String,Missing} | ||
654 | project_file = env_project_file(env) | ||
655 | if project_file isa String | ||
656 | proj = project_file_name_uuid(project_file, pkg.name) | ||
657 | if proj == pkg | ||
658 | # if `pkg` matches the project, return the project itself | ||
659 | return project_file_path(project_file) | ||
660 | end | ||
661 | mby_ext = project_file_ext_path(project_file, pkg.name) | ||
662 | mby_ext === nothing || return mby_ext | ||
663 | # look for manifest file and `where` stanza | ||
664 | return explicit_manifest_uuid_path(project_file, pkg) | ||
665 | elseif project_file | ||
666 | # if env names a directory, search it | ||
667 | return implicit_manifest_uuid_path(env, pkg) | ||
668 | end | ||
669 | return nothing | ||
670 | end | ||
671 | |||
672 | |||
673 | function find_ext_path(project_path::String, extname::String) | ||
674 | extfiledir = joinpath(project_path, "ext", extname, extname * ".jl") | ||
675 | isfile(extfiledir) && return extfiledir | ||
676 | return joinpath(project_path, "ext", extname * ".jl") | ||
677 | end | ||
678 | |||
679 | function project_file_ext_path(project_file::String, name::String) | ||
680 | d = parsed_toml(project_file) | ||
681 | p = project_file_path(project_file) | ||
682 | exts = get(d, "extensions", nothing)::Union{Dict{String, Any}, Nothing} | ||
683 | if exts !== nothing | ||
684 | if name in keys(exts) | ||
685 | return find_ext_path(p, name) | ||
686 | end | ||
687 | end | ||
688 | return nothing | ||
689 | end | ||
690 | |||
691 | # find project file's top-level UUID entry (or nothing) | ||
692 | function project_file_name_uuid(project_file::String, name::String)::PkgId | ||
693 | d = parsed_toml(project_file) | ||
694 | uuid′ = get(d, "uuid", nothing)::Union{String, Nothing} | ||
695 | uuid = uuid′ === nothing ? dummy_uuid(project_file) : UUID(uuid′) | ||
696 | name = get(d, "name", name)::String | ||
697 | return PkgId(uuid, name) | ||
698 | end | ||
699 | |||
700 | function project_file_path(project_file::String) | ||
701 | d = parsed_toml(project_file) | ||
702 | joinpath(dirname(project_file), get(d, "path", "")::String) | ||
703 | end | ||
704 | |||
705 | # find project file's corresponding manifest file | ||
706 | function project_file_manifest_path(project_file::String)::Union{Nothing,String} | ||
707 | @lock require_lock begin | ||
708 | cache = LOADING_CACHE[] | ||
709 | if cache !== nothing | ||
710 | manifest_path = get(cache.project_file_manifest_path, project_file, missing) | ||
711 | manifest_path === missing || return manifest_path | ||
712 | end | ||
713 | dir = abspath(dirname(project_file)) | ||
714 | d = parsed_toml(project_file) | ||
715 | explicit_manifest = get(d, "manifest", nothing)::Union{String, Nothing} | ||
716 | manifest_path = nothing | ||
717 | if explicit_manifest !== nothing | ||
718 | manifest_file = normpath(joinpath(dir, explicit_manifest)) | ||
719 | if isfile_casesensitive(manifest_file) | ||
720 | manifest_path = manifest_file | ||
721 | end | ||
722 | end | ||
723 | if manifest_path === nothing | ||
724 | for mfst in manifest_names | ||
725 | manifest_file = joinpath(dir, mfst) | ||
726 | if isfile_casesensitive(manifest_file) | ||
727 | manifest_path = manifest_file | ||
728 | break | ||
729 | end | ||
730 | end | ||
731 | end | ||
732 | if cache !== nothing | ||
733 | cache.project_file_manifest_path[project_file] = manifest_path | ||
734 | end | ||
735 | return manifest_path | ||
736 | end | ||
737 | end | ||
738 | |||
739 | # given a directory (implicit env from LOAD_PATH) and a name, | ||
740 | # check if it is an implicit package | ||
741 | function entry_point_and_project_file_inside(dir::String, name::String)::Union{Tuple{Nothing,Nothing},Tuple{String,Nothing},Tuple{String,String}} | ||
742 | path = normpath(joinpath(dir, "src", "$name.jl")) | ||
743 | isfile_casesensitive(path) || return nothing, nothing | ||
744 | for proj in project_names | ||
745 | project_file = normpath(joinpath(dir, proj)) | ||
746 | isfile_casesensitive(project_file) || continue | ||
747 | return path, project_file | ||
748 | end | ||
749 | return path, nothing | ||
750 | end | ||
751 | |||
752 | # given a project directory (implicit env from LOAD_PATH) and a name, | ||
753 | # find an entry point for `name`, and see if it has an associated project file | ||
754 | function entry_point_and_project_file(dir::String, name::String)::Union{Tuple{Nothing,Nothing},Tuple{String,Nothing},Tuple{String,String}} | ||
755 | path = normpath(joinpath(dir, "$name.jl")) | ||
756 | isfile_casesensitive(path) && return path, nothing | ||
757 | dir = joinpath(dir, name) | ||
758 | path, project_file = entry_point_and_project_file_inside(dir, name) | ||
759 | path === nothing || return path, project_file | ||
760 | dir = dir * ".jl" | ||
761 | path, project_file = entry_point_and_project_file_inside(dir, name) | ||
762 | path === nothing || return path, project_file | ||
763 | return nothing, nothing | ||
764 | end | ||
765 | |||
766 | # given a path and a name, return the entry point | ||
767 | function entry_path(path::String, name::String)::Union{Nothing,String} | ||
768 | isfile_casesensitive(path) && return normpath(path) | ||
769 | path = normpath(joinpath(path, "src", "$name.jl")) | ||
770 | isfile_casesensitive(path) && return path | ||
771 | return nothing # source not found | ||
772 | end | ||
773 | |||
774 | ## explicit project & manifest API ## | ||
775 | |||
776 | # find project file root or deps `name => uuid` mapping | ||
777 | # return `nothing` if `name` is not found | ||
778 | function explicit_project_deps_get(project_file::String, name::String)::Union{Nothing,UUID} | ||
779 | d = parsed_toml(project_file) | ||
780 | root_uuid = dummy_uuid(project_file) | ||
781 | if get(d, "name", nothing)::Union{String, Nothing} === name | ||
782 | uuid = get(d, "uuid", nothing)::Union{String, Nothing} | ||
783 | return uuid === nothing ? root_uuid : UUID(uuid) | ||
784 | end | ||
785 | deps = get(d, "deps", nothing)::Union{Dict{String, Any}, Nothing} | ||
786 | if deps !== nothing | ||
787 | uuid = get(deps, name, nothing)::Union{String, Nothing} | ||
788 | uuid === nothing || return UUID(uuid) | ||
789 | end | ||
790 | return nothing | ||
791 | end | ||
792 | |||
793 | function is_v1_format_manifest(raw_manifest::Dict{String}) | ||
794 | if haskey(raw_manifest, "manifest_format") | ||
795 | mf = raw_manifest["manifest_format"] | ||
796 | if mf isa Dict{String} && haskey(mf, "uuid") | ||
797 | # the off-chance where an old format manifest has a dep called "manifest_format" | ||
798 | return true | ||
799 | end | ||
800 | return false | ||
801 | else | ||
802 | return true | ||
803 | end | ||
804 | end | ||
805 | |||
806 | # returns a deps list for both old and new manifest formats | ||
807 | function get_deps(raw_manifest::Dict) | ||
808 | if is_v1_format_manifest(raw_manifest) | ||
809 | return raw_manifest | ||
810 | else | ||
811 | # if the manifest has no deps, there won't be a `deps` field | ||
812 | return get(Dict{String, Any}, raw_manifest, "deps")::Dict{String, Any} | ||
813 | end | ||
814 | end | ||
815 | |||
816 | # find `where` stanza and return the PkgId for `name` | ||
817 | # return `nothing` if it did not find `where` (indicating caller should continue searching) | ||
818 | function explicit_manifest_deps_get(project_file::String, where::PkgId, name::String)::Union{Nothing,PkgId} | ||
819 | manifest_file = project_file_manifest_path(project_file) | ||
820 | manifest_file === nothing && return nothing # manifest not found--keep searching LOAD_PATH | ||
821 | d = get_deps(parsed_toml(manifest_file)) | ||
822 | found_where = false | ||
823 | found_name = false | ||
824 | for (dep_name, entries) in d | ||
825 | entries::Vector{Any} | ||
826 | for entry in entries | ||
827 | entry = entry::Dict{String, Any} | ||
828 | uuid = get(entry, "uuid", nothing)::Union{String, Nothing} | ||
829 | uuid === nothing && continue | ||
830 | if UUID(uuid) === where.uuid | ||
831 | found_where = true | ||
832 | # deps is either a list of names (deps = ["DepA", "DepB"]) or | ||
833 | # a table of entries (deps = {"DepA" = "6ea...", "DepB" = "55d..."} | ||
834 | deps = get(entry, "deps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing} | ||
835 | if deps isa Vector{String} | ||
836 | found_name = name in deps | ||
837 | break | ||
838 | elseif deps isa Dict{String, Any} | ||
839 | deps = deps::Dict{String, Any} | ||
840 | for (dep, uuid) in deps | ||
841 | uuid::String | ||
842 | if dep === name | ||
843 | return PkgId(UUID(uuid), name) | ||
844 | end | ||
845 | end | ||
846 | end | ||
847 | else # Check for extensions | ||
848 | extensions = get(entry, "extensions", nothing) | ||
849 | if extensions !== nothing | ||
850 | if haskey(extensions, where.name) && where.uuid == uuid5(UUID(uuid), where.name) | ||
851 | found_where = true | ||
852 | if name == dep_name | ||
853 | return PkgId(UUID(uuid), name) | ||
854 | end | ||
855 | exts = extensions[where.name]::Union{String, Vector{String}} | ||
856 | if (exts isa String && name == exts) || (exts isa Vector{String} && name in exts) | ||
857 | weakdeps = get(entry, "weakdeps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing} | ||
858 | if weakdeps !== nothing | ||
859 | if weakdeps isa Vector{String} | ||
860 | found_name = name in weakdeps | ||
861 | break | ||
862 | elseif weakdeps isa Dict{String, Any} | ||
863 | weakdeps = weakdeps::Dict{String, Any} | ||
864 | for (dep, uuid) in weakdeps | ||
865 | uuid::String | ||
866 | if dep === name | ||
867 | return PkgId(UUID(uuid), name) | ||
868 | end | ||
869 | end | ||
870 | end | ||
871 | end | ||
872 | end | ||
873 | # `name` is not an ext, do standard lookup as if this was the parent | ||
874 | return identify_package(PkgId(UUID(uuid), dep_name), name) | ||
875 | end | ||
876 | end | ||
877 | end | ||
878 | end | ||
879 | end | ||
880 | found_where || return nothing | ||
881 | found_name || return PkgId(name) | ||
882 | # Only reach here if deps was not a dict which mean we have a unique name for the dep | ||
883 | name_deps = get(d, name, nothing)::Union{Nothing, Vector{Any}} | ||
884 | if name_deps === nothing || length(name_deps) != 1 | ||
885 | error("expected a single entry for $(repr(name)) in $(repr(project_file))") | ||
886 | end | ||
887 | entry = first(name_deps::Vector{Any})::Dict{String, Any} | ||
888 | uuid = get(entry, "uuid", nothing)::Union{String, Nothing} | ||
889 | uuid === nothing && return nothing | ||
890 | return PkgId(UUID(uuid), name) | ||
891 | end | ||
892 | |||
893 | # find `uuid` stanza, return the corresponding path | ||
894 | function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{Nothing,String,Missing} | ||
895 | manifest_file = project_file_manifest_path(project_file) | ||
896 | manifest_file === nothing && return nothing # no manifest, skip env | ||
897 | |||
898 | d = get_deps(parsed_toml(manifest_file)) | ||
899 | entries = get(d, pkg.name, nothing)::Union{Nothing, Vector{Any}} | ||
900 | if entries !== nothing | ||
901 | for entry in entries | ||
902 | entry = entry::Dict{String, Any} | ||
903 | uuid = get(entry, "uuid", nothing)::Union{Nothing, String} | ||
904 | uuid === nothing && continue | ||
905 | if UUID(uuid) === pkg.uuid | ||
906 | return explicit_manifest_entry_path(manifest_file, pkg, entry) | ||
907 | end | ||
908 | end | ||
909 | end | ||
910 | # Extensions | ||
911 | for (name, entries) in d | ||
912 | entries = entries::Vector{Any} | ||
913 | for entry in entries | ||
914 | uuid = get(entry, "uuid", nothing)::Union{Nothing, String} | ||
915 | extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}} | ||
916 | if extensions !== nothing && haskey(extensions, pkg.name) && uuid !== nothing && uuid5(UUID(uuid), pkg.name) == pkg.uuid | ||
917 | parent_path = locate_package(PkgId(UUID(uuid), name)) | ||
918 | if parent_path === nothing | ||
919 | error("failed to find source of parent package: \"$name\"") | ||
920 | end | ||
921 | p = normpath(dirname(parent_path), "..") | ||
922 | return find_ext_path(p, pkg.name) | ||
923 | end | ||
924 | end | ||
925 | end | ||
926 | return nothing | ||
927 | end | ||
928 | |||
929 | function explicit_manifest_entry_path(manifest_file::String, pkg::PkgId, entry::Dict{String,Any}) | ||
930 | path = get(entry, "path", nothing)::Union{Nothing, String} | ||
931 | if path !== nothing | ||
932 | path = normpath(abspath(dirname(manifest_file), path)) | ||
933 | return path | ||
934 | end | ||
935 | hash = get(entry, "git-tree-sha1", nothing)::Union{Nothing, String} | ||
936 | hash === nothing && return nothing | ||
937 | hash = SHA1(hash) | ||
938 | # Keep the 4 since it used to be the default | ||
939 | uuid = pkg.uuid::UUID # checked within `explicit_manifest_uuid_path` | ||
940 | for slug in (version_slug(uuid, hash), version_slug(uuid, hash, 4)) | ||
941 | for depot in DEPOT_PATH | ||
942 | path = joinpath(depot, "packages", pkg.name, slug) | ||
943 | ispath(path) && return abspath(path) | ||
944 | end | ||
945 | end | ||
946 | # no depot contains the package, return missing to stop looking | ||
947 | return missing | ||
948 | end | ||
949 | |||
950 | ## implicit project & manifest API ## | ||
951 | |||
952 | # look for an entry point for `name` from a top-level package (no environment) | ||
953 | # otherwise return `nothing` to indicate the caller should keep searching | ||
954 | function implicit_project_deps_get(dir::String, name::String)::Union{Nothing,PkgId} | ||
955 | path, project_file = entry_point_and_project_file(dir, name) | ||
956 | if project_file === nothing | ||
957 | path === nothing && return nothing | ||
958 | return PkgId(name) | ||
959 | end | ||
960 | proj = project_file_name_uuid(project_file, name) | ||
961 | proj.name == name || return nothing | ||
962 | return proj | ||
963 | end | ||
964 | |||
965 | # look for an entry-point for `name`, check that UUID matches | ||
966 | # if there's a project file, look up `name` in its deps and return that | ||
967 | # otherwise return `nothing` to indicate the caller should keep searching | ||
968 | function implicit_manifest_deps_get(dir::String, where::PkgId, name::String)::Union{Nothing,PkgId} | ||
969 | @assert where.uuid !== nothing | ||
970 | project_file = entry_point_and_project_file(dir, where.name)[2] | ||
971 | project_file === nothing && return nothing # a project file is mandatory for a package with a uuid | ||
972 | proj = project_file_name_uuid(project_file, where.name) | ||
973 | proj == where || return nothing # verify that this is the correct project file | ||
974 | # this is the correct project, so stop searching here | ||
975 | pkg_uuid = explicit_project_deps_get(project_file, name) | ||
976 | return PkgId(pkg_uuid, name) | ||
977 | end | ||
978 | |||
979 | # look for an entry-point for `pkg` and return its path if UUID matches | ||
980 | function implicit_manifest_uuid_path(dir::String, pkg::PkgId)::Union{Nothing,String} | ||
981 | path, project_file = entry_point_and_project_file(dir, pkg.name) | ||
982 | if project_file === nothing | ||
983 | pkg.uuid === nothing || return nothing | ||
984 | return path | ||
985 | end | ||
986 | proj = project_file_name_uuid(project_file, pkg.name) | ||
987 | proj == pkg || return nothing | ||
988 | return path | ||
989 | end | ||
990 | |||
991 | ## other code loading functionality ## | ||
992 | |||
993 | function find_source_file(path::AbstractString) | ||
994 | (isabspath(path) || isfile(path)) && return path | ||
995 | base_path = joinpath(Sys.BINDIR, DATAROOTDIR, "julia", "base", path) | ||
996 | return isfile(base_path) ? normpath(base_path) : nothing | ||
997 | end | ||
998 | |||
999 | cache_file_entry(pkg::PkgId) = joinpath( | ||
1000 | "compiled", | ||
1001 | "v$(VERSION.major).$(VERSION.minor)", | ||
1002 | pkg.uuid === nothing ? "" : pkg.name), | ||
1003 | pkg.uuid === nothing ? pkg.name : package_slug(pkg.uuid) | ||
1004 | |||
1005 | function find_all_in_cache_path(pkg::PkgId) | ||
1006 | paths = String[] | ||
1007 | entrypath, entryfile = cache_file_entry(pkg) | ||
1008 | for path in joinpath.(DEPOT_PATH, entrypath) | ||
1009 | isdir(path) || continue | ||
1010 | for file in readdir(path, sort = false) # no sort given we sort later | ||
1011 | if !((pkg.uuid === nothing && file == entryfile * ".ji") || | ||
1012 | (pkg.uuid !== nothing && startswith(file, entryfile * "_") && | ||
1013 | endswith(file, ".ji"))) | ||
1014 | continue | ||
1015 | end | ||
1016 | filepath = joinpath(path, file) | ||
1017 | isfile_casesensitive(filepath) && push!(paths, filepath) | ||
1018 | end | ||
1019 | end | ||
1020 | if length(paths) > 1 | ||
1021 | # allocating the sort vector is less expensive than using sort!(.. by=mtime), which would | ||
1022 | # call the relatively slow mtime multiple times per path | ||
1023 | p = sortperm(mtime.(paths), rev = true) | ||
1024 | return paths[p] | ||
1025 | else | ||
1026 | return paths | ||
1027 | end | ||
1028 | end | ||
1029 | |||
1030 | ocachefile_from_cachefile(cachefile) = string(chopsuffix(cachefile, ".ji"), ".", Base.Libc.dlext) | ||
1031 | cachefile_from_ocachefile(cachefile) = string(chopsuffix(cachefile, ".$(Base.Libc.dlext)"), ".ji") | ||
1032 | |||
1033 | |||
1034 | # use an Int counter so that nested @time_imports calls all remain open | ||
1035 | const TIMING_IMPORTS = Threads.Atomic{Int}(0) | ||
1036 | |||
1037 | # these return either the array of modules loaded from the path / content given | ||
1038 | # or an Exception that describes why it couldn't be loaded | ||
1039 | # and it reconnects the Base.Docs.META | ||
1040 | function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}, depmods::Vector{Any}) | ||
1041 | assert_havelock(require_lock) | ||
1042 | timing_imports = TIMING_IMPORTS[] > 0 | ||
1043 | try | ||
1044 | if timing_imports | ||
1045 | t_before = time_ns() | ||
1046 | cumulative_compile_timing(true) | ||
1047 | t_comp_before = cumulative_compile_time_ns() | ||
1048 | end | ||
1049 | |||
1050 | if ocachepath !== nothing | ||
1051 | @debug "Loading object cache file $ocachepath for $pkg" | ||
1052 | sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any, Cint, Cstring), ocachepath, depmods, false, pkg.name) | ||
1053 | else | ||
1054 | @debug "Loading cache file $path for $pkg" | ||
1055 | sv = ccall(:jl_restore_incremental, Any, (Cstring, Any, Cint, Cstring), path, depmods, false, pkg.name) | ||
1056 | end | ||
1057 | if isa(sv, Exception) | ||
1058 | return sv | ||
1059 | end | ||
1060 | |||
1061 | restored = register_restored_modules(sv, pkg, path) | ||
1062 | |||
1063 | for M in restored | ||
1064 | M = M::Module | ||
1065 | if parentmodule(M) === M && PkgId(M) == pkg | ||
1066 | if timing_imports | ||
1067 | elapsed = round((time_ns() - t_before) / 1e6, digits = 1) | ||
1068 | comp_time, recomp_time = cumulative_compile_time_ns() .- t_comp_before | ||
1069 | print(lpad(elapsed, 9), " ms ") | ||
1070 | parentid = get(EXT_PRIMED, pkg, nothing) | ||
1071 | if parentid !== nothing | ||
1072 | print(parentid.name, " → ") | ||
1073 | end | ||
1074 | print(pkg.name) | ||
1075 | if comp_time > 0 | ||
1076 | printstyled(" ", Ryu.writefixed(Float64(100 * comp_time / (elapsed * 1e6)), 2), "% compilation time", color = Base.info_color()) | ||
1077 | end | ||
1078 | if recomp_time > 0 | ||
1079 | perc = Float64(100 * recomp_time / comp_time) | ||
1080 | printstyled(" (", perc < 1 ? "<1" : Ryu.writefixed(perc, 0), "% recompilation)", color = Base.warn_color()) | ||
1081 | end | ||
1082 | println() | ||
1083 | end | ||
1084 | return M | ||
1085 | end | ||
1086 | end | ||
1087 | return ErrorException("Required dependency $pkg failed to load from a cache file.") | ||
1088 | |||
1089 | finally | ||
1090 | timing_imports && cumulative_compile_timing(false) | ||
1091 | end | ||
1092 | end | ||
1093 | |||
1094 | function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String) | ||
1095 | # This function is also used by PkgCacheInspector.jl | ||
1096 | restored = sv[1]::Vector{Any} | ||
1097 | for M in restored | ||
1098 | M = M::Module | ||
1099 | if isdefined(M, Base.Docs.META) && getfield(M, Base.Docs.META) !== nothing | ||
1100 | push!(Base.Docs.modules, M) | ||
1101 | end | ||
1102 | if parentmodule(M) === M | ||
1103 | register_root_module(M) | ||
1104 | end | ||
1105 | end | ||
1106 | |||
1107 | # Register this cache path now - If Requires.jl is loaded, Revise may end | ||
1108 | # up looking at the cache path during the init callback. | ||
1109 | get!(PkgOrigin, pkgorigins, pkg).cachepath = path | ||
1110 | |||
1111 | inits = sv[2]::Vector{Any} | ||
1112 | if !isempty(inits) | ||
1113 | unlock(require_lock) # temporarily _unlock_ during these callbacks | ||
1114 | try | ||
1115 | for (i, mod) in pairs(inits) | ||
1116 | run_module_init(mod, i) | ||
1117 | end | ||
1118 | finally | ||
1119 | lock(require_lock) | ||
1120 | end | ||
1121 | end | ||
1122 | return restored | ||
1123 | end | ||
1124 | |||
1125 | function run_module_init(mod::Module, i::Int=1) | ||
1126 | # `i` informs ordering for the `@time_imports` report formatting | ||
1127 | if TIMING_IMPORTS[] == 0 | ||
1128 | ccall(:jl_init_restored_module, Cvoid, (Any,), mod) | ||
1129 | else | ||
1130 | if isdefined(mod, :__init__) | ||
1131 | connector = i > 1 ? "├" : "┌" | ||
1132 | printstyled(" $connector ", color = :light_black) | ||
1133 | |||
1134 | elapsedtime = time_ns() | ||
1135 | cumulative_compile_timing(true) | ||
1136 | compile_elapsedtimes = cumulative_compile_time_ns() | ||
1137 | |||
1138 | ccall(:jl_init_restored_module, Cvoid, (Any,), mod) | ||
1139 | |||
1140 | elapsedtime = (time_ns() - elapsedtime) / 1e6 | ||
1141 | cumulative_compile_timing(false); | ||
1142 | comp_time, recomp_time = (cumulative_compile_time_ns() .- compile_elapsedtimes) ./ 1e6 | ||
1143 | |||
1144 | print(round(elapsedtime, digits=1), " ms $mod.__init__() ") | ||
1145 | if comp_time > 0 | ||
1146 | printstyled(Ryu.writefixed(Float64(100 * comp_time / elapsedtime), 2), "% compilation time", color = Base.info_color()) | ||
1147 | end | ||
1148 | if recomp_time > 0 | ||
1149 | perc = Float64(100 * recomp_time / comp_time) | ||
1150 | printstyled(" (", perc < 1 ? "<1" : Ryu.writefixed(perc, 0), "% recompilation)", color = Base.warn_color()) | ||
1151 | end | ||
1152 | println() | ||
1153 | end | ||
1154 | end | ||
1155 | end | ||
1156 | |||
1157 | function run_package_callbacks(modkey::PkgId) | ||
1158 | run_extension_callbacks(modkey) | ||
1159 | assert_havelock(require_lock) | ||
1160 | unlock(require_lock) | ||
1161 | try | ||
1162 | for callback in package_callbacks | ||
1163 | invokelatest(callback, modkey) | ||
1164 | end | ||
1165 | catch | ||
1166 | # Try to continue loading if a callback errors | ||
1167 | errs = current_exceptions() | ||
1168 | @error "Error during package callback" exception=errs | ||
1169 | finally | ||
1170 | lock(require_lock) | ||
1171 | end | ||
1172 | nothing | ||
1173 | end | ||
1174 | |||
1175 | |||
1176 | ############## | ||
1177 | # Extensions # | ||
1178 | ############## | ||
1179 | |||
1180 | mutable struct ExtensionId | ||
1181 | const id::PkgId | ||
1182 | const parentid::PkgId # just need the name, for printing | ||
1183 | ntriggers::Int # how many more packages must be defined until this is loaded | ||
1184 | end | ||
1185 | |||
1186 | const EXT_PRIMED = Dict{PkgId, PkgId}() # Extension -> Parent | ||
1187 | const EXT_DORMITORY = Dict{PkgId,Vector{ExtensionId}}() # Trigger -> Extensions that can be triggered by it | ||
1188 | const EXT_DORMITORY_FAILED = ExtensionId[] | ||
1189 | |||
1190 | function insert_extension_triggers(pkg::PkgId) | ||
1191 | pkg.uuid === nothing && return | ||
1192 | path_env_loc = locate_package_env(pkg) | ||
1193 | path_env_loc === nothing && return | ||
1194 | path, env_loc = path_env_loc | ||
1195 | if path === nothing || env_loc === nothing | ||
1196 | return | ||
1197 | end | ||
1198 | insert_extension_triggers(env_loc, pkg) | ||
1199 | end | ||
1200 | |||
1201 | function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missing} | ||
1202 | project_file = env_project_file(env) | ||
1203 | if project_file isa String | ||
1204 | # Look in project for extensions to insert | ||
1205 | proj_pkg = project_file_name_uuid(project_file, pkg.name) | ||
1206 | if pkg == proj_pkg | ||
1207 | d_proj = parsed_toml(project_file) | ||
1208 | weakdeps = get(d_proj, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}} | ||
1209 | extensions = get(d_proj, "extensions", nothing)::Union{Nothing, Dict{String, Any}} | ||
1210 | extensions === nothing && return | ||
1211 | weakdeps === nothing && return | ||
1212 | if weakdeps isa Dict{String, Any} | ||
1213 | return _insert_extension_triggers(pkg, extensions, weakdeps) | ||
1214 | end | ||
1215 | end | ||
1216 | |||
1217 | # Now look in manifest | ||
1218 | manifest_file = project_file_manifest_path(project_file) | ||
1219 | manifest_file === nothing && return | ||
1220 | d = get_deps(parsed_toml(manifest_file)) | ||
1221 | for (dep_name, entries) in d | ||
1222 | entries::Vector{Any} | ||
1223 | for entry in entries | ||
1224 | entry = entry::Dict{String, Any} | ||
1225 | uuid = get(entry, "uuid", nothing)::Union{String, Nothing} | ||
1226 | uuid === nothing && continue | ||
1227 | if UUID(uuid) == pkg.uuid | ||
1228 | weakdeps = get(entry, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}} | ||
1229 | extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}} | ||
1230 | extensions === nothing && return | ||
1231 | weakdeps === nothing && return | ||
1232 | if weakdeps isa Dict{String, Any} | ||
1233 | return _insert_extension_triggers(pkg, extensions, weakdeps) | ||
1234 | end | ||
1235 | |||
1236 | d_weakdeps = Dict{String, Any}() | ||
1237 | for (dep_name, entries) in d | ||
1238 | dep_name in weakdeps || continue | ||
1239 | entries::Vector{Any} | ||
1240 | if length(entries) != 1 | ||
1241 | error("expected a single entry for $(repr(dep_name)) in $(repr(project_file))") | ||
1242 | end | ||
1243 | entry = first(entries)::Dict{String, Any} | ||
1244 | uuid = entry["uuid"]::String | ||
1245 | d_weakdeps[dep_name] = uuid | ||
1246 | end | ||
1247 | @assert length(d_weakdeps) == length(weakdeps) | ||
1248 | return _insert_extension_triggers(pkg, extensions, d_weakdeps) | ||
1249 | end | ||
1250 | end | ||
1251 | end | ||
1252 | end | ||
1253 | return nothing | ||
1254 | end | ||
1255 | |||
1256 | function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}, weakdeps::Dict{String, Any}) | ||
1257 | for (ext, triggers) in extensions | ||
1258 | triggers = triggers::Union{String, Vector{String}} | ||
1259 | triggers isa String && (triggers = [triggers]) | ||
1260 | id = PkgId(uuid5(parent.uuid, ext), ext) | ||
1261 | if id in keys(EXT_PRIMED) || haskey(Base.loaded_modules, id) | ||
1262 | continue # extension is already primed or loaded, don't add it again | ||
1263 | end | ||
1264 | EXT_PRIMED[id] = parent | ||
1265 | gid = ExtensionId(id, parent, 1 + length(triggers)) | ||
1266 | trigger1 = get!(Vector{ExtensionId}, EXT_DORMITORY, parent) | ||
1267 | push!(trigger1, gid) | ||
1268 | for trigger in triggers | ||
1269 | # TODO: Better error message if this lookup fails? | ||
1270 | uuid_trigger = UUID(weakdeps[trigger]::String) | ||
1271 | trigger_id = PkgId(uuid_trigger, trigger) | ||
1272 | if !haskey(Base.loaded_modules, trigger_id) || haskey(package_locks, trigger_id) | ||
1273 | trigger1 = get!(Vector{ExtensionId}, EXT_DORMITORY, trigger_id) | ||
1274 | push!(trigger1, gid) | ||
1275 | else | ||
1276 | gid.ntriggers -= 1 | ||
1277 | end | ||
1278 | end | ||
1279 | end | ||
1280 | end | ||
1281 | |||
1282 | loading_extension::Bool = false | ||
1283 | precompiling_extension::Bool = false | ||
1284 | function run_extension_callbacks(extid::ExtensionId) | ||
1285 | assert_havelock(require_lock) | ||
1286 | succeeded = try | ||
1287 | # Used by Distributed to now load extensions in the package callback | ||
1288 | global loading_extension = true | ||
1289 | _require_prelocked(extid.id) | ||
1290 | @debug "Extension $(extid.id.name) of $(extid.parentid.name) loaded" | ||
1291 | true | ||
1292 | catch | ||
1293 | # Try to continue loading if loading an extension errors | ||
1294 | errs = current_exceptions() | ||
1295 | @error "Error during loading of extension $(extid.id.name) of $(extid.parentid.name), \ | ||
1296 | use `Base.retry_load_extensions()` to retry." exception=errs | ||
1297 | false | ||
1298 | finally | ||
1299 | global loading_extension = false | ||
1300 | end | ||
1301 | return succeeded | ||
1302 | end | ||
1303 | |||
1304 | function run_extension_callbacks(pkgid::PkgId) | ||
1305 | assert_havelock(require_lock) | ||
1306 | # take ownership of extids that depend on this pkgid | ||
1307 | extids = pop!(EXT_DORMITORY, pkgid, nothing) | ||
1308 | extids === nothing && return | ||
1309 | for extid in extids | ||
1310 | if extid.ntriggers > 0 | ||
1311 | # indicate pkgid is loaded | ||
1312 | extid.ntriggers -= 1 | ||
1313 | end | ||
1314 | if extid.ntriggers < 0 | ||
1315 | # indicate pkgid is loaded | ||
1316 | extid.ntriggers += 1 | ||
1317 | succeeded = false | ||
1318 | else | ||
1319 | succeeded = true | ||
1320 | end | ||
1321 | if extid.ntriggers == 0 | ||
1322 | # actually load extid, now that all dependencies are met, | ||
1323 | # and record the result | ||
1324 | succeeded = succeeded && run_extension_callbacks(extid) | ||
1325 | succeeded || push!(EXT_DORMITORY_FAILED, extid) | ||
1326 | end | ||
1327 | end | ||
1328 | return | ||
1329 | end | ||
1330 | |||
1331 | """ | ||
1332 | retry_load_extensions() | ||
1333 | |||
1334 | Loads all the (not yet loaded) extensions that have their extension-dependencies loaded. | ||
1335 | This is used in cases where the automatic loading of an extension failed | ||
1336 | due to some problem with the extension. Instead of restarting the Julia session, | ||
1337 | the extension can be fixed, and this function run. | ||
1338 | """ | ||
1339 | function retry_load_extensions() | ||
1340 | @lock require_lock begin | ||
1341 | # this copy is desired since run_extension_callbacks will release this lock | ||
1342 | # so this can still mutate the list to drop successful ones | ||
1343 | failed = copy(EXT_DORMITORY_FAILED) | ||
1344 | empty!(EXT_DORMITORY_FAILED) | ||
1345 | filter!(failed) do extid | ||
1346 | return !run_extension_callbacks(extid) | ||
1347 | end | ||
1348 | prepend!(EXT_DORMITORY_FAILED, failed) | ||
1349 | end | ||
1350 | return | ||
1351 | end | ||
1352 | |||
1353 | """ | ||
1354 | get_extension(parent::Module, extension::Symbol) | ||
1355 | |||
1356 | Return the module for `extension` of `parent` or return `nothing` if the extension is not loaded. | ||
1357 | """ | ||
1358 | get_extension(parent::Module, ext::Symbol) = get_extension(PkgId(parent), ext) | ||
1359 | function get_extension(parentid::PkgId, ext::Symbol) | ||
1360 | parentid.uuid === nothing && return nothing | ||
1361 | extid = PkgId(uuid5(parentid.uuid, string(ext)), string(ext)) | ||
1362 | return get(loaded_modules, extid, nothing) | ||
1363 | end | ||
1364 | |||
1365 | # End extensions | ||
1366 | |||
1367 | # should sync with the types of arguments of `stale_cachefile` | ||
1368 | const StaleCacheKey = Tuple{Base.PkgId, UInt128, String, String} | ||
1369 | |||
1370 | """ | ||
1371 | Base.isprecompiled(pkg::PkgId; ignore_loaded::Bool=false) | ||
1372 | |||
1373 | Returns whether a given PkgId within the active project is precompiled. | ||
1374 | |||
1375 | By default this check observes the same approach that code loading takes | ||
1376 | with respect to when different versions of dependencies are currently loaded | ||
1377 | to that which is expected. To ignore loaded modules and answer as if in a | ||
1378 | fresh julia session specify `ignore_loaded=true`. | ||
1379 | |||
1380 | !!! compat "Julia 1.10" | ||
1381 | This function requires at least Julia 1.10. | ||
1382 | """ | ||
1383 | function isprecompiled(pkg::PkgId; | ||
1384 | ignore_loaded::Bool=false, | ||
1385 | stale_cache::Dict{StaleCacheKey,Bool}=Dict{StaleCacheKey, Bool}(), | ||
1386 | cachepaths::Vector{String}=Base.find_all_in_cache_path(pkg), | ||
1387 | sourcepath::Union{String,Nothing}=Base.locate_package(pkg) | ||
1388 | ) | ||
1389 | isnothing(sourcepath) && error("Cannot locate source for $(repr(pkg))") | ||
1390 | for path_to_try in cachepaths | ||
1391 | staledeps = stale_cachefile(sourcepath, path_to_try, ignore_loaded = true) | ||
1392 | if staledeps === true | ||
1393 | continue | ||
1394 | end | ||
1395 | staledeps, _ = staledeps::Tuple{Vector{Any}, Union{Nothing, String}} | ||
1396 | # finish checking staledeps module graph | ||
1397 | for i in 1:length(staledeps) | ||
1398 | dep = staledeps[i] | ||
1399 | dep isa Module && continue | ||
1400 | modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128} | ||
1401 | modpaths = find_all_in_cache_path(modkey) | ||
1402 | for modpath_to_try in modpaths::Vector{String} | ||
1403 | stale_cache_key = (modkey, modbuild_id, modpath, modpath_to_try)::StaleCacheKey | ||
1404 | if get!(() -> stale_cachefile(stale_cache_key...; ignore_loaded) === true, | ||
1405 | stale_cache, stale_cache_key) | ||
1406 | continue | ||
1407 | end | ||
1408 | @goto check_next_dep | ||
1409 | end | ||
1410 | @goto check_next_path | ||
1411 | @label check_next_dep | ||
1412 | end | ||
1413 | try | ||
1414 | # update timestamp of precompilation file so that it is the first to be tried by code loading | ||
1415 | touch(path_to_try) | ||
1416 | catch ex | ||
1417 | # file might be read-only and then we fail to update timestamp, which is fine | ||
1418 | ex isa IOError || rethrow() | ||
1419 | end | ||
1420 | return true | ||
1421 | @label check_next_path | ||
1422 | end | ||
1423 | return false | ||
1424 | end | ||
1425 | |||
1426 | # loads a precompile cache file, after checking stale_cachefile tests | ||
1427 | function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt128) | ||
1428 | assert_havelock(require_lock) | ||
1429 | loaded = nothing | ||
1430 | if root_module_exists(modkey) | ||
1431 | loaded = root_module(modkey) | ||
1432 | else | ||
1433 | loaded = start_loading(modkey) | ||
1434 | if loaded === nothing | ||
1435 | try | ||
1436 | modpath = locate_package(modkey) | ||
1437 | modpath === nothing && return nothing | ||
1438 | set_pkgorigin_version_path(modkey, String(modpath)) | ||
1439 | loaded = _require_search_from_serialized(modkey, String(modpath), build_id) | ||
1440 | finally | ||
1441 | end_loading(modkey, loaded) | ||
1442 | end | ||
1443 | if loaded isa Module | ||
1444 | insert_extension_triggers(modkey) | ||
1445 | run_package_callbacks(modkey) | ||
1446 | end | ||
1447 | end | ||
1448 | end | ||
1449 | if !(loaded isa Module) || PkgId(loaded) != modkey | ||
1450 | return ErrorException("Required dependency $modkey failed to load from a cache file.") | ||
1451 | end | ||
1452 | return loaded | ||
1453 | end | ||
1454 | |||
1455 | # loads a precompile cache file, ignoring stale_cachefile tests | ||
1456 | # assuming all depmods are already loaded and everything is valid | ||
1457 | function _tryrequire_from_serialized(modkey::PkgId, path::String, ocachepath::Union{Nothing, String}, sourcepath::String, depmods::Vector{Any}) | ||
1458 | assert_havelock(require_lock) | ||
1459 | loaded = nothing | ||
1460 | if root_module_exists(modkey) | ||
1461 | loaded = root_module(modkey) | ||
1462 | else | ||
1463 | loaded = start_loading(modkey) | ||
1464 | if loaded === nothing | ||
1465 | try | ||
1466 | for i in 1:length(depmods) | ||
1467 | dep = depmods[i] | ||
1468 | dep isa Module && continue | ||
1469 | _, depkey, depbuild_id = dep::Tuple{String, PkgId, UInt128} | ||
1470 | @assert root_module_exists(depkey) | ||
1471 | dep = root_module(depkey) | ||
1472 | depmods[i] = dep | ||
1473 | end | ||
1474 | set_pkgorigin_version_path(modkey, sourcepath) | ||
1475 | loaded = _include_from_serialized(modkey, path, ocachepath, depmods) | ||
1476 | finally | ||
1477 | end_loading(modkey, loaded) | ||
1478 | end | ||
1479 | if loaded isa Module | ||
1480 | insert_extension_triggers(modkey) | ||
1481 | run_package_callbacks(modkey) | ||
1482 | end | ||
1483 | end | ||
1484 | end | ||
1485 | if !(loaded isa Module) || PkgId(loaded) != modkey | ||
1486 | return ErrorException("Required dependency $modkey failed to load from a cache file.") | ||
1487 | end | ||
1488 | return loaded | ||
1489 | end | ||
1490 | |||
1491 | # loads a precompile cache file, ignoring stale_cachefile tests | ||
1492 | # load the best available (non-stale) version of all dependent modules first | ||
1493 | function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}) | ||
1494 | assert_havelock(require_lock) | ||
1495 | local depmodnames | ||
1496 | io = open(path, "r") | ||
1497 | try | ||
1498 | iszero(isvalid_cache_header(io)) && return ArgumentError("Invalid header in cache file $path.") | ||
1499 | _, _, depmodnames, _, _, _, clone_targets, _ = parse_cache_header(io) | ||
1500 | pkgimage = !isempty(clone_targets) | ||
1501 | if pkgimage | ||
1502 | ocachepath !== nothing || return ArgumentError("Expected ocachepath to be provided") | ||
1503 | isfile(ocachepath) || return ArgumentError("Ocachepath $ocachepath is not a file.") | ||
1504 | ocachepath == ocachefile_from_cachefile(path) || return ArgumentError("$ocachepath is not the expected ocachefile") | ||
1505 | # TODO: Check for valid clone_targets? | ||
1506 | isvalid_pkgimage_crc(io, ocachepath) || return ArgumentError("Invalid checksum in cache file $ocachepath.") | ||
1507 | else | ||
1508 | @assert ocachepath === nothing | ||
1509 | end | ||
1510 | isvalid_file_crc(io) || return ArgumentError("Invalid checksum in cache file $path.") | ||
1511 | finally | ||
1512 | close(io) | ||
1513 | end | ||
1514 | ndeps = length(depmodnames) | ||
1515 | depmods = Vector{Any}(undef, ndeps) | ||
1516 | for i in 1:ndeps | ||
1517 | modkey, build_id = depmodnames[i] | ||
1518 | dep = _tryrequire_from_serialized(modkey, build_id) | ||
1519 | if !isa(dep, Module) | ||
1520 | return dep | ||
1521 | end | ||
1522 | depmods[i] = dep | ||
1523 | end | ||
1524 | # then load the file | ||
1525 | return _include_from_serialized(pkg, path, ocachepath, depmods) | ||
1526 | end | ||
1527 | |||
1528 | # returns `nothing` if require found a precompile cache for this sourcepath, but couldn't load it | ||
1529 | # returns the set of modules restored if the cache load succeeded | ||
1530 | @constprop :none function _require_search_from_serialized(pkg::PkgId, sourcepath::String, build_id::UInt128) | ||
1531 | assert_havelock(require_lock) | ||
1532 | paths = find_all_in_cache_path(pkg) | ||
1533 | for path_to_try in paths::Vector{String} | ||
1534 | staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try) | ||
1535 | if staledeps === true | ||
1536 | continue | ||
1537 | end | ||
1538 | staledeps, ocachefile = staledeps::Tuple{Vector{Any}, Union{Nothing, String}} | ||
1539 | # finish checking staledeps module graph | ||
1540 | for i in 1:length(staledeps) | ||
1541 | dep = staledeps[i] | ||
1542 | dep isa Module && continue | ||
1543 | modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128} | ||
1544 | modpaths = find_all_in_cache_path(modkey) | ||
1545 | for modpath_to_try in modpaths | ||
1546 | modstaledeps = stale_cachefile(modkey, modbuild_id, modpath, modpath_to_try) | ||
1547 | if modstaledeps === true | ||
1548 | continue | ||
1549 | end | ||
1550 | modstaledeps, modocachepath = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}} | ||
1551 | staledeps[i] = (modpath, modkey, modpath_to_try, modstaledeps, modocachepath) | ||
1552 | @goto check_next_dep | ||
1553 | end | ||
1554 | @debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache." | ||
1555 | @goto check_next_path | ||
1556 | @label check_next_dep | ||
1557 | end | ||
1558 | try | ||
1559 | touch(path_to_try) # update timestamp of precompilation file | ||
1560 | catch ex # file might be read-only and then we fail to update timestamp, which is fine | ||
1561 | ex isa IOError || rethrow() | ||
1562 | end | ||
1563 | # finish loading module graph into staledeps | ||
1564 | for i in 1:length(staledeps) | ||
1565 | dep = staledeps[i] | ||
1566 | dep isa Module && continue | ||
1567 | modpath, modkey, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, String, Vector{Any}, Union{Nothing, String}} | ||
1568 | dep = _tryrequire_from_serialized(modkey, modcachepath, modocachepath, modpath, modstaledeps) | ||
1569 | if !isa(dep, Module) | ||
1570 | @debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep | ||
1571 | @goto check_next_path | ||
1572 | end | ||
1573 | staledeps[i] = dep | ||
1574 | end | ||
1575 | restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps) | ||
1576 | isa(restored, Module) && return restored | ||
1577 | @debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored | ||
1578 | continue | ||
1579 | @label check_next_path | ||
1580 | end | ||
1581 | return nothing | ||
1582 | end | ||
1583 | |||
1584 | # to synchronize multiple tasks trying to import/using something | ||
1585 | const package_locks = Dict{PkgId,Pair{Task,Threads.Condition}}() | ||
1586 | |||
1587 | debug_loading_deadlocks::Bool = true # Enable a slightly more expensive, but more complete algorithm that can handle simultaneous tasks. | ||
1588 | # This only triggers if you have multiple tasks trying to load the same package at the same time, | ||
1589 | # so it is unlikely to make a difference normally. | ||
1590 | function start_loading(modkey::PkgId) | ||
1591 | # handle recursive calls to require | ||
1592 | assert_havelock(require_lock) | ||
1593 | loading = get(package_locks, modkey, nothing) | ||
1594 | if loading !== nothing | ||
1595 | # load already in progress for this module on the task | ||
1596 | task, cond = loading | ||
1597 | deps = String[modkey.name] | ||
1598 | pkgid = modkey | ||
1599 | assert_havelock(cond.lock) | ||
1600 | if debug_loading_deadlocks && current_task() !== task | ||
1601 | waiters = Dict{Task,Pair{Task,PkgId}}() # invert to track waiting tasks => loading tasks | ||
1602 | for each in package_locks | ||
1603 | cond2 = each[2][2] | ||
1604 | assert_havelock(cond2.lock) | ||
1605 | for waiting in cond2.waitq | ||
1606 | push!(waiters, waiting => (each[2][1] => each[1])) | ||
1607 | end | ||
1608 | end | ||
1609 | while true | ||
1610 | running = get(waiters, task, nothing) | ||
1611 | running === nothing && break | ||
1612 | task, pkgid = running | ||
1613 | push!(deps, pkgid.name) | ||
1614 | task === current_task() && break | ||
1615 | end | ||
1616 | end | ||
1617 | if current_task() === task | ||
1618 | others = String[modkey.name] # repeat this to emphasize the cycle here | ||
1619 | for each in package_locks # list the rest of the packages being loaded too | ||
1620 | if each[2][1] === task | ||
1621 | other = each[1].name | ||
1622 | other == modkey.name || other == pkgid.name || push!(others, other) | ||
1623 | end | ||
1624 | end | ||
1625 | msg = sprint(deps, others) do io, deps, others | ||
1626 | print(io, "deadlock detected in loading ") | ||
1627 | join(io, deps, " -> ") | ||
1628 | print(io, " -> ") | ||
1629 | join(io, others, " && ") | ||
1630 | end | ||
1631 | throw(ConcurrencyViolationError(msg)) | ||
1632 | end | ||
1633 | return wait(cond) | ||
1634 | end | ||
1635 | package_locks[modkey] = current_task() => Threads.Condition(require_lock) | ||
1636 | return | ||
1637 | end | ||
1638 | |||
1639 | function end_loading(modkey::PkgId, @nospecialize loaded) | ||
1640 | loading = pop!(package_locks, modkey) | ||
1641 | notify(loading[2], loaded, all=true) | ||
1642 | nothing | ||
1643 | end | ||
1644 | |||
1645 | # to notify downstream consumers that a module was successfully loaded | ||
1646 | # Callbacks take the form (mod::Base.PkgId) -> nothing. | ||
1647 | # WARNING: This is an experimental feature and might change later, without deprecation. | ||
1648 | const package_callbacks = Any[] | ||
1649 | # to notify downstream consumers that a file has been included into a particular module | ||
1650 | # Callbacks take the form (mod::Module, filename::String) -> nothing | ||
1651 | # WARNING: This is an experimental feature and might change later, without deprecation. | ||
1652 | const include_callbacks = Any[] | ||
1653 | |||
1654 | # used to optionally track dependencies when requiring a module: | ||
1655 | const _concrete_dependencies = Pair{PkgId,UInt128}[] # these dependency versions are "set in stone", and the process should try to avoid invalidating them | ||
1656 | const _require_dependencies = Any[] # a list of (mod, path, mtime) tuples that are the file dependencies of the module currently being precompiled | ||
1657 | const _track_dependencies = Ref(false) # set this to true to track the list of file dependencies | ||
1658 | function _include_dependency(mod::Module, _path::AbstractString) | ||
1659 | prev = source_path(nothing) | ||
1660 | if prev === nothing | ||
1661 | path = abspath(_path) | ||
1662 | else | ||
1663 | path = normpath(joinpath(dirname(prev), _path)) | ||
1664 | end | ||
1665 | if _track_dependencies[] | ||
1666 | @lock require_lock begin | ||
1667 | push!(_require_dependencies, (mod, path, mtime(path))) | ||
1668 | end | ||
1669 | end | ||
1670 | return path, prev | ||
1671 | end | ||
1672 | |||
1673 | """ | ||
1674 | include_dependency(path::AbstractString) | ||
1675 | |||
1676 | In a module, declare that the file, directory, or symbolic link specified by `path` | ||
1677 | (relative or absolute) is a dependency for precompilation; that is, the module will need | ||
1678 | to be recompiled if the modification time of `path` changes. | ||
1679 | |||
1680 | This is only needed if your module depends on a path that is not used via [`include`](@ref). It has | ||
1681 | no effect outside of compilation. | ||
1682 | """ | ||
1683 | function include_dependency(path::AbstractString) | ||
1684 | _include_dependency(Main, path) | ||
1685 | return nothing | ||
1686 | end | ||
1687 | |||
1688 | # we throw PrecompilableError when a module doesn't want to be precompiled | ||
1689 | import Core: PrecompilableError | ||
1690 | function show(io::IO, ex::PrecompilableError) | ||
1691 | print(io, "Declaring __precompile__(false) is not allowed in files that are being precompiled.") | ||
1692 | end | ||
1693 | precompilableerror(ex::PrecompilableError) = true | ||
1694 | precompilableerror(ex::WrappedException) = precompilableerror(ex.error) | ||
1695 | precompilableerror(@nospecialize ex) = false | ||
1696 | |||
1697 | # Call __precompile__(false) at the top of a tile prevent it from being precompiled (false) | ||
1698 | """ | ||
1699 | __precompile__(isprecompilable::Bool) | ||
1700 | |||
1701 | Specify whether the file calling this function is precompilable, defaulting to `true`. | ||
1702 | If a module or file is *not* safely precompilable, it should call `__precompile__(false)` in | ||
1703 | order to throw an error if Julia attempts to precompile it. | ||
1704 | """ | ||
1705 | @noinline function __precompile__(isprecompilable::Bool=true) | ||
1706 | if !isprecompilable && ccall(:jl_generating_output, Cint, ()) != 0 | ||
1707 | throw(PrecompilableError()) | ||
1708 | end | ||
1709 | nothing | ||
1710 | end | ||
1711 | |||
1712 | # require always works in Main scope and loads files from node 1 | ||
1713 | const toplevel_load = Ref(true) | ||
1714 | |||
1715 | const _require_world_age = Ref{UInt}(typemax(UInt)) | ||
1716 | |||
1717 | """ | ||
1718 | require(into::Module, module::Symbol) | ||
1719 | |||
1720 | This function is part of the implementation of [`using`](@ref) / [`import`](@ref), if a module is not | ||
1721 | already defined in `Main`. It can also be called directly to force reloading a module, | ||
1722 | regardless of whether it has been loaded before (for example, when interactively developing | ||
1723 | libraries). | ||
1724 | |||
1725 | Loads a source file, in the context of the `Main` module, on every active node, searching | ||
1726 | standard locations for files. `require` is considered a top-level operation, so it sets the | ||
1727 | current `include` path but does not use it to search for files (see help for [`include`](@ref)). | ||
1728 | This function is typically used to load library code, and is implicitly called by `using` to | ||
1729 | load packages. | ||
1730 | |||
1731 | When searching for files, `require` first looks for package code in the global array | ||
1732 | [`LOAD_PATH`](@ref). `require` is case-sensitive on all platforms, including those with | ||
1733 | case-insensitive filesystems like macOS and Windows. | ||
1734 | |||
1735 | For more details regarding code loading, see the manual sections on [modules](@ref modules) and | ||
1736 | [parallel computing](@ref code-availability). | ||
1737 | """ | ||
1738 | function require(into::Module, mod::Symbol) | ||
1739 | if _require_world_age[] != typemax(UInt) | ||
1740 | Base.invoke_in_world(_require_world_age[], __require, into, mod) | ||
1741 | else | ||
1742 | @invokelatest __require(into, mod) | ||
1743 | end | ||
1744 | end | ||
1745 | |||
1746 | function __require(into::Module, mod::Symbol) | ||
1747 | @lock require_lock begin | ||
1748 | LOADING_CACHE[] = LoadingCache() | ||
1749 | try | ||
1750 | uuidkey_env = identify_package_env(into, String(mod)) | ||
1751 | # Core.println("require($(PkgId(into)), $mod) -> $uuidkey_env") | ||
1752 | if uuidkey_env === nothing | ||
1753 | where = PkgId(into) | ||
1754 | if where.uuid === nothing | ||
1755 | hint, dots = begin | ||
1756 | if isdefined(into, mod) && getfield(into, mod) isa Module | ||
1757 | true, "." | ||
1758 | elseif isdefined(parentmodule(into), mod) && getfield(parentmodule(into), mod) isa Module | ||
1759 | true, ".." | ||
1760 | else | ||
1761 | false, "" | ||
1762 | end | ||
1763 | end | ||
1764 | hint_message = hint ? ", maybe you meant `import/using $(dots)$(mod)`" : "" | ||
1765 | start_sentence = hint ? "Otherwise, run" : "Run" | ||
1766 | throw(ArgumentError(""" | ||
1767 | Package $mod not found in current path$hint_message. | ||
1768 | - $start_sentence `import Pkg; Pkg.add($(repr(String(mod))))` to install the $mod package.""")) | ||
1769 | else | ||
1770 | throw(ArgumentError(""" | ||
1771 | Package $(where.name) does not have $mod in its dependencies: | ||
1772 | - You may have a partially installed environment. Try `Pkg.instantiate()` | ||
1773 | to ensure all packages in the environment are installed. | ||
1774 | - Or, if you have $(where.name) checked out for development and have | ||
1775 | added $mod as a dependency but haven't updated your primary | ||
1776 | environment's manifest file, try `Pkg.resolve()`. | ||
1777 | - Otherwise you may need to report an issue with $(where.name)""")) | ||
1778 | end | ||
1779 | end | ||
1780 | uuidkey, env = uuidkey_env | ||
1781 | if _track_dependencies[] | ||
1782 | push!(_require_dependencies, (into, binpack(uuidkey), 0.0)) | ||
1783 | end | ||
1784 | return _require_prelocked(uuidkey, env) | ||
1785 | finally | ||
1786 | LOADING_CACHE[] = nothing | ||
1787 | end | ||
1788 | end | ||
1789 | end | ||
1790 | |||
1791 | require(uuidkey::PkgId) = @lock require_lock _require_prelocked(uuidkey) | ||
1792 | |||
1793 | const REPL_PKGID = PkgId(UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL") | ||
1794 | |||
1795 | function _require_prelocked(uuidkey::PkgId, env=nothing) | ||
1796 | if _require_world_age[] != typemax(UInt) | ||
1797 | Base.invoke_in_world(_require_world_age[], __require_prelocked, uuidkey, env) | ||
1798 | else | ||
1799 | @invokelatest __require_prelocked(uuidkey, env) | ||
1800 | end | ||
1801 | end | ||
1802 | |||
1803 | function __require_prelocked(uuidkey::PkgId, env=nothing) | ||
1804 | assert_havelock(require_lock) | ||
1805 | if !root_module_exists(uuidkey) | ||
1806 | newm = _require(uuidkey, env) | ||
1807 | if newm === nothing | ||
1808 | error("package `$(uuidkey.name)` did not define the expected \ | ||
1809 | module `$(uuidkey.name)`, check for typos in package module name") | ||
1810 | end | ||
1811 | insert_extension_triggers(uuidkey) | ||
1812 | # After successfully loading, notify downstream consumers | ||
1813 | run_package_callbacks(uuidkey) | ||
1814 | if uuidkey == REPL_PKGID | ||
1815 | REPL_MODULE_REF[] = newm | ||
1816 | end | ||
1817 | else | ||
1818 | newm = root_module(uuidkey) | ||
1819 | end | ||
1820 | return newm | ||
1821 | end | ||
1822 | |||
1823 | mutable struct PkgOrigin | ||
1824 | path::Union{String,Nothing} | ||
1825 | cachepath::Union{String,Nothing} | ||
1826 | version::Union{VersionNumber,Nothing} | ||
1827 | end | ||
1828 | PkgOrigin() = PkgOrigin(nothing, nothing, nothing) | ||
1829 | const pkgorigins = Dict{PkgId,PkgOrigin}() | ||
1830 | |||
1831 | const loaded_modules = Dict{PkgId,Module}() | ||
1832 | const loaded_modules_order = Vector{Module}() | ||
1833 | const module_keys = IdDict{Module,PkgId}() # the reverse | ||
1834 | |||
1835 | is_root_module(m::Module) = @lock require_lock haskey(module_keys, m) | ||
1836 | root_module_key(m::Module) = @lock require_lock module_keys[m] | ||
1837 | |||
1838 | @constprop :none function register_root_module(m::Module) | ||
1839 | # n.b. This is called from C after creating a new module in `Base.__toplevel__`, | ||
1840 | # instead of adding them to the binding table there. | ||
1841 | @lock require_lock begin | ||
1842 | key = PkgId(m, String(nameof(m))) | ||
1843 | if haskey(loaded_modules, key) | ||
1844 | oldm = loaded_modules[key] | ||
1845 | if oldm !== m | ||
1846 | if (0 != ccall(:jl_generating_output, Cint, ())) && (JLOptions().incremental != 0) | ||
1847 | error("Replacing module `$(key.name)`") | ||
1848 | else | ||
1849 | @warn "Replacing module `$(key.name)`" | ||
1850 | end | ||
1851 | end | ||
1852 | end | ||
1853 | push!(loaded_modules_order, m) | ||
1854 | loaded_modules[key] = m | ||
1855 | module_keys[m] = key | ||
1856 | end | ||
1857 | nothing | ||
1858 | end | ||
1859 | |||
1860 | register_root_module(Core) | ||
1861 | register_root_module(Base) | ||
1862 | register_root_module(Main) | ||
1863 | |||
1864 | # This is used as the current module when loading top-level modules. | ||
1865 | # It has the special behavior that modules evaluated in it get added | ||
1866 | # to the loaded_modules table instead of getting bindings. | ||
1867 | baremodule __toplevel__ | ||
1868 | using Base | ||
1869 | end | ||
1870 | |||
1871 | # get a top-level Module from the given key | ||
1872 | root_module(key::PkgId) = @lock require_lock loaded_modules[key] | ||
1873 | function root_module(where::Module, name::Symbol) | ||
1874 | key = identify_package(where, String(name)) | ||
1875 | key isa PkgId || throw(KeyError(name)) | ||
1876 | return root_module(key) | ||
1877 | end | ||
1878 | maybe_root_module(key::PkgId) = @lock require_lock get(loaded_modules, key, nothing) | ||
1879 | |||
1880 | root_module_exists(key::PkgId) = @lock require_lock haskey(loaded_modules, key) | ||
1881 | loaded_modules_array() = @lock require_lock copy(loaded_modules_order) | ||
1882 | |||
1883 | function unreference_module(key::PkgId) | ||
1884 | if haskey(loaded_modules, key) | ||
1885 | m = pop!(loaded_modules, key) | ||
1886 | # need to ensure all modules are GC rooted; will still be referenced | ||
1887 | # in module_keys | ||
1888 | end | ||
1889 | end | ||
1890 | |||
1891 | # whoever takes the package_locks[pkg] must call this function immediately | ||
1892 | function set_pkgorigin_version_path(pkg::PkgId, path::Union{String,Nothing}) | ||
1893 | assert_havelock(require_lock) | ||
1894 | pkgorigin = get!(PkgOrigin, pkgorigins, pkg) | ||
1895 | if path !== nothing | ||
1896 | # Pkg needs access to the version of packages in the sysimage. | ||
1897 | if Core.Compiler.generating_sysimg() | ||
1898 | pkgorigin.version = get_pkgversion_from_path(joinpath(dirname(path), "..")) | ||
1899 | end | ||
1900 | end | ||
1901 | pkgorigin.path = path | ||
1902 | nothing | ||
1903 | end | ||
1904 | |||
1905 | # A hook to allow code load to use Pkg.precompile | ||
1906 | const PKG_PRECOMPILE_HOOK = Ref{Function}() | ||
1907 | |||
1908 | # Returns `nothing` or the new(ish) module | ||
1909 | function _require(pkg::PkgId, env=nothing) | ||
1910 | assert_havelock(require_lock) | ||
1911 | loaded = start_loading(pkg) | ||
1912 | loaded === nothing || return loaded | ||
1913 | |||
1914 | last = toplevel_load[] | ||
1915 | try | ||
1916 | toplevel_load[] = false | ||
1917 | # perform the search operation to select the module file require intends to load | ||
1918 | path = locate_package(pkg, env) | ||
1919 | if path === nothing | ||
1920 | throw(ArgumentError(""" | ||
1921 | Package $pkg is required but does not seem to be installed: | ||
1922 | - Run `Pkg.instantiate()` to install all recorded dependencies. | ||
1923 | """)) | ||
1924 | end | ||
1925 | set_pkgorigin_version_path(pkg, path) | ||
1926 | |||
1927 | pkg_precompile_attempted = false # being safe to avoid getting stuck in a Pkg.precompile loop | ||
1928 | |||
1929 | # attempt to load the module file via the precompile cache locations | ||
1930 | if JLOptions().use_compiled_modules != 0 | ||
1931 | @label load_from_cache | ||
1932 | m = _require_search_from_serialized(pkg, path, UInt128(0)) | ||
1933 | if m isa Module | ||
1934 | return m | ||
1935 | end | ||
1936 | end | ||
1937 | |||
1938 | # if the module being required was supposed to have a particular version | ||
1939 | # but it was not handled by the precompile loader, complain | ||
1940 | for (concrete_pkg, concrete_build_id) in _concrete_dependencies | ||
1941 | if pkg == concrete_pkg | ||
1942 | @warn """Module $(pkg.name) with build ID $((UUID(concrete_build_id))) is missing from the cache. | ||
1943 | This may mean $pkg does not support precompilation but is imported by a module that does.""" | ||
1944 | if JLOptions().incremental != 0 | ||
1945 | # during incremental precompilation, this should be fail-fast | ||
1946 | throw(PrecompilableError()) | ||
1947 | end | ||
1948 | end | ||
1949 | end | ||
1950 | |||
1951 | if JLOptions().use_compiled_modules != 0 | ||
1952 | if (0 == ccall(:jl_generating_output, Cint, ())) || (JLOptions().incremental != 0) | ||
1953 | if !pkg_precompile_attempted && isinteractive() && isassigned(PKG_PRECOMPILE_HOOK) | ||
1954 | pkg_precompile_attempted = true | ||
1955 | unlock(require_lock) | ||
1956 | try | ||
1957 | @invokelatest PKG_PRECOMPILE_HOOK[](pkg.name, _from_loading = true) | ||
1958 | finally | ||
1959 | lock(require_lock) | ||
1960 | end | ||
1961 | @goto load_from_cache | ||
1962 | end | ||
1963 | # spawn off a new incremental pre-compile task for recursive `require` calls | ||
1964 | cachefile_or_module = maybe_cachefile_lock(pkg, path) do | ||
1965 | # double-check now that we have lock | ||
1966 | m = _require_search_from_serialized(pkg, path, UInt128(0)) | ||
1967 | m isa Module && return m | ||
1968 | compilecache(pkg, path) | ||
1969 | end | ||
1970 | cachefile_or_module isa Module && return cachefile_or_module::Module | ||
1971 | cachefile = cachefile_or_module | ||
1972 | if isnothing(cachefile) # maybe_cachefile_lock returns nothing if it had to wait for another process | ||
1973 | @goto load_from_cache # the new cachefile will have the newest mtime so will come first in the search | ||
1974 | elseif isa(cachefile, Exception) | ||
1975 | if precompilableerror(cachefile) | ||
1976 | verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug | ||
1977 | @logmsg verbosity "Skipping precompilation since __precompile__(false). Importing $pkg." | ||
1978 | else | ||
1979 | @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m | ||
1980 | end | ||
1981 | # fall-through to loading the file locally if not incremental | ||
1982 | else | ||
1983 | cachefile, ocachefile = cachefile::Tuple{String, Union{Nothing, String}} | ||
1984 | m = _tryrequire_from_serialized(pkg, cachefile, ocachefile) | ||
1985 | if !isa(m, Module) | ||
1986 | @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m | ||
1987 | else | ||
1988 | return m | ||
1989 | end | ||
1990 | end | ||
1991 | if JLOptions().incremental != 0 | ||
1992 | # during incremental precompilation, this should be fail-fast | ||
1993 | throw(PrecompilableError()) | ||
1994 | end | ||
1995 | end | ||
1996 | end | ||
1997 | |||
1998 | # just load the file normally via include | ||
1999 | # for unknown dependencies | ||
2000 | uuid = pkg.uuid | ||
2001 | uuid = (uuid === nothing ? (UInt64(0), UInt64(0)) : convert(NTuple{2, UInt64}, uuid)) | ||
2002 | old_uuid = ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), __toplevel__) | ||
2003 | if uuid !== old_uuid | ||
2004 | ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), __toplevel__, uuid) | ||
2005 | end | ||
2006 | unlock(require_lock) | ||
2007 | try | ||
2008 | include(__toplevel__, path) | ||
2009 | loaded = get(loaded_modules, pkg, nothing) | ||
2010 | finally | ||
2011 | lock(require_lock) | ||
2012 | if uuid !== old_uuid | ||
2013 | ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), __toplevel__, old_uuid) | ||
2014 | end | ||
2015 | end | ||
2016 | finally | ||
2017 | toplevel_load[] = last | ||
2018 | end_loading(pkg, loaded) | ||
2019 | end | ||
2020 | return loaded | ||
2021 | end | ||
2022 | |||
2023 | # Only used from test/precompile.jl | ||
2024 | function _require_from_serialized(uuidkey::PkgId, path::String, ocachepath::Union{String, Nothing}) | ||
2025 | @lock require_lock begin | ||
2026 | set_pkgorigin_version_path(uuidkey, nothing) | ||
2027 | newm = _tryrequire_from_serialized(uuidkey, path, ocachepath) | ||
2028 | newm isa Module || throw(newm) | ||
2029 | insert_extension_triggers(uuidkey) | ||
2030 | # After successfully loading, notify downstream consumers | ||
2031 | run_package_callbacks(uuidkey) | ||
2032 | return newm | ||
2033 | end | ||
2034 | end | ||
2035 | |||
2036 | |||
2037 | |||
2038 | # relative-path load | ||
2039 | |||
2040 | """ | ||
2041 | include_string([mapexpr::Function,] m::Module, code::AbstractString, filename::AbstractString="string") | ||
2042 | |||
2043 | Like [`include`](@ref), except reads code from the given string rather than from a file. | ||
2044 | |||
2045 | The optional first argument `mapexpr` can be used to transform the included code before | ||
2046 | it is evaluated: for each parsed expression `expr` in `code`, the `include_string` function | ||
2047 | actually evaluates `mapexpr(expr)`. If it is omitted, `mapexpr` defaults to [`identity`](@ref). | ||
2048 | |||
2049 | !!! compat "Julia 1.5" | ||
2050 | Julia 1.5 is required for passing the `mapexpr` argument. | ||
2051 | """ | ||
2052 |
288 (100 %)
samples spent in include_string
function include_string(mapexpr::Function, mod::Module, code::AbstractString,
288 (100 %) (incl.) when called from #invokelatest#2 line 887 |
||
2053 | filename::AbstractString="string") | ||
2054 | loc = LineNumberNode(1, Symbol(filename)) | ||
2055 | try | ||
2056 | ast = Meta.parseall(code, filename=filename) | ||
2057 | @assert Meta.isexpr(ast, :toplevel) | ||
2058 | result = nothing | ||
2059 | line_and_ex = Expr(:toplevel, loc, nothing) | ||
2060 | for ex in ast.args | ||
2061 | if ex isa LineNumberNode | ||
2062 | loc = ex | ||
2063 | line_and_ex.args[1] = ex | ||
2064 | continue | ||
2065 | end | ||
2066 | ex = mapexpr(ex) | ||
2067 | # Wrap things to be eval'd in a :toplevel expr to carry line | ||
2068 | # information as part of the expr. | ||
2069 | line_and_ex.args[2] = ex | ||
2070 | 288 (100 %) |
288 (100 %)
samples spent calling
eval
result = Core.eval(mod, line_and_ex)
|
|
2071 | end | ||
2072 | return result | ||
2073 | catch exc | ||
2074 | # TODO: Now that stacktraces are more reliable we should remove | ||
2075 | # LoadError and expose the real error type directly. | ||
2076 | rethrow(LoadError(filename, loc.line, exc)) | ||
2077 | end | ||
2078 | end | ||
2079 | |||
2080 | include_string(m::Module, txt::AbstractString, fname::AbstractString="string") = | ||
2081 | include_string(identity, m, txt, fname) | ||
2082 | |||
2083 | function source_path(default::Union{AbstractString,Nothing}="") | ||
2084 | s = current_task().storage | ||
2085 | if s !== nothing | ||
2086 | s = s::IdDict{Any,Any} | ||
2087 | if haskey(s, :SOURCE_PATH) | ||
2088 | return s[:SOURCE_PATH]::Union{Nothing,String} | ||
2089 | end | ||
2090 | end | ||
2091 | return default | ||
2092 | end | ||
2093 | |||
2094 | function source_dir() | ||
2095 | p = source_path(nothing) | ||
2096 | return p === nothing ? pwd() : dirname(p) | ||
2097 | end | ||
2098 | |||
2099 | """ | ||
2100 | Base.include([mapexpr::Function,] m::Module, path::AbstractString) | ||
2101 | |||
2102 | Evaluate the contents of the input source file in the global scope of module `m`. | ||
2103 | Every module (except those defined with [`baremodule`](@ref)) has its own | ||
2104 | definition of `include` omitting the `m` argument, which evaluates the file in that module. | ||
2105 | Returns the result of the last evaluated expression of the input file. During including, | ||
2106 | a task-local include path is set to the directory containing the file. Nested calls to | ||
2107 | `include` will search relative to that path. This function is typically used to load source | ||
2108 | interactively, or to combine files in packages that are broken into multiple source files. | ||
2109 | |||
2110 | The optional first argument `mapexpr` can be used to transform the included code before | ||
2111 | it is evaluated: for each parsed expression `expr` in `path`, the `include` function | ||
2112 | actually evaluates `mapexpr(expr)`. If it is omitted, `mapexpr` defaults to [`identity`](@ref). | ||
2113 | |||
2114 | !!! compat "Julia 1.5" | ||
2115 | Julia 1.5 is required for passing the `mapexpr` argument. | ||
2116 | """ | ||
2117 | Base.include # defined in Base.jl | ||
2118 | |||
2119 | # Full include() implementation which is used after bootstrap | ||
2120 | function _include(mapexpr::Function, mod::Module, _path::AbstractString) | ||
2121 | @noinline # Workaround for module availability in _simplify_include_frames | ||
2122 | path, prev = _include_dependency(mod, _path) | ||
2123 | for callback in include_callbacks # to preserve order, must come before eval in include_string | ||
2124 | invokelatest(callback, mod, path) | ||
2125 | end | ||
2126 | code = read(path, String) | ||
2127 | tls = task_local_storage() | ||
2128 | tls[:SOURCE_PATH] = path | ||
2129 | try | ||
2130 | return include_string(mapexpr, mod, code, path) | ||
2131 | finally | ||
2132 | if prev === nothing | ||
2133 | delete!(tls, :SOURCE_PATH) | ||
2134 | else | ||
2135 | tls[:SOURCE_PATH] = prev | ||
2136 | end | ||
2137 | end | ||
2138 | end | ||
2139 | |||
2140 | """ | ||
2141 | evalfile(path::AbstractString, args::Vector{String}=String[]) | ||
2142 | |||
2143 | Load the file into an anonymous module using [`include`](@ref), evaluate all expressions, | ||
2144 | and return the value of the last expression. | ||
2145 | The optional `args` argument can be used to set the input arguments of the script (i.e. the global `ARGS` variable). | ||
2146 | Note that definitions (e.g. methods, globals) are evaluated in the anonymous module and do not affect the current module. | ||
2147 | |||
2148 | # Example | ||
2149 | |||
2150 | ```jldoctest | ||
2151 | julia> write("testfile.jl", \"\"\" | ||
2152 | @show ARGS | ||
2153 | 1 + 1 | ||
2154 | \"\"\"); | ||
2155 | |||
2156 | julia> x = evalfile("testfile.jl", ["ARG1", "ARG2"]); | ||
2157 | ARGS = ["ARG1", "ARG2"] | ||
2158 | |||
2159 | julia> x | ||
2160 | 2 | ||
2161 | |||
2162 | julia> rm("testfile.jl") | ||
2163 | ``` | ||
2164 | """ | ||
2165 | function evalfile(path::AbstractString, args::Vector{String}=String[]) | ||
2166 | return Core.eval(Module(:__anon__), | ||
2167 | Expr(:toplevel, | ||
2168 | :(const ARGS = $args), | ||
2169 | :(eval(x) = $(Expr(:core, :eval))(__anon__, x)), | ||
2170 | :(include(x) = $(Expr(:top, :include))(__anon__, x)), | ||
2171 | :(include(mapexpr::Function, x) = $(Expr(:top, :include))(mapexpr, __anon__, x)), | ||
2172 | :(include($path)))) | ||
2173 | end | ||
2174 | evalfile(path::AbstractString, args::Vector) = evalfile(path, String[args...]) | ||
2175 | |||
2176 | function load_path_setup_code(load_path::Bool=true) | ||
2177 | code = """ | ||
2178 | append!(empty!(Base.DEPOT_PATH), $(repr(map(abspath, DEPOT_PATH)))) | ||
2179 | append!(empty!(Base.DL_LOAD_PATH), $(repr(map(abspath, DL_LOAD_PATH)))) | ||
2180 | """ | ||
2181 | if load_path | ||
2182 | load_path = map(abspath, Base.load_path()) | ||
2183 | path_sep = Sys.iswindows() ? ';' : ':' | ||
2184 | any(path -> path_sep in path, load_path) && | ||
2185 | error("LOAD_PATH entries cannot contain $(repr(path_sep))") | ||
2186 | code *= """ | ||
2187 | append!(empty!(Base.LOAD_PATH), $(repr(load_path))) | ||
2188 | ENV["JULIA_LOAD_PATH"] = $(repr(join(load_path, Sys.iswindows() ? ';' : ':'))) | ||
2189 | Base.set_active_project(nothing) | ||
2190 | """ | ||
2191 | end | ||
2192 | return code | ||
2193 | end | ||
2194 | |||
2195 | # this is called in the external process that generates precompiled package files | ||
2196 | function include_package_for_output(pkg::PkgId, input::String, depot_path::Vector{String}, dl_load_path::Vector{String}, load_path::Vector{String}, | ||
2197 | concrete_deps::typeof(_concrete_dependencies), source::Union{Nothing,String}) | ||
2198 | append!(empty!(Base.DEPOT_PATH), depot_path) | ||
2199 | append!(empty!(Base.DL_LOAD_PATH), dl_load_path) | ||
2200 | append!(empty!(Base.LOAD_PATH), load_path) | ||
2201 | ENV["JULIA_LOAD_PATH"] = join(load_path, Sys.iswindows() ? ';' : ':') | ||
2202 | set_active_project(nothing) | ||
2203 | Base._track_dependencies[] = true | ||
2204 | get!(Base.PkgOrigin, Base.pkgorigins, pkg).path = input | ||
2205 | append!(empty!(Base._concrete_dependencies), concrete_deps) | ||
2206 | uuid_tuple = pkg.uuid === nothing ? (UInt64(0), UInt64(0)) : convert(NTuple{2, UInt64}, pkg.uuid) | ||
2207 | |||
2208 | ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, uuid_tuple) | ||
2209 | if source !== nothing | ||
2210 | task_local_storage()[:SOURCE_PATH] = source | ||
2211 | end | ||
2212 | |||
2213 | ccall(:jl_set_newly_inferred, Cvoid, (Any,), Core.Compiler.newly_inferred) | ||
2214 | Core.Compiler.track_newly_inferred.x = true | ||
2215 | try | ||
2216 | Base.include(Base.__toplevel__, input) | ||
2217 | catch ex | ||
2218 | precompilableerror(ex) || rethrow() | ||
2219 | @debug "Aborting `create_expr_cache'" exception=(ErrorException("Declaration of __precompile__(false) not allowed"), catch_backtrace()) | ||
2220 | exit(125) # we define status = 125 means PrecompileableError | ||
2221 | finally | ||
2222 | Core.Compiler.track_newly_inferred.x = false | ||
2223 | end | ||
2224 | end | ||
2225 | |||
2226 | const PRECOMPILE_TRACE_COMPILE = Ref{String}() | ||
2227 | function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::Union{Nothing, String}, | ||
2228 | concrete_deps::typeof(_concrete_dependencies), internal_stderr::IO = stderr, internal_stdout::IO = stdout) | ||
2229 | @nospecialize internal_stderr internal_stdout | ||
2230 | rm(output, force=true) # Remove file if it exists | ||
2231 | output_o === nothing || rm(output_o, force=true) | ||
2232 | depot_path = map(abspath, DEPOT_PATH) | ||
2233 | dl_load_path = map(abspath, DL_LOAD_PATH) | ||
2234 | load_path = map(abspath, Base.load_path()) | ||
2235 | path_sep = Sys.iswindows() ? ';' : ':' | ||
2236 | any(path -> path_sep in path, load_path) && | ||
2237 | error("LOAD_PATH entries cannot contain $(repr(path_sep))") | ||
2238 | |||
2239 | deps_strs = String[] | ||
2240 | function pkg_str(_pkg::PkgId) | ||
2241 | if _pkg.uuid === nothing | ||
2242 | "Base.PkgId($(repr(_pkg.name)))" | ||
2243 | else | ||
2244 | "Base.PkgId(Base.UUID(\"$(_pkg.uuid)\"), $(repr(_pkg.name)))" | ||
2245 | end | ||
2246 | end | ||
2247 | for (pkg, build_id) in concrete_deps | ||
2248 | push!(deps_strs, "$(pkg_str(pkg)) => $(repr(build_id))") | ||
2249 | end | ||
2250 | |||
2251 | if output_o !== nothing | ||
2252 | cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) | ||
2253 | opt_level = Base.JLOptions().opt_level | ||
2254 | opts = `-O$(opt_level) --output-o $(output_o) --output-ji $(output) --output-incremental=yes` | ||
2255 | else | ||
2256 | cpu_target = nothing | ||
2257 | opts = `-O0 --output-ji $(output) --output-incremental=yes` | ||
2258 | end | ||
2259 | |||
2260 | deps_eltype = sprint(show, eltype(concrete_deps); context = :module=>nothing) | ||
2261 | deps = deps_eltype * "[" * join(deps_strs, ",") * "]" | ||
2262 | trace = isassigned(PRECOMPILE_TRACE_COMPILE) ? `--trace-compile=$(PRECOMPILE_TRACE_COMPILE[])` : `` | ||
2263 | io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd) $(opts) | ||
2264 | --startup-file=no --history-file=no --warn-overwrite=yes | ||
2265 | --color=$(have_color === nothing ? "auto" : have_color ? "yes" : "no") | ||
2266 | $trace | ||
2267 | -`, | ||
2268 | "OPENBLAS_NUM_THREADS" => 1, | ||
2269 | "JULIA_NUM_THREADS" => 1), | ||
2270 | stderr = internal_stderr, stdout = internal_stdout), | ||
2271 | "w", stdout) | ||
2272 | # write data over stdin to avoid the (unlikely) case of exceeding max command line size | ||
2273 | write(io.in, """ | ||
2274 | empty!(Base.EXT_DORMITORY) # If we have a custom sysimage with `EXT_DORMITORY` prepopulated | ||
2275 | Base.precompiling_extension = $(loading_extension) | ||
2276 | Base.include_package_for_output($(pkg_str(pkg)), $(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)), | ||
2277 | $(repr(load_path)), $deps, $(repr(source_path(nothing)))) | ||
2278 | """) | ||
2279 | close(io.in) | ||
2280 | return io | ||
2281 | end | ||
2282 | |||
2283 | function compilecache_dir(pkg::PkgId) | ||
2284 | entrypath, entryfile = cache_file_entry(pkg) | ||
2285 | return joinpath(DEPOT_PATH[1], entrypath) | ||
2286 | end | ||
2287 | |||
2288 | function compilecache_path(pkg::PkgId, prefs_hash::UInt64; project::String=something(Base.active_project(), ""))::String | ||
2289 | entrypath, entryfile = cache_file_entry(pkg) | ||
2290 | cachepath = joinpath(DEPOT_PATH[1], entrypath) | ||
2291 | isdir(cachepath) || mkpath(cachepath) | ||
2292 | if pkg.uuid === nothing | ||
2293 | abspath(cachepath, entryfile) * ".ji" | ||
2294 | else | ||
2295 | crc = _crc32c(project) | ||
2296 | crc = _crc32c(unsafe_string(JLOptions().image_file), crc) | ||
2297 | crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc) | ||
2298 | crc = _crc32c(ccall(:jl_cache_flags, UInt8, ()), crc) | ||
2299 | |||
2300 | cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) | ||
2301 | if cpu_target === nothing | ||
2302 | cpu_target = unsafe_string(JLOptions().cpu_target) | ||
2303 | end | ||
2304 | crc = _crc32c(cpu_target, crc) | ||
2305 | |||
2306 | crc = _crc32c(prefs_hash, crc) | ||
2307 | project_precompile_slug = slug(crc, 5) | ||
2308 | abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji")) | ||
2309 | end | ||
2310 | end | ||
2311 | |||
2312 | """ | ||
2313 | Base.compilecache(module::PkgId) | ||
2314 | |||
2315 | Creates a precompiled cache file for a module and all of its dependencies. | ||
2316 | This can be used to reduce package load times. Cache files are stored in | ||
2317 | `DEPOT_PATH[1]/compiled`. See [Module initialization and precompilation](@ref) | ||
2318 | for important notes. | ||
2319 | """ | ||
2320 | function compilecache(pkg::PkgId, internal_stderr::IO = stderr, internal_stdout::IO = stdout) | ||
2321 | @nospecialize internal_stderr internal_stdout | ||
2322 | path = locate_package(pkg) | ||
2323 | path === nothing && throw(ArgumentError("$pkg not found during precompilation")) | ||
2324 | return compilecache(pkg, path, internal_stderr, internal_stdout) | ||
2325 | end | ||
2326 | |||
2327 | const MAX_NUM_PRECOMPILE_FILES = Ref(10) | ||
2328 | |||
2329 | function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, internal_stdout::IO = stdout, | ||
2330 | keep_loaded_modules::Bool = true) | ||
2331 | |||
2332 | @nospecialize internal_stderr internal_stdout | ||
2333 | # decide where to put the resulting cache file | ||
2334 | cachepath = compilecache_dir(pkg) | ||
2335 | |||
2336 | # build up the list of modules that we want the precompile process to preserve | ||
2337 | concrete_deps = copy(_concrete_dependencies) | ||
2338 | if keep_loaded_modules | ||
2339 | for mod in loaded_modules_array() | ||
2340 | if !(mod === Main || mod === Core || mod === Base) | ||
2341 | push!(concrete_deps, PkgId(mod) => module_build_id(mod)) | ||
2342 | end | ||
2343 | end | ||
2344 | end | ||
2345 | # run the expression and cache the result | ||
2346 | verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug | ||
2347 | @logmsg verbosity "Precompiling $pkg" | ||
2348 | |||
2349 | # create a temporary file in `cachepath` directory, write the cache in it, | ||
2350 | # write the checksum, _and then_ atomically move the file to `cachefile`. | ||
2351 | mkpath(cachepath) | ||
2352 | cache_objects = JLOptions().use_pkgimages != 0 | ||
2353 | tmppath, tmpio = mktemp(cachepath) | ||
2354 | |||
2355 | if cache_objects | ||
2356 | tmppath_o, tmpio_o = mktemp(cachepath) | ||
2357 | tmppath_so, tmpio_so = mktemp(cachepath) | ||
2358 | else | ||
2359 | tmppath_o = nothing | ||
2360 | end | ||
2361 | local p | ||
2362 | try | ||
2363 | close(tmpio) | ||
2364 | if cache_objects | ||
2365 | close(tmpio_o) | ||
2366 | close(tmpio_so) | ||
2367 | end | ||
2368 | p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, internal_stderr, internal_stdout) | ||
2369 | |||
2370 | if success(p) | ||
2371 | if cache_objects | ||
2372 | # Run linker over tmppath_o | ||
2373 | Linking.link_image(tmppath_o, tmppath_so) | ||
2374 | end | ||
2375 | |||
2376 | # Read preferences hash back from .ji file (we can't precompute because | ||
2377 | # we don't actually know what the list of compile-time preferences are without compiling) | ||
2378 | prefs_hash = preferences_hash(tmppath) | ||
2379 | cachefile = compilecache_path(pkg, prefs_hash) | ||
2380 | ocachefile = cache_objects ? ocachefile_from_cachefile(cachefile) : nothing | ||
2381 | |||
2382 | # append checksum for so to the end of the .ji file: | ||
2383 | crc_so = UInt32(0) | ||
2384 | if cache_objects | ||
2385 | crc_so = open(_crc32c, tmppath_so, "r") | ||
2386 | end | ||
2387 | |||
2388 | # append extra crc to the end of the .ji file: | ||
2389 | open(tmppath, "r+") do f | ||
2390 | if iszero(isvalid_cache_header(f)) | ||
2391 | error("Invalid header for $pkg in new cache file $(repr(tmppath)).") | ||
2392 | end | ||
2393 | seekend(f) | ||
2394 | write(f, crc_so) | ||
2395 | seekstart(f) | ||
2396 | write(f, _crc32c(f)) | ||
2397 | end | ||
2398 | |||
2399 | # inherit permission from the source file (and make them writable) | ||
2400 | chmod(tmppath, filemode(path) & 0o777 | 0o200) | ||
2401 | |||
2402 | # prune the directory with cache files | ||
2403 | if pkg.uuid !== nothing | ||
2404 | entrypath, entryfile = cache_file_entry(pkg) | ||
2405 | cachefiles = filter!(x -> startswith(x, entryfile * "_") && endswith(x, ".ji"), readdir(cachepath)) | ||
2406 | if length(cachefiles) >= MAX_NUM_PRECOMPILE_FILES[] | ||
2407 | idx = findmin(mtime.(joinpath.(cachepath, cachefiles)))[2] | ||
2408 | evicted_cachefile = joinpath(cachepath, cachefiles[idx]) | ||
2409 | @debug "Evicting file from cache" evicted_cachefile | ||
2410 | rm(evicted_cachefile; force=true) | ||
2411 | try | ||
2412 | rm(ocachefile_from_cachefile(evicted_cachefile); force=true) | ||
2413 | @static if Sys.isapple() | ||
2414 | rm(ocachefile_from_cachefile(evicted_cachefile) * ".dSYM"; force=true, recursive=true) | ||
2415 | end | ||
2416 | catch e | ||
2417 | e isa IOError || rethrow() | ||
2418 | end | ||
2419 | end | ||
2420 | end | ||
2421 | |||
2422 | if cache_objects | ||
2423 | try | ||
2424 | rename(tmppath_so, ocachefile::String; force=true) | ||
2425 | catch e | ||
2426 | e isa IOError || rethrow() | ||
2427 | isfile(ocachefile::String) || rethrow() | ||
2428 | # Windows prevents renaming a file that is in use so if there is a Julia session started | ||
2429 | # with a package image loaded, we cannot rename that file. | ||
2430 | # The code belows append a `_i` to the name of the cache file where `i` is the smallest number such that | ||
2431 | # that cache file does not exist. | ||
2432 | ocachename, ocacheext = splitext(ocachefile::String) | ||
2433 | old_cachefiles = Set(readdir(cachepath)) | ||
2434 | num = 1 | ||
2435 | while true | ||
2436 | ocachefile = ocachename * "_$num" * ocacheext | ||
2437 | in(basename(ocachefile), old_cachefiles) || break | ||
2438 | num += 1 | ||
2439 | end | ||
2440 | # TODO: Risk for a race here if some other process grabs this name before us | ||
2441 | cachefile = cachefile_from_ocachefile(ocachefile) | ||
2442 | rename(tmppath_so, ocachefile::String; force=true) | ||
2443 | end | ||
2444 | @static if Sys.isapple() | ||
2445 | run(`$(Linking.dsymutil()) $ocachefile`, Base.DevNull(), Base.DevNull(), Base.DevNull()) | ||
2446 | end | ||
2447 | end | ||
2448 | # this is atomic according to POSIX (not Win32): | ||
2449 | rename(tmppath, cachefile; force=true) | ||
2450 | return cachefile, ocachefile | ||
2451 | end | ||
2452 | finally | ||
2453 | rm(tmppath, force=true) | ||
2454 | if cache_objects | ||
2455 | rm(tmppath_o::String, force=true) | ||
2456 | rm(tmppath_so, force=true) | ||
2457 | end | ||
2458 | end | ||
2459 | if p.exitcode == 125 | ||
2460 | return PrecompilableError() | ||
2461 | else | ||
2462 | error("Failed to precompile $pkg to $(repr(tmppath)).") | ||
2463 | end | ||
2464 | end | ||
2465 | |||
2466 | function module_build_id(m::Module) | ||
2467 | hi, lo = ccall(:jl_module_build_id, NTuple{2,UInt64}, (Any,), m) | ||
2468 | return (UInt128(hi) << 64) | lo | ||
2469 | end | ||
2470 | |||
2471 | function isvalid_cache_header(f::IOStream) | ||
2472 | pkgimage = Ref{UInt8}() | ||
2473 | checksum = ccall(:jl_read_verify_header, UInt64, (Ptr{Cvoid}, Ptr{UInt8}, Ptr{Int64}, Ptr{Int64}), f.ios, pkgimage, Ref{Int64}(), Ref{Int64}()) # returns checksum id or zero | ||
2474 | |||
2475 | if !iszero(checksum) && pkgimage[] != 0 | ||
2476 | @debug "Cache header was for pkgimage" | ||
2477 | return UInt64(0) # We somehow read the header for a pkgimage and not a ji | ||
2478 | end | ||
2479 | return checksum | ||
2480 | end | ||
2481 | isvalid_file_crc(f::IOStream) = (_crc32c(seekstart(f), filesize(f) - 4) == read(f, UInt32)) | ||
2482 | |||
2483 | function isvalid_pkgimage_crc(f::IOStream, ocachefile::String) | ||
2484 | seekstart(f) # TODO necessary | ||
2485 | seek(f, filesize(f) - 8) | ||
2486 | expected_crc_so = read(f, UInt32) | ||
2487 | crc_so = open(_crc32c, ocachefile, "r") | ||
2488 | expected_crc_so == crc_so | ||
2489 | end | ||
2490 | |||
2491 | struct CacheHeaderIncludes | ||
2492 | id::PkgId | ||
2493 | filename::String | ||
2494 | mtime::Float64 | ||
2495 | modpath::Vector{String} # seemingly not needed in Base, but used by Revise | ||
2496 | end | ||
2497 | |||
2498 | function parse_cache_header(f::IO) | ||
2499 | flags = read(f, UInt8) | ||
2500 | modules = Vector{Pair{PkgId, UInt64}}() | ||
2501 | while true | ||
2502 | n = read(f, Int32) | ||
2503 | n == 0 && break | ||
2504 | sym = String(read(f, n)) # module name | ||
2505 | uuid = UUID((read(f, UInt64), read(f, UInt64))) # pkg UUID | ||
2506 | build_id = read(f, UInt64) # build UUID (mostly just a timestamp) | ||
2507 | push!(modules, PkgId(uuid, sym) => build_id) | ||
2508 | end | ||
2509 | totbytes = read(f, Int64) # total bytes for file dependencies + preferences | ||
2510 | # read the list of requirements | ||
2511 | # and split the list into include and requires statements | ||
2512 | includes = CacheHeaderIncludes[] | ||
2513 | requires = Pair{PkgId, PkgId}[] | ||
2514 | while true | ||
2515 | n2 = read(f, Int32) | ||
2516 | totbytes -= 4 | ||
2517 | if n2 == 0 | ||
2518 | break | ||
2519 | end | ||
2520 | depname = String(read(f, n2)) | ||
2521 | totbytes -= n2 | ||
2522 | mtime = read(f, Float64) | ||
2523 | totbytes -= 8 | ||
2524 | n1 = read(f, Int32) | ||
2525 | totbytes -= 4 | ||
2526 | # map ids to keys | ||
2527 | modkey = (n1 == 0) ? PkgId("") : modules[n1].first | ||
2528 | modpath = String[] | ||
2529 | if n1 != 0 | ||
2530 | # determine the complete module path | ||
2531 | while true | ||
2532 | n1 = read(f, Int32) | ||
2533 | totbytes -= 4 | ||
2534 | if n1 == 0 | ||
2535 | break | ||
2536 | end | ||
2537 | push!(modpath, String(read(f, n1))) | ||
2538 | totbytes -= n1 | ||
2539 | end | ||
2540 | end | ||
2541 | if depname[1] == '\0' | ||
2542 | push!(requires, modkey => binunpack(depname)) | ||
2543 | else | ||
2544 | push!(includes, CacheHeaderIncludes(modkey, depname, mtime, modpath)) | ||
2545 | end | ||
2546 | end | ||
2547 | prefs = String[] | ||
2548 | while true | ||
2549 | n2 = read(f, Int32) | ||
2550 | totbytes -= 4 | ||
2551 | if n2 == 0 | ||
2552 | break | ||
2553 | end | ||
2554 | push!(prefs, String(read(f, n2))) | ||
2555 | totbytes -= n2 | ||
2556 | end | ||
2557 | prefs_hash = read(f, UInt64) | ||
2558 | totbytes -= 8 | ||
2559 | srctextpos = read(f, Int64) | ||
2560 | totbytes -= 8 | ||
2561 | @assert totbytes == 0 "header of cache file appears to be corrupt (totbytes == $(totbytes))" | ||
2562 | # read the list of modules that are required to be present during loading | ||
2563 | required_modules = Vector{Pair{PkgId, UInt128}}() | ||
2564 | while true | ||
2565 | n = read(f, Int32) | ||
2566 | n == 0 && break | ||
2567 | sym = String(read(f, n)) # module name | ||
2568 | uuid = UUID((read(f, UInt64), read(f, UInt64))) # pkg UUID | ||
2569 | build_id = UInt128(read(f, UInt64)) << 64 | ||
2570 | build_id |= read(f, UInt64) | ||
2571 | push!(required_modules, PkgId(uuid, sym) => build_id) | ||
2572 | end | ||
2573 | l = read(f, Int32) | ||
2574 | clone_targets = read(f, l) | ||
2575 | |||
2576 | return modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags | ||
2577 | end | ||
2578 | |||
2579 | function parse_cache_header(cachefile::String; srcfiles_only::Bool=false) | ||
2580 | io = open(cachefile, "r") | ||
2581 | try | ||
2582 | iszero(isvalid_cache_header(io)) && throw(ArgumentError("Invalid header in cache file $cachefile.")) | ||
2583 | ret = parse_cache_header(io) | ||
2584 | srcfiles_only || return ret | ||
2585 | _, (includes, _), _, srctextpos, _... = ret | ||
2586 | srcfiles = srctext_files(io, srctextpos) | ||
2587 | delidx = Int[] | ||
2588 | for (i, chi) in enumerate(includes) | ||
2589 | chi.filename ∈ srcfiles || push!(delidx, i) | ||
2590 | end | ||
2591 | deleteat!(includes, delidx) | ||
2592 | return ret | ||
2593 | finally | ||
2594 | close(io) | ||
2595 | end | ||
2596 | end | ||
2597 | |||
2598 | preferences_hash(f::IO) = parse_cache_header(f)[6] | ||
2599 | function preferences_hash(cachefile::String) | ||
2600 | io = open(cachefile, "r") | ||
2601 | try | ||
2602 | if iszero(isvalid_cache_header(io)) | ||
2603 | throw(ArgumentError("Invalid header in cache file $cachefile.")) | ||
2604 | end | ||
2605 | return preferences_hash(io) | ||
2606 | finally | ||
2607 | close(io) | ||
2608 | end | ||
2609 | end | ||
2610 | |||
2611 | function cache_dependencies(f::IO) | ||
2612 | _, (includes, _), modules, _... = parse_cache_header(f) | ||
2613 | return modules, map(chi -> (chi.filename, chi.mtime), includes) # return just filename and mtime | ||
2614 | end | ||
2615 | |||
2616 | function cache_dependencies(cachefile::String) | ||
2617 | io = open(cachefile, "r") | ||
2618 | try | ||
2619 | iszero(isvalid_cache_header(io)) && throw(ArgumentError("Invalid header in cache file $cachefile.")) | ||
2620 | return cache_dependencies(io) | ||
2621 | finally | ||
2622 | close(io) | ||
2623 | end | ||
2624 | end | ||
2625 | |||
2626 | function read_dependency_src(io::IO, filename::AbstractString) | ||
2627 | srctextpos = parse_cache_header(io)[4] | ||
2628 | srctextpos == 0 && error("no source-text stored in cache file") | ||
2629 | seek(io, srctextpos) | ||
2630 | return _read_dependency_src(io, filename) | ||
2631 | end | ||
2632 | |||
2633 | function _read_dependency_src(io::IO, filename::AbstractString) | ||
2634 | while !eof(io) | ||
2635 | filenamelen = read(io, Int32) | ||
2636 | filenamelen == 0 && break | ||
2637 | fn = String(read(io, filenamelen)) | ||
2638 | len = read(io, UInt64) | ||
2639 | if fn == filename | ||
2640 | return String(read(io, len)) | ||
2641 | end | ||
2642 | seek(io, position(io) + len) | ||
2643 | end | ||
2644 | error(filename, " is not stored in the source-text cache") | ||
2645 | end | ||
2646 | |||
2647 | function read_dependency_src(cachefile::String, filename::AbstractString) | ||
2648 | io = open(cachefile, "r") | ||
2649 | try | ||
2650 | iszero(isvalid_cache_header(io)) && throw(ArgumentError("Invalid header in cache file $cachefile.")) | ||
2651 | return read_dependency_src(io, filename) | ||
2652 | finally | ||
2653 | close(io) | ||
2654 | end | ||
2655 | end | ||
2656 | |||
2657 | function srctext_files(f::IO, srctextpos::Int64) | ||
2658 | files = Set{String}() | ||
2659 | srctextpos == 0 && return files | ||
2660 | seek(f, srctextpos) | ||
2661 | while !eof(f) | ||
2662 | filenamelen = read(f, Int32) | ||
2663 | filenamelen == 0 && break | ||
2664 | fn = String(read(f, filenamelen)) | ||
2665 | len = read(f, UInt64) | ||
2666 | push!(files, fn) | ||
2667 | seek(f, position(f) + len) | ||
2668 | end | ||
2669 | return files | ||
2670 | end | ||
2671 | |||
2672 | # Test to see if this UUID is mentioned in this `Project.toml`; either as | ||
2673 | # the top-level UUID (e.g. that of the project itself), as a dependency, | ||
2674 | # or as an extra/weakdep for Preferences. | ||
2675 | function get_uuid_name(project::Dict{String, Any}, uuid::UUID) | ||
2676 | uuid_p = get(project, "uuid", nothing)::Union{Nothing, String} | ||
2677 | name = get(project, "name", nothing)::Union{Nothing, String} | ||
2678 | if name !== nothing && uuid_p !== nothing && UUID(uuid_p) == uuid | ||
2679 | return name | ||
2680 | end | ||
2681 | deps = get(project, "deps", nothing)::Union{Nothing, Dict{String, Any}} | ||
2682 | if deps !== nothing | ||
2683 | for (k, v) in deps | ||
2684 | if uuid == UUID(v::String) | ||
2685 | return k | ||
2686 | end | ||
2687 | end | ||
2688 | end | ||
2689 | for subkey in ("deps", "extras", "weakdeps") | ||
2690 | subsection = get(project, subkey, nothing)::Union{Nothing, Dict{String, Any}} | ||
2691 | if subsection !== nothing | ||
2692 | for (k, v) in subsection | ||
2693 | if uuid == UUID(v::String) | ||
2694 | return k | ||
2695 | end | ||
2696 | end | ||
2697 | end | ||
2698 | end | ||
2699 | return nothing | ||
2700 | end | ||
2701 | |||
2702 | function get_uuid_name(project_toml::String, uuid::UUID) | ||
2703 | project = parsed_toml(project_toml) | ||
2704 | return get_uuid_name(project, uuid) | ||
2705 | end | ||
2706 | |||
2707 | # If we've asked for a specific UUID, this function will extract the prefs | ||
2708 | # for that particular UUID. Otherwise, it returns all preferences. | ||
2709 | function filter_preferences(prefs::Dict{String, Any}, pkg_name) | ||
2710 | if pkg_name === nothing | ||
2711 | return prefs | ||
2712 | else | ||
2713 | return get(Dict{String, Any}, prefs, pkg_name)::Dict{String, Any} | ||
2714 | end | ||
2715 | end | ||
2716 | |||
2717 | function collect_preferences(project_toml::String, uuid::Union{UUID,Nothing}) | ||
2718 | # We'll return a list of dicts to be merged | ||
2719 | dicts = Dict{String, Any}[] | ||
2720 | |||
2721 | project = parsed_toml(project_toml) | ||
2722 | pkg_name = nothing | ||
2723 | if uuid !== nothing | ||
2724 | # If we've been given a UUID, map that to the name of the package as | ||
2725 | # recorded in the preferences section. If we can't find that mapping, | ||
2726 | # exit out, as it means there's no way preferences can be set for that | ||
2727 | # UUID, as we only allow actual dependencies to have preferences set. | ||
2728 | pkg_name = get_uuid_name(project, uuid) | ||
2729 | if pkg_name === nothing | ||
2730 | return dicts | ||
2731 | end | ||
2732 | end | ||
2733 | |||
2734 | # Look first inside of `Project.toml` to see we have preferences embedded within there | ||
2735 | proj_preferences = get(Dict{String, Any}, project, "preferences")::Dict{String, Any} | ||
2736 | push!(dicts, filter_preferences(proj_preferences, pkg_name)) | ||
2737 | |||
2738 | # Next, look for `(Julia)LocalPreferences.toml` files next to this `Project.toml` | ||
2739 | project_dir = dirname(project_toml) | ||
2740 | for name in preferences_names | ||
2741 | toml_path = joinpath(project_dir, name) | ||
2742 | if isfile(toml_path) | ||
2743 | prefs = parsed_toml(toml_path) | ||
2744 | push!(dicts, filter_preferences(prefs, pkg_name)) | ||
2745 | |||
2746 | # If we find `JuliaLocalPreferences.toml`, don't look for `LocalPreferences.toml` | ||
2747 | break | ||
2748 | end | ||
2749 | end | ||
2750 | |||
2751 | return dicts | ||
2752 | end | ||
2753 | |||
2754 | """ | ||
2755 | recursive_prefs_merge(base::Dict, overrides::Dict...) | ||
2756 | |||
2757 | Helper function to merge preference dicts recursively, honoring overrides in nested | ||
2758 | dictionaries properly. | ||
2759 | """ | ||
2760 | function recursive_prefs_merge(base::Dict{String, Any}, overrides::Dict{String, Any}...) | ||
2761 | new_base = Base._typeddict(base, overrides...) | ||
2762 | |||
2763 | for override in overrides | ||
2764 | # Clear entries are keys that should be deleted from any previous setting. | ||
2765 | override_clear = get(override, "__clear__", nothing) | ||
2766 | if override_clear isa Vector{String} | ||
2767 | for k in override_clear | ||
2768 | delete!(new_base, k) | ||
2769 | end | ||
2770 | end | ||
2771 | |||
2772 | for (k, override_k) in override | ||
2773 | # Note that if `base` has a mapping that is _not_ a `Dict`, and `override` | ||
2774 | new_base_k = get(new_base, k, nothing) | ||
2775 | if new_base_k isa Dict{String, Any} && override_k isa Dict{String, Any} | ||
2776 | new_base[k] = recursive_prefs_merge(new_base_k, override_k) | ||
2777 | else | ||
2778 | new_base[k] = override_k | ||
2779 | end | ||
2780 | end | ||
2781 | end | ||
2782 | return new_base | ||
2783 | end | ||
2784 | |||
2785 | function get_preferences(uuid::Union{UUID,Nothing} = nothing) | ||
2786 | merged_prefs = Dict{String,Any}() | ||
2787 | for env in reverse(load_path()) | ||
2788 | project_toml = env_project_file(env) | ||
2789 | if !isa(project_toml, String) | ||
2790 | continue | ||
2791 | end | ||
2792 | |||
2793 | # Collect all dictionaries from the current point in the load path, then merge them in | ||
2794 | dicts = collect_preferences(project_toml, uuid) | ||
2795 | merged_prefs = recursive_prefs_merge(merged_prefs, dicts...) | ||
2796 | end | ||
2797 | return merged_prefs | ||
2798 | end | ||
2799 | |||
2800 | function get_preferences_hash(uuid::Union{UUID, Nothing}, prefs_list::Vector{String}) | ||
2801 | # Start from a predictable hash point to ensure that the same preferences always | ||
2802 | # hash to the same value, modulo changes in how Dictionaries are hashed. | ||
2803 | h = UInt(0) | ||
2804 | uuid === nothing && return UInt64(h) | ||
2805 | |||
2806 | # Load the preferences | ||
2807 | prefs = get_preferences(uuid) | ||
2808 | |||
2809 | # Walk through each name that's called out as a compile-time preference | ||
2810 | for name in prefs_list | ||
2811 | prefs_value = get(prefs, name, nothing) | ||
2812 | if prefs_value !== nothing | ||
2813 | h = hash(prefs_value, h)::UInt | ||
2814 | end | ||
2815 | end | ||
2816 | # We always return a `UInt64` so that our serialization format is stable | ||
2817 | return UInt64(h) | ||
2818 | end | ||
2819 | |||
2820 | get_preferences_hash(m::Module, prefs_list::Vector{String}) = get_preferences_hash(PkgId(m).uuid, prefs_list) | ||
2821 | |||
2822 | # This is how we keep track of who is using what preferences at compile-time | ||
2823 | const COMPILETIME_PREFERENCES = Dict{UUID,Set{String}}() | ||
2824 | |||
2825 | # In `Preferences.jl`, if someone calls `load_preference(@__MODULE__, key)` while we're precompiling, | ||
2826 | # we mark that usage as a usage at compile-time and call this method, so that at the end of `.ji` generation, | ||
2827 | # we can record the list of compile-time preferences and embed that into the `.ji` header | ||
2828 | function record_compiletime_preference(uuid::UUID, key::String) | ||
2829 | pref = get!(Set{String}, COMPILETIME_PREFERENCES, uuid) | ||
2830 | push!(pref, key) | ||
2831 | return nothing | ||
2832 | end | ||
2833 | get_compiletime_preferences(uuid::UUID) = collect(get(Vector{String}, COMPILETIME_PREFERENCES, uuid)) | ||
2834 | get_compiletime_preferences(m::Module) = get_compiletime_preferences(PkgId(m).uuid) | ||
2835 | get_compiletime_preferences(::Nothing) = String[] | ||
2836 | |||
2837 | function check_clone_targets(clone_targets) | ||
2838 | rejection_reason = ccall(:jl_check_pkgimage_clones, Any, (Ptr{Cchar},), clone_targets) | ||
2839 | if rejection_reason !== nothing | ||
2840 | return rejection_reason | ||
2841 | end | ||
2842 | end | ||
2843 | |||
2844 | struct CacheFlags | ||
2845 | # OOICCDDP - see jl_cache_flags | ||
2846 | use_pkgimages::Bool | ||
2847 | debug_level::Int | ||
2848 | check_bounds::Int | ||
2849 | inline::Bool | ||
2850 | opt_level::Int | ||
2851 | |||
2852 | function CacheFlags(f::UInt8) | ||
2853 | use_pkgimages = Bool(f & 1) | ||
2854 | debug_level = Int((f >> 1) & 3) | ||
2855 | check_bounds = Int((f >> 3) & 3) | ||
2856 | inline = Bool((f >> 5) & 1) | ||
2857 | opt_level = Int((f >> 6) & 3) # define OPT_LEVEL in statiddata_utils | ||
2858 | new(use_pkgimages, debug_level, check_bounds, inline, opt_level) | ||
2859 | end | ||
2860 | end | ||
2861 | CacheFlags(f::Int) = CacheFlags(UInt8(f)) | ||
2862 | CacheFlags() = CacheFlags(ccall(:jl_cache_flags, UInt8, ())) | ||
2863 | |||
2864 | function show(io::IO, cf::CacheFlags) | ||
2865 | print(io, "use_pkgimages = ", cf.use_pkgimages) | ||
2866 | print(io, ", debug_level = ", cf.debug_level) | ||
2867 | print(io, ", check_bounds = ", cf.check_bounds) | ||
2868 | print(io, ", inline = ", cf.inline) | ||
2869 | print(io, ", opt_level = ", cf.opt_level) | ||
2870 | end | ||
2871 | |||
2872 | struct ImageTarget | ||
2873 | name::String | ||
2874 | flags::Int32 | ||
2875 | ext_features::String | ||
2876 | features_en::Vector{UInt8} | ||
2877 | features_dis::Vector{UInt8} | ||
2878 | end | ||
2879 | |||
2880 | function parse_image_target(io::IO) | ||
2881 | flags = read(io, Int32) | ||
2882 | nfeature = read(io, Int32) | ||
2883 | feature_en = read(io, 4*nfeature) | ||
2884 | feature_dis = read(io, 4*nfeature) | ||
2885 | name_len = read(io, Int32) | ||
2886 | name = String(read(io, name_len)) | ||
2887 | ext_features_len = read(io, Int32) | ||
2888 | ext_features = String(read(io, ext_features_len)) | ||
2889 | ImageTarget(name, flags, ext_features, feature_en, feature_dis) | ||
2890 | end | ||
2891 | |||
2892 | function parse_image_targets(targets::Vector{UInt8}) | ||
2893 | io = IOBuffer(targets) | ||
2894 | ntargets = read(io, Int32) | ||
2895 | targets = Vector{ImageTarget}(undef, ntargets) | ||
2896 | for i in 1:ntargets | ||
2897 | targets[i] = parse_image_target(io) | ||
2898 | end | ||
2899 | return targets | ||
2900 | end | ||
2901 | |||
2902 | function current_image_targets() | ||
2903 | targets = @ccall jl_reflect_clone_targets()::Vector{UInt8} | ||
2904 | return parse_image_targets(targets) | ||
2905 | end | ||
2906 | |||
2907 | struct FeatureName | ||
2908 | name::Cstring | ||
2909 | bit::UInt32 # bit index into a `uint32_t` array; | ||
2910 | llvmver::UInt32 # 0 if it is available on the oldest LLVM version we support | ||
2911 | end | ||
2912 | |||
2913 | function feature_names() | ||
2914 | fnames = Ref{Ptr{FeatureName}}() | ||
2915 | nf = Ref{Csize_t}() | ||
2916 | @ccall jl_reflect_feature_names(fnames::Ptr{Ptr{FeatureName}}, nf::Ptr{Csize_t})::Cvoid | ||
2917 | if fnames[] == C_NULL | ||
2918 | @assert nf[] == 0 | ||
2919 | return Vector{FeatureName}(undef, 0) | ||
2920 | end | ||
2921 | Base.unsafe_wrap(Array, fnames[], nf[], own=false) | ||
2922 | end | ||
2923 | |||
2924 | function test_feature(features::Vector{UInt8}, feat::FeatureName) | ||
2925 | bitidx = feat.bit | ||
2926 | u8idx = div(bitidx, 8) + 1 | ||
2927 | bit = bitidx % 8 | ||
2928 | return (features[u8idx] & (1 << bit)) != 0 | ||
2929 | end | ||
2930 | |||
2931 | function show(io::IO, it::ImageTarget) | ||
2932 | print(io, it.name) | ||
2933 | if !isempty(it.ext_features) | ||
2934 | print(io, ",", it.ext_features) | ||
2935 | end | ||
2936 | print(io, "; flags=", it.flags) | ||
2937 | print(io, "; features_en=(") | ||
2938 | first = true | ||
2939 | for feat in feature_names() | ||
2940 | if test_feature(it.features_en, feat) | ||
2941 | name = Base.unsafe_string(feat.name) | ||
2942 | if first | ||
2943 | first = false | ||
2944 | print(io, name) | ||
2945 | else | ||
2946 | print(io, ", ", name) | ||
2947 | end | ||
2948 | end | ||
2949 | end | ||
2950 | print(io, ")") | ||
2951 | # Is feature_dis useful? | ||
2952 | end | ||
2953 | |||
2954 | # Set by FileWatching.__init__() | ||
2955 | global mkpidlock_hook | ||
2956 | global trymkpidlock_hook | ||
2957 | global parse_pidfile_hook | ||
2958 | |||
2959 | # The preferences hash is only known after precompilation so just assume no preferences. | ||
2960 | # Also ignore the active project, which means that if all other conditions are equal, | ||
2961 | # the same package cannot be precompiled from different projects and/or different preferences at the same time. | ||
2962 | compilecache_pidfile_path(pkg::PkgId) = compilecache_path(pkg, UInt64(0); project="") * ".pidfile" | ||
2963 | |||
2964 | const compilecache_pidlock_stale_age = 10 | ||
2965 | |||
2966 | # Allows processes to wait if another process is precompiling a given source already. | ||
2967 | # The lock file mtime will be updated when held at most every `stale_age/2` seconds, with expected | ||
2968 | # variance of 10 seconds or more being infrequent but not unusual. | ||
2969 | # After `stale_age` seconds beyond the mtime of the lock file, the lock file is deleted and | ||
2970 | # precompilation will proceed if the locking process no longer exists or after `stale_age * 5` | ||
2971 | # seconds if the process does still exist. | ||
2972 | # If the lock is held by another host, it will conservatively wait `stale_age * 5` | ||
2973 | # seconds since processes cannot be checked remotely | ||
2974 | function maybe_cachefile_lock(f, pkg::PkgId, srcpath::String; stale_age=compilecache_pidlock_stale_age) | ||
2975 | if @isdefined(mkpidlock_hook) && @isdefined(trymkpidlock_hook) && @isdefined(parse_pidfile_hook) | ||
2976 | pidfile = compilecache_pidfile_path(pkg) | ||
2977 | cachefile = invokelatest(trymkpidlock_hook, f, pidfile; stale_age) | ||
2978 | if cachefile === false | ||
2979 | pid, hostname, age = invokelatest(parse_pidfile_hook, pidfile) | ||
2980 | verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug | ||
2981 | if isempty(hostname) || hostname == gethostname() | ||
2982 | @logmsg verbosity "Waiting for another process (pid: $pid) to finish precompiling $pkg. Pidfile: $pidfile" | ||
2983 | else | ||
2984 | @logmsg verbosity "Waiting for another machine (hostname: $hostname, pid: $pid) to finish precompiling $pkg. Pidfile: $pidfile" | ||
2985 | end | ||
2986 | # wait until the lock is available, but don't actually acquire it | ||
2987 | # returning nothing indicates a process waited for another | ||
2988 | return invokelatest(mkpidlock_hook, Returns(nothing), pidfile; stale_age) | ||
2989 | end | ||
2990 | return cachefile | ||
2991 | else | ||
2992 | # for packages loaded before FileWatching.__init__() | ||
2993 | f() | ||
2994 | end | ||
2995 | end | ||
2996 | # returns true if it "cachefile.ji" is stale relative to "modpath.jl" and build_id for modkey | ||
2997 | # otherwise returns the list of dependencies to also check | ||
2998 | @constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false) | ||
2999 | return stale_cachefile(PkgId(""), UInt128(0), modpath, cachefile; ignore_loaded) | ||
3000 | end | ||
3001 | @constprop :none function stale_cachefile(modkey::PkgId, build_id::UInt128, modpath::String, cachefile::String; ignore_loaded::Bool = false) | ||
3002 | io = open(cachefile, "r") | ||
3003 | try | ||
3004 | checksum = isvalid_cache_header(io) | ||
3005 | if iszero(checksum) | ||
3006 | @debug "Rejecting cache file $cachefile due to it containing an invalid cache header" | ||
3007 | return true # invalid cache file | ||
3008 | end | ||
3009 | modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags = parse_cache_header(io) | ||
3010 | if isempty(modules) | ||
3011 | return true # ignore empty file | ||
3012 | end | ||
3013 | if ccall(:jl_match_cache_flags, UInt8, (UInt8,), flags) == 0 | ||
3014 | @debug """ | ||
3015 | Rejecting cache file $cachefile for $modkey since the flags are mismatched | ||
3016 | current session: $(CacheFlags()) | ||
3017 | cache file: $(CacheFlags(flags)) | ||
3018 | """ | ||
3019 | return true | ||
3020 | end | ||
3021 | pkgimage = !isempty(clone_targets) | ||
3022 | if pkgimage | ||
3023 | ocachefile = ocachefile_from_cachefile(cachefile) | ||
3024 | if JLOptions().use_pkgimages == 0 | ||
3025 | # presence of clone_targets means native code cache | ||
3026 | @debug "Rejecting cache file $cachefile for $modkey since it would require usage of pkgimage" | ||
3027 | return true | ||
3028 | end | ||
3029 | rejection_reasons = check_clone_targets(clone_targets) | ||
3030 | if !isnothing(rejection_reasons) | ||
3031 | @debug("Rejecting cache file $cachefile for $modkey:", | ||
3032 | Reasons=rejection_reasons, | ||
3033 | var"Image Targets"=parse_image_targets(clone_targets), | ||
3034 | var"Current Targets"=current_image_targets()) | ||
3035 | return true | ||
3036 | end | ||
3037 | if !isfile(ocachefile) | ||
3038 | @debug "Rejecting cache file $cachefile for $modkey since pkgimage $ocachefile was not found" | ||
3039 | return true | ||
3040 | end | ||
3041 | else | ||
3042 | ocachefile = nothing | ||
3043 | end | ||
3044 | id = first(modules) | ||
3045 | if id.first != modkey && modkey != PkgId("") | ||
3046 | @debug "Rejecting cache file $cachefile for $modkey since it is for $id instead" | ||
3047 | return true | ||
3048 | end | ||
3049 | if build_id != UInt128(0) | ||
3050 | id_build = (UInt128(checksum) << 64) | id.second | ||
3051 | if id_build != build_id | ||
3052 | @debug "Ignoring cache file $cachefile for $modkey ($((UUID(id_build)))) since it is does not provide desired build_id ($((UUID(build_id))))" | ||
3053 | return true | ||
3054 | end | ||
3055 | end | ||
3056 | id = id.first | ||
3057 | modules = Dict{PkgId, UInt64}(modules) | ||
3058 | |||
3059 | # Check if transitive dependencies can be fulfilled | ||
3060 | ndeps = length(required_modules) | ||
3061 | depmods = Vector{Any}(undef, ndeps) | ||
3062 | for i in 1:ndeps | ||
3063 | req_key, req_build_id = required_modules[i] | ||
3064 | # Module is already loaded | ||
3065 | if root_module_exists(req_key) | ||
3066 | M = root_module(req_key) | ||
3067 | if PkgId(M) == req_key && module_build_id(M) === req_build_id | ||
3068 | depmods[i] = M | ||
3069 | elseif ignore_loaded | ||
3070 | # Used by Pkg.precompile given that there it's ok to precompile different versions of loaded packages | ||
3071 | @goto locate_branch | ||
3072 | else | ||
3073 | @debug "Rejecting cache file $cachefile because module $req_key is already loaded and incompatible." | ||
3074 | return true # Won't be able to fulfill dependency | ||
3075 | end | ||
3076 | else | ||
3077 | @label locate_branch | ||
3078 | path = locate_package(req_key) | ||
3079 | if path === nothing | ||
3080 | @debug "Rejecting cache file $cachefile because dependency $req_key not found." | ||
3081 | return true # Won't be able to fulfill dependency | ||
3082 | end | ||
3083 | depmods[i] = (path, req_key, req_build_id) | ||
3084 | end | ||
3085 | end | ||
3086 | |||
3087 | # check if this file is going to provide one of our concrete dependencies | ||
3088 | # or if it provides a version that conflicts with our concrete dependencies | ||
3089 | # or neither | ||
3090 | skip_timecheck = false | ||
3091 | for (req_key, req_build_id) in _concrete_dependencies | ||
3092 | build_id = get(modules, req_key, UInt64(0)) | ||
3093 | if build_id !== UInt64(0) | ||
3094 | build_id |= UInt128(checksum) << 64 | ||
3095 | if build_id === req_build_id | ||
3096 | skip_timecheck = true | ||
3097 | break | ||
3098 | end | ||
3099 | @debug "Rejecting cache file $cachefile because it provides the wrong build_id (got $((UUID(build_id)))) for $req_key (want $(UUID(req_build_id)))" | ||
3100 | return true # cachefile doesn't provide the required version of the dependency | ||
3101 | end | ||
3102 | end | ||
3103 | |||
3104 | # now check if this file is fresh relative to its source files | ||
3105 | if !skip_timecheck | ||
3106 | if !samefile(includes[1].filename, modpath) && !samefile(fixup_stdlib_path(includes[1].filename), modpath) | ||
3107 | @debug "Rejecting cache file $cachefile because it is for file $(includes[1].filename) not file $modpath" | ||
3108 | return true # cache file was compiled from a different path | ||
3109 | end | ||
3110 | for (modkey, req_modkey) in requires | ||
3111 | # verify that `require(modkey, name(req_modkey))` ==> `req_modkey` | ||
3112 | if identify_package(modkey, req_modkey.name) != req_modkey | ||
3113 | @debug "Rejecting cache file $cachefile because uuid mapping for $modkey => $req_modkey has changed" | ||
3114 | return true | ||
3115 | end | ||
3116 | end | ||
3117 | for chi in includes | ||
3118 | f, ftime_req = chi.filename, chi.mtime | ||
3119 | if !ispath(f) | ||
3120 | _f = fixup_stdlib_path(f) | ||
3121 | if isfile(_f) && startswith(_f, Sys.STDLIB) | ||
3122 | # mtime is changed by extraction | ||
3123 | @debug "Skipping mtime check for file $f used by $cachefile, since it is a stdlib" | ||
3124 | continue | ||
3125 | end | ||
3126 | @debug "Rejecting stale cache file $cachefile because file $f does not exist" | ||
3127 | return true | ||
3128 | end | ||
3129 | ftime = mtime(f) | ||
3130 | is_stale = ( ftime != ftime_req ) && | ||
3131 | ( ftime != floor(ftime_req) ) && # Issue #13606, PR #13613: compensate for Docker images rounding mtimes | ||
3132 | ( ftime != ceil(ftime_req) ) && # PR: #47433 Compensate for CirceCI's truncating of timestamps in its caching | ||
3133 | ( ftime != trunc(ftime_req, digits=6) ) && # Issue #20837, PR #20840: compensate for GlusterFS truncating mtimes to microseconds | ||
3134 | ( ftime != 1.0 ) && # PR #43090: provide compatibility with Nix mtime. | ||
3135 | !( 0 < (ftime_req - ftime) < 1e-6 ) # PR #45552: Compensate for Windows tar giving mtimes that may be incorrect by up to one microsecond | ||
3136 | if is_stale | ||
3137 | @debug "Rejecting stale cache file $cachefile (mtime $ftime_req) because file $f (mtime $ftime) has changed" | ||
3138 | return true | ||
3139 | end | ||
3140 | end | ||
3141 | end | ||
3142 | |||
3143 | if !isvalid_file_crc(io) | ||
3144 | @debug "Rejecting cache file $cachefile because it has an invalid checksum" | ||
3145 | return true | ||
3146 | end | ||
3147 | |||
3148 | if pkgimage | ||
3149 | if !isvalid_pkgimage_crc(io, ocachefile::String) | ||
3150 | @debug "Rejecting cache file $cachefile because $ocachefile has an invalid checksum" | ||
3151 | return true | ||
3152 | end | ||
3153 | end | ||
3154 | |||
3155 | curr_prefs_hash = get_preferences_hash(id.uuid, prefs) | ||
3156 | if prefs_hash != curr_prefs_hash | ||
3157 | @debug "Rejecting cache file $cachefile because preferences hash does not match 0x$(string(prefs_hash, base=16)) != 0x$(string(curr_prefs_hash, base=16))" | ||
3158 | return true | ||
3159 | end | ||
3160 | |||
3161 | return depmods, ocachefile # fresh cachefile | ||
3162 | finally | ||
3163 | close(io) | ||
3164 | end | ||
3165 | end | ||
3166 | |||
3167 | """ | ||
3168 | @__FILE__ -> String | ||
3169 | |||
3170 | Expand to a string with the path to the file containing the | ||
3171 | macrocall, or an empty string if evaluated by `julia -e <expr>`. | ||
3172 | Return `nothing` if the macro was missing parser source information. | ||
3173 | Alternatively see [`PROGRAM_FILE`](@ref). | ||
3174 | """ | ||
3175 | macro __FILE__() | ||
3176 | __source__.file === nothing && return nothing | ||
3177 | return String(__source__.file::Symbol) | ||
3178 | end | ||
3179 | |||
3180 | """ | ||
3181 | @__DIR__ -> String | ||
3182 | |||
3183 | Expand to a string with the absolute path to the directory of the file | ||
3184 | containing the macrocall. | ||
3185 | Return the current working directory if run from a REPL or if evaluated by `julia -e <expr>`. | ||
3186 | """ | ||
3187 | macro __DIR__() | ||
3188 | __source__.file === nothing && return nothing | ||
3189 | _dirname = dirname(String(__source__.file::Symbol)) | ||
3190 | return isempty(_dirname) ? pwd() : abspath(_dirname) | ||
3191 | end | ||
3192 | |||
3193 | """ | ||
3194 | precompile(f, argtypes::Tuple{Vararg{Any}}) | ||
3195 | |||
3196 | Compile the given function `f` for the argument tuple (of types) `argtypes`, but do not execute it. | ||
3197 | """ | ||
3198 | function precompile(@nospecialize(f), @nospecialize(argtypes::Tuple)) | ||
3199 | precompile(Tuple{Core.Typeof(f), argtypes...}) | ||
3200 | end | ||
3201 | |||
3202 | const ENABLE_PRECOMPILE_WARNINGS = Ref(false) | ||
3203 | function precompile(@nospecialize(argt::Type)) | ||
3204 | ret = ccall(:jl_compile_hint, Int32, (Any,), argt) != 0 | ||
3205 | if !ret && ENABLE_PRECOMPILE_WARNINGS[] | ||
3206 | @warn "Inactive precompile statement" maxlog=100 form=argt _module=nothing _file=nothing _line=0 | ||
3207 | end | ||
3208 | return ret | ||
3209 | end | ||
3210 | |||
3211 | # Variants that work for `invoke`d calls for which the signature may not be sufficient | ||
3212 | precompile(mi::Core.MethodInstance, world::UInt=get_world_counter()) = | ||
3213 | (ccall(:jl_compile_method_instance, Cvoid, (Any, Any, UInt), mi, C_NULL, world); return true) | ||
3214 | |||
3215 | """ | ||
3216 | precompile(f, argtypes::Tuple{Vararg{Any}}, m::Method) | ||
3217 | |||
3218 | Precompile a specific method for the given argument types. This may be used to precompile | ||
3219 | a different method than the one that would ordinarily be chosen by dispatch, thus | ||
3220 | mimicking `invoke`. | ||
3221 | """ | ||
3222 | function precompile(@nospecialize(f), @nospecialize(argtypes::Tuple), m::Method) | ||
3223 | precompile(Tuple{Core.Typeof(f), argtypes...}, m) | ||
3224 | end | ||
3225 | |||
3226 | function precompile(@nospecialize(argt::Type), m::Method) | ||
3227 | atype, sparams = ccall(:jl_type_intersection_with_env, Any, (Any, Any), argt, m.sig)::SimpleVector | ||
3228 | mi = Core.Compiler.specialize_method(m, atype, sparams) | ||
3229 | return precompile(mi) | ||
3230 | end | ||
3231 | |||
3232 | precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), Nothing)) | ||
3233 | precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String)) | ||
3234 | precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), IO, IO)) | ||
3235 | precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), IO, IO)) |