Skip to content

Commit

Permalink
require: refactor error handling
Browse files Browse the repository at this point in the history 
it was a redundant to repeat the name->file mapping in require_from_serialized,
factor it out to manage more of the error handling at one time
(even though this approach requires the duplication of slightly more code)
  • Loading branch information
@vtjnash
vtjnash committed Jan 2, 2018
1 parent 2cc82d2 commit f532093
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 70 deletions.
167 changes: 100 additions & 67 deletions base/loading.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,11 +104,10 @@ function find_source_file(path::String)
return isfile(base_path) ? base_path : nothing
end

function find_all_in_cache_path(mod::Symbol)
name = string(mod)
function find_all_in_cache_path(name::String)
paths = String[]
for prefix in LOAD_CACHE_PATH
path = joinpath(prefix, name*".ji")
path = joinpath(prefix, name * ".ji")
if isfile_casesensitive(path)
push!(paths, path)
end
Expand All @@ -118,22 +117,38 @@ end

# these return either the array of modules loaded from the path / content given
# or an Exception that describes why it couldn't be loaded
function _include_from_serialized(content::Vector{UInt8}, depmods::Vector{Module})
return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8}, Int, Any), content, sizeof(content), depmods)
end
function _include_from_serialized(path::String, depmods::Vector{Module})
return ccall(:jl_restore_incremental, Any, (Cstring, Any), path, depmods)
end

# returns an array of modules loaded, or an Exception that describes why it failed
# and it reconnects the Base.Docs.META
function _require_from_serialized(mod::Symbol, path_to_try::String)
return _require_from_serialized(mod, path_to_try, parse_cache_header(path_to_try)[3])
function _include_from_serialized(path::String, depmods::Vector{Any})
restored = ccall(:jl_restore_incremental, Any, (Cstring, Any), path, depmods)
if !isa(restored, Exception)
for M in restored::Vector{Any}
M = M::Module
if isdefined(M, Base.Docs.META)
push!(Base.Docs.modules, M)
end
if module_parent(M) === M
register_root_module(module_name(M), M)
end
end
end
return restored
end
function _require_from_serialized(mod::Symbol, path_to_try::String, depmodnames::Vector{Pair{Symbol, UInt64}})
# load all of the dependent modules


function _require_from_serialized(path::String)
# loads a precompile cache file, ignoring stale_cachfile tests
# load all of the dependent modules first
local depmodnames
io = open(path, "r")
try
isvalid_cache_header(io) || return ArgumentError("Invalid header in cache file $path.")
depmodnames = parse_cache_header(io)[3]
isvalid_file_crc(io) || return ArgumentError("Invalid checksum in cache file $path.")
finally
close(io)
end
ndeps = length(depmodnames)
depmods = Vector{Module}(uninitialized, ndeps)
depmods = Vector{Any}(uninitialized, ndeps)
for i in 1:ndeps
modname, uuid = depmodnames[i]
if root_module_exists(modname)
Expand All @@ -159,42 +174,45 @@ function _require_from_serialized(mod::Symbol, path_to_try::String, depmodnames:
end
isassigned(depmods, i) || return ErrorException("Required dependency $modname failed to load from a cache file.")
end
# now load the path_to_try.ji file
restored = _include_from_serialized(path_to_try, depmods)
if !isa(restored, Exception)
for M in restored::Vector{Any}
M = M::Module
if isdefined(M, Base.Docs.META)
push!(Base.Docs.modules, M)
end
if module_parent(M) === M
register_root_module(module_name(M), M)
end
end
end
return restored
# then load the file
return _include_from_serialized(path, depmods)
end

# returns `true` if require found a precompile cache for this mod, but couldn't load it
# returns `true` if require found a precompile cache for this sourcepath, but couldn't load it
# returns `false` if the module isn't known to be precompilable
# returns the set of modules restored if the cache load succeeded
function _require_search_from_serialized(mod::Symbol, sourcepath::String)
paths = find_all_in_cache_path(mod)
paths = find_all_in_cache_path(String(mod)) # cache files for sourcepath are stored keyed by the `mod` symbol name
for path_to_try in paths::Vector{String}
deps = stale_cachefile(sourcepath, path_to_try)
if deps === true
continue
end
restored = _require_from_serialized(mod, path_to_try, deps)
if isa(restored, Exception)
if isa(restored, ErrorException)
# can't use this cache due to a module uuid mismatch,
# defer reporting error until after trying all of the possible matches
@debug "Failed to load $path_to_try because $(restored.msg)"
# finish loading module graph into deps
for i in 1:length(deps)
dep = deps[i]
dep isa Module && continue
modpath, modname, uuid = dep::Tuple{String, Symbol, UInt64}
reqmod = _require_search_from_serialized(modname, modpath)
if !isa(reqmod, Bool)
for M in reqmod::Vector{Any}
if module_name(M) === modname && module_uuid(M) === uuid
deps[i] = M
break
end
end
for callback in package_callbacks
invokelatest(callback, modname)
end
end
if !isa(deps[i], Module)
@debug "Required dependency $modname failed to load from cache file for $modpath."
continue
end
@warn "Deserialization checks failed while attempting to load cache from $path_to_try"
throw(restored)
end
restored = _include_from_serialized(path_to_try, deps)
if isa(restored, Exception)
@debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored
else
return restored
end
Expand Down Expand Up @@ -420,7 +438,7 @@ function _require(mod::Symbol)
# spawn off a new incremental pre-compile task for recursive `require` calls
# or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable)
cachefile = compilecache(mod)
m = _require_from_serialized(mod, cachefile)
m = _require_from_serialized(cachefile)
if isa(m, Exception)
@warn "The call to compilecache failed to create a usable precompiled cache file for module $name" exception=m
# fall-through, TODO: disable __precompile__(true) error so that the normal include will succeed
Expand All @@ -440,7 +458,7 @@ function _require(mod::Symbol)
end
# the file requested `__precompile__`, so try to build a cache file and use that
cachefile = compilecache(mod)
m = _require_from_serialized(mod, cachefile)
m = _require_from_serialized(cachefile)
if isa(m, Exception)
@warn """Module `$mod` declares `__precompile__(true)` but `require` failed
to create a usable precompiled cache file""" exception=m
Expand Down Expand Up @@ -634,7 +652,8 @@ end

module_uuid(m::Module) = ccall(:jl_module_uuid, UInt64, (Any,), m)

isvalid_cache_header(f::IOStream) = 0 != ccall(:jl_read_verify_header, Cint, (Ptr{Cvoid},), f.ios)
isvalid_cache_header(f::IOStream) = (0 != ccall(:jl_read_verify_header, Cint, (Ptr{Cvoid},), f.ios))
isvalid_file_crc(f::IOStream) = (_crc32c(seekstart(f), filesize(f) - 4) == ntoh(read(f, UInt32)))

function parse_cache_header(f::IO)
modules = Vector{Pair{Symbol, UInt64}}()
Expand Down Expand Up @@ -738,60 +757,74 @@ function stale_cachefile(modpath::String, cachefile::String)
@debug "Rejecting cache file $cachefile due to it containing an invalid cache header"
return true # invalid cache file
end

modules, files, required_modules = parse_cache_header(io)
modules = Dict{Symbol, UInt64}(modules)

# Check if transitive dependencies can be fullfilled
for (mod, uuid_req) in required_modules
ndeps = length(required_modules)
depmods = Vector{Any}(uninitialized, ndeps)
for i in 1:ndeps
mod, uuid_req = required_modules[i]
# Module is already loaded
if root_module_exists(mod)
continue
end
name = string(mod)
path = find_package(name)
if path === nothing
@debug "Rejecting cache file $cachefile because dependency $name not found."
return true # Won't be able to fullfill dependency
M = root_module(mod)
if module_name(M) === mod && module_uuid(M) === uuid_req
depmods[i] = M
else
@debug "Rejecting cache file $cachefile because module $name is already loaded and incompatible."
return true # Won't be able to fulfill dependency
end
else
name = string(mod)
path = find_package(name)
if path === nothing
@debug "Rejecting cache file $cachefile because dependency $name not found."
return true # Won't be able to fulfill dependency
end
depmods[i] = (String(path), mod, uuid_req)
end
end

# check if this file is going to provide one of our concrete dependencies
# or if it provides a version that conflicts with our concrete dependencies
# or neither
skip_timecheck = false
for (mod, uuid_req) in _concrete_dependencies
uuid = get(modules, mod, UInt64(0))
if uuid !== UInt64(0)
if uuid === uuid_req
return required_modules # this is the file we want
skip_timecheck = true
break
end
@debug "Rejecting cache file $cachefile because it provides the wrong uuid (got $uuid) for $mod (want $uuid_req)"
return true # cachefile doesn't provide the required version of the dependency
end
end

# now check if this file is fresh relative to its source files
if !samefile(files[1][2], modpath)
@debug "Rejecting cache file $cachefile because it is for file $(files[1][2])) not file $modpath"
return true # cache file was compiled from a different path
end
for (_, f, ftime_req) in files
# Issue #13606: compensate for Docker images rounding mtimes
# Issue #20837: compensate for GlusterFS truncating mtimes to microseconds
ftime = mtime(f)
if ftime != ftime_req && ftime != floor(ftime_req) && ftime != trunc(ftime_req, 6)
@debug "Rejecting stale cache file $cachefile (mtime $ftime_req) because file $f (mtime $ftime) has changed"
return true
if !skip_timecheck
if !samefile(files[1][2], modpath)
@debug "Rejecting cache file $cachefile because it is for file $(files[1][2])) not file $modpath"
return true # cache file was compiled from a different path
end
for (_, f, ftime_req) in files
# Issue #13606: compensate for Docker images rounding mtimes
# Issue #20837: compensate for GlusterFS truncating mtimes to microseconds
ftime = mtime(f)
if ftime != ftime_req && ftime != floor(ftime_req) && ftime != trunc(ftime_req, 6)
@debug "Rejecting stale cache file $cachefile (mtime $ftime_req) because file $f (mtime $ftime) has changed"
return true
end
end
end

# finally, verify that the cache file has a valid checksum
crc = _crc32c(seekstart(io), filesize(io)-4)
if crc != ntoh(read(io, UInt32))
if !isvalid_file_crc(io)
@debug "Rejecting cache file $cachefile because it has an invalid checksum"
return true
end

return required_modules # fresh cachefile
return depmods # fresh cachefile
finally
close(io)
end
Expand Down
6 changes: 3 additions & 3 deletions test/compile.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ try
__precompile__(true)
module $Foo_module
using $FooBase_module, $FooBase_module.typeA
import $FooBase_module, $FooBase_module.typeA
import $Foo2_module: $Foo2_module, override
import $FooBase_module.hash
import Test
Expand Down Expand Up @@ -174,8 +174,8 @@ try
cachefile = joinpath(dir, "$Foo_module.ji")
# use _require_from_serialized to ensure that the test fails if
# the module doesn't reload from the image:
@test_logs (:warn,"Replacing module `$Foo_module`") begin
ms = Base._require_from_serialized(Foo_module, cachefile)
@test_logs (:warn, "Replacing module `$Foo_module`") begin
ms = Base._require_from_serialized(cachefile)
@test isa(ms, Array{Any,1})
end

Expand Down

0 comments on commit f532093

Please sign in to comment.