Skip to content

Commit

Permalink
[loading]: Rework preferences loading
Browse files Browse the repository at this point in the history 
Implements the `Preferences` loading framework as outlined in [0]. The
most drastic change is that the list of compile-time preferences is no
longer sequestered within its own dictionary, but is instead
autodetected at compile-time and communicated back to the compiler.
This list of compile-time preferences is now embedded as an array of
strings that the loader must load, then index into the preferences
dictionary with that list to check the preferences hash.

In a somewhat bizarre turn of events, because we want the `.ji` filename
to incorporate the preferences hash, and because we can't know how to
generate the hash until after we've precompiled, I had to move the `.ji`
filename generation step to _after_ we precompile the `.ji` file.

[0]: #37791 (comment)
  • Loading branch information
@staticfloat
staticfloat committed Oct 1, 2020
1 parent 2fe2b43 commit f510352
Show file tree
Hide file tree
Showing 2 changed files with 207 additions and 72 deletions.
248 changes: 181 additions & 67 deletions base/loading.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,8 @@ end

const project_names = ("JuliaProject.toml", "Project.toml")
const manifest_names = ("JuliaManifest.toml", "Manifest.toml")
const preferences_names = ("JuliaPreferences.toml", "Preferences.toml")
const local_preferences_names = ("LocalJuliaPreferences.toml", "LocalPreferences.toml")

# classify the LOAD_PATH entry to be one of:
# - `false`: nonexistant / nothing to see here
Expand Down Expand Up @@ -322,31 +324,6 @@ function manifest_deps_get(env::String, where::PkgId, name::String, cache::TOMLC
return nothing
end

function uuid_in_environment(project_file::String, uuid::UUID, cache::TOMLCache)
# First, check to see if we're looking for the environment itself
proj_uuid = get(parsed_toml(cache, project_file), "uuid", nothing)
if proj_uuid !== nothing && UUID(proj_uuid) == uuid
return true
end

# Check to see if there's a Manifest.toml associated with this project
manifest_file = project_file_manifest_path(project_file, cache)
if manifest_file === nothing
return false
end
manifest = parsed_toml(cache, manifest_file)
for (dep_name, entries) in manifest
for entry in entries
entry_uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
if uuid !== nothing && UUID(entry_uuid) == uuid
return true
end
end
end
# If all else fails, return `false`
return false
end

function manifest_uuid_path(env::String, pkg::PkgId, cache::TOMLCache)::Union{Nothing,String}
project_file = env_project_file(env)
if project_file isa String
Expand Down Expand Up @@ -1220,7 +1197,12 @@ end
@assert precompile(create_expr_cache, (PkgId, String, String, typeof(_concrete_dependencies), Bool))
@assert precompile(create_expr_cache, (PkgId, String, String, typeof(_concrete_dependencies), Bool))

function compilecache_path(pkg::PkgId, cache::TOMLCache)::String
function compilecache_dir(pkg::PkgId)
entrypath, entryfile = cache_file_entry(pkg)
return joinpath(DEPOT_PATH[1], entrypath)
end

function compilecache_path(pkg::PkgId, prefs_hash::UInt64, cache::TOMLCache)::String
entrypath, entryfile = cache_file_entry(pkg)
cachepath = joinpath(DEPOT_PATH[1], entrypath)
isdir(cachepath) || mkpath(cachepath)
Expand All @@ -1230,7 +1212,7 @@ function compilecache_path(pkg::PkgId, cache::TOMLCache)::String
crc = _crc32c(something(Base.active_project(), ""))
crc = _crc32c(unsafe_string(JLOptions().image_file), crc)
crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc)
crc = _crc32c(get_preferences_hash(pkg.uuid, cache), crc)
crc = _crc32c(prefs_hash, crc)
project_precompile_slug = slug(crc, 5)
abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji"))
end
Expand All @@ -1254,17 +1236,8 @@ const MAX_NUM_PRECOMPILE_FILES = 10

function compilecache(pkg::PkgId, path::String, cache::TOMLCache = TOMLCache(), show_errors::Bool = true)
# decide where to put the resulting cache file
cachefile = compilecache_path(pkg, cache)
cachepath = dirname(cachefile)
# prune the directory with cache files
if pkg.uuid !== nothing
entrypath, entryfile = cache_file_entry(pkg)
cachefiles = filter!(x -> startswith(x, entryfile * "_"), readdir(cachepath))
if length(cachefiles) >= MAX_NUM_PRECOMPILE_FILES
idx = findmin(mtime.(joinpath.(cachepath, cachefiles)))[2]
rm(joinpath(cachepath, cachefiles[idx]))
end
end
cachepath = compilecache_dir(pkg)

# build up the list of modules that we want the precompile process to preserve
concrete_deps = copy(_concrete_dependencies)
for (key, mod) in loaded_modules
Expand All @@ -1278,6 +1251,7 @@ function compilecache(pkg::PkgId, path::String, cache::TOMLCache = TOMLCache(),

# create a temporary file in `cachepath` directory, write the cache in it,
# write the checksum, _and then_ atomically move the file to `cachefile`.
mkpath(cachepath)
tmppath, tmpio = mktemp(cachepath)
local p
try
Expand All @@ -1291,6 +1265,21 @@ function compilecache(pkg::PkgId, path::String, cache::TOMLCache = TOMLCache(),
# inherit permission from the source file
chmod(tmppath, filemode(path) & 0o777)

# Read preferences hash back from .ji file (we can't precompute because
# we don't actually know what the list of compile-time preferences are without compiling)
prefs_hash = preferences_hash(tmppath)
cachefile = compilecache_path(pkg, prefs_hash, cache)

# prune the directory with cache files
if pkg.uuid !== nothing
entrypath, entryfile = cache_file_entry(pkg)
cachefiles = filter!(x -> startswith(x, entryfile * "_"), readdir(cachepath))
if length(cachefiles) >= MAX_NUM_PRECOMPILE_FILES
idx = findmin(mtime.(joinpath.(cachepath, cachefiles)))[2]
rm(joinpath(cachepath, cachefiles[idx]))
end
end

# this is atomic according to POSIX:
rename(tmppath, cachefile; force=true)
return cachefile
Expand Down Expand Up @@ -1334,7 +1323,10 @@ function parse_cache_header(f::IO)
requires = Pair{PkgId, PkgId}[]
while true
n2 = read(f, Int32)
n2 == 0 && break
if n2 == 0
totbytes -= 4
break
end
depname = String(read(f, n2))
mtime = read(f, Float64)
n1 = read(f, Int32)
Expand All @@ -1358,10 +1350,20 @@ function parse_cache_header(f::IO)
end
totbytes -= 4 + 4 + n2 + 8
end
prefs = String[]
while true
n2 = read(f, Int32)
if n2 == 0
totbytes -= 4
break
end
push!(prefs, String(read(f, n2)))
totbytes -= 4 + n2
end
prefs_hash = read(f, UInt64)
totbytes -= 8
@assert totbytes == 12 "header of cache file appears to be corrupt"
srctextpos = read(f, Int64)
totbytes -= 8 + 8
@assert totbytes == 0 "header of cache file appears to be corrupt"
# read the list of modules that are required to be present during loading
required_modules = Vector{Pair{PkgId, UInt64}}()
while true
Expand All @@ -1372,7 +1374,7 @@ function parse_cache_header(f::IO)
build_id = read(f, UInt64) # build id
push!(required_modules, PkgId(uuid, sym) => build_id)
end
return modules, (includes, requires), required_modules, srctextpos, prefs_hash
return modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash
end

function parse_cache_header(cachefile::String; srcfiles_only::Bool=false)
Expand All @@ -1381,21 +1383,37 @@ function parse_cache_header(cachefile::String; srcfiles_only::Bool=false)
!isvalid_cache_header(io) && throw(ArgumentError("Invalid header in cache file $cachefile."))
ret = parse_cache_header(io)
srcfiles_only || return ret
modules, (includes, requires), required_modules, srctextpos, prefs_hash = ret
modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash = ret
srcfiles = srctext_files(io, srctextpos)
delidx = Int[]
for (i, chi) in enumerate(includes)
chi.filename ∈ srcfiles || push!(delidx, i)
end
deleteat!(includes, delidx)
return modules, (includes, requires), required_modules, srctextpos, prefs_hash
return modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash
finally
close(io)
end
end



preferences_hash(f::IO) = parse_cache_header(f)[end]
function preferences_hash(cachefile::String)
io = open(cachefile, "r")
try
if !isvalid_cache_header(io)
throw(ArgumentError("Invalid header in cache file $cachefile."))
end
return preferences_hash(io)
finally
close(io)
end
end


function cache_dependencies(f::IO)
defs, (includes, requires), modules, srctextpos, prefs_hash = parse_cache_header(f)
defs, (includes, requires), modules, srctextpos, prefs, prefs_hash = parse_cache_header(f)
return modules, map(chi -> (chi.filename, chi.mtime), includes) # return just filename and mtime
end

Expand All @@ -1410,7 +1428,7 @@ function cache_dependencies(cachefile::String)
end

function read_dependency_src(io::IO, filename::AbstractString)
modules, (includes, requires), required_modules, srctextpos, prefs_hash = parse_cache_header(io)
modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash = parse_cache_header(io)
srctextpos == 0 && error("no source-text stored in cache file")
seek(io, srctextpos)
return _read_dependency_src(io, filename)
Expand Down Expand Up @@ -1455,36 +1473,132 @@ function srctext_files(f::IO, srctextpos::Int64)
return files
end

# Find the Project.toml that we should load/store to for Preferences
function get_preferences_project_path(uuid::UUID, cache::TOMLCache = TOMLCache())
function get_uuid_name(project_toml::String, uuid::UUID, cache::TOMLCache = TOMLCache())
d = parsed_toml(cache, project_toml)

# Test to see if this UUID is mentioned in this `Project.toml`; either as
# the top-level UUID (e.g. that of the project itself) or as a dependency.
if haskey(d, "uuid") && haskey(d, "name") && UUID(d["uuid"]) == uuid
return d["name"]
elseif haskey(d, "deps")
# Search for the UUID as one of the deps
for (k, v) in d["deps"]
if v == uuid
return k
end
end
end

return nothing
end

function collect_preferences!(dicts::Vector{Dict}, project_toml::String,
pkg_name::String, cache::TOMLCache = TOMLCache())
# Look first for `Preferences.toml`, then for `LocalPreferences.toml`
for names_collection in (preferences_names, local_preferences_names)
for name in names_collection
toml_path = joinpath(project_dir, name)
if isfile(toml_path)
prefs = TOML.parsefile(toml_path)
push!(dicts, get(prefs, pkg_name, Dict()))

# If we find `JuliaPreferences.toml`, don't look for `Preferences.toml`,
# but do look for `Local(Julia)Preferences.toml`
break
end
end
end

return dicts
end

function get_clear_keys(d::Dict)
if haskey(d, "__clear_keys__") && isa(d["__clear_keys__"], Vector)
return d["__clear_keys__"]
end
return String[]
end

"""
recursive_merge(base::Dict, overrides::Dict...)
Helper function to merge preference dicts recursively, honoring overrides in nested
dictionaries properly.
"""
function recursive_merge(base::Dict, overrides::Dict...)
new_base = Base._typeddict(base, overrides...)

for override in overrides
# Clear keys are keys that should be deleted from any previous setting.
clear_keys = get_clear_keys(base)
for k in clear_keys
delete!(new_base, k)
end

for (k, v) in override
# Note that if `base` has a mapping that is _not_ a `Dict`, and `override`
if haskey(new_base, k) && isa(new_base[k], Dict) && isa(override[k], Dict)
new_base[k] = recursive_merge(new_base[k], override[k])
else
new_base[k] = override[k]
end
end
end
return new_base
end

function get_preferences(uuid::UUID, cache::TOMLCache = TOMLCache())
merged_prefs = Dict{String,Any}()
for env in load_path()
project_file = env_project_file(env)
if !isa(project_file, String)
project_toml = env_project_file(env)
if !isa(project_toml, String)
continue
end
if uuid_in_environment(project_file, uuid, cache)
return project_file

pkg_name = get_uuid_name(project_toml, uuid, cache)
if pkg_name !== nothing
project_dir = dirname(project_toml)
recursive_merge(merged_prefs, collect_preferences!(merged_prefs, project_dir, pkg_name, cache)...)
end
end
return nothing
return merged_prefs
end

function get_preferences(uuid::UUID, cache::TOMLCache = TOMLCache();
prefs_key::String = "compile-preferences")
project_path = get_preferences_project_path(uuid, cache)
if project_path !== nothing
preferences = get(parsed_toml(cache, project_path), prefs_key, Dict{String,Any}())
if haskey(preferences, string(uuid))
return preferences[string(uuid)]
function get_preferences_hash(uuid::UUID, prefs_list::Vector{String}, cache::TOMLCache = TOMLCache())
# Start from the "null" hash
h = get_preferences_hash(nothing, prefs_list, cache)

# Load the preferences
prefs = get_preferences(uuid, cache)

# Walk through each name that's called out as a compile-time preference
for name in prefs_list
if haskey(prefs, name)
h = hash(prefs[name], h)
end
end
# Fall back to default value of "no preferences".
return Dict{String,Any}()
return h
end
get_preferences_hash(uuid::UUID, cache::TOMLCache = TOMLCache()) = UInt64(hash(get_preferences(uuid, cache)))
get_preferences_hash(m::Module, cache::TOMLCache = TOMLCache()) = get_preferences_hash(PkgId(m).uuid, cache)
get_preferences_hash(::Nothing, cache::TOMLCache = TOMLCache()) = UInt64(hash(Dict{String,Any}()))
get_preferences_hash(m::Module, prefs_list::Vector{String}, cache::TOMLCache = TOMLCache()) = get_preferences_hash(PkgId(m).uuid, prefs_list, cache)
get_preferences_hash(::Nothing, prefs_list::Vector{String}, cache::TOMLCache = TOMLCache()) = UInt64(0x6e65726566657250)

# This is how we keep track of who is using what preferences at compile-time
const COMPILETIME_PREFERENCES = Dict{UUID,Set{String}}()

# In `Preferences.jl`, if someone calls `load_preference(@__MODULE__, key)` while we're precompiling,
# we mark that usage as a usage at compile-time and call this method, so that at the end of `.ji` generation,
# we can
function record_compiletime_preference(uuid::UUID, key::String)
if !haskey(COMPILETIME_PREFERENCES, uuid)
COMPILETIME_PREFERENCES[uuid] = Set((key,))
else
push!(COMPILETIME_PREFERENCES[uuid], key)
end
return nothing
end
get_compiletime_preferences(uuid::UUID) = get(COMPILETIME_PREFERENCES, uuid, String[])
get_compiletime_preferences(m::Module) = get_compiletime_preferences(PkgId(m).uuid)
get_compiletime_preferences(::Nothing) = String[]

# returns true if it "cachefile.ji" is stale relative to "modpath.jl"
# otherwise returns the list of dependencies to also check
Expand All @@ -1496,7 +1610,7 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache)
@debug "Rejecting cache file $cachefile due to it containing an invalid cache header"
return true # invalid cache file
end
modules, (includes, requires), required_modules, srctextpos, prefs_hash = parse_cache_header(io)
modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash = parse_cache_header(io)
id = isempty(modules) ? nothing : first(modules).first
modules = Dict{PkgId, UInt64}(modules)

Expand Down Expand Up @@ -1572,7 +1686,7 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache)
end

if isa(id, PkgId)
curr_prefs_hash = get_preferences_hash(id.uuid, cache)
curr_prefs_hash = get_preferences_hash(id.uuid, prefs, cache)
if prefs_hash != curr_prefs_hash
@debug "Rejecting cache file $cachefile because preferences hash does not match 0x$(string(prefs_hash, base=16)) != 0x$(string(curr_prefs_hash, base=16))"
return true
Expand Down
Loading

0 comments on commit f510352

Please sign in to comment.