Merge pull request #18230 from JuliaLang/jn/release-0.4-cache-compile…

…-dep-tracking

[release 0.4] cache compile dep tracking

base/loading.jl

@@ -28,9 +28,9 @@ end
 
 function find_in_node_path(name, srcpath, node::Int=1)
     if myid() == node
-        find_in_path(name, srcpath)
+        return find_in_path(name, srcpath)
     else
-        remotecall_fetch(node, find_in_path, name, srcpath)
+        return remotecall_fetch(node, find_in_path, name, srcpath)
     end
 end
 
@@ -39,7 +39,7 @@ function find_source_file(file)
     file2 = find_in_path(file)
     file2 !== nothing && return file2
     file2 = joinpath(JULIA_HOME, DATAROOTDIR, "julia", "base", file)
-    isfile(file2) ? file2 : nothing
+    return isfile(file2) ? file2 : nothing
 end
 
 function find_all_in_cache_path(mod::Symbol)
@@ -51,18 +51,22 @@ function find_all_in_cache_path(mod::Symbol)
             push!(paths, path)
         end
     end
-    paths
+    return paths
 end
 
 function _include_from_serialized(content::Vector{UInt8})
-    return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8},Int), content, sizeof(content))
+    return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8}, Int), content, sizeof(content))
+end
+
+function _include_from_serialized(path::ByteString)
+    return ccall(:jl_restore_incremental, Any, (Cstring,), path)
 end
 
 # returns an array of modules loaded, or nothing if failed
 function _require_from_serialized(node::Int, mod::Symbol, path_to_try::ByteString, toplevel_load::Bool)
-    restored = nothing
+    local restored = nothing
+    local content::Vector{UInt8}
     if toplevel_load && myid() == 1 && nprocs() > 1
-        recompile_stale(mod, path_to_try)
         # broadcast top-level import/using from node 1 (only)
         if node == myid()
             content = open(readbytes, path_to_try)
@@ -80,13 +84,11 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::ByteStrin
             end
         end
     elseif node == myid()
-        myid() == 1 && recompile_stale(mod, path_to_try)
-        restored = ccall(:jl_restore_incremental, Any, (Ptr{UInt8},), path_to_try)
+        restored = _include_from_serialized(path_to_try)
     else
         content = remotecall_fetch(node, open, readbytes, path_to_try)
         restored = _include_from_serialized(content)
     end
-    # otherwise, continue search
 
     if restored !== nothing
         for M in restored
@@ -98,22 +100,28 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::ByteStrin
     return restored
 end
 
-function _require_from_serialized(node::Int, mod::Symbol, toplevel_load::Bool)
+# returns `true` if require found a precompile cache for this mod, but couldn't load it
+# returns `false` if the module isn't known to be precompilable
+# returns the set of modules restored if the cache load succeeded
+function _require_search_from_serialized(node::Int, mod::Symbol, sourcepath::ByteString, toplevel_load::Bool)
     if node == myid()
         paths = find_all_in_cache_path(mod)
     else
         paths = @fetchfrom node find_all_in_cache_path(mod)
     end
-    sort!(paths, by=mtime, rev=true) # try newest cachefiles first
+
     for path_to_try in paths
+        if stale_cachefile(sourcepath, path_to_try)
+            continue
+        end
         restored = _require_from_serialized(node, mod, path_to_try, toplevel_load)
         if restored === nothing
             warn("deserialization checks failed while attempting to load cache from $path_to_try")
         else
             return restored
         end
     end
-    return nothing
+    return !isempty(paths)
 end
 
 # to synchronize multiple tasks trying to import/using something
@@ -232,21 +240,27 @@ function require(mod::Symbol)
     last = toplevel_load::Bool
     try
         toplevel_load = false
-        if nothing !== _require_from_serialized(1, mod, last)
-            return
+        name = string(mod)
+        path = find_in_node_path(name, nothing, 1)
+        if path === nothing
+            throw(ArgumentError("module $name not found in current path.\nRun `Pkg.add(\"$name\")` to install the $name package."))
         end
-        if JLOptions().incremental != 0
-            # spawn off a new incremental precompile task from node 1 for recursive `require` calls
+
+        doneprecompile = _require_search_from_serialized(1, mod, path, last)
+        if !isa(doneprecompile, Bool)
+            return # success
+        elseif doneprecompile === true || JLOptions().incremental != 0
+            # spawn off a new incremental pre-compile task from node 1 for recursive `require` calls
+            # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable)
             cachefile = compilecache(mod)
             if nothing === _require_from_serialized(1, mod, cachefile, last)
-                warn("require failed to create a precompiled cache file")
+                warn("compilecache failed to create a usable precompiled cache file for module $name.")
+            else
+                return # success
             end
-            return
         end
+        # fall-through to attempting to load the source file
 
-        name = string(mod)
-        path = find_in_node_path(name, nothing, 1)
-        path === nothing && throw(ArgumentError("$name not found in path"))
         try
             if last && myid() == 1 && nprocs() > 1
                 # include on node 1 first to check for PrecompilableErrors
@@ -259,13 +273,12 @@ function require(mod::Symbol)
                 eval(Main, :(Base.include_from_node1($path)))
             end
         catch ex
-            if !precompilableerror(ex, true)
+            if doneprecompile === true || !precompilableerror(ex, true)
                 rethrow() # rethrow non-precompilable=true errors
             end
-            isinteractive() && info("Precompiling module $mod...")
             cachefile = compilecache(mod)
             if nothing === _require_from_serialized(1, mod, cachefile, last)
-                error("__precompile__(true) but require failed to create a precompiled cache file")
+                error("module $mod declares __precompile__(true) but require failed to create a usable precompiled cache file.")
             end
         end
     finally
@@ -396,6 +409,13 @@ function compilecache(name::ByteString)
         mkpath(cachepath)
     end
     cachefile = abspath(cachepath, name*".ji")
+    if isinteractive()
+        if isfile(cachepath)
+            info("Recompiling stale cache file $cachefile for module $name.")
+        else
+            info("Precompiling module $name.")
+        end
+    end
     if !success(create_expr_cache(path, cachefile))
         error("Failed to precompile $name to $cachefile")
     end
@@ -445,7 +465,7 @@ function stale_cachefile(modpath, cachefile)
         if files[1][1] != modpath
             return true # cache file was compiled from a different path
         end
-        for (f,ftime) in files
+        for (f, ftime) in files
             # Issue #13606: compensate for Docker images rounding mtimes
             if mtime(f) ∉ (ftime, floor(ftime))
                 return true
@@ -465,14 +485,3 @@ function stale_cachefile(modpath, cachefile)
         close(io)
     end
 end
-
-function recompile_stale(mod, cachefile)
-    path = find_in_path(string(mod), nothing)
-    if path === nothing
-        error("module $mod not found in current path; you should rm(\"$(escape_string(cachefile))\") to remove the orphaned cache file")
-    end
-    if stale_cachefile(path, cachefile)
-        info("Recompiling stale cache file $cachefile for module $mod.")
-        compilecache(mod)
-    end
-end

doc/manual/modules.rst

@@ -294,12 +294,22 @@ initialization steps that must occur at *runtime* from steps that can
 occur at *compile time*.  For this purpose, Julia allows you to define
 an ``__init__()`` function in your module that executes any
 initialization steps that must occur at runtime.
+This function will not be called during compilation
+(``--output-*`` or ``__precompile__()``).
+You may, of course, call it manually if necessary,
+but the default is to assume this function deals with computing state for
+the local machine, which does not need to be -- or even should not be --
+captured in the compiled image.
+It will be called after the module is loaded into a process,
+including if it is being loaded into an incremental compile
+(``--output-incremental=yes``), but not if it is being loaded
+into a full-compilation process.
 
 In particular, if you define a ``function __init__()`` in a module,
 then Julia will call ``__init__()`` immediately *after* the module is
 loaded (e.g., by ``import``, ``using``, or ``require``) at runtime for
 the *first* time (i.e., ``__init__`` is only called once, and only
-after all statements in the module have been executed).  Because it is
+after all statements in the module have been executed). Because it is
 called after the module is fully imported, any submodules or other
 imported modules have their ``__init__`` functions called *before* the
 ``__init__`` of the enclosing module.
@@ -308,7 +318,7 @@ Two typical uses of ``__init__`` are calling runtime initialization
 functions of external C libraries and initializing global constants
 that involve pointers returned by external libraries.  For example,
 suppose that we are calling a C library ``libfoo`` that requires us
-to call a ``foo_init()`` initialization function at runtime.   Suppose
+to call a ``foo_init()`` initialization function at runtime. Suppose
 that we also want to define a global constant ``foo_data_ptr`` that
 holds the return value of a ``void *foo_data()`` function defined by
 ``libfoo`` — this constant must be initialized at runtime (not at compile

test/compile.jl

@@ -42,7 +42,7 @@ try
     # the module doesn't load from the image:
     try
         redirected_stderr()
-        @test nothing !== Base._require_from_serialized(myid(), Foo_module, #=broadcast-load=#false)
+        @test nothing !== Base._require_search_from_serialized(myid(), Foo_module, Foo_file, #=broadcast-load=#false)
     finally
         redirect_stderr(olderr)
     end
@@ -88,14 +88,15 @@ try
               end
               """)
     end
-    Base.compilecache("FooBar")
     sleep(2)
+
+    Base.compilecache("FooBar")
     @test isfile(joinpath(dir, "FooBar.ji"))
+    @test !Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji"))
 
     touch(FooBar_file)
     insert!(Base.LOAD_CACHE_PATH, 1, dir2)
-    Base.recompile_stale(:FooBar, joinpath(dir, "FooBar.ji"))
-    sleep(2)
+    Base.compilecache("FooBar")
     @test isfile(joinpath(dir2, "FooBar.ji"))
     @test Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji"))
     @test !Base.stale_cachefile(FooBar_file, joinpath(dir2, "FooBar.ji"))