compilethis(iscompilable) function for automated opt-in to module com…

…pilation on import (closes JuliaLang#12462)

NEWS.md

@@ -28,11 +28,17 @@ New language features
   * The syntax `function foo end` can be used to introduce a generic function without
     yet adding any methods ([#8283]).
 
-  * Incremental compilation of modules: `Base.compile(module::Symbol)` imports the named module,
-    but instead of loading it into the current session saves the result of compiling it in
+  * Incremental compilation of modules: call ``compilethis()`` at the top of a
+    module file to automatically compile it when it is imported ([#12491]), or manually
+    run `Base.compile(modulename)`. The resulting compiled `.ji` file is saved in
     `~/.julia/lib/v0.4` ([#8745]).
 
-      * See manual section on `Module initialization and precompilation` (under `Modules`) for details and errata.
+      * See manual section on `Module initialization and precompilation` (under `Modules`) for details and errata.  In particular, to be safely compilable a module may need an `__init__` function to separate code that must be executed at runtime rather than compile-time.  Modules that are *not* compilable should call `compilethis(false)`.
+
+      * The compiled `.ji` file includes a list of dependencies (modules and files that
+        were imported/included at compile-time), and the module is automatically recompiled
+        upon `import` when any of its dependencies have changed.  Explicit dependencies
+        on other files can be declared with `include_dependency(path)` ([#12458]).
 
       * New option `--output-incremental={yes|no}` added to invoke the equivalent of ``Base.compile`` from the command line.
 
@@ -1557,3 +1563,5 @@ Too numerous to mention.
 [#12137]: https://github.com/JuliaLang/julia/issues/12137
 [#12162]: https://github.com/JuliaLang/julia/issues/12162
 [#12393]: https://github.com/JuliaLang/julia/issues/12393
+[#12458]: https://github.com/JuliaLang/julia/issues/12458
+[#12491]: https://github.com/JuliaLang/julia/issues/12491

base/docs/helpdb.jl

@@ -14579,6 +14579,26 @@ used via `include`.   It has no effect outside of compilation.
 """
 include_dependency
 
+doc"""
+```rst
+::
+           __compile__(iscompilable::Bool=true)
+
+Specify whether the file calling this function is compilable.  If
+``iscompilable`` is ``true``, then ``__compile__`` throws an exception
+*unless* the file is being compiled, and in a module file it causes
+the module to be automatically compiled when it is imported.
+Typically, ``__compile__()`` should occur before the ``module``
+declaration in the file, or better yet ``VERSION >= v"0.4-" &&
+__compile__()`` in order to be backward-compatible with Julia 0.3.
+
+If a module or file is *not* safely compilable, it should call
+``__compile__(false)`` in order to throw an error if Julia attempts
+to compile it.
+```
+"""
+__compile__
+
 doc"""
 ```rst
 ::

base/exports.jl

@@ -1101,6 +1101,7 @@ export
     workspace,
 
 # loading source files
+    __compile__,
     evalfile,
     include,
     include_string,

base/loading.jl

@@ -131,6 +131,30 @@ function include_dependency(path::AbstractString)
     return nothing
 end
 
+# We throw CompilableError(true) when a module wants to be compiled but isn't,
+# and CompilableError(false) when a module doesn't want to be compiled but is
+immutable CompilableError <: Exception
+    iscompilable::Bool
+end
+function show(io::IO, ex::CompilableError)
+    if ex.iscompilable
+        print(io, "__compile__(true) is only allowed in module files being imported")
+    else
+        print(io, "__compile__(false) is not allowed in files that are being compiled")
+    end
+end
+compilableerror(ex::CompilableError, c) = ex.iscompilable == c
+compilableerror(ex::LoadError, c) = compilableerror(ex.error, c)
+compilableerror(ex, c) = false
+
+# put at the top of a file to force it to be compiled (true), or
+# to be prevent it from being compiled (false).
+function __compile__(iscompilable::Bool=true)
+    if myid() == 1 && iscompilable != (0 != ccall(:jl_generating_output, Cint, ()))
+        throw(CompilableError(iscompilable))
+    end
+end
+
 # require always works in Main scope and loads files from node 1
 toplevel_load = true
 function require(mod::Symbol)
@@ -166,13 +190,26 @@ function require(mod::Symbol)
         name = string(mod)
         path = find_in_node_path(name, source_dir(), 1)
         path === nothing && throw(ArgumentError("$name not found in path"))
-        if last && myid() == 1 && nprocs() > 1
-            # broadcast top-level import/using from node 1 (only)
-            content = open(readall, path)
-            refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in procs() ]
-            for r in refs; wait(r); end
-        else
-            eval(Main, :(Base.include_from_node1($path)))
+        try
+            if last && myid() == 1 && nprocs() > 1
+                # include on node 1 first to check for CompilableErrors
+                eval(Main, :(Base.include_from_node1($path)))
+
+                # broadcast top-level import/using from node 1 (only)
+                refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in filter(x -> x != 1, procs()) ]
+                for r in refs; wait(r); end
+            else
+                eval(Main, :(Base.include_from_node1($path)))
+            end
+        catch ex
+            if !compilableerror(ex, true)
+                rethrow() # rethrow non-compilable=true errors
+            end
+            isinteractive() && info("Compiling module $mod...")
+            cachefile = compile(mod)
+            if nothing === _require_from_serialized(1, mod, cachefile, last)
+                error("__compile__(true) but require failed to create a precompiled cache file")
+            end
         end
     finally
         toplevel_load = last
@@ -294,7 +331,9 @@ function compile(name::ByteString)
         mkpath(cachepath)
     end
     cachefile = abspath(cachepath, name*".ji")
-    create_expr_cache(path, cachefile)
+    if !success(create_expr_cache(path, cachefile))
+        error("Failed to compile $name to $cachefile")
+    end
     return cachefile
 end
 

doc/manual/modules.rst

@@ -263,9 +263,22 @@ incremental compile and custom system image.
 To create a custom system image that can be used to start julia with the -J option,
 recompile Julia after modifying the file ``base/userimg.jl`` to require the desired modules.
 
-To create an incremental precompiled module file,
-call ``Base.compile(modulename::Symbol)``.
-The resulting cache files will be stored in ``Base.LOAD_CACHE_PATH[1]``.
+To create an incremental precompiled module file, add
+``__compile__()`` at the top of your module file (before the
+``module`` starts).  This will cause it to be automatically compiled
+the first time it is imported.  Alternatively, you can manually call
+``Base.compile(modulename)``.  The resulting cache files will be
+stored in ``Base.LOAD_CACHE_PATH[1]``.  Subsequently, the module is
+automatically recompiled upon ``import`` whenever any of its
+dependencies change; dependencies are modules it imports, the Julia
+build, files it includes, or explicit dependencies declared by
+``include_dependency(path)`` in the module file(s).  Compiling a
+module also recursively compiles any modules that are imported
+therein.  If you know that it is *not* safe to compile your module
+(for the reasons described below), you should put
+``__compile__(false)`` in the module file to cause ``Base.compile`` to
+throw an error (and thereby prevent the module from being imported by
+any other compiled module).
 
 In order to make your module work with precompilation,
 however, you may need to change your module to explicitly separate any

test/compile.jl

@@ -7,10 +7,11 @@ insert!(LOAD_PATH, 1, dir)
 insert!(Base.LOAD_CACHE_PATH, 1, dir)
 Foo_module = :Foo4b3a94a1a081a8cb
 try
-    file = joinpath(dir, "$Foo_module.jl")
+    Foo_file = joinpath(dir, "$Foo_module.jl")
 
-    open(file, "w") do f
+    open(Foo_file, "w") do f
         print(f, """
+              compilethis(true)
               module $Foo_module
               @doc "foo function" foo(x) = x + 1
               include_dependency("foo.jl")
@@ -22,10 +23,17 @@ try
               """)
     end
 
-    cachefile = Base.compile(Foo_module)
+    if myid() == 1
+        @test_throws Base.CompilableError compilethis(true)
+        @test_throws LoadError include(Foo_file) # from compilethis(true)
+    end
+
+    Base.require(Foo_module)
+    cachefile = joinpath(dir, "$Foo_module.ji")
 
     # use _require_from_serialized to ensure that the test fails if
     # the module doesn't load from the image:
+    println(STDERR, "\nNOTE: The following 'replacing module' warning indicates normal operation:")
     @test nothing !== Base._require_from_serialized(myid(), Foo_module, true)
 
     let Foo = eval(Main, Foo_module)
@@ -39,9 +47,19 @@ try
         deps = Base.cache_dependencies(cachefile)
         @test sort(deps[1]) == map(s -> (s, Base.module_uuid(eval(s))),
                                    [:Base,:Core,:Main])
-        @test sort(deps[2]) == [file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")]
+        @test sort(deps[2]) == [Foo_file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")]
     end
 
+    Baz_file = joinpath(dir, "Baz.jl")
+    open(Baz_file, "w") do f
+        print(f, """
+              compilethis(false)
+              module Baz
+              end
+              """)
+    end
+    println(STDERR, "\nNOTE: The following 'LoadError: compilethis(false)' indicates normal operation")
+    @test_throws ErrorException Base.compile("Baz") # from compilethis(false)
 finally
     splice!(Base.LOAD_CACHE_PATH, 1)
     splice!(LOAD_PATH, 1)