diff --git a/NEWS.md b/NEWS.md index 458bd5d60d89f..8e86d4000f6bb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -28,13 +28,19 @@ New language features * The syntax `function foo end` can be used to introduce a generic function without yet adding any methods ([#8283]). - * Incremental compilation of modules: `Base.compile(module::Symbol)` imports the named module, - but instead of loading it into the current session saves the result of compiling it in + * Incremental precompilation of modules: call ``__precompile__()`` at the top of a + module file to automatically precompile it when it is imported ([#12491]), or manually + run `Base.compile(modulename)`. The resulting precompiled `.ji` file is saved in `~/.julia/lib/v0.4` ([#8745]). - * See manual section on `Module initialization and precompilation` (under `Modules`) for details and errata. + * See manual section on `Module initialization and precompilation` (under `Modules`) for details and errata. In particular, to be safely precompilable a module may need an `__init__` function to separate code that must be executed at runtime rather than precompile-time. Modules that are *not* precompilable should call `__precompile__(false)`. - * New option `--output-incremental={yes|no}` added to invoke the equivalent of ``Base.compile`` from the command line. + * The precompiled `.ji` file includes a list of dependencies (modules and files that + were imported/included at precompile-time), and the module is automatically recompiled + upon `import` when any of its dependencies have changed. Explicit dependencies + on other files can be declared with `include_dependency(path)` ([#12458]). + + * New option `--output-incremental={yes|no}` added to invoke the equivalent of ``Base.compilecache`` from the command line. * The syntax `new{parameters...}(...)` can be used in constructors to specify parameters for the type to be constructed ([#8135]). @@ -1557,3 +1563,5 @@ Too numerous to mention. [#12137]: https://github.com/JuliaLang/julia/issues/12137 [#12162]: https://github.com/JuliaLang/julia/issues/12162 [#12393]: https://github.com/JuliaLang/julia/issues/12393 +[#12458]: https://github.com/JuliaLang/julia/issues/12458 +[#12491]: https://github.com/JuliaLang/julia/issues/12491 diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 1952da6fea3da..8d345889ebc12 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -8472,12 +8472,12 @@ whos doc""" ```rst :: - compile(module::Symbol) + Base.compilecache(module::Symbol) Creates a precompiled cache file for module (see help for ``require``) and all of its dependencies. This can be used to reduce package load times. Cache files are stored in LOAD_CACHE_PATH[1], which defaults to `~/.julia/lib/VERSION`. See the manual section `Module initialization and precompilation` (under `Modules`) for important notes. ``` """ -compile +compilecache doc""" ```rst @@ -14579,6 +14579,26 @@ used via `include`. It has no effect outside of compilation. """ include_dependency +doc""" +```rst +:: + __precompile__(isprecompilable::Bool=true) + +Specify whether the file calling this function is precompilable. If +``isprecompilable`` is ``true``, then ``__precompile__`` throws an exception +*unless* the file is being precompiled, and in a module file it causes +the module to be automatically precompiled when it is imported. +Typically, ``__precompile__()`` should occur before the ``module`` +declaration in the file, or better yet ``VERSION >= v"0.4" && +__precompile__()`` in order to be backward-compatible with Julia 0.3. + +If a module or file is *not* safely precompilable, it should call +``__precompile__(false)`` in order to throw an error if Julia attempts +to precompile it. +``` +""" +__precompile__ + doc""" ```rst :: diff --git a/base/exports.jl b/base/exports.jl index 55d20fd323891..30f7b2d57da00 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1101,6 +1101,7 @@ export workspace, # loading source files + __precompile__, evalfile, include, include_string, diff --git a/base/loading.jl b/base/loading.jl index 0ad0223aeb89f..28dfc2564f36d 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -131,6 +131,30 @@ function include_dependency(path::AbstractString) return nothing end +# We throw PrecompilableError(true) when a module wants to be precompiled but isn't, +# and PrecompilableError(false) when a module doesn't want to be precompiled but is +immutable PrecompilableError <: Exception + isprecompilable::Bool +end +function show(io::IO, ex::PrecompilableError) + if ex.isprecompilable + print(io, "__precompile__(true) is only allowed in module files being imported") + else + print(io, "__precompile__(false) is not allowed in files that are being precompiled") + end +end +precompilableerror(ex::PrecompilableError, c) = ex.isprecompilable == c +precompilableerror(ex::LoadError, c) = precompilableerror(ex.error, c) +precompilableerror(ex, c) = false + +# put at the top of a file to force it to be precompiled (true), or +# to be prevent it from being precompiled (false). +function __precompile__(isprecompilable::Bool=true) + if myid() == 1 && isprecompilable != (0 != ccall(:jl_generating_output, Cint, ())) + throw(PrecompilableError(isprecompilable)) + end +end + # require always works in Main scope and loads files from node 1 toplevel_load = true function require(mod::Symbol) @@ -155,8 +179,8 @@ function require(mod::Symbol) return end if JLOptions().incremental != 0 - # spawn off a new incremental compile task from node 1 for recursive `require` calls - cachefile = compile(mod) + # spawn off a new incremental precompile task from node 1 for recursive `require` calls + cachefile = compilecache(mod) if nothing === _require_from_serialized(1, mod, cachefile, last) warn("require failed to create a precompiled cache file") end @@ -166,13 +190,26 @@ function require(mod::Symbol) name = string(mod) path = find_in_node_path(name, source_dir(), 1) path === nothing && throw(ArgumentError("$name not found in path")) - if last && myid() == 1 && nprocs() > 1 - # broadcast top-level import/using from node 1 (only) - content = open(readall, path) - refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in procs() ] - for r in refs; wait(r); end - else - eval(Main, :(Base.include_from_node1($path))) + try + if last && myid() == 1 && nprocs() > 1 + # include on node 1 first to check for PrecompilableErrors + eval(Main, :(Base.include_from_node1($path))) + + # broadcast top-level import/using from node 1 (only) + refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in filter(x -> x != 1, procs()) ] + for r in refs; wait(r); end + else + eval(Main, :(Base.include_from_node1($path))) + end + catch ex + if !precompilableerror(ex, true) + rethrow() # rethrow non-precompilable=true errors + end + isinteractive() && info("Precompiling module $mod...") + cachefile = compilecache(mod) + if nothing === _require_from_serialized(1, mod, cachefile, last) + error("__precompile__(true) but require failed to create a precompiled cache file") + end end finally toplevel_load = last @@ -284,9 +321,9 @@ function create_expr_cache(input::AbstractString, output::AbstractString) return pobj end -compile(mod::Symbol) = compile(string(mod)) -function compile(name::ByteString) - myid() == 1 || error("can only compile from node 1") +compilecache(mod::Symbol) = compilecache(string(mod)) +function compilecache(name::ByteString) + myid() == 1 || error("can only precompile from node 1") path = find_in_path(name) path === nothing && throw(ArgumentError("$name not found in path")) cachepath = LOAD_CACHE_PATH[1] @@ -294,7 +331,9 @@ function compile(name::ByteString) mkpath(cachepath) end cachefile = abspath(cachepath, name*".ji") - create_expr_cache(path, cachefile) + if !success(create_expr_cache(path, cachefile)) + error("Failed to precompile $name to $cachefile") + end return cachefile end diff --git a/doc/manual/modules.rst b/doc/manual/modules.rst index 2b81c0158fe2d..a1221a4167e5b 100644 --- a/doc/manual/modules.rst +++ b/doc/manual/modules.rst @@ -263,9 +263,22 @@ incremental compile and custom system image. To create a custom system image that can be used to start julia with the -J option, recompile Julia after modifying the file ``base/userimg.jl`` to require the desired modules. -To create an incremental precompiled module file, -call ``Base.compile(modulename::Symbol)``. -The resulting cache files will be stored in ``Base.LOAD_CACHE_PATH[1]``. +To create an incremental precompiled module file, add +``__precompile__()`` at the top of your module file (before the +``module`` starts). This will cause it to be automatically compiled +the first time it is imported. Alternatively, you can manually call +``Base.compilecache(modulename)``. The resulting cache files will be +stored in ``Base.LOAD_CACHE_PATH[1]``. Subsequently, the module is +automatically recompiled upon ``import`` whenever any of its +dependencies change; dependencies are modules it imports, the Julia +build, files it includes, or explicit dependencies declared by +``include_dependency(path)`` in the module file(s). Precompiling a +module also recursively precompiles any modules that are imported +therein. If you know that it is *not* safe to precompile your module +(for the reasons described below), you should put +``__precompile__(false)`` in the module file to cause ``Base.compilecache`` to +throw an error (and thereby prevent the module from being imported by +any other precompiled module). In order to make your module work with precompilation, however, you may need to change your module to explicitly separate any @@ -379,15 +392,15 @@ Other known potential failure scenarios include: # or move the assignment into the runtime: __init__() = global mystdout = Base.STDOUT #= also works =# -Several additional restrictions are placed on the operations that can be done while compiling code +Several additional restrictions are placed on the operations that can be done while precompiling code to help the user avoid other wrong-behavior situations: 1. Calling ``eval`` to cause a side-effect in another module. - This will also cause a warning to be emitted when the incremental compile flag is set. + This will also cause a warning to be emitted when the incremental precompile flag is set. 2. ``global const`` statements from local scope after ``__init__()`` has been started (see issue #12010 for plans to add an error for this) -3. Replacing a module (or calling ``workspace()``) is a runtime error while doing an incremental compile. +3. Replacing a module (or calling ``workspace()``) is a runtime error while doing an incremental precompile. A few other points to be aware of: diff --git a/src/dump.c b/src/dump.c index a523e5cf24053..e41f52efb05c5 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1657,7 +1657,7 @@ static void jl_reinit_item(ios_t *f, jl_value_t *v, int how) { jl_errorf("invalid redefinition of constant %s", mod->name->name); // this also throws } if (jl_generating_output() && jl_options.incremental) { - jl_errorf("cannot replace module %s during incremental compile", mod->name->name); + jl_errorf("cannot replace module %s during incremental precompile", mod->name->name); } jl_printf(JL_STDERR, "WARNING: replacing module %s\n", mod->name->name); } diff --git a/test/compile.jl b/test/compile.jl index 4f70e1877d1af..4bc8d57dcc05f 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -7,10 +7,11 @@ insert!(LOAD_PATH, 1, dir) insert!(Base.LOAD_CACHE_PATH, 1, dir) Foo_module = :Foo4b3a94a1a081a8cb try - file = joinpath(dir, "$Foo_module.jl") + Foo_file = joinpath(dir, "$Foo_module.jl") - open(file, "w") do f + open(Foo_file, "w") do f print(f, """ + __precompile__(true) module $Foo_module @doc "foo function" foo(x) = x + 1 include_dependency("foo.jl") @@ -22,10 +23,17 @@ try """) end - cachefile = Base.compile(Foo_module) + if myid() == 1 + @test_throws Base.PrecompilableError __precompile__(true) + @test_throws LoadError include(Foo_file) # from __precompile__(true) + end + + Base.require(Foo_module) + cachefile = joinpath(dir, "$Foo_module.ji") # use _require_from_serialized to ensure that the test fails if # the module doesn't load from the image: + println(STDERR, "\nNOTE: The following 'replacing module' warning indicates normal operation:") @test nothing !== Base._require_from_serialized(myid(), Foo_module, true) let Foo = eval(Main, Foo_module) @@ -39,9 +47,19 @@ try deps = Base.cache_dependencies(cachefile) @test sort(deps[1]) == map(s -> (s, Base.module_uuid(eval(s))), [:Base,:Core,:Main]) - @test sort(deps[2]) == [file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")] + @test sort(deps[2]) == [Foo_file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")] end + Baz_file = joinpath(dir, "Baz.jl") + open(Baz_file, "w") do f + print(f, """ + __precompile__(false) + module Baz + end + """) + end + println(STDERR, "\nNOTE: The following 'LoadError: __precompile__(false)' indicates normal operation") + @test_throws ErrorException Base.compilecache("Baz") # from __precompile__(false) finally splice!(Base.LOAD_CACHE_PATH, 1) splice!(LOAD_PATH, 1)