From b01b9fca5b415cdcf132b50c3ee6a267878da444 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Thu, 6 Aug 2015 11:08:30 -0400 Subject: [PATCH] compilethis(iscompilable) function for automated opt-in to module compilation on import (closes #12462) --- NEWS.md | 14 ++++++++--- base/docs/helpdb.jl | 20 +++++++++++++++ base/exports.jl | 1 + base/loading.jl | 55 ++++++++++++++++++++++++++++++++++++------ doc/manual/modules.rst | 19 ++++++++++++--- test/compile.jl | 26 +++++++++++++++++--- 6 files changed, 117 insertions(+), 18 deletions(-) diff --git a/NEWS.md b/NEWS.md index 458bd5d60d89fa..d110dbf9f75486 100644 --- a/NEWS.md +++ b/NEWS.md @@ -28,11 +28,17 @@ New language features * The syntax `function foo end` can be used to introduce a generic function without yet adding any methods ([#8283]). - * Incremental compilation of modules: `Base.compile(module::Symbol)` imports the named module, - but instead of loading it into the current session saves the result of compiling it in + * Incremental compilation of modules: call ``compilethis()`` at the top of a + module file to automatically compile it when it is imported ([#12491]), or manually + run `Base.compile(modulename)`. The resulting compiled `.ji` file is saved in `~/.julia/lib/v0.4` ([#8745]). - * See manual section on `Module initialization and precompilation` (under `Modules`) for details and errata. + * See manual section on `Module initialization and precompilation` (under `Modules`) for details and errata. In particular, to be safely compilable a module may need an `__init__` function to separate code that must be executed at runtime rather than compile-time. Modules that are *not* compilable should call `compilethis(false)`. + + * The compiled `.ji` file includes a list of dependencies (modules and files that + were imported/included at compile-time), and the module is automatically recompiled + upon `import` when any of its dependencies have changed. Explicit dependencies + on other files can be declared with `include_dependency(path)` ([#12458]). * New option `--output-incremental={yes|no}` added to invoke the equivalent of ``Base.compile`` from the command line. @@ -1557,3 +1563,5 @@ Too numerous to mention. [#12137]: https://github.com/JuliaLang/julia/issues/12137 [#12162]: https://github.com/JuliaLang/julia/issues/12162 [#12393]: https://github.com/JuliaLang/julia/issues/12393 +[#12458]: https://github.com/JuliaLang/julia/issues/12458 +[#12491]: https://github.com/JuliaLang/julia/issues/12491 diff --git a/base/docs/helpdb.jl b/base/docs/helpdb.jl index 1952da6fea3dae..93886742fb193f 100644 --- a/base/docs/helpdb.jl +++ b/base/docs/helpdb.jl @@ -14579,6 +14579,26 @@ used via `include`. It has no effect outside of compilation. """ include_dependency +doc""" +```rst +:: + __compile__(iscompilable::Bool=true) + +Specify whether the file calling this function is compilable. If +``iscompilable`` is ``true``, then ``__compile__`` throws an exception +*unless* the file is being compiled, and in a module file it causes +the module to be automatically compiled when it is imported. +Typically, ``__compile__()`` should occur before the ``module`` +declaration in the file, or better yet ``VERSION >= v"0.4-" && +__compile__()`` in order to be backward-compatible with Julia 0.3. + +If a module or file is *not* safely compilable, it should call +``__compile__(false)`` in order to throw an error if Julia attempts +to compile it. +``` +""" +__compile__ + doc""" ```rst :: diff --git a/base/exports.jl b/base/exports.jl index 55d20fd3238913..72bc1436ed1f5f 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1101,6 +1101,7 @@ export workspace, # loading source files + __compile__, evalfile, include, include_string, diff --git a/base/loading.jl b/base/loading.jl index 0ad0223aeb89fd..2cfc831456e0bc 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -131,6 +131,30 @@ function include_dependency(path::AbstractString) return nothing end +# We throw CompilableError(true) when a module wants to be compiled but isn't, +# and CompilableError(false) when a module doesn't want to be compiled but is +immutable CompilableError <: Exception + iscompilable::Bool +end +function show(io::IO, ex::CompilableError) + if ex.iscompilable + print(io, "__compile__(true) is only allowed in module files being imported") + else + print(io, "__compile__(false) is not allowed in files that are being compiled") + end +end +compilableerror(ex::CompilableError, c) = ex.iscompilable == c +compilableerror(ex::LoadError, c) = compilableerror(ex.error, c) +compilableerror(ex, c) = false + +# put at the top of a file to force it to be compiled (true), or +# to be prevent it from being compiled (false). +function __compile__(iscompilable::Bool=true) + if myid() == 1 && iscompilable != (0 != ccall(:jl_generating_output, Cint, ())) + throw(CompilableError(iscompilable)) + end +end + # require always works in Main scope and loads files from node 1 toplevel_load = true function require(mod::Symbol) @@ -166,13 +190,26 @@ function require(mod::Symbol) name = string(mod) path = find_in_node_path(name, source_dir(), 1) path === nothing && throw(ArgumentError("$name not found in path")) - if last && myid() == 1 && nprocs() > 1 - # broadcast top-level import/using from node 1 (only) - content = open(readall, path) - refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in procs() ] - for r in refs; wait(r); end - else - eval(Main, :(Base.include_from_node1($path))) + try + if last && myid() == 1 && nprocs() > 1 + # include on node 1 first to check for CompilableErrors + eval(Main, :(Base.include_from_node1($path))) + + # broadcast top-level import/using from node 1 (only) + refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in filter(x -> x != 1, procs()) ] + for r in refs; wait(r); end + else + eval(Main, :(Base.include_from_node1($path))) + end + catch ex + if !compilableerror(ex, true) + rethrow() # rethrow non-compilable=true errors + end + isinteractive() && info("Compiling module $mod...") + cachefile = compile(mod) + if nothing === _require_from_serialized(1, mod, cachefile, last) + error("__compile__(true) but require failed to create a precompiled cache file") + end end finally toplevel_load = last @@ -294,7 +331,9 @@ function compile(name::ByteString) mkpath(cachepath) end cachefile = abspath(cachepath, name*".ji") - create_expr_cache(path, cachefile) + if !success(create_expr_cache(path, cachefile)) + error("Failed to compile $name to $cachefile") + end return cachefile end diff --git a/doc/manual/modules.rst b/doc/manual/modules.rst index 2b81c0158fe2dc..777f9a15f1713b 100644 --- a/doc/manual/modules.rst +++ b/doc/manual/modules.rst @@ -263,9 +263,22 @@ incremental compile and custom system image. To create a custom system image that can be used to start julia with the -J option, recompile Julia after modifying the file ``base/userimg.jl`` to require the desired modules. -To create an incremental precompiled module file, -call ``Base.compile(modulename::Symbol)``. -The resulting cache files will be stored in ``Base.LOAD_CACHE_PATH[1]``. +To create an incremental precompiled module file, add +``__compile__()`` at the top of your module file (before the +``module`` starts). This will cause it to be automatically compiled +the first time it is imported. Alternatively, you can manually call +``Base.compile(modulename)``. The resulting cache files will be +stored in ``Base.LOAD_CACHE_PATH[1]``. Subsequently, the module is +automatically recompiled upon ``import`` whenever any of its +dependencies change; dependencies are modules it imports, the Julia +build, files it includes, or explicit dependencies declared by +``include_dependency(path)`` in the module file(s). Compiling a +module also recursively compiles any modules that are imported +therein. If you know that it is *not* safe to compile your module +(for the reasons described below), you should put +``__compile__(false)`` in the module file to cause ``Base.compile`` to +throw an error (and thereby prevent the module from being imported by +any other compiled module). In order to make your module work with precompilation, however, you may need to change your module to explicitly separate any diff --git a/test/compile.jl b/test/compile.jl index 4f70e1877d1afd..74218533f90333 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -7,10 +7,11 @@ insert!(LOAD_PATH, 1, dir) insert!(Base.LOAD_CACHE_PATH, 1, dir) Foo_module = :Foo4b3a94a1a081a8cb try - file = joinpath(dir, "$Foo_module.jl") + Foo_file = joinpath(dir, "$Foo_module.jl") - open(file, "w") do f + open(Foo_file, "w") do f print(f, """ + compilethis(true) module $Foo_module @doc "foo function" foo(x) = x + 1 include_dependency("foo.jl") @@ -22,10 +23,17 @@ try """) end - cachefile = Base.compile(Foo_module) + if myid() == 1 + @test_throws Base.CompilableError compilethis(true) + @test_throws LoadError include(Foo_file) # from compilethis(true) + end + + Base.require(Foo_module) + cachefile = joinpath(dir, "$Foo_module.ji") # use _require_from_serialized to ensure that the test fails if # the module doesn't load from the image: + println(STDERR, "\nNOTE: The following 'replacing module' warning indicates normal operation:") @test nothing !== Base._require_from_serialized(myid(), Foo_module, true) let Foo = eval(Main, Foo_module) @@ -39,9 +47,19 @@ try deps = Base.cache_dependencies(cachefile) @test sort(deps[1]) == map(s -> (s, Base.module_uuid(eval(s))), [:Base,:Core,:Main]) - @test sort(deps[2]) == [file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")] + @test sort(deps[2]) == [Foo_file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")] end + Baz_file = joinpath(dir, "Baz.jl") + open(Baz_file, "w") do f + print(f, """ + compilethis(false) + module Baz + end + """) + end + println(STDERR, "\nNOTE: The following 'LoadError: compilethis(false)' indicates normal operation") + @test_throws ErrorException Base.compile("Baz") # from compilethis(false) finally splice!(Base.LOAD_CACHE_PATH, 1) splice!(LOAD_PATH, 1)