optimizer: support callsite annotations of @inline and @noinline

NEWS.md

@@ -11,6 +11,7 @@ New language features
   as `.&&` and `.||`. ([#39594])
 * `⫪` (U+2AEA, `\Top`, `\downvDash`) and `⫫` (U+2AEB, `\Bot`, `\upvDash`, `\indep`)
   may now be used as binary operators with comparison precedence. ([#39403])
+* `@noinline` can now be used at function callsites and `@inline` and `@noinline` can be used in `do` blocks. ([#40754]).
 
 Language changes
 ----------------

base/compiler/abstractinterpretation.jl

@@ -1784,7 +1784,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState)
                     if isa(fname, Slot)
                         changes = StateUpdate(fname, VarState(Any, false), changes, false)
                     end
-                elseif hd === :inbounds || hd === :meta || hd === :loopinfo || hd === :code_coverage_effect
+                elseif hd === :inbounds || hd === :meta || hd === :loopinfo || hd === :code_coverage_effect || hd === :noinline
                     # these do not generate code
                 else
                     t = abstract_eval_statement(interp, stmt, changes, frame)

base/compiler/optimize.jl

@@ -135,6 +135,8 @@ const SLOT_USEDUNDEF    = 32 # slot has uses that might raise UndefVarError
 # This statement was marked as @inbounds by the user. If replaced by inlining,
 # any contained boundschecks may be removed
 const IR_FLAG_INBOUNDS       = 0x01
+# This statement was marked as @noinline by the user
+const IR_FLAG_NOINLINE       = 0x01 << 7
 # This statement may be removed if its result is unused. In particular it must
 # thus be both pure and effect free.
 const IR_FLAG_EFFECT_FREE    = 0x01 << 4

base/compiler/ssair/driver.jl

@@ -79,6 +79,7 @@ function convert_to_ircode(ci::CodeInfo, code::Vector{Any}, coverage::Bool, narg
     renumber_ir_elements!(code, changemap, labelmap)
 
     inbounds_depth = 0 # Number of stacked inbounds
+    disable_inline = false # whether or not to disable inline optimization
     meta = Any[]
     flags = fill(0x00, length(code))
     for i = 1:length(code)
@@ -93,6 +94,10 @@ function convert_to_ircode(ci::CodeInfo, code::Vector{Any}, coverage::Bool, narg
                 inbounds_depth -= 1
             end
             stmt = nothing
+        elseif isexpr(stmt, :noinline)
+            arg1 = stmt.args[1]
+            disable_inline = arg1
+            stmt = nothing
         else
             stmt = normalize(stmt, meta)
         end
@@ -101,6 +106,9 @@ function convert_to_ircode(ci::CodeInfo, code::Vector{Any}, coverage::Bool, narg
             if inbounds_depth > 0
                 flags[i] |= IR_FLAG_INBOUNDS
             end
+            if disable_inline
+                flags[i] |= IR_FLAG_NOINLINE
+            end
         end
     end
     strip_trailing_junk!(ci, code, stmtinfo, flags)

base/compiler/ssair/inlining.jl

@@ -790,7 +790,7 @@ function validate_sparams(sparams::SimpleVector)
 end
 
 function analyze_method!(match::MethodMatch, atypes::Vector{Any},
-                         state::InliningState, @nospecialize(stmttyp))
+                         state::InliningState, @nospecialize(stmttyp), dont_inline::Bool)
     method = match.method
     methsig = method.sig
 
@@ -810,7 +810,7 @@ function analyze_method!(match::MethodMatch, atypes::Vector{Any},
     validate_sparams(match.sparams) || return nothing
 
 
-    if !state.params.inlining
+    if !state.params.inlining || dont_inline
         return compileable_specialization(state.et, match)
     end
 
@@ -1049,10 +1049,20 @@ is_builtin(s::Signature) =
     isa(s.f, Builtin) ||
     s.ft ⊑ Builtin
 
+function check_noinline_flag(ir::IRCode, idx::Int)
+    try
+        (ir.stmts[idx][:flag] & IR_FLAG_NOINLINE) != 0
+    catch BoundsError
+        false
+    end
+
+end
+
 function inline_invoke!(ir::IRCode, idx::Int, sig::Signature, info::InvokeCallInfo,
         state::InliningState, todo::Vector{Pair{Int, Any}})
     stmt = ir.stmts[idx][:inst]
     calltype = ir.stmts[idx][:type]
+    dont_inline = check_noinline_flag(ir, idx)
 
     if !info.match.fully_covers
         # TODO: We could union split out the signature check and continue on
@@ -1064,7 +1074,7 @@ function inline_invoke!(ir::IRCode, idx::Int, sig::Signature, info::InvokeCallIn
     atypes = atypes[4:end]
     pushfirst!(atypes, atype0)
 
-    result = analyze_method!(info.match, atypes, state, calltype)
+    result = analyze_method!(info.match, atypes, state, calltype, dont_inline)
     handle_single_case!(ir, stmt, idx, result, true, todo)
     return nothing
 end
@@ -1164,6 +1174,7 @@ function analyze_single_call!(ir::IRCode, todo::Vector{Pair{Int, Any}}, idx::Int
     signature_union = Union{}
     only_method = nothing  # keep track of whether there is one matching method
     too_many = false
+    dont_inline = check_noinline_flag(ir, idx)
     local meth
     local fully_covered = true
     for i in 1:length(infos)
@@ -1192,7 +1203,7 @@ function analyze_single_call!(ir::IRCode, todo::Vector{Pair{Int, Any}}, idx::Int
                 fully_covered = false
                 continue
             end
-            case = analyze_method!(match, sig.atypes, state, calltype)
+            case = analyze_method!(match, sig.atypes, state, calltype, dont_inline)
             if case === nothing
                 fully_covered = false
                 continue
@@ -1219,7 +1230,7 @@ function analyze_single_call!(ir::IRCode, todo::Vector{Pair{Int, Any}}, idx::Int
             match = meth[1]
         end
         fully_covered = true
-        case = analyze_method!(match, sig.atypes, state, calltype)
+        case = analyze_method!(match, sig.atypes, state, calltype, dont_inline)
         case === nothing && return
         push!(cases, Pair{Any,Any}(match.spec_types, case))
     end
@@ -1280,6 +1291,7 @@ function assemble_inline_todo!(ir::IRCode, state::InliningState)
         stmt = ir.stmts[idx][:inst]
         calltype = ir.stmts[idx][:type]
         info = ir.stmts[idx][:info]
+        dont_inline = check_noinline_flag(ir, idx)
 
         # Check whether this call was @pure and evaluates to a constant
         if info isa MethodResultPure
@@ -1308,7 +1320,7 @@ function assemble_inline_todo!(ir::IRCode, state::InliningState)
         end
 
         if isa(info, OpaqueClosureCallInfo)
-            result = analyze_method!(info.match, sig.atypes, state, calltype)
+            result = analyze_method!(info.match, sig.atypes, state, calltype, dont_inline)
             handle_single_case!(ir, stmt, idx, result, false, todo)
             continue
         end

base/compiler/utilities.jl

@@ -59,7 +59,8 @@ end
 
 # Meta expression head, these generally can't be deleted even when they are
 # in a dead branch but can be ignored when analyzing uses/liveness.
-is_meta_expr_head(head::Symbol) = (head === :inbounds || head === :boundscheck || head === :meta || head === :loopinfo)
+is_meta_expr_head(head::Symbol) = (head === :inbounds || head === :boundscheck || head === :meta ||
+    head === :loopinfo || head === :noinline)
 
 sym_isless(a::Symbol, b::Symbol) = ccall(:strcmp, Int32, (Ptr{UInt8}, Ptr{UInt8}), a, b) < 0
 

base/compiler/validation.jl

@@ -16,6 +16,7 @@ const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange}(
     :leave => 1:1,
     :pop_exception => 1:1,
     :inbounds => 1:1,
+    :noinline => 1:1,
     :boundscheck => 0:0,
     :copyast => 1:1,
     :meta => 0:typemax(Int),
@@ -141,7 +142,7 @@ function validate_code!(errors::Vector{>:InvalidCodeError}, c::CodeInfo, is_top_
                 head === :const || head === :enter || head === :leave || head === :pop_exception ||
                 head === :method || head === :global || head === :static_parameter ||
                 head === :new || head === :splatnew || head === :thunk || head === :loopinfo ||
-                head === :throw_undef_if_not || head === :code_coverage_effect
+                head === :throw_undef_if_not || head === :code_coverage_effect || head === :noinline
                 validate_val!(x)
             else
                 # TODO: nothing is actually in statement position anymore

base/expr.jl

@@ -184,7 +184,12 @@ Give a hint to the compiler that this function is worth inlining.
 
 Small functions typically do not need the `@inline` annotation,
 as the compiler does it automatically. By using `@inline` on bigger functions,
-an extra nudge can be given to the compiler to inline it.
+an extra nudge can be given to the compiler to inline it. `@inline` can
+only be used at function definitions.
+
+If `@inline` is used at the beginning of a `do` block of a function
+call, the `do` block will be marked for inlining.
+
 This is shown in the following example:
 
 ```julia
@@ -193,34 +198,89 @@ This is shown in the following example:
         Function Definition
     =#
 end
+
+bigfunction2() do
+    @inline
+    #=
+        Do Body
+    #=
 ```
+
+!!! compat "Julia 1.7"
+    Usage in `do` blocks requires at least Julia 1.7
 """
 macro inline(ex)
     esc(isa(ex, Expr) ? pushmeta!(ex, :inline) : ex)
 end
 
+macro inline()
+    Expr(:meta, :inline)
+end
+
+
 """
     @noinline
 
 Give a hint to the compiler that it should not inline a function.
 
 Small functions are typically inlined automatically.
 By using `@noinline` on small functions, auto-inlining can be
-prevented. This is shown in the following example:
+prevented. `@noinline` can be used at function definitions,
+function calls, and in `do` blocks.
+
+If `@noinline` is used at the beginning of a `do` block of a function
+call, the `do` block won't be inlined but the function itself still
+could be. Similarily, if `@noinline` is specified immediately before a
+function call with a `do` block, that function won't be inlined
+but the `do` block body could possibly be.
+
+This is shown in the following examples:
 
 ```julia
 @noinline function smallfunction(x)
     #=
         Function Definition
     =#
 end
+
+@noinline previouslydefinedfunction(x)
+
+f() do
+    @noinline
+    #=
+        Do Body
+    #=
 ```
 
 !!! note
     If the function is trivial (for example returning a constant) it might get inlined anyway.
+
+!!! compat "Julia 1.7"
+    Callsite usage requires at least Julia 1.7
+
+!!! compat "Julia 1.7"
+    Usage in `do` blocks requires at least Julia 1.7
 """
 macro noinline(ex)
-    esc(isa(ex, Expr) ? pushmeta!(ex, :noinline) : ex)
+    if isa(ex, Expr)
+        if ex.head === :function || is_short_function_def(ex) || ex.head === :->
+            # function definition noinline
+            esc(pushmeta!(ex, :noinline))
+        else
+            # callsite noinline
+            return Expr(:block,
+                        Expr(:noinline, true),
+                        Expr(:local, Expr(:(=), :val, esc(ex))),
+                        Expr(:noinline, false),
+                        :val)
+        end
+    else
+        esc(ex)
+    end
+end
+
+macro noinline()
+    Expr(:meta, :noinline)
 end
 
 """

base/meta.jl

@@ -413,6 +413,7 @@ end
 
 _instantiate_type_in_env(x, spsig, spvals) = ccall(:jl_instantiate_type_in_env, Any, (Any, Any, Ptr{Any}), x, spsig, spvals)
 
-is_meta_expr_head(head::Symbol) = (head === :inbounds || head === :boundscheck || head === :meta || head === :loopinfo)
+is_meta_expr_head(head::Symbol) = (head === :inbounds || head === :boundscheck || head === :meta
+                                   || head === :loopinfo || head === :noinline)
 
 end # module

src/ast.scm

@@ -276,7 +276,7 @@
 ;; predicates and accessors
 
 (define (quoted? e)
-  (memq (car e) '(quote top core globalref outerref line break inert meta inbounds loopinfo)))
+  (memq (car e) '(quote top core globalref outerref line break inert meta inbounds noinline loopinfo)))
 (define (quotify e) `',e)
 (define (unquote e)
   (if (and (pair? e) (memq (car e) '(quote inert)))

src/codegen.cpp

@@ -4273,7 +4273,7 @@ static void emit_stmtpos(jl_codectx_t &ctx, jl_value_t *expr, int ssaval_result)
     jl_value_t **args = (jl_value_t**)jl_array_data(ex->args);
     jl_sym_t *head = ex->head;
     if (head == meta_sym || head == inbounds_sym || head == coverageeffect_sym
-            || head == aliasscope_sym || head == popaliasscope_sym) {
+            || head == aliasscope_sym || head == popaliasscope_sym || head == noinline_sym) {
         // some expression types are metadata and can be ignored
         // in statement position
         return;
@@ -4704,7 +4704,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval)
     }
     else if (head == leave_sym || head == coverageeffect_sym
             || head == pop_exception_sym || head == enter_sym || head == inbounds_sym
-            || head == aliasscope_sym || head == popaliasscope_sym) {
+            || head == aliasscope_sym || head == popaliasscope_sym || head == noinline_sym) {
         jl_errorf("Expr(:%s) in value position", jl_symbol_name(head));
     }
     else if (head == boundscheck_sym) {

src/interpreter.c

@@ -303,7 +303,7 @@ static jl_value_t *eval_value(jl_value_t *e, interpreter_state *s)
         return jl_true;
     }
     else if (head == meta_sym || head == coverageeffect_sym || head == inbounds_sym || head == loopinfo_sym ||
-             head == aliasscope_sym || head == popaliasscope_sym) {
+             head == aliasscope_sym || head == popaliasscope_sym || head == noinline_sym) {
         return jl_nothing;
     }
     else if (head == gc_preserve_begin_sym || head == gc_preserve_end_sym) {

src/julia-syntax.scm

@@ -3336,7 +3336,7 @@ f(x) = yt(x)
          thunk with-static-parameters toplevel-only
          global globalref outerref const-if-global thismodule
          const null true false ssavalue isdefined toplevel module lambda error
-         gc_preserve_begin gc_preserve_end import using export)))
+         gc_preserve_begin gc_preserve_end import using export noinline)))
 
 (define (local-in? s lam)
   (or (assq s (car  (lam:vinfo lam)))
@@ -4428,7 +4428,7 @@ f(x) = yt(x)
                (cons (car e) args)))
 
             ;; metadata expressions
-            ((line meta inbounds loopinfo gc_preserve_end aliasscope popaliasscope)
+            ((line meta inbounds loopinfo gc_preserve_end aliasscope popaliasscope noinline)
              (let ((have-ret? (and (pair? code) (pair? (car code)) (eq? (caar code) 'return))))
                (cond ((eq? (car e) 'line)
                       (set! current-loc e)
@@ -4573,7 +4573,7 @@ f(x) = yt(x)
           (begin (set! linetable (cons (make-lineinfo name file line) linetable))
                  (set! current-loc 1)))
       (if (or reachable
-              (and (pair? e) (memq (car e) '(meta inbounds gc_preserve_begin gc_preserve_end aliasscope popaliasscope))))
+              (and (pair? e) (memq (car e) '(meta inbounds gc_preserve_begin gc_preserve_end aliasscope popaliasscope noinline))))
           (begin (set! code (cons e code))
                  (set! i (+ i 1))
                  (set! locs (cons current-loc locs)))))

src/macroexpand.scm

@@ -352,7 +352,7 @@
                                    ,(resolve-expansion-vars-with-new-env (caddr arg) env m parent-scope inarg))))
                              (else
                               `(global ,(resolve-expansion-vars-with-new-env arg env m parent-scope inarg))))))
-           ((using import export meta line inbounds boundscheck loopinfo) (map unescape e))
+           ((using import export meta line noinline inbounds boundscheck loopinfo) (map unescape e))
            ((macrocall) e) ; invalid syntax anyways, so just act like it's quoted.
            ((symboliclabel) e)
            ((symbolicgoto) e)

src/method.c

@@ -65,7 +65,8 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve
             e->head == quote_sym || e->head == inert_sym ||
             e->head == meta_sym || e->head == inbounds_sym ||
             e->head == boundscheck_sym || e->head == loopinfo_sym ||
-            e->head == aliasscope_sym || e->head == popaliasscope_sym) {
+            e->head == aliasscope_sym || e->head == popaliasscope_sym ||
+            e->head == noinline_sym) {
             // ignore these
         }
         else {