Report index
StatProfilerHTML.jl report
Generated on Thu, 21 Dec 2023 12:59:22
File source code
Line Exclusive Inclusive Code
1 # This file is a part of Julia. License is MIT: https://julialang.org/license
2
3 # Support for @simd for
4
5 module SimdLoop
6
7 export @simd, simd_outer_range, simd_inner_length, simd_index
8
9 # Error thrown from ill-formed uses of @simd
10 struct SimdError <: Exception
11 msg::String
12 end
13
14 # Parse iteration space expression
15 # symbol '=' range
16 # symbol 'in' range
17 function parse_iteration_space(x)
18 (isa(x, Expr) && (x.head === :(=) || x.head === :in)) || throw(SimdError("= or in expected"))
19 length(x.args) == 2 || throw(SimdError("simd range syntax is wrong"))
20 isa(x.args[1], Symbol) || throw(SimdError("simd loop index must be a symbol"))
21 x.args # symbol, range
22 end
23
24 # reject invalid control flow statements in @simd loop body
25 function check_body!(x::Expr)
26 if x.head === :break || x.head === :continue
27 throw(SimdError("$(x.head) is not allowed inside a @simd loop body"))
28 elseif x.head === :macrocall && x.args[1] === Symbol("@goto")
29 throw(SimdError("@goto is not allowed inside a @simd loop body"))
30 end
31 for arg in x.args
32 check_body!(arg)
33 end
34 return true
35 end
36 check_body!(x::QuoteNode) = check_body!(x.value)
37 check_body!(x) = true
38
39 # @simd splits a for loop into two loops: an outer scalar loop and
40 # an inner loop marked with :loopinfo. The simd_... functions define
41 # the splitting.
42 # Custom iterators that do not support random access cannot support
43 # vectorization. In order to be compatible with `@simd` annotated loops,
44 #they should override `simd_inner_length(v::MyIter, j) = 1`,
45 #`simd_outer_range(v::MyIter) = v`, and `simd_index(v::MyIter, j, i) = j`.
46
47 # Get range for outer loop.
48 simd_outer_range(r) = 0:0
49
50 # Get trip count for inner loop.
51 @inline simd_inner_length(r, j) = Base.length(r)
52
53 # Construct user-level element from original range, outer loop index j, and inner loop index i.
54 @inline simd_index(r, j, i) = (@inbounds ret = r[i+firstindex(r)]; ret)
55
56 # Compile Expr x in context of @simd.
57 function compile(x, ivdep)
58 (isa(x, Expr) && x.head === :for) || throw(SimdError("for loop expected"))
59 length(x.args) == 2 || throw(SimdError("1D for loop expected"))
60 check_body!(x)
61
62 var,range = parse_iteration_space(x.args[1])
63 r = gensym("r") # Range value
64 j = gensym("i") # Iteration variable for outer loop
65 n = gensym("n") # Trip count for inner loop
66 i = gensym("i") # Trip index for inner loop
67 quote
68 # Evaluate range value once, to enhance type and data flow analysis by optimizers.
69 let $r = $range
70 for $j in Base.simd_outer_range($r)
71 let $n = Base.simd_inner_length($r,$j)
72 if zero($n) < $n
73 # Lower loop in way that seems to work best for LLVM 3.3 vectorizer.
74 let $i = zero($n)
75 9 (3 %)
9 (3 %) samples spent in macro expansion
9 (100 %) (incl.) when called from copyto! line 1003
9 (100 %) samples spent calling <
while $i < $n
76 local $var = Base.simd_index($r,$j,$i)
77 48 (17 %)
48 (17 %) samples spent in macro expansion
48 (100 %) (incl.) when called from copyto! line 1003
48 (100 %) samples spent calling macro expansion
$(x.args[2]) # Body of loop
78 13 (5 %)
13 (5 %) samples spent in macro expansion
13 (100 %) (incl.) when called from copyto! line 1003
13 (100 %) samples spent calling +
$i += 1
79 $(Expr(:loopinfo, Symbol("julia.simdloop"), ivdep)) # Mark loop as SIMD loop
80 end
81 end
82 end
83 end
84 end
85 end
86 nothing
87 end
88 end
89
90 """
91 @simd
92
93 Annotate a `for` loop to allow the compiler to take extra liberties to allow loop re-ordering
94
95 !!! warning
96 This feature is experimental and could change or disappear in future versions of Julia.
97 Incorrect use of the `@simd` macro may cause unexpected results.
98
99 The object iterated over in a `@simd for` loop should be a one-dimensional range.
100 By using `@simd`, you are asserting several properties of the loop:
101
102 * It is safe to execute iterations in arbitrary or overlapping order, with special consideration for reduction variables.
103 * Floating-point operations on reduction variables can be reordered or contracted, possibly causing different results than without `@simd`.
104
105 In many cases, Julia is able to automatically vectorize inner for loops without the use of `@simd`.
106 Using `@simd` gives the compiler a little extra leeway to make it possible in more situations. In
107 either case, your inner loop should have the following properties to allow vectorization:
108
109 * The loop must be an innermost loop
110 * The loop body must be straight-line code. Therefore, [`@inbounds`](@ref) is
111 currently needed for all array accesses. The compiler can sometimes turn
112 short `&&`, `||`, and `?:` expressions into straight-line code if it is safe
113 to evaluate all operands unconditionally. Consider using the [`ifelse`](@ref)
114 function instead of `?:` in the loop if it is safe to do so.
115 * Accesses must have a stride pattern and cannot be "gathers" (random-index
116 reads) or "scatters" (random-index writes).
117 * The stride should be unit stride.
118
119 !!! note
120 The `@simd` does not assert by default that the loop is completely free of loop-carried
121 memory dependencies, which is an assumption that can easily be violated in generic code.
122 If you are writing non-generic code, you can use `@simd ivdep for ... end` to also assert that:
123
124 * There exists no loop-carried memory dependencies
125 * No iteration ever waits on a previous iteration to make forward progress.
126 """
127 macro simd(forloop)
128 esc(compile(forloop, nothing))
129 end
130
131 macro simd(ivdep, forloop)
132 if ivdep === :ivdep
133 esc(compile(forloop, Symbol("julia.ivdep")))
134 else
135 throw(SimdError("Only ivdep is valid as the first argument to @simd"))
136 end
137 end
138
139 end # module SimdLoop