Line | Exclusive | Inclusive | Code |
---|---|---|---|
1 | # This file is a part of Julia. License is MIT: https://julialang.org/license | ||
2 | |||
3 | # Support for @simd for | ||
4 | |||
5 | module SimdLoop | ||
6 | |||
7 | export @simd, simd_outer_range, simd_inner_length, simd_index | ||
8 | |||
9 | # Error thrown from ill-formed uses of @simd | ||
10 | struct SimdError <: Exception | ||
11 | msg::String | ||
12 | end | ||
13 | |||
14 | # Parse iteration space expression | ||
15 | # symbol '=' range | ||
16 | # symbol 'in' range | ||
17 | function parse_iteration_space(x) | ||
18 | (isa(x, Expr) && (x.head === :(=) || x.head === :in)) || throw(SimdError("= or in expected")) | ||
19 | length(x.args) == 2 || throw(SimdError("simd range syntax is wrong")) | ||
20 | isa(x.args[1], Symbol) || throw(SimdError("simd loop index must be a symbol")) | ||
21 | x.args # symbol, range | ||
22 | end | ||
23 | |||
24 | # reject invalid control flow statements in @simd loop body | ||
25 | function check_body!(x::Expr) | ||
26 | if x.head === :break || x.head === :continue | ||
27 | throw(SimdError("$(x.head) is not allowed inside a @simd loop body")) | ||
28 | elseif x.head === :macrocall && x.args[1] === Symbol("@goto") | ||
29 | throw(SimdError("@goto is not allowed inside a @simd loop body")) | ||
30 | end | ||
31 | for arg in x.args | ||
32 | check_body!(arg) | ||
33 | end | ||
34 | return true | ||
35 | end | ||
36 | check_body!(x::QuoteNode) = check_body!(x.value) | ||
37 | check_body!(x) = true | ||
38 | |||
39 | # @simd splits a for loop into two loops: an outer scalar loop and | ||
40 | # an inner loop marked with :loopinfo. The simd_... functions define | ||
41 | # the splitting. | ||
42 | # Custom iterators that do not support random access cannot support | ||
43 | # vectorization. In order to be compatible with `@simd` annotated loops, | ||
44 | #they should override `simd_inner_length(v::MyIter, j) = 1`, | ||
45 | #`simd_outer_range(v::MyIter) = v`, and `simd_index(v::MyIter, j, i) = j`. | ||
46 | |||
47 | # Get range for outer loop. | ||
48 | simd_outer_range(r) = 0:0 | ||
49 | |||
50 | # Get trip count for inner loop. | ||
51 | @inline simd_inner_length(r, j) = Base.length(r) | ||
52 | |||
53 | # Construct user-level element from original range, outer loop index j, and inner loop index i. | ||
54 | @inline simd_index(r, j, i) = (@inbounds ret = r[i+firstindex(r)]; ret) | ||
55 | |||
56 | # Compile Expr x in context of @simd. | ||
57 | function compile(x, ivdep) | ||
58 | (isa(x, Expr) && x.head === :for) || throw(SimdError("for loop expected")) | ||
59 | length(x.args) == 2 || throw(SimdError("1D for loop expected")) | ||
60 | check_body!(x) | ||
61 | |||
62 | var,range = parse_iteration_space(x.args[1]) | ||
63 | r = gensym("r") # Range value | ||
64 | j = gensym("i") # Iteration variable for outer loop | ||
65 | n = gensym("n") # Trip count for inner loop | ||
66 | i = gensym("i") # Trip index for inner loop | ||
67 | quote | ||
68 | # Evaluate range value once, to enhance type and data flow analysis by optimizers. | ||
69 | let $r = $range | ||
70 | for $j in Base.simd_outer_range($r) | ||
71 | let $n = Base.simd_inner_length($r,$j) | ||
72 | if zero($n) < $n | ||
73 | # Lower loop in way that seems to work best for LLVM 3.3 vectorizer. | ||
74 | let $i = zero($n) | ||
75 | 9 (3 %) |
9 (100 %)
samples spent calling
<
while $i < $n
|
|
76 | local $var = Base.simd_index($r,$j,$i) | ||
77 | 48 (17 %) |
48 (100 %)
samples spent calling
macro expansion
$(x.args[2]) # Body of loop
|
|
78 | 13 (5 %) |
13 (100 %)
samples spent calling
+
$i += 1
|
|
79 | $(Expr(:loopinfo, Symbol("julia.simdloop"), ivdep)) # Mark loop as SIMD loop | ||
80 | end | ||
81 | end | ||
82 | end | ||
83 | end | ||
84 | end | ||
85 | end | ||
86 | nothing | ||
87 | end | ||
88 | end | ||
89 | |||
90 | """ | ||
91 | @simd | ||
92 | |||
93 | Annotate a `for` loop to allow the compiler to take extra liberties to allow loop re-ordering | ||
94 | |||
95 | !!! warning | ||
96 | This feature is experimental and could change or disappear in future versions of Julia. | ||
97 | Incorrect use of the `@simd` macro may cause unexpected results. | ||
98 | |||
99 | The object iterated over in a `@simd for` loop should be a one-dimensional range. | ||
100 | By using `@simd`, you are asserting several properties of the loop: | ||
101 | |||
102 | * It is safe to execute iterations in arbitrary or overlapping order, with special consideration for reduction variables. | ||
103 | * Floating-point operations on reduction variables can be reordered or contracted, possibly causing different results than without `@simd`. | ||
104 | |||
105 | In many cases, Julia is able to automatically vectorize inner for loops without the use of `@simd`. | ||
106 | Using `@simd` gives the compiler a little extra leeway to make it possible in more situations. In | ||
107 | either case, your inner loop should have the following properties to allow vectorization: | ||
108 | |||
109 | * The loop must be an innermost loop | ||
110 | * The loop body must be straight-line code. Therefore, [`@inbounds`](@ref) is | ||
111 | currently needed for all array accesses. The compiler can sometimes turn | ||
112 | short `&&`, `||`, and `?:` expressions into straight-line code if it is safe | ||
113 | to evaluate all operands unconditionally. Consider using the [`ifelse`](@ref) | ||
114 | function instead of `?:` in the loop if it is safe to do so. | ||
115 | * Accesses must have a stride pattern and cannot be "gathers" (random-index | ||
116 | reads) or "scatters" (random-index writes). | ||
117 | * The stride should be unit stride. | ||
118 | |||
119 | !!! note | ||
120 | The `@simd` does not assert by default that the loop is completely free of loop-carried | ||
121 | memory dependencies, which is an assumption that can easily be violated in generic code. | ||
122 | If you are writing non-generic code, you can use `@simd ivdep for ... end` to also assert that: | ||
123 | |||
124 | * There exists no loop-carried memory dependencies | ||
125 | * No iteration ever waits on a previous iteration to make forward progress. | ||
126 | """ | ||
127 | macro simd(forloop) | ||
128 | esc(compile(forloop, nothing)) | ||
129 | end | ||
130 | |||
131 | macro simd(ivdep, forloop) | ||
132 | if ivdep === :ivdep | ||
133 | esc(compile(forloop, Symbol("julia.ivdep"))) | ||
134 | else | ||
135 | throw(SimdError("Only ivdep is valid as the first argument to @simd")) | ||
136 | end | ||
137 | end | ||
138 | |||
139 | end # module SimdLoop |