Runtime activity in mode (#1816)

* Runtime activity in mode * fixup * cr * rules * fix fwd * fr * fr * fr * fr * fr * Update test_forward.jl * Update test_forward.jl * fix * fix * inv * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * improve gradient
EnzymeAD · Sep 15, 2024 · ffcb7dd · ffcb7dd
1 parent 0b6effa
commit ffcb7dd
Show file tree

Hide file tree

Showing 33 changed files with 495 additions and 392 deletions.
diff --git a/Project.toml b/Project.toml
@@ -6,6 +6,7 @@ version = "0.13.0"
 [deps]
 CEnum = "fa961155-64e5-5f13-b03f-caf6b980ea82"
 EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869"
+EnzymeTestUtils = "12d8515a-0907-448a-8884-5fe00fdf1c5a"
 Enzyme_jll = "7cc45869-7501-5eee-bdea-0790c847d4ef"
 GPUCompiler = "61eb1bfa-7361-4325-ad38-22787b887f55"
 LLVM = "929cbde3-209d-540e-8aea-75f648917ca0"
@@ -16,11 +17,25 @@ Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 
+[weakdeps]
+BFloat16s = "ab4f0b2a-ad5b-11e8-123f-65d77653426b"
+ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
+LogExpFunctions = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
+SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+
+[extensions]
+EnzymeBFloat16sExt = "BFloat16s"
+EnzymeChainRulesCoreExt = "ChainRulesCore"
+EnzymeLogExpFunctionsExt = "LogExpFunctions"
+EnzymeSpecialFunctionsExt = "SpecialFunctions"
+EnzymeStaticArraysExt = "StaticArrays"
+
 [compat]
 BFloat16s = "0.2, 0.3, 0.4, 0.5"
 CEnum = "0.4, 0.5"
 ChainRulesCore = "1"
-EnzymeCore = "0.7.8"
+EnzymeCore = "0.8"
 Enzyme_jll = "0.0.146, 0.0.148"
 GPUCompiler = "0.21, 0.22, 0.23, 0.24, 0.25, 0.26, 0.27"
 LLVM = "6.1, 7, 8, =9.0"
@@ -31,23 +46,9 @@ SpecialFunctions = "1, 2"
 StaticArrays = "1"
 julia = "1.10"
 
-[extensions]
-EnzymeBFloat16sExt = "BFloat16s"
-EnzymeChainRulesCoreExt = "ChainRulesCore"
-EnzymeLogExpFunctionsExt = "LogExpFunctions"
-EnzymeSpecialFunctionsExt = "SpecialFunctions"
-EnzymeStaticArraysExt = "StaticArrays"
-
 [extras]
 BFloat16s = "ab4f0b2a-ad5b-11e8-123f-65d77653426b"
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
 LogExpFunctions = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
 SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
-
-[weakdeps]
-BFloat16s = "ab4f0b2a-ad5b-11e8-123f-65d77653426b"
-ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-LogExpFunctions = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
-SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b"
-StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
diff --git a/docs/src/faq.md b/docs/src/faq.md
@@ -268,7 +268,7 @@ Enzyme.autodiff(Reverse, f, Active(1.2), Const(Vector{Float64}(undef, 1)), Const
 ((0.0, nothing, nothing, nothing),)
 ```
 
-Passing in a dupliacted (e.g. differentiable) variable for `tmp` now leads to the correct answer.
+Passing in a duplicated (e.g. differentiable) variable for `tmp` now leads to the correct answer.
 
 ```jldoctest storage
 Enzyme.autodiff(Reverse, f, Active(1.2), Duplicated(Vector{Float64}(undef, 1), zeros(1)), Const(1), Const(5))  # Correct (returns 10.367999999999999 == 1.2^4 * 5)
@@ -278,9 +278,11 @@ Enzyme.autodiff(Reverse, f, Active(1.2), Duplicated(Vector{Float64}(undef, 1), z
 ((10.367999999999999, nothing, nothing, nothing),)
 ```
 
-However, even if we ignore the semantic guarantee provided by marking `tmp` as constant, another issue arises. When computing the original function, intermediate computations (like in `f` above) can use `tmp` for temporary storage. When computing the derivative, Enzyme also needs additional temporary storage space for the corresponding derivative variables as well. If `tmp` is marked as Const, Enzyme does not have any temporary storage space for the derivatives!
+## Runtime Activity
 
-Recent versions of Enzyme will attempt to error when they detect these latter types of situations, which we will refer to as `activity unstable`. This term is chosen to mirror the Julia notion of type-unstable code (e.g. where a type is not known at compile time). If an expression is activity unstable, it could either be constant, or active, depending on data not known at compile time. For example, consider the following:
+When computing the derivative of mutable variables, Enzyme also needs additional temporary storage space for the corresponding derivative variables. If an argument `tmp` is marked as Const, Enzyme does not have any temporary storage space for the derivatives!
+
+Enzyme will error when they detect these latter types of situations, which we will refer to as `activity unstable`. This term is chosen to mirror the Julia notion of type-unstable code (e.g. where a type is not known at compile time). If an expression is activity unstable, it could either be constant, or active, depending on data not known at compile time. For example, consider the following:
 
 ```julia
 function g(cond, active_var, constant_var)
@@ -293,7 +295,7 @@ end
 Enzyme.autodiff(Forward, g, Const(condition), Duplicated(x, dx), Const(y))
 ```
 
-The returned value here could either by constant or duplicated, depending on the runtime-defined value of `cond`. If `cond` is true, Enzyme simply returns the shadow of `active_var` as the derivative. However, if `cond` is false, there is no derivative shadow for `constant_var` and Enzyme will throw a "Mismatched activity" error. For some simple types, e.g. a float Enzyme can circumvent this issue, for example by returning the float 0. Similarly, for some types like the Symbol type, which are never differentiable, such a shadow value will never be used, and Enzyme can return the original "primal" value as its derivative.  However, for arbitrary data structures, Enzyme presently has no generic mechanism to resolve this.
+The returned value here could either by constant or duplicated, depending on the runtime-defined value of `cond`. If `cond` is true, Enzyme simply returns the shadow of `active_var` as the derivative. However, if `cond` is false, there is no derivative shadow for `constant_var` and Enzyme will throw a `EnzymeRuntimeActivityError` error. For some simple types, e.g. a float Enzyme can circumvent this issue, for example by returning the float 0. Similarly, for some types like the Symbol type, which are never differentiable, such a shadow value will never be used, and Enzyme can return the original "primal" value as its derivative.  However, for arbitrary data structures, Enzyme presently has no generic mechanism to resolve this.
 
 For example consider a third function:
 ```julia
@@ -308,13 +310,17 @@ Enzyme provides a nice utility `Enzyme.make_zero` which takes a data structure a
 
 If one created a new zero'd copy of each return from `g`, this would mean that the derivative `dresult` would have one copy made for the first element, and a second copy made for the second element. This could lead to incorrect results, and is unfortunately not a general resolution. However, for non-mutable variables (e.g. like floats) or non-differrentiable types (e.g. like Symbols) this problem can never arise.
 
-Instead, Enzyme has a special mode known as "Runtime Activity" which can handle these types of situations. It can come with a minor performance reduction, and is therefore off by default. It can be enabled with `Enzyme.API.runtimeActivity!(true)` right after importing Enzyme for the first time. 
+Instead, Enzyme has a special mode known as "Runtime Activity" which can handle these types of situations. It can come with a minor performance reduction, and is therefore off by default. It can be enabled with by setting runtime activity to true in a desired differentiation mode.
 
 The way Enzyme's runtime activity resolves this issue is to return the original primal variable as the derivative whenever it needs to denote the fact that a variable is a constant. As this issue can only arise with mutable variables, they must be represented in memory via a pointer. All addtional loads and stores will now be modified to first check if the primal pointer is the same as the shadow pointer, and if so, treat it as a constant. Note that this check is not saying that the same arrays contain the same values, but rather the same backing memory represents both the primal and the shadow (e.g. `a === b` or equivalently `pointer(a) == pointer(b)`). 
 
 Enabling runtime activity does therefore, come with a sharp edge, which is that if the computed derivative of a function is mutable, one must also check to see if the primal and shadow represent the same pointer, and if so the true derivative of the function is actually zero.
 
-Generally, the preferred solution to these type of activity unstable codes should be to make your variables all activity-stable (e.g. always containing differentiable memory or always containing non-differentiable memory). However, with care, Enzyme does support "Runtime Activity" as a way to differentiate these programs without having to modify your code.
+Generally, the preferred solution to these type of activity unstable codes should be to make your variables all activity-stable (e.g. always containing differentiable memory or always containing non-differentiable memory). However, with care, Enzyme does support "Runtime Activity" as a way to differentiate these programs without having to modify your code. One can enable runtime activity for your code by changing the mode, such as
+
+```julia
+Enzyme.autodiff(set_runtime_activity(Forward), h, Const(condition), Duplicated(x, dx), Const(y))
+```
 
 ## Mixed activity
 

diff --git a/examples/custom_rule.jl b/examples/custom_rule.jl
@@ -57,14 +57,15 @@ using .EnzymeRules
 
 # In this section, we write a simple forward rule to start out:
 
-function forward(func::Const{typeof(f)}, ::Type{<:Duplicated}, y::Duplicated, x::Duplicated)
+function forward(config::FwdConfig, func::Const{typeof(f)}, ::Type{<:Duplicated}, y::Duplicated, x::Duplicated)
     println("Using custom rule!")
     ret = func.val(y.val, x.val)
     y.dval .= 2 .* x.val .* x.dval
     return Duplicated(ret, sum(y.dval)) 
 end
 
 # In the signature of our rule, we have made use of `Enzyme`'s activity annotations. Let's break down each one:
+# - the [`FwdConfig`](@ref) configuration passes certain compile-time information about differentiation procedure (the width, and if we're using runtime activity),
 # - the [`Const`](@ref) annotation on `f` indicates that we accept a function `f` that does not have a derivative component,
 #   which makes sense since `f` is not a closure with data that could be differentiated. 
 # - the [`Duplicated`](@ref) annotation given in the second argument annotates the return value of `f`. This means that
@@ -96,7 +97,7 @@ g(y, x) = f(y, x)^2 # function to differentiate
 # To squeeze out the last drop of performance, the below rule avoids computing the output of the original function and 
 # just computes its derivative.
 
-function forward(func::Const{typeof(f)}, ::Type{<:DuplicatedNoNeed}, y::Duplicated, x::Duplicated)
+function forward(config, func::Const{typeof(f)}, ::Type{<:DuplicatedNoNeed}, y::Duplicated, x::Duplicated)
     println("Using custom rule with DuplicatedNoNeed output.")
     y.val .= x.val.^2 
     y.dval .= 2 .* x.val .* x.dval
@@ -127,7 +128,7 @@ dy = [0.0, 0.0]
 
 Base.delete_method.(methods(forward, (Const{typeof(f)}, Vararg{Any}))) # delete our old rules
 
-function forward(func::Const{typeof(f)}, RT::Type{<:Union{Const, DuplicatedNoNeed, Duplicated}}, 
+function forward(config, func::Const{typeof(f)}, RT::Type{<:Union{Const, DuplicatedNoNeed, Duplicated}}, 
                  y::Union{Const, Duplicated}, x::Union{Const, Duplicated})
     println("Using our general custom rule!")
     y.val .= x.val.^2 

diff --git a/ext/EnzymeChainRulesCoreExt.jl b/ext/EnzymeChainRulesCoreExt.jl
@@ -54,7 +54,7 @@ function Enzyme._import_frule(fn, tys...)
     end
 
     quote
-        function EnzymeRules.forward(fn::FA, ::Type{RetAnnotation}, $(exprs...); kwargs...) where {RetAnnotation, FA<:Annotation{<:$(esc(fn))}, $(anns...)}
+        function EnzymeRules.forward(config, fn::FA, ::Type{RetAnnotation}, $(exprs...); kwargs...) where {RetAnnotation, FA<:Annotation{<:$(esc(fn))}, $(anns...)}
             batchsize = same_or_one(1, $(vals...))
             if batchsize == 1
                 dfn = fn isa Const ? $ChainRulesCore.NoTangent() : fn.dval

diff --git a/lib/EnzymeCore/Project.toml b/lib/EnzymeCore/Project.toml
@@ -1,11 +1,11 @@
 name = "EnzymeCore"
 uuid = "f151be2c-9106-41f4-ab19-57ee4f262869"
 authors = ["William Moses <[email protected]>", "Valentin Churavy <[email protected]>"]
-version = "0.7.8"
+version = "0.8.0"
 
 [compat]
 Adapt = "3, 4"
-julia = "1.6"
+julia = "1.10"
 
 [deps]
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"