diff --git a/benchmark/hessian.jl b/benchmark/hessian.jl index 46cee269..501c0ebd 100644 --- a/benchmark/hessian.jl +++ b/benchmark/hessian.jl @@ -9,10 +9,10 @@ Test cases taken from the article: > https://www.tandfonline.com/doi/full/10.1080/10556788.2018.1480625 =# -function hessbench(method) +function hessbench(detector) suite = BenchmarkGroup() - suite["ArrowHead"] = hessbench_arrowhead(method) - suite["RandomSparsity"] = hessbench_randomsparsity(method) + suite["ArrowHead"] = hessbench_arrowhead(detector) + suite["RandomSparsity"] = hessbench_randomsparsity(detector) return suite end @@ -32,7 +32,7 @@ function (ah::ArrowHead)(x::AbstractVector) end end -function hessbench_arrowhead(method) +function hessbench_arrowhead(detector) suite = BenchmarkGroup() # Commented-out cases (N, K) are included in the JuMP paper linked above, # but excluded from to accelerate the benchmark suite. @@ -48,7 +48,7 @@ function hessbench_arrowhead(method) ] x = rand(N) f = ArrowHead(K) - suite["N=$N, K=$K"] = @benchmarkable hessian_sparsity($f, $x, $method) + suite["N=$N, K=$K"] = @benchmarkable hessian_sparsity($f, $x, $detector) end return suite end @@ -70,7 +70,7 @@ function (rs::RandomSparsity)(x::AbstractVector) end end -function hessbench_randomsparsity(method) +function hessbench_randomsparsity(detector) suite = BenchmarkGroup() # Commented-out cases (N, K) are included in the JuMP paper linked above, # but excluded from to accelerate the benchmark suite. @@ -86,7 +86,7 @@ function hessbench_randomsparsity(method) ] x = rand(N) f = RandomSparsity(N, K) - suite["N=$N, K=$K"] = @benchmarkable hessian_sparsity($f, $x, $method) + suite["N=$N, K=$K"] = @benchmarkable hessian_sparsity($f, $x, $detector) end return suite end diff --git a/benchmark/jacobian.jl b/benchmark/jacobian.jl index 0e114a87..40f15949 100644 --- a/benchmark/jacobian.jl +++ b/benchmark/jacobian.jl @@ -7,11 +7,11 @@ using SparseArrays: sprand using SimpleDiffEq: ODEProblem, solve, SimpleEuler using Flux: Conv -function jacbench(method) +function jacbench(detector) suite = BenchmarkGroup() - suite["SparseMul"] = jacbench_sparsemul(method) - suite["Brusselator"] = jacbench_brusselator(method) - suite["Conv"] = jacbench_conv(method) + suite["SparseMul"] = jacbench_sparsemul(detector) + suite["Brusselator"] = jacbench_brusselator(detector) + suite["Conv"] = jacbench_conv(detector) return suite end @@ -35,13 +35,13 @@ function (ism::IteratedSparseMul)(x::AbstractVector) return y end -function jacbench_sparsemul(method) +function jacbench_sparsemul(detector) suite = BenchmarkGroup() for n in [50], p in [0.01, 0.25], depth in [5] x = rand(n) f = IteratedSparseMul(; n, p, depth) suite["n=$n, p=$p, depth=$depth"] = @benchmarkable jacobian_sparsity( - $f, $x, $method + $f, $x, $detector ) end return suite @@ -49,20 +49,20 @@ end ## Brusselator -function jacbench_brusselator(method) +function jacbench_brusselator(detector) suite = BenchmarkGroup() for N in (6, 24) f! = Brusselator!(N) x = rand(N, N, 2) y = similar(x) - suite["operator"]["N=$N"] = @benchmarkable jacobian_sparsity($f!, $y, $x, $method) + suite["operator"]["N=$N"] = @benchmarkable jacobian_sparsity($f!, $y, $x, $detector) solver = SimpleEuler() prob = ODEProblem(brusselator_2d_loop!, x, (0.0, 1.0), f!.params) function brusselator_ode_solve(x) return solve(ODEProblem(brusselator_2d_loop!, x, (0.0, 1.0), f!.params), solver; dt=0.5).u[end] end suite["ODE"]["N=$N"] = @benchmarkable jacobian_sparsity( - $brusselator_ode_solve, $x, $method + $brusselator_ode_solve, $x, $detector ) end return suite @@ -70,13 +70,13 @@ end ## Convolution -function jacbench_conv(method) +function jacbench_conv(detector) # TODO: benchmark local sparsity tracers on LeNet-5 CNN layer = Conv((5, 5), 3 => 2) suite = BenchmarkGroup() for N in (28, 128) suite["N=$N"] = @benchmarkable jacobian_sparsity( - $layer, $(rand(N, N, 3, 1)), $method + $layer, $(rand(N, N, 3, 1)), $detector ) end return suite diff --git a/docs/src/user/global_vs_local.md b/docs/src/user/global_vs_local.md index fa7c9a6d..77ac87ec 100644 --- a/docs/src/user/global_vs_local.md +++ b/docs/src/user/global_vs_local.md @@ -24,14 +24,14 @@ These are computed by [`TracerLocalSparsityDetector`](@ref): ```@repl localvsglobal using SparseConnectivityTracer -method = TracerLocalSparsityDetector(); +detector = TracerLocalSparsityDetector(); f(x) = x[1]*x[2]; -jacobian_sparsity(f, [1, 1], method) -jacobian_sparsity(f, [0, 1], method) -jacobian_sparsity(f, [1, 0], method) -jacobian_sparsity(f, [0, 0], method) +jacobian_sparsity(f, [1, 1], detector) +jacobian_sparsity(f, [0, 1], detector) +jacobian_sparsity(f, [1, 0], detector) +jacobian_sparsity(f, [0, 0], detector) ``` In contrast to this, [`TracerSparsityDetector`](@ref) computes a conservative union over all sparsity patterns in $\mathbf{x} \in \mathbb{R}^2$. @@ -39,13 +39,13 @@ The resulting **global** pattern therefore does not depend on the input. All of the following function calls are equivalent: ```@repl localvsglobal -method = TracerSparsityDetector(); +detector = TracerSparsityDetector() -jacobian_sparsity(f, [1, 1], method) -jacobian_sparsity(f, [0, 1], method) -jacobian_sparsity(f, [1, 0], method) -jacobian_sparsity(f, [0, 0], method) -jacobian_sparsity(f, rand(2), method) +jacobian_sparsity(f, [1, 1], detector) +jacobian_sparsity(f, [0, 1], detector) +jacobian_sparsity(f, [1, 0], detector) +jacobian_sparsity(f, [0, 0], detector) +jacobian_sparsity(f, rand(2), detector) ``` !!! tip "Global vs. Local" diff --git a/docs/src/user/limitations.md b/docs/src/user/limitations.md index 01b34177..14a673bc 100644 --- a/docs/src/user/limitations.md +++ b/docs/src/user/limitations.md @@ -24,7 +24,7 @@ it must be written generically enough to accept numbers of type `T<:Real` as (or ```@example notgeneric using SparseConnectivityTracer - method = TracerSparsityDetector() + detector = TracerSparsityDetector() relu_bad(x::AbstractFloat) = max(zero(x), x) outer_function_bad(xs) = sum(relu_bad, xs) @@ -39,7 +39,7 @@ it must be written generically enough to accept numbers of type `T<:Real` as (or outer_function_bad(xs) - jacobian_sparsity(outer_function_bad, xs, method) + jacobian_sparsity(outer_function_bad, xs, detector) ``` This is easily fixed by loosening type restrictions or adding an additional methods on `Real`: @@ -51,7 +51,7 @@ it must be written generically enough to accept numbers of type `T<:Real` as (or ``` ```@repl notgeneric - jacobian_sparsity(outer_function_good, xs, method) + jacobian_sparsity(outer_function_good, xs, detector) ``` ## Limited control flow diff --git a/test/brusselator.jl b/test/brusselator.jl index 51e897f1..7b1597ec 100644 --- a/test/brusselator.jl +++ b/test/brusselator.jl @@ -9,18 +9,18 @@ using Test # Load definitions of GRADIENT_TRACERS, GRADIENT_PATTERNS, HESSIAN_TRACERS and HESSIAN_PATTERNS include("tracers_definitions.jl") -function test_brusselator(method::AbstractSparsityDetector) +function test_brusselator(detector::AbstractSparsityDetector) N = 6 f! = Brusselator!(N) x = rand(N, N, 2) y = similar(x) - J = ADTypes.jacobian_sparsity(f!, y, x, method) + J = ADTypes.jacobian_sparsity(f!, y, x, detector) @test_reference "references/pattern/jacobian/Brusselator.txt" BitMatrix(J) end @testset "$T" for T in GRADIENT_TRACERS - method = TracerSparsityDetector(; gradient_tracer_type=T) - test_brusselator(method) + detector = TracerSparsityDetector(; gradient_tracer_type=T) + test_brusselator(detector) yield() end diff --git a/test/ext/test_LogExpFunctions.jl b/test/ext/test_LogExpFunctions.jl index 7bc8ee9f..a06a2730 100644 --- a/test/ext/test_LogExpFunctions.jl +++ b/test/ext/test_LogExpFunctions.jl @@ -28,8 +28,8 @@ lef_1_to_1 = union(lef_1_to_1_pos_input, lef_1_to_1_neg_input) lef_2_to_1 = (xlogy, xlog1py, xexpy, logaddexp, logsubexp) @testset "Jacobian Global" begin - method = TracerSparsityDetector() - J(f, x) = jacobian_sparsity(f, x, method) + detector = TracerSparsityDetector() + J(f, x) = jacobian_sparsity(f, x, detector) @testset "1-to-1 functions" begin @testset "$f" for f in lef_1_to_1 @@ -44,8 +44,8 @@ lef_2_to_1 = (xlogy, xlog1py, xexpy, logaddexp, logsubexp) end @testset "Jacobian Local" begin - method = TracerLocalSparsityDetector() - J(f, x) = jacobian_sparsity(f, x, method) + detector = TracerLocalSparsityDetector() + J(f, x) = jacobian_sparsity(f, x, detector) @testset "1-to-1 functions" begin @testset "$f" for f in lef_1_to_1_pos_input @@ -63,8 +63,8 @@ end end @testset "Hessian Global" begin - method = TracerSparsityDetector() - H(f, x) = hessian_sparsity(f, x, method) + detector = TracerSparsityDetector() + H(f, x) = hessian_sparsity(f, x, detector) @testset "1-to-1 functions" begin @testset "$f" for f in lef_1_to_1 @@ -79,8 +79,8 @@ end end @testset "Hessian Local" begin - method = TracerLocalSparsityDetector() - H(f, x) = hessian_sparsity(f, x, method) + detector = TracerLocalSparsityDetector() + H(f, x) = hessian_sparsity(f, x, detector) @testset "1-to-1 functions" begin @testset "$f" for f in lef_1_to_1_pos_input diff --git a/test/ext/test_NNlib.jl b/test/ext/test_NNlib.jl index a349737f..f27e0771 100644 --- a/test/ext/test_NNlib.jl +++ b/test/ext/test_NNlib.jl @@ -32,8 +32,8 @@ NNLIB_ACTIVATIONS_F = ( NNLIB_ACTIVATIONS = union(NNLIB_ACTIVATIONS_S, NNLIB_ACTIVATIONS_F) @testset "Jacobian Global" begin - method = TracerSparsityDetector() - J(f, x) = jacobian_sparsity(f, x, method) + detector = TracerSparsityDetector() + J(f, x) = jacobian_sparsity(f, x, detector) @testset "$f" for f in NNLIB_ACTIVATIONS @test J(f, 1) ≈ [1;;] @@ -41,8 +41,8 @@ NNLIB_ACTIVATIONS = union(NNLIB_ACTIVATIONS_S, NNLIB_ACTIVATIONS_F) end @testset "Jacobian Local" begin - method = TracerLocalSparsityDetector() - J(f, x) = jacobian_sparsity(f, x, method) + detector = TracerLocalSparsityDetector() + J(f, x) = jacobian_sparsity(f, x, detector) @test J(NNlib.relu, -1) ≈ [0;;] @test J(NNlib.relu, 1) ≈ [1;;] @@ -82,8 +82,8 @@ end end @testset "Global Hessian" begin - method = TracerSparsityDetector() - H(f, x) = hessian_sparsity(f, x, method) + detector = TracerSparsityDetector() + H(f, x) = hessian_sparsity(f, x, detector) @testset "First-order differentiable" begin @testset "$f" for f in NNLIB_ACTIVATIONS_F @@ -98,8 +98,8 @@ end end @testset "Local Hessian" begin - method = TracerLocalSparsityDetector() - H(f, x) = hessian_sparsity(f, x, method) + detector = TracerLocalSparsityDetector() + H(f, x) = hessian_sparsity(f, x, detector) @test H(NNlib.relu, -1) ≈ [0;;] @test H(NNlib.relu, 1) ≈ [0;;] diff --git a/test/ext/test_NaNMath.jl b/test/ext/test_NaNMath.jl index 15041421..f9f2aebc 100644 --- a/test/ext/test_NaNMath.jl +++ b/test/ext/test_NaNMath.jl @@ -22,8 +22,8 @@ nan_1_to_1 = ( ) @testset "Jacobian Global" begin - method = TracerSparsityDetector() - J(f, x) = jacobian_sparsity(f, x, method) + detector = TracerSparsityDetector() + J(f, x) = jacobian_sparsity(f, x, detector) @testset "1-to-1 functions" begin @testset "$f" for f in nan_1_to_1 @@ -38,8 +38,8 @@ nan_1_to_1 = ( end @testset "Jacobian Local" begin - method = TracerLocalSparsityDetector() - J(f, x) = jacobian_sparsity(f, x, method) + detector = TracerLocalSparsityDetector() + J(f, x) = jacobian_sparsity(f, x, detector) @testset "2-to-1 functions" begin @test J(x -> NaNMath.max(x[1], x[2]), [1.0, 2.0, 0.0]) == [0 1 0] @@ -50,8 +50,8 @@ end end @testset "Hessian Global" begin - method = TracerSparsityDetector() - H(f, x) = hessian_sparsity(f, x, method) + detector = TracerSparsityDetector() + H(f, x) = hessian_sparsity(f, x, detector) @testset "1-to-1 functions" begin @testset "$f" for f in nan_1_to_1 @@ -66,8 +66,8 @@ end end @testset "Hessian Local" begin - method = TracerLocalSparsityDetector() - H(f, x) = hessian_sparsity(f, x, method) + detector = TracerLocalSparsityDetector() + H(f, x) = hessian_sparsity(f, x, detector) @testset "2-to-1 functions" begin @test H(x -> NaNMath.max(x[1], x[2]), [1.0, 2.0, 0.0]) == zeros(Bool, 3, 3) diff --git a/test/ext/test_SpecialFunctions.jl b/test/ext/test_SpecialFunctions.jl index 3108e077..9b8d9a03 100644 --- a/test/ext/test_SpecialFunctions.jl +++ b/test/ext/test_SpecialFunctions.jl @@ -7,8 +7,8 @@ using Test include("../tracers_definitions.jl") @testset "Jacobian Global" begin - method = TracerSparsityDetector() - J(f, x) = jacobian_sparsity(f, x, method) + detector = TracerSparsityDetector() + J(f, x) = jacobian_sparsity(f, x, detector) @test J(x -> erf(x[1]), rand(2)) == [1 0] @test J(x -> beta(x[1], x[2]), rand(3)) == [1 1 0] @@ -16,13 +16,13 @@ end # TODO: add tests # @testset "Jacobian Local" begin -# method = TracerLocalSparsityDetector() -# J(f, x) = jacobian_sparsity(f, x, method) +# detector = TracerLocalSparsityDetector() +# J(f, x) = jacobian_sparsity(f, x, detector) # end @testset "Global Hessian" begin - method = TracerSparsityDetector() - H(f, x) = hessian_sparsity(f, x, method) + detector = TracerSparsityDetector() + H(f, x) = hessian_sparsity(f, x, detector) @test H(x -> erf(x[1]), rand(2)) == [ 1 0 @@ -37,6 +37,6 @@ end # TODO: add tests # @testset "Local Hessian" begin -# method = TracerLocalSparsityDetector() -# H(f, x) = hessian_sparsity(f, x, method) +# detector = TracerLocalSparsityDetector() +# H(f, x) = hessian_sparsity(f, x, detector) # end diff --git a/test/flux.jl b/test/flux.jl index d6db1b3d..afdb1d9e 100644 --- a/test/flux.jl +++ b/test/flux.jl @@ -56,25 +56,25 @@ const BIAS_FLUX = [0.1] const LAYER = Conv(WEIGHTS_FLUX, BIAS_FLUX) # Conv((2, 2), 2 => 1) const LAYER_RELU = Conv(WEIGHTS_FLUX, BIAS_FLUX, relu) # Conv((2, 2), 2 => 1, relu) -function test_flux_conv(method::AbstractSparsityDetector) - J = jacobian_sparsity(LAYER, INPUT_FLUX, method) +function test_flux_conv(detector::AbstractSparsityDetector) + J = jacobian_sparsity(LAYER, INPUT_FLUX, detector) @test_reference "references/pattern/jacobian/NNlib/conv.txt" BitMatrix(J) end -function test_flux_conv_local(method::AbstractSparsityDetector) - J = jacobian_sparsity(LAYER_RELU, INPUT_FLUX, method) +function test_flux_conv_local(detector::AbstractSparsityDetector) + J = jacobian_sparsity(LAYER_RELU, INPUT_FLUX, detector) @test_reference "references/pattern/jacobian/NNlib/conv_relu.txt" BitMatrix(J) end @testset "Global" begin @testset "$T" for T in GRADIENT_TRACERS - method = TracerSparsityDetector(; gradient_tracer_type=T) - test_flux_conv(method) + detector = TracerSparsityDetector(; gradient_tracer_type=T) + test_flux_conv(detector) end end @testset "Local" begin @testset "$T" for T in GRADIENT_TRACERS - method = TracerLocalSparsityDetector(; gradient_tracer_type=T) - test_flux_conv(method) - test_flux_conv_local(method) + detector = TracerLocalSparsityDetector(; gradient_tracer_type=T) + test_flux_conv(detector) + test_flux_conv_local(detector) end end diff --git a/test/test_arrays.jl b/test/test_arrays.jl index 405f0679..3f069d80 100644 --- a/test/test_arrays.jl +++ b/test/test_arrays.jl @@ -45,13 +45,13 @@ pow3(A) = A^3 # Testing utilities # #===================# -method = TracerSparsityDetector() +detector = TracerSparsityDetector() allone(A) = all(isone, A) allzero(A) = all(iszero, A) # Short-hand for Jacobian pattern of `x -> sum(f(A))` -Jsum(f, A) = jacobian_sparsity(SumOutputs(f), A, method) +Jsum(f, A) = jacobian_sparsity(SumOutputs(f), A, detector) # Test whether all entries in Jacobian are zero testJ0(f, A) = @testset "Jacobian" begin @test allzero(Jsum(f, A)) @@ -75,7 +75,7 @@ function testJ1(f, A::Diagonal) end # Short-hand for Hessian pattern of `x -> sum(f(A))` -Hsum(f, A) = hessian_sparsity(SumOutputs(f), A, method) +Hsum(f, A) = hessian_sparsity(SumOutputs(f), A, detector) # Test whether all entries in Hessian are zero testH0(f, A) = @testset "Hessian" begin @test allzero(Hsum(f, A)) diff --git a/test/test_gradient.jl b/test/test_gradient.jl index f8230cc6..02e6989e 100644 --- a/test/test_gradient.jl +++ b/test/test_gradient.jl @@ -13,14 +13,14 @@ REAL_TYPES = (Float64, Int, Bool, UInt8, Float16, Rational{Int}) # These exists to be able to quickly run tests in the REPL. # NOTE: J gets overwritten inside the testsets. -method = TracerSparsityDetector() -J(f, x) = jacobian_sparsity(f, x, method) +detector = TracerSparsityDetector() +J(f, x) = jacobian_sparsity(f, x, detector) @testset "Jacobian Global" begin @testset "$P" for P in GRADIENT_PATTERNS T = GradientTracer{P} - method = TracerSparsityDetector(; gradient_tracer_type=T) - J(f, x) = jacobian_sparsity(f, x, method) + detector = TracerSparsityDetector(; gradient_tracer_type=T) + J(f, x) = jacobian_sparsity(f, x, detector) @testset "Trivial examples" begin f(x) = [x[1]^2, 2 * x[1] * x[2]^2, sin(x[3])] @@ -126,8 +126,8 @@ end @testset "Jacobian Local" begin @testset "$P" for P in GRADIENT_PATTERNS T = GradientTracer{P} - method = TracerLocalSparsityDetector(; gradient_tracer_type=T) - J(f, x) = jacobian_sparsity(f, x, method) + detector = TracerLocalSparsityDetector(; gradient_tracer_type=T) + J(f, x) = jacobian_sparsity(f, x, detector) @testset "Trivial examples" begin diff --git a/test/test_hessian.jl b/test/test_hessian.jl index 7f639db8..c628ef75 100644 --- a/test/test_hessian.jl +++ b/test/test_hessian.jl @@ -10,8 +10,8 @@ REAL_TYPES = (Float64, Int, Bool, UInt8, Float16, Rational{Int}) # These exists to be able to quickly run tests in the REPL. # NOTE: H gets overwritten inside the testsets. -method = TracerSparsityDetector() -H(f, x) = hessian_sparsity(f, x, method) +detector = TracerSparsityDetector() +H(f, x) = hessian_sparsity(f, x, detector) P = first(HESSIAN_PATTERNS) T = HessianTracer{P} @@ -20,8 +20,8 @@ D = Dual{Int,T} @testset "Global Hessian" begin @testset "$P" for P in HESSIAN_PATTERNS T = HessianTracer{P} - method = TracerSparsityDetector(; hessian_tracer_type=T) - H(f, x) = hessian_sparsity(f, x, method) + detector = TracerSparsityDetector(; hessian_tracer_type=T) + H(f, x) = hessian_sparsity(f, x, detector) @testset "Trivial examples" begin @test H(identity, rand()) ≈ [0;;] @@ -266,8 +266,8 @@ end @testset "Local Hessian" begin @testset "$P" for P in HESSIAN_PATTERNS T = HessianTracer{P} - method = TracerLocalSparsityDetector(; hessian_tracer_type=T) - H(f, x) = hessian_sparsity(f, x, method) + detector = TracerLocalSparsityDetector(; hessian_tracer_type=T) + H(f, x) = hessian_sparsity(f, x, detector) @testset "Trivial examples" begin f1(x) = x[1] + x[2] * x[3] + 1 / x[4] + x[2] * max(x[1], x[5])