Skip to content

Commit

Permalink
Implement a 10 min timeout
Browse files Browse the repository at this point in the history
  • Loading branch information
@avik-pal
avik-pal committed Dec 14, 2023
1 parent d4a4192 commit 9422d0b
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 70 deletions.
6 changes: 2 additions & 4 deletions benchmarks/NonlinearProblem/Manifest.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1348,11 +1348,9 @@ version = "0.1.1"

[[deps.NonlinearSolve]]
deps = ["ADTypes", "ArrayInterface", "ConcreteStructs", "DiffEqBase", "EnumX", "FastBroadcast", "FiniteDiff", "ForwardDiff", "LazyArrays", "LineSearches", "LinearAlgebra", "LinearSolve", "MaybeInplace", "PrecompileTools", "Printf", "RecursiveArrayTools", "Reexport", "SciMLBase", "SciMLOperators", "SimpleNonlinearSolve", "SparseArrays", "SparseDiffTools", "StaticArrays", "UnPack"]
git-tree-sha1 = "4101ffb9e1f79027edb98120b12e03593281e3d6"
repo-rev = "ap/approx_sparsity"
repo-url = "https://github.com/SciML/NonlinearSolve.jl.git"
git-tree-sha1 = "767100434dc775c993082e70e453ba7bd9866afa"
uuid = "8913a72c-1f9b-4ce2-8d82-65094dcecaec"
version = "3.0.2"
version = "3.1.0"

[deps.NonlinearSolve.extensions]
NonlinearSolveBandedMatricesExt = "BandedMatrices"
Expand Down
119 changes: 53 additions & 66 deletions benchmarks/NonlinearProblem/bruss.jmd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,26 @@ using BenchmarkTools
RUS = RadiusUpdateSchemes;
```

Define a utility to timeout the benchmark after a certain time.

```julia
macro timeout(seconds, expr, fail = nothing)
quote
tsk = @task $(esc(expr))
schedule(tsk)
Timer($(esc(seconds))) do timer
istaskdone(tsk) || Base.throwto(tsk, InterruptException())
end
try
fetch(tsk)
catch err
Base.printstyled("Timed Out.\n"; color = :red)
$(esc(fail))
end
end
end
```

Define the Brussletor problem.

```julia
Expand Down Expand Up @@ -81,7 +101,12 @@ solver_names = ["NewtonRaphson (Dense)", "NewtonRaphson (Approximate Sparse)",
"TrustRegion (Approximate Sparse)", "TrustRegion (Exact Sparse)",
"KINSOL", "CMINPACK", "NLsolveJL (Trust Region)", "SimpleNewtonRaphson",
"SimpleTrustRegion"]
runtimes_scaling = zeros(length(solver_names), length(Ns))
runtimes_scaling = zeros(length(solver_names), length(Ns)) .- 1
prob_alg_mapping = [
(:dense, NewtonRaphson()), (:approx, NewtonRaphson()), (:exact, NewtonRaphson()),
(:dense, TrustRegion()), (:approx, TrustRegion()), (:exact, TrustRegion()),
(:dense, KINSOL()), (:dense, CMINPACK()), (:dense, NLsolveJL()),
(:dense, SimpleNewtonRaphson()), (:dense, SimpleTrustRegion())]

for (i, N) in enumerate(Ns)
prob_dense = generate_brusselator_problem(N; abstol=1e-6, reltol=1e-6)
Expand All @@ -92,71 +117,33 @@ for (i, N) in enumerate(Ns)

println("Benchmarking N = $N")

sol = solve(prob_dense, NewtonRaphson())
runtimes_scaling[1, i] = @belapsed solve($prob_dense, $NewtonRaphson())
println(" Dense (Newton Raphson): ", runtimes_scaling[1, i],
" resid = ", norm(sol.resid), " retcode = ", sol.retcode,
" stats = ", sol.stats)

sol = solve(prob_approx_sparse, NewtonRaphson())
runtimes_scaling[2, i] = @belapsed solve($prob_approx_sparse, $NewtonRaphson())
println(" Approximate Sparse (Newton Raphson): ", runtimes_scaling[2, i],
" resid = ", norm(sol.resid), " retcode = ", sol.retcode,
" stats = ", sol.stats)

sol = solve(prob_exact_sparse, NewtonRaphson())
runtimes_scaling[3, i] = @belapsed solve($prob_exact_sparse, $NewtonRaphson())
println(" Exact Sparse (Newton Raphson): ", runtimes_scaling[3, i],
" resid = ", norm(sol.resid), " retcode = ", sol.retcode,
" stats = ", sol.stats)

sol = solve(prob_dense, TrustRegion())
runtimes_scaling[4, i] = @belapsed solve($prob_dense, $TrustRegion())
println(" Dense (Trust Region): ", runtimes_scaling[4, i],
" resid = ", norm(sol.resid), " retcode = ", sol.retcode,
" stats = ", sol.stats)

sol = solve(prob_approx_sparse, TrustRegion())
runtimes_scaling[5, i] = @belapsed solve($prob_approx_sparse, $TrustRegion())
println(" Approximate Sparse (Trust Region): ", runtimes_scaling[5, i],
" resid = ", norm(sol.resid), " retcode = ", sol.retcode,
" stats = ", sol.stats)

sol = solve(prob_exact_sparse, TrustRegion())
runtimes_scaling[6, i] = @belapsed solve($prob_exact_sparse, $TrustRegion())
println(" Exact Sparse (Trust Region): ", runtimes_scaling[6, i],
" resid = ", norm(sol.resid), " retcode = ", sol.retcode,
" stats = ", sol.stats)

sol = solve(prob_dense, KINSOL())
runtimes_scaling[7, i] = @belapsed solve($prob_dense, $KINSOL())
println(" KINSOL: ", runtimes_scaling[7, i],
" resid = ", norm(sol.resid), " retcode = ", sol.retcode,
" stats = ", sol.stats)

sol = solve(prob_dense, CMINPACK())
runtimes_scaling[8, i] = @belapsed solve($prob_dense, $CMINPACK())
println(" CMINPACK: ", runtimes_scaling[8, i],
" resid = ", norm(sol.resid), " retcode = ", sol.retcode,
" stats = ", sol.stats)

sol = solve(prob_dense, NLsolveJL())
runtimes_scaling[9, i] = @belapsed solve($prob_dense, $NLsolveJL())
println(" NLsolveJL: ", runtimes_scaling[9, i],
" resid = ", norm(sol.resid), " retcode = ", sol.retcode,
" stats = ", sol.stats)

sol = solve(prob_dense, SimpleNewtonRaphson())
runtimes_scaling[10, i] = @belapsed solve($prob_dense, $SimpleNewtonRaphson())
println(" SimpleNewtonRaphson: ", runtimes_scaling[10, i],
" resid = ", norm(sol.resid), " retcode = ", sol.retcode,
" stats = ", sol.stats)

sol = solve(prob_dense, SimpleTrustRegion())
runtimes_scaling[11, i] = @belapsed solve($prob_dense, $SimpleTrustRegion())
println(" SimpleTrustRegion: ", runtimes_scaling[11, i],
" resid = ", norm(sol.resid), " retcode = ", sol.retcode,
" stats = ", sol.stats)
for (j, (ptype, alg)) in enumerate(prob_alg_mapping)
if ptype == :dense
prob = prob_dense
elseif ptype == :approx
prob = prob_approx_sparse
elseif ptype == :exact
prob = prob_exact_sparse
end

if j > 1 && runtimes_scaling[j - 1, i] == -1
# The last benchmark failed so skip this too
runtimes_scaling[j, i] = NaN
println(" $(solver_names[j]): Would Have Timed out")
else
@timeout 1000 begin
sol = solve(prob, alg)
runtimes_scaling[j, i] = @belapsed solve($prob, $alg)
println(" $(solver_names[j]): ", runtimes_scaling[j, i],
" resid = ", norm(sol.resid), " retcode = ", sol.retcode,
" stats = ", sol.stats)
end
if runtimes_scaling[j, i] == -1
println(" $(solver_names[j]): Timed out")
runtimes_scaling[j, i] = NaN
end
end
end

println()
end
Expand Down

0 comments on commit 9422d0b

Please sign in to comment.