Merge branch 'master' into onlyreal

FluxML · Mar 26, 2019 · 4fbf7ab · 4fbf7ab
2 parents ad8b04f + 2bd7e8a
commit 4fbf7ab
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 1 deletion.
diff --git a/.codecov.yml b/.codecov.yml
@@ -0,0 +1 @@
+comment: false
diff --git a/src/activation.jl b/src/activation.jl
@@ -116,7 +116,7 @@ softsign(x::Real) = x / (one(x) + abs(x))
 
 See [Deep Sparse Rectifier Neural Networks](http://proceedings.mlr.press/v15/glorot11a/glorot11a.pdf).
 """
-softplus(x::Real) = log1p(exp(x))
+softplus(x::Real) = ifelse(x > 0, x + log1p(exp(-x)), log1p(exp(x)))
 
 # Provide an informative error message if activation functions are called with an array
 for f in (:σ, :σ_stable, :logσ, :relu, :leakyrelu, :elu, :gelu, :swish, :selu, :softsign, :softplus)

diff --git a/test/activation.jl b/test/activation.jl
@@ -45,6 +45,8 @@ end
   @test gelu(0.0) == 0.0
   @test swish(0.0) == 0.0
   @test softplus(0.0) ≈ log(2.0)
+  @test softplus(1e8) ≈ 1e8
+  @test softplus(-1e8) ≈ 0.0
   @test softsign(0.0) == 0.0
   @test selu(0.0) == 0.0