From bdf167dcd96425af9ff735896942a2df7b779895 Mon Sep 17 00:00:00 2001
From: pevnak <pevnak@gmail.com>
Date: Mon, 30 Jul 2018 11:51:01 +0200
Subject: [PATCH] fixed softplus

---
 src/activation.jl  | 2 +-
 test/activation.jl | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/activation.jl b/src/activation.jl
index 9f45b25d4..75956f7ed 100644
--- a/src/activation.jl
+++ b/src/activation.jl
@@ -102,4 +102,4 @@ softsign(x) = x / (one(x) + abs(x))
 
 See [Deep Sparse Rectifier Neural Networks](http://proceedings.mlr.press/v15/glorot11a/glorot11a.pdf).
 """
-softplus(x) = log1p(exp(x))
+softplus(x) = ifelse(x > 0, x + log1p(exp(-x)), log1p(exp(x)))
diff --git a/test/activation.jl b/test/activation.jl
index d3400a257..4bce7fbfb 100644
--- a/test/activation.jl
+++ b/test/activation.jl
@@ -44,6 +44,8 @@ end
   @test elu(0.0) == 0.0
   @test swish(0.0) == 0.0
   @test softplus(0.0) ≈ log(2.0)
+  @test softplus(1e8) ≈ 1e8
+  @test softplus(-1e8) ≈ 0.0
   @test softsign(0.0) == 0.0
   @test selu(0.0) == 0.0