update

src/broadcast.jl

@@ -32,6 +32,7 @@ broadcast_ops = [
     # "fdim",
     ("invxback","invxback","(-xi*yi*yi)"),
     ("reluback","reluback","(yi>0?xi:0)"),
+    ("eluback", "eluback", "ifelse(yi>0,dyi,yi+1)"),
     ("sigmback","sigmback","(xi*yi*(1-yi))"),
     ("tanhback","tanhback","(xi*(1-yi*yi))"),
     ("rpow","rpow","pow(yi,xi)"),   # need this for Array.^Scalar

src/unary.jl

@@ -45,6 +45,7 @@ unary_ops = [
 # "normcdfinv",
 # "rcbrt",
 ("relu", "relu", "(xi>0?xi:0)"),
+("elu", "elu", "(xi>0?xi:exp(xi)-1)"),
 # "rint",
 "round",
 # "rsqrt",
@@ -99,6 +100,7 @@ end
 for (f,g,y,dx) in
     ((:invx, :invxback, :(one(T)/xi), :(-yi*yi*dyi)),
      (:relu, :reluback, :(max(zero(T),xi)), :(ifelse(yi>0,dyi,zero(T)))),
+     (:elu, :eluback, :(ifelse(xi>0,xi,exp(xi)-1)), :(ifelse(yi>0,dyi,yi+1))),
      (:tanx, :tanhback, :(tanh(xi)), :(dyi*(one(T)-yi*yi))),
      (:sigm, :sigmback, 
       # Numerically stable implementation from
@@ -153,21 +155,6 @@ broadcast(::typeof(+), a::KnetArray)=a
 +(a::KnetArray)=a
 -(a::KnetArray)=broadcast(-,a)
 
-"""
-    elu(x, alpha=1)
-
-Exponential Linear Unit. Returns
-`max(0,x) + alpha*(exp(min(x,0)) - 1)
-
-Paper Ref. :
-"Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs) (ICLR 2016)"
-"""
-function elu(x, alpha=1)
-    p = relu(x)
-    m = -relu(-x)
-    return p + alpha*(exp(m) - 1)
-end
-
 """
     selu(x)
 
@@ -180,8 +167,8 @@ Self-Normalizing Neural Networks
 https://arxiv.org/abs/1706.02515
 """
 function selu(x)
-    alpha = 1.6732632f
-    scale = 1.0507009f
+    alpha = 1.6732632f0
+    scale = 1.0507009f0
     p = relu(x)
     m = -relu(-x)
     return scale*(p + alpha*(exp(m) - 1))