Leakyrelu

src/conversions.jl

@@ -101,7 +101,7 @@ function from_onnx_conv(attrs::Dict; pooling=false)
     # manually calculating the padding for :autopad
     if haskey(attrs, :auto_pad)
         if !(attrs[:auto_pad] in ["SAME_LOWER", "SAME_UPPER", "VALID","NOTSET"])
-        error("auto_pad $(attrs[:auto_pad]) isn't supported;{SAME_LOWER, SAME_UPPER, VALID, NOTSET}")
+            error("auto_pad $(attrs[:auto_pad]) isn't supported;{SAME_LOWER, SAME_UPPER, VALID, NOTSET}")
         end
 
         if attrs[:auto_pad]=="SAME_LOWER"||attrs[:auto_pad]=="SAME_UPPER"

src/load.jl

@@ -134,6 +134,11 @@ function load_node!(tape::Tape, ::OpConfig{:ONNX, :Relu}, args::VarVec, attrs::A
     return push_call!(tape, relu, args[1])
 end
 
+function load_node!(tape::Tape, ::OpConfig{:ONNX, :LeakyRelu}, args::VarVec, attrs::AttrDict)
+    haskey
+    return push_call!(tape, leakyrelu, args[1]; (;a = get(attrs,:alpha, 0.01))...) #default value 
+end
+
 function load_node!(tape::Tape, ::OpConfig{:ONNX, :Elu}, args::VarVec, attrs::AttrDict)
     return push_call!(tape, elu, args[1])
 end

src/ops.jl

@@ -49,6 +49,7 @@ add(xs...) = .+(xs...)
 sub(xs...) = .-(xs...)
 mul(xs...) = .*(xs...)
 relu(x) = NNlib.relu.(x)
+leakyrelu(x;a = 0.01) = NNlib.leakyrelu.(x,a)
 elu(x) = NNlib.elu.(x)
 tanh(x) = Base.tanh.(x)
 maxpool(x; kernel, pad = 0, stride = 1) = NNlib.maxpool(x, kernel; pad = pad, stride = stride)

src/save.jl

@@ -197,6 +197,12 @@ function save_node!(g::GraphProto, ::OpConfig{:ONNX, typeof(relu)}, op::Umlaut.C
     push!(g.node, nd)
 end
 
+function save_node!(g::GraphProto, ::@opconfig_kw(:ONNX, leakyrelu), op::Umlaut.Call)
+    kw_dict = kwargs2dict(op)
+    nd = NodeProto("LeakyRelu", op, Dict(:alpha=>kw_dict[:a]))
+    push!(g.node, nd)
+end
+
 function save_node!(g::GraphProto, ::OpConfig{:ONNX, typeof(elu)}, op::Umlaut.Call)
     nd = NodeProto("Elu", op)
     push!(g.node, nd)
@@ -372,4 +378,4 @@ function save(filename::String, tape::Tape{ONNXCtx})
     open(filename, "w") do io
         save(io, tape)
     end
-end
+end

test/backend.jl

@@ -100,11 +100,16 @@ const ONNX_RELEASE_URL = "https://github.com/ordicker/ONNXBackendTests.jl/releas
                         "test_max_uint64",
                         #"test_max_uint8",
                         "test_basic_conv_without_padding",
-                        "test_conv_with_autopad_same"]
+                        "test_conv_with_autopad_same",
+                        "test_relu",
+                        "test_leakyrelu",
+                        "test_leakyrelu_default",
+                        "test_leakyrelu_example"]
+
             onnx_output = outputs(prefix*dirname)[1] # TODO: some tests have more than 1 output
             model, inputs = load_model_and_inputs(prefix*dirname)
             julia_output = play!(model, inputs...)
-            @test onnx_output==julia_output
+            @test onnx_output≈julia_output
             # save and eval test
             ort_test(model, inputs...) #from ort.jl
         end