Skip to content

Commit

Permalink
Merged PR 1138: Add activation functions: Elu, LeakyRelu, Thresholded…
Browse files Browse the repository at this point in the history 
…Relu

Add activation functions: Elu, LeakyRelu, ThresholdedRelu

Related work items: #30
  • Loading branch information
Ke Deng committed Mar 29, 2018
1 parent abf2ee0 commit 33d78bc
Show file tree
Hide file tree
Showing 12 changed files with 202 additions and 175 deletions.
1 change: 1 addition & 0 deletions lotus/core/framework/kernel_def_builder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ class KernelDef {
return *this;
}

// allowing output j to reuse memory of input i
KernelDef& MayInplace(int i, int j) {
// TODO: validate inputs.
inplace_map_.push_back({i, j});
Expand Down
21 changes: 21 additions & 0 deletions lotus/core/providers/cpu/activation/activations.cc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#include "core/providers/cpu/activation/activations.h"

namespace Lotus {

#define REGISTER_UNARY_ELEMENTWISE_KERNEL(x) \
REGISTER_KERNEL(KernelDef(#x) \
.Domain(LotusIR::kOnnxDomain) \
.SinceVersion(1, 2) \
.Provider(LotusIR::kCpuExecutionProvider) \
.MayInplace(0, 0) \
.TypeConstraint("T", DataTypeImpl::GetTensorType<float>()), \
x<float>)

REGISTER_UNARY_ELEMENTWISE_KERNEL(Elu);
REGISTER_UNARY_ELEMENTWISE_KERNEL(LeakyRelu);
REGISTER_UNARY_ELEMENTWISE_KERNEL(Relu);
REGISTER_UNARY_ELEMENTWISE_KERNEL(Sigmoid);
REGISTER_UNARY_ELEMENTWISE_KERNEL(Tanh);
REGISTER_UNARY_ELEMENTWISE_KERNEL(ThresholdedRelu);

} // namespace Lotus
47 changes: 47 additions & 0 deletions lotus/core/providers/cpu/activation/activations.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#ifndef CORE_PROVIDERS_CPU_ACTIVATION_RELU_H
#define CORE_PROVIDERS_CPU_ACTIVATION_RELU_H

#include "core/common/common.h"
#include "core/framework/op_kernel.h"
#include "core/util/math_cpuonly.h"

namespace Lotus {

DECLARE_EIGEN_UNARY_ELEMENTWISE_KERNEL(Elu,
{
EIGEN_X_VAR(xm);
EIGEN_Y = (xm >= 0).select(xm, Attr("alpha") * (xm.exp() - 1));
},
{"alpha"})

DECLARE_EIGEN_UNARY_ELEMENTWISE_KERNEL(LeakyRelu,
{
EIGEN_X_VAR(xm);
EIGEN_Y = (xm >= 0).select(xm, Attr("alpha") * xm);
},
{"alpha"})

DECLARE_EIGEN_UNARY_ELEMENTWISE_KERNEL(Relu,
{ EIGEN_Y = EIGEN_X.cwiseMax(0); },
{})

DECLARE_EIGEN_UNARY_ELEMENTWISE_KERNEL(Sigmoid,
{
EIGEN_X_VAR(xm);
EIGEN_Y_VAR(ym);
ym = (xm >= 0).select(1 / (1. + (-xm.abs()).exp()), 1 - 1 / (1. + (-xm.abs()).exp()));
},
{})

DECLARE_EIGEN_UNARY_ELEMENTWISE_KERNEL(Tanh, { EIGEN_Y = EIGEN_X.tanh(); }, {})

DECLARE_EIGEN_UNARY_ELEMENTWISE_KERNEL(ThresholdedRelu,
{
EIGEN_X_VAR(xm);
EIGEN_Y = (xm >= Attr("alpha")).select(xm, 0);
},
{"alpha"})

} // namespace Lotus

#endif // !CORE_PROVIDERS_CPU_ACTIVATION_RELU_H
12 changes: 0 additions & 12 deletions lotus/core/providers/cpu/activation/relu.cc
View file

This file was deleted.

13 changes: 0 additions & 13 deletions lotus/core/providers/cpu/activation/relu.h
View file

This file was deleted.

12 changes: 0 additions & 12 deletions lotus/core/providers/cpu/activation/sigmoid.cc
View file

This file was deleted.

19 changes: 0 additions & 19 deletions lotus/core/providers/cpu/activation/sigmoid.h
View file

This file was deleted.

11 changes: 0 additions & 11 deletions lotus/core/providers/cpu/activation/tanh.cc
View file

This file was deleted.

13 changes: 0 additions & 13 deletions lotus/core/providers/cpu/activation/tanh.h
View file

This file was deleted.

1 change: 1 addition & 0 deletions lotus/core/providers/cpu/math/clip.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ REGISTER_KERNEL(KernelDef("Clip")
.Domain(LotusIR::kOnnxDomain)
.SinceVersion(1, 2)
.Provider(LotusIR::kCpuExecutionProvider)
.MayInplace(0, 0)
.TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Clip<float>);
} // namespace Lotus
188 changes: 99 additions & 89 deletions lotus/core/util/math_cpuonly.h
View file
Original file line number Diff line number Diff line change
@@ -1,90 +1,100 @@
/**
* Derived from caffe2, need copy right announcement here.
*/

/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifndef LOTUS_UTILS_MATH_CPU_H_
#define LOTUS_UTILS_MATH_CPU_H_

#include "Eigen/Core"
#include "Eigen/Dense"

namespace Lotus {

// Common Eigen types that we will often use
template <typename T>
using EigenMatrixMap =
Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic> >;
template <typename T>
using EigenArrayMap =
Eigen::Map<Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic> >;
template <typename T>
using EigenVectorMap = Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, 1> >;
template <typename T>
using EigenVectorArrayMap = Eigen::Map<Eigen::Array<T, Eigen::Dynamic, 1> >;
template <typename T>
using ConstEigenMatrixMap =
Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic> >;
template <typename T>
using ConstEigenArrayMap =
Eigen::Map<const Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic> >;
template <typename T>
using ConstEigenVectorMap =
Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, 1> >;
template <typename T>
using ConstEigenVectorArrayMap =
Eigen::Map<const Eigen::Array<T, Eigen::Dynamic, 1> >;

class CPUMathUtil {
public:
/*CPUMathUtil contains some help method like generate a
random seed. We only need a single instance for it.*/
static CPUMathUtil& Instance() {
static CPUMathUtil p;
return p;
}
//todo: the random generate interface.
private:
CPUMathUtil() {}
};

#define EIGEN_Y EigenVectorArrayMap<T>(Y->mutable_data<T>(), Y->shape().Size())
#define EIGEN_X ConstEigenVectorArrayMap<T>(X->data<T>(), X->shape().Size())
#define EIGEN_X_VAR(var) ConstEigenVectorArrayMap<T> var(X->data<T>(), X->shape().Size())
#define EIGEN_Y_VAR(var) EigenVectorArrayMap<T> var(Y->mutable_data<T>(), Y->shape().Size())

#define DECLARE_EIGEN_UNARY_ELEMENTWISE_KERNEL(class_name, func) \
template <typename T> \
class class_name final : public OpKernel { \
public: \
static const char* TypeTraits() { \
return #class_name; \
} \
\
class_name(const OpKernelInfo& info) : OpKernel(info) {} \
\
Status compute(OpKernelContext* context) const override { \
const Tensor* X = context->template input<Tensor>(0); \
Tensor* Y = context->output(0, X->shape()); \
func; \
return Status::OK(); \
} \
};

} // namespace Lotus
/**
* Derived from caffe2, need copy right announcement here.
*/

/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifndef LOTUS_UTILS_MATH_CPU_H_
#define LOTUS_UTILS_MATH_CPU_H_

#include "Eigen/Core"
#include "Eigen/Dense"

namespace Lotus {

// Common Eigen types that we will often use
template <typename T>
using EigenMatrixMap =
Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic> >;
template <typename T>
using EigenArrayMap =
Eigen::Map<Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic> >;
template <typename T>
using EigenVectorMap = Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, 1> >;
template <typename T>
using EigenVectorArrayMap = Eigen::Map<Eigen::Array<T, Eigen::Dynamic, 1> >;
template <typename T>
using ConstEigenMatrixMap =
Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic> >;
template <typename T>
using ConstEigenArrayMap =
Eigen::Map<const Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic> >;
template <typename T>
using ConstEigenVectorMap =
Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, 1> >;
template <typename T>
using ConstEigenVectorArrayMap =
Eigen::Map<const Eigen::Array<T, Eigen::Dynamic, 1> >;

class CPUMathUtil {
public:
/*CPUMathUtil contains some help method like generate a
random seed. We only need a single instance for it.*/
static CPUMathUtil& Instance() {
static CPUMathUtil p;
return p;
}
//todo: the random generate interface.
private:
CPUMathUtil() {}
};

#define EIGEN_Y EigenVectorArrayMap<T>(Y->mutable_data<T>(), Y->shape().Size())
#define EIGEN_X ConstEigenVectorArrayMap<T>(X->data<T>(), X->shape().Size())
#define EIGEN_X_VAR(var) ConstEigenVectorArrayMap<T> var(X->data<T>(), X->shape().Size())
#define EIGEN_Y_VAR(var) EigenVectorArrayMap<T> var(Y->mutable_data<T>(), Y->shape().Size())

#define DECLARE_EIGEN_UNARY_ELEMENTWISE_KERNEL(class_name, func, attrs) \
template <typename T> \
class class_name final : public OpKernel { \
public: \
static const char* TypeTraits() { \
return #class_name; \
} \
\
class_name(const OpKernelInfo& info) : OpKernel(info) { \
for (auto name : std::vector<std::string>(attrs)) { \
T value; \
LOTUS_ENFORCE(op_kernel_info_.GetAttr<T>(name, &value).IsOK()); \
attr_.insert(std::make_pair(std::string(name), value)); \
} \
} \
\
Status compute(OpKernelContext* context) const override { \
const Tensor* X = context->template input<Tensor>(0); \
Tensor* Y = context->output(0, X->shape()); \
func; \
return Status::OK(); \
} \
\
private: \
std::unordered_map<std::string, T> attr_; \
T Attr(const char* name) const { return attr_.at(std::string(name)); } \
};

} // namespace Lotus
#endif
Loading

0 comments on commit 33d78bc

Please sign in to comment.