Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature inception layer #48

Merged
merged 32 commits into from
Dec 29, 2017
Merged
Changes from 1 commit
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
dbd1893
Added InceptionInfo struct
Sergio0694 Dec 23, 2017
fd7f9ec
Temp move to APIs folder
Sergio0694 Dec 24, 2017
2cc4879
Name switched back to original (path fix)
Sergio0694 Dec 24, 2017
efd16e4
Merge pull request #46 from Sergio0694/tweak_case-path
Sergio0694 Dec 24, 2017
ed56e1f
Initial inception layer structure added
Sergio0694 Dec 24, 2017
9c9a79c
Added inception cuDNN base initialization
Sergio0694 Dec 24, 2017
9bc9e80
Added inception layer weights initialization, minor changes
Sergio0694 Dec 24, 2017
de8a9e7
Inception constructors and Clone method implemented
Sergio0694 Dec 24, 2017
f41a4a8
Inception layer forward method implemented (WIP)
Sergio0694 Dec 24, 2017
127e241
Minor code improvements to the CuDnn layers
Sergio0694 Dec 24, 2017
612ba59
Merge pull request #47 from Sergio0694/dev
Sergio0694 Dec 24, 2017
c16c249
Minor bug fixes
Sergio0694 Dec 24, 2017
6a82479
Minor improvements to the Tensor struct
Sergio0694 Dec 24, 2017
657c277
InceptionInfo struct improved, minor changes
Sergio0694 Dec 25, 2017
801b4ca
Inception layer forward method implemented
Sergio0694 Dec 25, 2017
7e6366b
Inception backpropagation 90% completed
Sergio0694 Dec 25, 2017
d1de1ab
GPU copy to rows and rows allocation methods improved
Sergio0694 Dec 26, 2017
07af79e
Inception layer delta loading fixed
Sergio0694 Dec 26, 2017
c6a84a6
Inception layer implementation finished (WIP)
Sergio0694 Dec 26, 2017
ceae801
Inception layer public API and serialization methods added
Sergio0694 Dec 26, 2017
393d01a
Minor fixes to the inception layer (WIP)
Sergio0694 Dec 26, 2017
84502d3
Ooops!
Sergio0694 Dec 26, 2017
bfe4a04
More fixes to the inception layer (WIP)
Sergio0694 Dec 26, 2017
096994a
Fixed convolution output size
Sergio0694 Dec 26, 2017
fa6c36d
Added initial inception layer tests
Sergio0694 Dec 26, 2017
b3e136c
Minor bug fixes, inception 5x5 test added
Sergio0694 Dec 27, 2017
c4a3966
Added inception layer pooling test, minor bug fixes
Sergio0694 Dec 27, 2017
ffa603e
Minor layer tweaks
Sergio0694 Dec 29, 2017
2ef571a
ContentEquals method improved with relative threshold
Sergio0694 Dec 29, 2017
968eba6
Inception layer bug fixes, more tests added
Sergio0694 Dec 29, 2017
fb738d1
CuDnnPoolingLayer backpropagation switched to cuDNN
Sergio0694 Dec 29, 2017
b84d991
Inception layer pool gradient test added
Sergio0694 Dec 29, 2017
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Added inception layer weights initialization, minor changes
Sergio0694 committed Dec 24, 2017
commit 9bc9e809968476d04c30e5f29fce918796107bee
Original file line number Diff line number Diff line change
@@ -59,7 +59,7 @@ public ref readonly TensorInfo KernelInfo

public ConvolutionalLayer(in TensorInfo input, in ConvolutionInfo operation, (int X, int Y) kernelSize, int kernels, ActivationFunctionType activation, BiasInitializationMode biasMode)
: base(input, new TensorInfo(input.Height - kernelSize.X + 1, input.Width - kernelSize.Y + 1, kernels),
WeightsProvider.NewConvolutionalKernels(input.Channels, kernelSize.X, kernelSize.Y, kernels),
WeightsProvider.NewConvolutionalKernels(input, kernelSize.X, kernelSize.Y, kernels),
WeightsProvider.NewBiases(kernels, biasMode), activation)
{
_OperationInfo = operation;
Original file line number Diff line number Diff line change
@@ -22,7 +22,7 @@ internal class FullyConnectedLayer : WeightedLayerBase

public FullyConnectedLayer(in TensorInfo input, int neurons, ActivationFunctionType activation, WeightsInitializationMode weightsMode, BiasInitializationMode biasMode)
: base(input, TensorInfo.CreateLinear(neurons),
WeightsProvider.NewFullyConnectedWeights(input.Size, neurons, weightsMode),
WeightsProvider.NewFullyConnectedWeights(input, neurons, weightsMode),
WeightsProvider.NewBiases(neurons, biasMode), activation) { }

public FullyConnectedLayer(in TensorInfo input, int neurons, [NotNull] float[] weights, [NotNull] float[] biases, ActivationFunctionType activation)
Original file line number Diff line number Diff line change
@@ -13,35 +13,34 @@ namespace NeuralNetworkNET.Networks.Implementations.Layers.Helpers
internal static class WeightsProvider
{
/// <summary>
/// Creates a weight matrix for a fully connected layer
/// Creates a weights vector for a fully connected layer
/// </summary>
/// <param name="inputs">The input neurons</param>
/// <param name="inputs">The layer inputs</param>
/// <param name="outputs">The output neurons</param>
/// <param name="mode">The initialization mode for the weights</param>
[Pure, NotNull]
public static unsafe float[] NewFullyConnectedWeights(int inputs, int outputs, WeightsInitializationMode mode)
public static unsafe float[] NewFullyConnectedWeights(in TensorInfo input, int outputs, WeightsInitializationMode mode)
{
if (inputs <= 0 || outputs <= 0) throw new ArgumentOutOfRangeException("The inputs and outputs must be positive numbers");
float[] weights = new float[inputs * outputs];
float[] weights = new float[input.Size * outputs];
fixed (float* pw = weights)
{
Tensor.Reshape(pw, inputs, outputs, out Tensor wTensor);
Tensor.Reshape(pw, input.Size, outputs, out Tensor wTensor);
switch (mode)
{
case WeightsInitializationMode.LeCunUniform:
KerasWeightsProvider.FillWithLeCunUniform(wTensor, inputs);
KerasWeightsProvider.FillWithLeCunUniform(wTensor, input.Size);
break;
case WeightsInitializationMode.GlorotNormal:
KerasWeightsProvider.FillWithGlorotNormal(wTensor, inputs, outputs);
KerasWeightsProvider.FillWithGlorotNormal(wTensor, input.Size, outputs);
break;
case WeightsInitializationMode.GlorotUniform:
KerasWeightsProvider.FillWithGlorotUniform(wTensor, inputs, outputs);
KerasWeightsProvider.FillWithGlorotUniform(wTensor, input.Size, outputs);
break;
case WeightsInitializationMode.HeEtAlNormal:
KerasWeightsProvider.FillWithHeEtAlNormal(wTensor, inputs);
KerasWeightsProvider.FillWithHeEtAlNormal(wTensor, input.Size);
break;
case WeightsInitializationMode.HeEtAlUniform:
KerasWeightsProvider.FillWithHeEtAlUniform(wTensor, inputs);
KerasWeightsProvider.FillWithHeEtAlUniform(wTensor, input.Size);
break;
default: throw new ArgumentOutOfRangeException(nameof(mode), "Unsupported weights initialization mode");
}
@@ -50,21 +49,49 @@ public static unsafe float[] NewFullyConnectedWeights(int inputs, int outputs, W
}

/// <summary>
/// Creates a weight matrix for a convolutional layer
/// Creates a weights vector for a convolutional layer
/// </summary>
/// <param name="inputDepth">The depth of the input volume</param>
/// <param name="input">The layer inputs</param>
/// <param name="kernelsHeight">The height of each kernel</param>
/// <param name="kernelsWidth">The width of each kernel</param>
/// <param name="kernels">The number of kernels in the layer</param>
[Pure, NotNull]
public static unsafe float[] NewConvolutionalKernels(int inputDepth, int kernelsHeight, int kernelsWidth, int kernels)
public static unsafe float[] NewConvolutionalKernels(in TensorInfo input, int kernelsHeight, int kernelsWidth, int kernels)
{
if (kernels <= 0) throw new ArgumentOutOfRangeException(nameof(kernels), "The number of kernels must be positive");
float[] weights = new float[kernels * kernelsHeight * kernelsWidth * inputDepth];
float[] weights = new float[kernels * kernelsHeight * kernelsWidth * input.Channels];
fixed (float* pw = weights)
{
Tensor.Reshape(pw, 1, weights.Length, out Tensor wTensor);
KerasWeightsProvider.FillWithHeEtAlUniform(wTensor, inputDepth * kernelsHeight * kernelsWidth);
KerasWeightsProvider.FillWithHeEtAlUniform(wTensor, input.Channels * kernelsHeight * kernelsWidth);
}
return weights;
}

/// <summary>
/// Creates a new mixed weights vector for an inception layer
/// </summary>
/// <param name="input">The layer inputs</param>
/// <param name="info">The info on the target inception layer</param>
[Pure, NotNull]
public static unsafe float[] NewInceptionWeights(in TensorInfo input, in InceptionInfo info)
{
int
_1x1Length = input.Channels * info.Primary1x1ConvolutionKernels,
_3x3Length = 3 * 3 * info.Primary1x1ConvolutionKernels * info.Secondary3x3ConvolutionKernels,
_5x5Length = 5 * 5 * info.Primary1x1ConvolutionKernels * info.Secondary5x5ConvolutionKernels,
secondary1x1Length = input.Channels * info.Chained1x1AfterPoolingConvolutionKernels;
float[] weights = new float[_1x1Length + _3x3Length + _5x5Length + secondary1x1Length];
fixed (float* pw = weights)
{
Tensor.Reshape(pw, 1, _1x1Length, out Tensor wTensor);
KerasWeightsProvider.FillWithHeEtAlUniform(wTensor, input.Channels);
Tensor.Reshape(pw + _1x1Length, 1, _3x3Length, out wTensor);
KerasWeightsProvider.FillWithHeEtAlUniform(wTensor, 3 * 3 * info.Primary1x1ConvolutionKernels);
Tensor.Reshape(pw + _1x1Length + _3x3Length, 1, _5x5Length, out wTensor);
KerasWeightsProvider.FillWithHeEtAlUniform(wTensor, 5 * 5 * info.Primary1x1ConvolutionKernels);
Tensor.Reshape(pw + _1x1Length + _3x3Length + _5x5Length, 1, secondary1x1Length, out wTensor);
KerasWeightsProvider.FillWithHeEtAlUniform(wTensor, input.Channels);
}
return weights;
}
34 changes: 17 additions & 17 deletions Unit/NeuralNetwork.NET.Cuda.Unit/CuDnnLayersTest.cs
Original file line number Diff line number Diff line change
@@ -73,7 +73,7 @@ private static unsafe void TestGradient(WeightedLayerBase cpu, WeightedLayerBase
[TestMethod]
public void FullyConnectedForward()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(400, 250, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 250);
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(400), 250, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 250);
FullyConnectedLayer
cpu = new FullyConnectedLayer(TensorInfo.CreateLinear(250), 127, ActivationFunctionType.LeCunTanh, WeightsInitializationMode.GlorotNormal, BiasInitializationMode.Gaussian),
gpu = new CuDnnFullyConnectedLayer(cpu.InputInfo, cpu.OutputInfo.Size, cpu.Weights, cpu.Biases, cpu.ActivationFunctionType);
@@ -84,8 +84,8 @@ public void FullyConnectedForward()
public void FullyConnectedBackward()
{
float[,]
delta_1 = WeightsProvider.NewFullyConnectedWeights(400, 127, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 127),
z = WeightsProvider.NewFullyConnectedWeights(400, 250, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 250);
delta_1 = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(400), 127, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 127),
z = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(400), 250, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 250);
FullyConnectedLayer
cpu = new FullyConnectedLayer(TensorInfo.CreateLinear(250), 127, ActivationFunctionType.LeCunTanh, WeightsInitializationMode.GlorotNormal, BiasInitializationMode.Gaussian),
gpu = new CuDnnFullyConnectedLayer(cpu.InputInfo, cpu.OutputInfo.Size, cpu.Weights, cpu.Biases, cpu.ActivationFunctionType);
@@ -96,8 +96,8 @@ public void FullyConnectedBackward()
public void FullyConnectedGradient()
{
float[,]
x = WeightsProvider.NewFullyConnectedWeights(400, 250, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 250),
delta = WeightsProvider.NewFullyConnectedWeights(400, 127, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 127);
x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(400), 250, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 250),
delta = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(400), 127, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 127);
FullyConnectedLayer
cpu = new FullyConnectedLayer(TensorInfo.CreateLinear(250), 127, ActivationFunctionType.LeCunTanh, WeightsInitializationMode.GlorotNormal, BiasInitializationMode.Gaussian),
gpu = new CuDnnFullyConnectedLayer(cpu.InputInfo, cpu.OutputInfo.Size, cpu.Weights, cpu.Biases, cpu.ActivationFunctionType);
@@ -111,7 +111,7 @@ public void FullyConnectedGradient()
[TestMethod]
public void SoftmaxForward()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(400, 250, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 250);
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(400), 250, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 250);
SoftmaxLayer
cpu = new SoftmaxLayer(TensorInfo.CreateLinear(250), 127, WeightsInitializationMode.GlorotNormal, BiasInitializationMode.Gaussian),
gpu = new CuDnnSoftmaxLayer(cpu.InputInfo, cpu.OutputInfo.Size, cpu.Weights, cpu.Biases);
@@ -122,8 +122,8 @@ public void SoftmaxForward()
public void SoftmaxBackward()
{
float[,]
delta_1 = WeightsProvider.NewFullyConnectedWeights(400, 127, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 127),
z = WeightsProvider.NewFullyConnectedWeights(400, 250, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 250);
delta_1 = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(400), 127, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 127),
z = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(400), 250, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 250);
SoftmaxLayer
cpu = new SoftmaxLayer(TensorInfo.CreateLinear(250), 127, WeightsInitializationMode.GlorotNormal, BiasInitializationMode.Gaussian),
gpu = new CuDnnSoftmaxLayer(cpu.InputInfo, cpu.OutputInfo.Size, cpu.Weights, cpu.Biases);
@@ -134,8 +134,8 @@ public void SoftmaxBackward()
public void SoftmaxGradient()
{
float[,]
a = WeightsProvider.NewFullyConnectedWeights(400, 250, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 250),
delta = WeightsProvider.NewFullyConnectedWeights(400, 127, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 127);
a = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(400), 250, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 250),
delta = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(400), 127, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 127);
SoftmaxLayer
cpu = new SoftmaxLayer(TensorInfo.CreateLinear(250), 127, WeightsInitializationMode.GlorotNormal, BiasInitializationMode.Gaussian),
gpu = new CuDnnSoftmaxLayer(cpu.InputInfo, cpu.OutputInfo.Size, cpu.Weights, cpu.Biases);
@@ -146,7 +146,7 @@ public void SoftmaxGradient()
public unsafe void SoftmaxBackwardOutput()
{
float[,]
x = WeightsProvider.NewFullyConnectedWeights(400, 250, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 250),
x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(400), 250, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 250),
y = new float[400, 127];
for (int i = 0; i < 400; i++)
y[i, ThreadSafeRandom.NextInt(max: 127)] = 1;
@@ -175,7 +175,7 @@ public unsafe void SoftmaxBackwardOutput()
[TestMethod]
public void ConvolutionForward()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(127, 58 * 58 * 3, WeightsInitializationMode.GlorotNormal).AsMatrix(127, 58 * 58 * 3);
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(127), 58 * 58 * 3, WeightsInitializationMode.GlorotNormal).AsMatrix(127, 58 * 58 * 3);
ConvolutionalLayer
cpu = new ConvolutionalLayer(new TensorInfo(58, 58, 3), ConvolutionInfo.Default, (5, 5), 20, ActivationFunctionType.LeakyReLU, BiasInitializationMode.Gaussian),
gpu = new CuDnnConvolutionalLayer(cpu.InputInfo, ConvolutionInfo.Default, cpu.KernelInfo, cpu.OutputInfo, cpu.Weights, cpu.Biases, cpu.ActivationFunctionType);
@@ -186,8 +186,8 @@ public void ConvolutionForward()
public unsafe void ConvolutionBackward()
{
float[,]
delta_1 = WeightsProvider.NewFullyConnectedWeights(127, 54 * 54 * 20, WeightsInitializationMode.GlorotNormal).AsMatrix(127, 54 * 54 * 20),
z = WeightsProvider.NewFullyConnectedWeights(127, 58 * 58 * 3, WeightsInitializationMode.GlorotNormal).AsMatrix(127, 58 * 58 * 3);
delta_1 = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(127), 54 * 54 * 20, WeightsInitializationMode.GlorotNormal).AsMatrix(127, 54 * 54 * 20),
z = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(127), 58 * 58 * 3, WeightsInitializationMode.GlorotNormal).AsMatrix(127, 58 * 58 * 3);
ConvolutionalLayer
cpu = new ConvolutionalLayer(new TensorInfo(58, 58, 3), ConvolutionInfo.Default, (5, 5), 20, ActivationFunctionType.LeCunTanh, BiasInitializationMode.Gaussian),
gpu = new CuDnnConvolutionalLayer(cpu.InputInfo, ConvolutionInfo.Default, cpu.KernelInfo, cpu.OutputInfo, cpu.Weights, cpu.Biases, ActivationFunctionType.LeCunTanh);
@@ -228,7 +228,7 @@ public void ConvolutionGradient()
[TestMethod]
public void PoolingForward()
{
float[,] x = WeightsProvider.NewFullyConnectedWeights(400, 58 * 58 * 3, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 58 * 58 * 3);
float[,] x = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(400), 58 * 58 * 3, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 58 * 58 * 3);
PoolingLayer
cpu = new PoolingLayer(new TensorInfo(58, 58, 3), PoolingInfo.Default, ActivationFunctionType.LeakyReLU),
gpu = new CuDnnPoolingLayer(cpu.InputInfo, PoolingInfo.Default, ActivationFunctionType.LeakyReLU);
@@ -239,8 +239,8 @@ public void PoolingForward()
public void PoolingBackward()
{
float[,]
delta_1 = WeightsProvider.NewFullyConnectedWeights(400, 29 * 29 * 3, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 29 * 29 * 3),
z = WeightsProvider.NewFullyConnectedWeights(400, 58 * 58 * 3, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 58 * 58 * 3);
delta_1 = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(400), 29 * 29 * 3, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 29 * 29 * 3),
z = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(400), 58 * 58 * 3, WeightsInitializationMode.GlorotNormal).AsMatrix(400, 58 * 58 * 3);
PoolingLayer
cpu = new PoolingLayer(new TensorInfo(58, 58, 3), PoolingInfo.Default, ActivationFunctionType.LeakyReLU),
gpu = new CuDnnPoolingLayer(cpu.InputInfo, PoolingInfo.Default, ActivationFunctionType.LeakyReLU);
6 changes: 2 additions & 4 deletions Unit/NeuralNetwork.NET.Unit/SerializationTest.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
using System;
using System.IO;
using System.Linq;
using System.IO;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using NeuralNetworkNET.APIs;
using NeuralNetworkNET.APIs.Enums;
@@ -50,7 +48,7 @@ public void StreamSerialize()
{
using (MemoryStream stream = new MemoryStream())
{
float[] w = WeightsProvider.NewFullyConnectedWeights(784, 30, WeightsInitializationMode.GlorotNormal);
float[] w = WeightsProvider.NewFullyConnectedWeights(TensorInfo.CreateLinear(784), 30, WeightsInitializationMode.GlorotNormal);
stream.WriteShuffled(w);
Assert.IsTrue(stream.Position == sizeof(float) * w.Length);
stream.Seek(0, SeekOrigin.Begin);